@dxos/plugin-transcription 0.8.2-main.fbd8ed0 → 0.8.2-staging.7ac8446

package/src/components/transcription.stories.tsx

@@ -16,148 +16,84 @@ import React, {
   useState,
 } from 'react';
 
-import { Events, IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
-import { withPluginManager } from '@dxos/app-framework/testing';
-import { AIServiceEdgeClient } from '@dxos/assistant';
-import { AI_SERVICE_ENDPOINT } from '@dxos/assistant/testing';
-import { Filter, MemoryQueue } from '@dxos/echo-db';
-import { create, createQueueDxn } from '@dxos/echo-schema';
+import { createStatic } from '@dxos/echo-schema';
 import { type DXN } from '@dxos/keys';
 import { log } from '@dxos/log';
-import { ClientPlugin } from '@dxos/plugin-client';
-import { PreviewPlugin } from '@dxos/plugin-preview';
-import { SpacePlugin } from '@dxos/plugin-space';
-import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
-import { ThemePlugin } from '@dxos/plugin-theme';
-import { useQueue, useSpace } from '@dxos/react-client/echo';
+import { Config } from '@dxos/react-client';
+import { useQueue } from '@dxos/react-client/echo';
+import { withClientProvider } from '@dxos/react-client/testing';
 import { IconButton, Toolbar } from '@dxos/react-ui';
 import { ScrollContainer } from '@dxos/react-ui-components';
-import { defaultTx } from '@dxos/react-ui-theme';
-import { DataType } from '@dxos/schema';
-import { seedTestData, Testing } from '@dxos/schema/testing';
 import { withLayout, withTheme } from '@dxos/storybook-utils';
 
-import { renderMarkdown, Transcript } from './Transcript';
-import { TranscriptionPlugin } from '../TranscriptionPlugin';
-import { processTranscriptMessage } from '../entity-extraction';
-import { useAudioFile, useAudioTrack, useQueueModelAdapter, useTranscriber } from '../hooks';
-import { type SerializationModel } from '../model';
-import { TestItem } from '../testing';
+import { Transcript } from './Transcript';
+import { useAudioFile, useAudioTrack, useTranscriber } from '../hooks';
 import { type TranscriberParams } from '../transcriber';
+import { TranscriptBlock } from '../types';
+import { randomQueueDxn } from '../util';
 
 const TranscriptionStory: FC<{
-  model: SerializationModel<DataType.Message>;
-  running: boolean;
-  onRunningChange: Dispatch<SetStateAction<boolean>>;
-}> = ({ model, running, onRunningChange }) => {
-  const space = useSpace();
-
+  playing: boolean;
+  setPlaying: Dispatch<SetStateAction<boolean>>;
+  blocks?: TranscriptBlock[];
+}> = ({ playing, setPlaying, blocks }) => {
   return (
     <div className='flex flex-col w-[30rem]'>
       <Toolbar.Root>
         <IconButton
           iconOnly
-          icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
-          label={running ? 'Pause' : 'Play'}
-          onClick={() => onRunningChange((running) => !running)}
+          icon={playing ? 'ph--pause--regular' : 'ph--play--regular'}
+          label={playing ? 'Pause' : 'Play'}
+          onClick={() => setPlaying((playing) => !playing)}
         />
       </Toolbar.Root>
       <ScrollContainer>
-        <Transcript space={space} model={model} attendableId='story' />
+        <Transcript blocks={blocks} attendableId='story' />
       </ScrollContainer>
     </div>
   );
 };
 
-const aiService = new AIServiceEdgeClient({
-  endpoint: AI_SERVICE_ENDPOINT.REMOTE,
-});
-
-const Microphone = ({ entityExtraction }: { entityExtraction?: boolean }) => {
-  const [running, setRunning] = useState(false);
+const Microphone = () => {
+  const [playing, setPlaying] = useState(false);
 
   // Audio.
-  const track = useAudioTrack(running);
+  const track = useAudioTrack(playing);
 
   // Queue.
-  const queueDxn = useMemo(() => createQueueDxn(), []);
-  const queue = useMemo(() => new MemoryQueue<DataType.Message>(queueDxn), [queueDxn]);
-  const model = useQueueModelAdapter(renderMarkdown([]), queue);
-  const space = useSpace();
+  const queueDxn = useMemo(() => randomQueueDxn(), []);
+  const queue = useQueue<TranscriptBlock>(queueDxn, { pollInterval: 500 });
 
   // Transcriber.
   const handleSegments = useCallback<TranscriberParams['onSegments']>(
-    async (blocks) => {
-      const message = create(DataType.Message, { sender: { name: 'You' }, created: new Date().toISOString(), blocks });
-
-      if (entityExtraction) {
-        if (!space) {
-          log.warn('no space');
-          return;
-        }
-        // TODO(dmaretskyi): Move to vector search index.
-        const { objects } = await space.db
-          .query(
-            Filter.or(
-              Filter.type(DataType.Person),
-              Filter.type(DataType.Organization),
-              Filter.type(Testing.DocumentType),
-            ),
-          )
-          .run();
-
-        log.info('context', { objects });
-
-        const result = await processTranscriptMessage({
-          message,
-          aiService,
-          context: {
-            objects,
-          },
-          options: {
-            fallbackToRaw: true,
-          },
-        });
-        void queue.append([result.message]);
-      } else {
-        void queue.append([message]);
-      }
+    async (segments) => {
+      const block = createStatic(TranscriptBlock, { segments });
+      queue?.append([block]);
     },
-    [queue, space],
+    [queue],
   );
-
-  const config = useRef<TranscriberParams['config']>({
-    transcribeAfterChunksAmount: 20,
-    prefixBufferChunksAmount: 5,
-  });
-
-  const transcriber = useTranscriber({
-    audioStreamTrack: track,
-    onSegments: handleSegments,
-    config: config.current,
-  });
-  const [isOpen, setIsOpen] = useState(false);
+  const transcriber = useTranscriber({ audioStreamTrack: track, onSegments: handleSegments });
   useEffect(() => {
-    void transcriber?.open().then(() => setIsOpen(true));
+    void transcriber?.open();
     return () => {
-      void transcriber?.close().then(() => setIsOpen(false));
+      void transcriber?.close();
     };
   }, [transcriber]);
 
   // Manage transcription state.
   useEffect(() => {
-    if (running && transcriber?.isOpen) {
-      transcriber?.startChunksRecording();
-    } else if (!running) {
-      transcriber?.stopChunksRecording();
+    if (playing && transcriber?.isOpen) {
+      void transcriber?.startChunksRecording();
+    } else if (!playing) {
+      void transcriber?.stopChunksRecording();
     }
-  }, [transcriber, running, isOpen]);
+  }, [transcriber, playing, transcriber?.isOpen]);
 
-  return <TranscriptionStory model={model} running={running} onRunningChange={setRunning} />;
+  return <TranscriptionStory playing={playing} setPlaying={setPlaying} blocks={queue?.items} />;
 };
 
 const AudioFile = ({ queueDxn, audioUrl }: { queueDxn: DXN; audioUrl: string; transcriptUrl: string }) => {
-  const [running, setRunning] = useState(false);
+  const [playing, setPlaying] = useState(false);
 
   // Audio.
   const audioElement = useRef<HTMLAudioElement>(null);
@@ -174,20 +110,19 @@ const AudioFile = ({ queueDxn, audioUrl }: { queueDxn: DXN; audioUrl: string; tr
       return;
     }
 
-    if (running) {
+    if (playing) {
       void audio.play();
     } else {
       void audio.pause();
     }
-  }, [audio, running]);
+  }, [audio, playing]);
 
   // Transcriber.
-  const queue = useQueue<DataType.Message>(queueDxn, { pollInterval: 500 });
-  const model = useQueueModelAdapter(renderMarkdown([]), queue);
+  const queue = useQueue<TranscriptBlock>(queueDxn, { pollInterval: 500 });
   const handleSegments = useCallback<TranscriberParams['onSegments']>(
-    async (blocks) => {
-      const message = create(DataType.Message, { sender: { name: 'You' }, created: new Date().toISOString(), blocks });
-      void queue?.append([message]);
+    async (segments) => {
+      const block = createStatic(TranscriptBlock, { authorName: 'test', authorHue: 'cyan', segments });
+      queue?.append([block]);
     },
     [queue],
   );
@@ -198,15 +133,15 @@ const AudioFile = ({ queueDxn, audioUrl }: { queueDxn: DXN; audioUrl: string; tr
   });
 
   useEffect(() => {
-    if (transcriber && running) {
-      void transcriber.open().then(manageChunkRecording);
-    } else if (!running) {
+    if (transcriber && playing) {
+      void transcriber.open();
+    } else if (!playing) {
       transcriber?.stopChunksRecording();
       void transcriber?.close();
     }
-  }, [transcriber, running]);
+  }, [transcriber, playing]);
 
-  const manageChunkRecording = () => {
+  useEffect(() => {
     if (track?.readyState === 'live' && transcriber?.isOpen) {
       log.info('starting transcription');
       transcriber.startChunksRecording();
@@ -214,80 +149,46 @@ const AudioFile = ({ queueDxn, audioUrl }: { queueDxn: DXN; audioUrl: string; tr
       log.info('stopping transcription');
       transcriber.stopChunksRecording();
     }
-  };
-  useEffect(() => {
-    manageChunkRecording();
   }, [transcriber, track?.readyState, transcriber?.isOpen]);
 
-  return <TranscriptionStory model={model} running={running} onRunningChange={setRunning} />;
+  return <TranscriptionStory playing={playing} setPlaying={setPlaying} blocks={queue?.items} />;
 };
 
 const meta: Meta<typeof AudioFile> = {
   title: 'plugins/plugin-transcription/transcription',
   decorators: [
-    withPluginManager({
-      plugins: [
-        ThemePlugin({ tx: defaultTx }),
-        StorybookLayoutPlugin(),
-        ClientPlugin({
-          types: [TestItem, DataType.Person, DataType.Organization, Testing.DocumentType],
-          onClientInitialized: async (_, client) => {
-            await client.halo.createIdentity();
-            await client.spaces.waitUntilReady();
-            await client.spaces.default.waitUntilReady();
-            await seedTestData(client.spaces.default);
+    withClientProvider({
+      config: new Config({
+        runtime: {
+          client: { edgeFeatures: { signaling: true } },
+          services: {
+            edge: { url: 'https://edge.dxos.workers.dev/' },
+            iceProviders: [{ urls: 'https://edge.dxos.workers.dev/ice' }],
           },
-        }),
-        SpacePlugin(),
-        SettingsPlugin(),
-        PreviewPlugin(),
-        IntentPlugin(),
-        TranscriptionPlugin(),
-      ],
-      fireEvents: [Events.SetupAppGraph],
-      // capabilities: [
-      //   contributes(
-      //     Capabilities.ReactSurface,
-      //     createSurface({
-      //       id: 'preview-test',
-      //       role: 'preview',
-      //       component: ({ data }) => {
-      //         const schema = getSchema(data);
-      //         if (!schema) {
-      //           return null;
-      //         }
-
-      //         return <Form schema={schema} values={data} />;
-      //       },
-      //     }),
-      //   ),
-      // ],
+        },
+      }),
     }),
     withTheme,
-    withLayout({ fullscreen: true, classNames: 'justify-center' }),
+    withLayout({
+      tooltips: true,
+      fullscreen: true,
+      classNames: 'justify-center',
+    }),
   ],
 };
 
 export default meta;
 
-export const Default: StoryObj<typeof Microphone> = {
-  render: Microphone,
-  args: {
-    entityExtraction: false,
-  },
-};
+type Story = StoryObj<typeof AudioFile>;
 
-export const EntityExtraction: StoryObj<typeof Microphone> = {
+export const Default: Story = {
   render: Microphone,
-  args: {
-    entityExtraction: true,
-  },
 };
 
-export const File: StoryObj<typeof AudioFile> = {
+export const File: Story = {
   render: AudioFile,
   args: {
-    queueDxn: createQueueDxn(),
+    queueDxn: randomQueueDxn(),
     // https://learnenglish.britishcouncil.org/general-english/audio-zone/living-london
     transcriptUrl: 'https://dxos.network/audio-london.txt',
     audioUrl: 'https://dxos.network/audio-london.m4a',

package/src/hooks/index.ts

@@ -4,6 +4,5 @@
 
 export * from './useAudioFile';
 export * from './useAudioTrack';
-export * from './useQueueModelAdapter';
 export * from './useTranscriber';
 export * from './useVoiceInput';

package/src/hooks/useTranscriber.ts

@@ -15,7 +15,6 @@ import { type Transcriber } from '../transcriber';
 export const useTranscriber = ({
   audioStreamTrack,
   onSegments,
-  config,
 }: Partial<TranscriptionCapabilities.GetTranscriberProps>) => {
   const [getTranscriber] = useCapabilities(TranscriptionCapabilities.Transcriber);
 
@@ -25,8 +24,8 @@ export const useTranscriber = ({
       return undefined;
     }
 
-    return getTranscriber({ audioStreamTrack, onSegments, config });
-  }, [audioStreamTrack, onSegments, getTranscriber, config]);
+    return getTranscriber({ audioStreamTrack, onSegments });
+  }, [audioStreamTrack, onSegments, getTranscriber]);
 
   useEffect(() => {
     return () => {

package/src/transcriber/{transcriber.browser.test.ts → transcriber.test.ts}

@@ -9,11 +9,11 @@ import { WaveFile } from 'wavefile';
 
 import { Trigger } from '@dxos/async';
 import { log } from '@dxos/log';
-import { type DataType } from '@dxos/schema';
 import { openAndClose } from '@dxos/test-utils';
 import { trace, TRACE_PROCESSOR } from '@dxos/tracing';
 
 import { type AudioChunk, type AudioRecorder, Transcriber } from '../transcriber';
+import { type TranscriptSegment } from '../types';
 import { mergeFloat64Arrays } from '../util';
 
 // TODO(burdon): ReferenceError: Worker is not defined (Can only run with web workers).
@@ -93,7 +93,7 @@ describe.skip('Transcriber', () => {
   });
 
   test.skip('transcription of audio recording', { timeout: 10_000 }, async () => {
-    const trigger = new Trigger<DataType.MessageBlock.Transcription[]>({ autoReset: true });
+    const trigger = new Trigger<TranscriptSegment[]>({ autoReset: true });
     const recorder = new MockAudioRecorder({
       buffer: await readFile('test.wav'),
       chunkDuration: 3_000,
@@ -127,7 +127,7 @@ describe.skip('Transcriber', () => {
   });
 
   test.skip('transcription of audio recording with overlapping chunks', { timeout: 20_000 }, async () => {
-    const trigger = new Trigger<DataType.MessageBlock.Transcription[]>({ autoReset: true });
+    const trigger = new Trigger<TranscriptSegment[]>({ autoReset: true });
     const recorder = new MockAudioRecorder({
       buffer: await readFile('test.wav'),
       chunkDuration: 3_000,

package/src/transcriber/transcriber.ts

@@ -7,11 +7,10 @@ import { WaveFile } from 'wavefile';
 import { DeferredTask, Trigger } from '@dxos/async';
 import { type Context, LifecycleState, Resource } from '@dxos/context';
 import { log } from '@dxos/log';
-import { type DataType } from '@dxos/schema';
 import { trace } from '@dxos/tracing';
 
 import { type AudioRecorder, type AudioChunk } from './audio-recorder';
-import { TRANSCRIPTION_URL } from '../types';
+import { TRANSCRIPTION_URL, type TranscriptSegment } from '../types';
 import { mergeFloat64Arrays } from '../util';
 
 type WhisperWord = {
@@ -65,7 +64,7 @@ export type TranscribeConfig = {
 export type TranscriberParams = {
   config: TranscribeConfig;
   recorder: AudioRecorder;
-  onSegments: (segments: DataType.MessageBlock.Transcription[]) => Promise<void>;
+  onSegments: (segments: TranscriptSegment[]) => Promise<void>;
 };
 
 /**
@@ -94,7 +93,6 @@ export class Transcriber extends Resource {
   }
 
   protected override async _open(ctx: Context) {
-    log.info('opening');
     this._recorder.setOnChunk((chunk) => this._saveAudioChunk(chunk));
     await this._recorder.start();
     this._transcribeTask = new DeferredTask(ctx, async () => this._transcribe());
@@ -102,18 +100,15 @@ export class Transcriber extends Resource {
   }
 
   protected override async _close() {
-    log.info('closing');
     this._recording = false;
     this._transcribeTask = undefined;
     await this._recorder.stop();
   }
 
   startChunksRecording() {
-    log.info('starting');
     if (this._lifecycleState !== LifecycleState.OPEN) {
       return;
     }
-
     this._recording = true;
   }
 
@@ -121,10 +116,8 @@ export class Transcriber extends Resource {
     if (this._lifecycleState !== LifecycleState.OPEN || !this._recording) {
       return;
     }
-
     this._recording = false;
     this._transcribeTask?.schedule();
-    log.info('stopped');
   }
 
   private _saveAudioChunk(chunk: AudioChunk) {
@@ -201,25 +194,24 @@ export class Transcriber extends Resource {
 
     if (!response.ok) {
       this._audioChunks = [];
-      throw new Error(`Transcription failed: ${response.statusText}`);
+      throw new Error('Failed to transcribe');
     }
 
-    const { segments } = (await response.json()) as {
+    const {
+      segments,
+    }: {
       segments: WhisperSegment[];
-    };
+    } = await response.json();
 
     log.info('transcription response', {
-      segments: segments.length,
+      segments,
       string: segments.map((segments) => segments.text).join(' '),
     });
 
     return segments;
   }
 
-  private _alignSegments(
-    segments: WhisperSegment[],
-    originalChunks: AudioChunk[],
-  ): DataType.MessageBlock.Transcription[] {
+  private _alignSegments(segments: WhisperSegment[], originalChunks: AudioChunk[]): TranscriptSegment[] {
     // Absolute zero for all relative timestamps in the segments.
     const zeroTimestamp = originalChunks.at(0)!.timestamp;
 
@@ -248,9 +240,8 @@ export class Transcriber extends Resource {
 
     // Add absolute timestamp to each segment.
     return filteredSegments.map((segment) => ({
-      type: 'transcription',
-      started: new Date(zeroTimestamp + segment.start * 1_000).toISOString(),
-      text: segment.text.trim(),
+      started: new Date(zeroTimestamp + segment.start * 1_000),
+      text: segment.text,
     }));
   }
 }