@dxos/plugin-transcription 0.8.3 → 0.8.4-main.03d5cd7b56

package/src/stories/MicrophoneTranscription.stories.tsx

@@ -0,0 +1,264 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type Meta, type StoryObj } from '@storybook/react-vite';
+import * as Effect from 'effect/Effect';
+import React, { useCallback, useEffect, useMemo, useState } from 'react';
+
+import { withPluginManager } from '@dxos/app-framework/testing';
+import { AppActivationEvents } from '@dxos/app-toolkit';
+import {
+  type ExtractionFunction,
+  extractionAnthropicFunction,
+  extractionNerFunction,
+  getNer,
+  processTranscriptMessage,
+} from '@dxos/assistant/extraction';
+import { Filter, type Obj } from '@dxos/echo';
+import { MemoryQueue } from '@dxos/echo-db';
+import { createQueueDXN } from '@dxos/echo/internal';
+import { invariant } from '@dxos/invariant';
+import { log } from '@dxos/log';
+import { ClientPlugin } from '@dxos/plugin-client/plugin';
+import { initializeIdentity } from '@dxos/plugin-client/testing';
+import { PreviewPlugin } from '@dxos/plugin-preview/testing';
+import { StorybookPlugin, corePlugins } from '@dxos/plugin-testing';
+import { IndexKind, useSpaces } from '@dxos/react-client/echo';
+import { withLayout } from '@dxos/react-ui/testing';
+import { TestSchema } from '@dxos/schema/testing';
+import { Message, Organization, Person } from '@dxos/types';
+import { seedTestData } from '@dxos/types/testing';
+
+import { useAudioTrack, useQueueModelAdapter, useTranscriber } from '#hooks';
+import { TestItem } from '#testing';
+
+import { type MediaStreamRecorderProps, type TranscriberProps } from '../transcriber';
+import { TranscriptionPlugin } from '../TranscriptionPlugin';
+import { renderByline } from '../util';
+import { TranscriptionStory } from './TranscriptionStory';
+import { useIsSpeaking } from './useIsSpeaking';
+
+const TRANSCRIBER_CONFIG = {
+  transcribeAfterChunksAmount: 50,
+  prefixBufferChunksAmount: 10,
+};
+
+const RECORDER_CONFIG = {
+  interval: 200,
+};
+
+type DefaultStoryProps = {
+  detectSpeaking?: boolean;
+  transcriberConfig: TranscriberProps['config'];
+  recorderConfig: MediaStreamRecorderProps['config'];
+  audioConstraints?: MediaTrackConstraints;
+  entityExtraction: 'none' | 'ner' | 'llm';
+};
+
+const DefaultStory = ({
+  detectSpeaking,
+  entityExtraction = 'none',
+  transcriberConfig,
+  recorderConfig,
+  audioConstraints,
+}: DefaultStoryProps) => {
+  const [running, setRunning] = useState(false);
+
+  // Audio.
+  const track = useAudioTrack(running, audioConstraints);
+
+  // Speaking.
+  const isSpeaking = detectSpeaking ? useIsSpeaking(track) : true;
+
+  // Queue.
+  // TODO(dmaretskyi): Use space.queues.create() instead.
+  const queueDxn = useMemo(() => createQueueDXN(), []);
+  const queue = useMemo(() => new MemoryQueue<Message.Message>(queueDxn), [queueDxn]);
+  const model = useQueueModelAdapter(renderByline([]), queue);
+  const [space] = useSpaces();
+
+  useEffect(() => {
+    if (!space) {
+      log.warn('no space');
+    }
+  }, [space]);
+
+  // Entity extraction.
+  const { extractionFunction, objects } = useMemo(() => {
+    if (!space) {
+      log.warn('no space');
+      return {};
+    }
+
+    let extractionFunction: ExtractionFunction | undefined;
+    let objects: Promise<Obj.Unknown[]> | undefined;
+
+    if (entityExtraction === 'ner') {
+      // Init model loading. Takes time.
+      void getNer();
+      extractionFunction = extractionNerFunction;
+    } else if (entityExtraction === 'llm') {
+      extractionFunction = extractionAnthropicFunction;
+      objects = space.db
+        .query(
+          Filter.or(
+            Filter.type(Person.Person),
+            Filter.type(Organization.Organization),
+            Filter.type(TestSchema.DocumentType),
+          ),
+        )
+        .run();
+    }
+
+    return { extractionFunction, objects };
+  }, [entityExtraction, space]);
+
+  // Transcriber.
+  const handleSegments = useCallback<TranscriberProps['onSegments']>(
+    async (blocks) => {
+      const message = Message.make({
+        sender: { name: 'You' },
+        created: new Date().toISOString(),
+        blocks,
+      });
+      if (!space) {
+        void queue.append([message]);
+        return;
+      }
+
+      if (entityExtraction !== 'none') {
+        invariant(extractionFunction, 'extractionFunction is required');
+        const result = await processTranscriptMessage({
+          function: extractionFunction,
+          input: {
+            message,
+            objects: await objects,
+          },
+          options: {
+            fallbackToRaw: true,
+            timeout: 30_000,
+          },
+        });
+        void queue.append([result.message]);
+      } else {
+        void queue.append([message]);
+      }
+    },
+    [queue, space],
+  );
+
+  const transcriber = useTranscriber({
+    audioStreamTrack: track,
+    onSegments: handleSegments,
+    transcriberConfig,
+    recorderConfig,
+  });
+  const [isOpen, setIsOpen] = useState(false);
+  useEffect(() => {
+    void transcriber?.open().then(() => setIsOpen(true));
+    return () => {
+      void transcriber?.close().then(() => setIsOpen(false));
+    };
+  }, [transcriber]);
+
+  // Manage transcription state.
+  useEffect(() => {
+    if (running && transcriber?.isOpen && isSpeaking) {
+      transcriber?.startChunksRecording();
+    } else if (!running || !isSpeaking) {
+      transcriber?.stopChunksRecording();
+    }
+  }, [transcriber, running, isOpen, isSpeaking]);
+
+  return <TranscriptionStory model={model} running={running} onRunningChange={setRunning} />;
+};
+
+const meta = {
+  title: 'plugins/plugin-transcription/components/MicrophoneTranscription',
+  render: DefaultStory,
+  decorators: [
+    withLayout({ layout: 'column' }),
+    withPluginManager({
+      plugins: [
+        ...corePlugins(),
+        StorybookPlugin({}),
+        ClientPlugin({
+          types: [TestItem, Person.Person, Organization.Organization, TestSchema.DocumentType],
+          onClientInitialized: ({ client }) =>
+            Effect.gen(function* () {
+              const { personalSpace } = yield* initializeIdentity(client);
+              // TODO(mykola): Make API easier to use.
+              // TODO(mykola): Delete after enabling vector indexing by default.
+              // Enable vector indexing.
+              yield* Effect.promise(() =>
+                client.services.services.QueryService!.setConfig({
+                  enabled: true,
+                  indexes: [
+                    //
+                    { kind: IndexKind.Kind.SCHEMA_MATCH },
+                    { kind: IndexKind.Kind.GRAPH },
+                    { kind: IndexKind.Kind.VECTOR },
+                  ],
+                }),
+              );
+              yield* Effect.promise(() => client.services.services.QueryService!.reindex());
+              yield* Effect.promise(() => seedTestData(personalSpace));
+            }),
+        }),
+
+        PreviewPlugin(),
+        TranscriptionPlugin(),
+      ],
+      fireEvents: [AppActivationEvents.SetupAppGraph],
+    }),
+  ],
+} satisfies Meta<typeof DefaultStory>;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+export const Default: Story = {
+  args: {
+    detectSpeaking: false,
+    entityExtraction: 'none',
+    transcriberConfig: TRANSCRIBER_CONFIG,
+    recorderConfig: RECORDER_CONFIG,
+    audioConstraints: {
+      echoCancellation: false,
+      noiseSuppression: false,
+      autoGainControl: false,
+    },
+  },
+};
+
+// NOTE: We are running out of free quota on hugging face entity extraction.
+// TODO(mykola): Fix hugging face quota issues.
+// export const EntityExtraction: Story = {
+//   args: {
+//     detectSpeaking: true,
+//     entityExtraction: 'ner',
+//     transcriberConfig: TRANSCRIBER_CONFIG,
+//     recorderConfig: RECORDER_CONFIG,
+//     audioConstraints: {
+//       echoCancellation: true,
+//       noiseSuppression: true,
+//       autoGainControl: true,
+//     },
+//   },
+// };
+
+export const SpeechDetection: Story = {
+  args: {
+    detectSpeaking: true,
+    entityExtraction: 'none',
+    transcriberConfig: TRANSCRIBER_CONFIG,
+    recorderConfig: RECORDER_CONFIG,
+    audioConstraints: {
+      echoCancellation: true,
+      noiseSuppression: true,
+      autoGainControl: true,
+    },
+  },
+};

package/src/stories/TranscriptionStory.tsx

@@ -0,0 +1,93 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import '@dxos-theme';
+
+import React, {
+  type ChangeEvent,
+  type Dispatch,
+  type FC,
+  type ReactNode,
+  type RefObject,
+  type SetStateAction,
+  useRef,
+} from 'react';
+
+import { IconButton, ScrollContainer, Toolbar } from '@dxos/react-ui';
+import { type Message } from '@dxos/types';
+
+import { Transcription } from '../components';
+import { type SerializationModel } from '../model';
+
+/**
+ * Story wrapper that renders a transcript with playback controls, an optional audio file
+ * upload affordance, and a toolbar slot for additional controls.
+ *
+ * @param onUpload - Callback fired when the user picks a local audio file via the upload button.
+ * @param uploadAccept - HTML `accept` filter for the hidden file input. Defaults to `audio/*`.
+ * @param toolbarSlot - Extra content rendered after the upload button inside the toolbar.
+ */
+export const TranscriptionStory: FC<{
+  model: SerializationModel<Message.Message>;
+  disabled?: boolean;
+  running: boolean;
+  onRunningChange: Dispatch<SetStateAction<boolean>>;
+  audioRef?: RefObject<HTMLAudioElement | null>;
+  onUpload?: (file: File) => void;
+  uploadAccept?: string;
+  toolbarSlot?: ReactNode;
+}> = ({ model, running, onRunningChange, audioRef, disabled, onUpload, uploadAccept = 'audio/*', toolbarSlot }) => {
+  const fileInputRef = useRef<HTMLInputElement>(null);
+
+  const handleFileChange = (event: ChangeEvent<HTMLInputElement>) => {
+    const file = event.target.files?.[0];
+    if (file && onUpload) {
+      onUpload(file);
+    }
+    // Reset so the same file can be re-selected.
+    event.target.value = '';
+  };
+
+  return (
+    <>
+      {audioRef && <audio ref={audioRef} autoPlay />}
+      <Toolbar.Root>
+        <IconButton
+          iconOnly
+          disabled={disabled}
+          icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
+          label={running ? 'Pause' : 'Play'}
+          onClick={() => onRunningChange((running) => !running)}
+        />
+        {onUpload && (
+          <>
+            <input
+              ref={fileInputRef}
+              type='file'
+              accept={uploadAccept}
+              disabled={disabled}
+              className='hidden'
+              onChange={handleFileChange}
+            />
+            <IconButton
+              iconOnly
+              disabled={disabled}
+              icon='ph--upload--regular'
+              label='Upload audio'
+              onClick={() => fileInputRef.current?.click()}
+            />
+          </>
+        )}
+        {toolbarSlot}
+      </Toolbar.Root>
+      <ScrollContainer.Root pin>
+        <ScrollContainer.Content>
+          <ScrollContainer.Viewport>
+            <Transcription model={model} />
+          </ScrollContainer.Viewport>
+        </ScrollContainer.Content>
+      </ScrollContainer.Root>
+    </>
+  );
+};

package/src/testing/testing.ts

@@ -2,24 +2,22 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Schema } from 'effect';
+import * as Schema from 'effect/Schema';
 import { useEffect, useMemo, useState } from 'react';
 
-import { EdgeAiServiceClient } from '@dxos/ai';
-import { AI_SERVICE_ENDPOINT } from '@dxos/ai/testing';
-import { extractionAnthropicFn, processTranscriptMessage } from '@dxos/assistant';
+import { extractionAnthropicFunction, processTranscriptMessage } from '@dxos/assistant/extraction';
 import { scheduleTaskInterval } from '@dxos/async';
 import { Filter, type Queue } from '@dxos/client/echo';
 import { Context } from '@dxos/context';
 import { type Key, Obj, Ref, Type } from '@dxos/echo';
-import { createQueueDXN } from '@dxos/echo-schema';
-import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
+import { createQueueDXN } from '@dxos/echo/internal';
 import { IdentityDid } from '@dxos/keys';
 import { log } from '@dxos/log';
-import { faker } from '@dxos/random';
-import { useQueue, type Space } from '@dxos/react-client/echo';
-import { DataType } from '@dxos/schema';
-import { Testing, seedTestData } from '@dxos/schema/testing';
+import { random } from '@dxos/random';
+import { type Space, useQueue } from '@dxos/react-client/echo';
+import { TestSchema } from '@dxos/schema/testing';
+import { type ContentBlock, Message, Organization, Person } from '@dxos/types';
+import { seedTestData } from '@dxos/types/testing';
 
 // TODO(burdon): Reconcile with plugin-markdown. Move to @dxos/schema/testing.
 export const TestItem = Schema.Struct({
@@ -32,15 +30,15 @@ export const TestItem = Schema.Struct({
     description: 'Product description',
   }),
 }).pipe(
-  Type.Obj({
-    typename: 'dxos.org/type/Test',
+  Type.object({
+    typename: 'org.dxos.type.test',
     version: '0.1.0',
   }),
 );
 
 // TODO(wittjosiah): Make builder generic and reuse for all message types.
 abstract class AbstractMessageBuilder {
-  abstract createMessage(numSegments?: number): Promise<DataType.Message>;
+  abstract createMessage(numSegments?: number): Promise<Message.Message>;
 }
 
 /**
@@ -51,8 +49,8 @@ export class MessageBuilder extends AbstractMessageBuilder {
 
   users = Array.from({ length: 5 }, () => ({
     identityDid: IdentityDid.random().toString(),
-    name: faker.person.fullName(),
-    email: faker.internet.email(),
+    name: random.person.fullName(),
+    email: random.internet.email(),
   }));
 
   start = new Date(Date.now() - 24 * 60 * 60 * 10_000);
@@ -61,27 +59,32 @@ export class MessageBuilder extends AbstractMessageBuilder {
     super();
   }
 
-  override async createMessage(numSegments = 1): Promise<DataType.Message> {
-    return Obj.make(DataType.Message, {
+  override async createMessage(numSegments = 1): Promise<Message.Message> {
+    return Obj.make(Message.Message, {
       created: this.next().toISOString(),
-      sender: faker.helpers.arrayElement(this.users),
+      sender: random.helpers.arrayElement(this.users),
       blocks: Array.from({ length: numSegments }).map(() => this.createBlock()),
     });
   }
 
-  createBlock(): DataType.MessageBlock.Transcription {
-    let text = faker.lorem.paragraph();
+  createBlock(): ContentBlock.Transcript {
+    let text = random.lorem.paragraph();
     if (this._space) {
-      const label = faker.commerce.productName();
-      const obj = this._space.db.add(Obj.make(TestItem, { title: label, description: faker.lorem.paragraph() }));
+      const label = random.commerce.productName();
+      const obj = this._space.db.add(
+        Obj.make(TestItem, {
+          title: label,
+          description: random.lorem.paragraph(),
+        }),
+      );
       const dxn = Ref.make(obj).dxn.toString();
       const words = text.split(' ');
-      words.splice(Math.floor(Math.random() * words.length), 0, `[${label}][${dxn}]`);
+      words.splice(Math.floor(Math.random() * words.length), 0, `[${label}](${dxn})`);
       text = words.join(' ');
     }
 
     return {
-      type: 'transcription',
+      _tag: 'transcript',
       started: this.next().toISOString(),
       text,
     };
@@ -95,15 +98,9 @@ export class MessageBuilder extends AbstractMessageBuilder {
 
 // TODO(burdon): Reconcile with BlockBuilder.
 class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
-  AiService = new EdgeAiServiceClient({
-    endpoint: AI_SERVICE_ENDPOINT.REMOTE,
-  });
-
-  executor = new FunctionExecutor(new ServiceContainer().setServices({ ai: { client: this.AiService } }));
-
   space: Space | undefined;
   currentMessage: number = 0;
-  transcriptMessages: DataType.Message[] = [];
+  transcriptMessages: Message.Message[] = [];
 
   async connect(space: Space): Promise<void> {
     this.space = space;
@@ -111,14 +108,18 @@ class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
     this.transcriptMessages = transcriptMessages;
   }
 
-  override async createMessage(): Promise<DataType.Message> {
+  override async createMessage(): Promise<Message.Message> {
     if (!this.space) {
       throw new Error('Space not connected');
     }
 
-    const { objects } = await this.space.db
+    const objects = await this.space.db
       .query(
-        Filter.or(Filter.type(DataType.Person), Filter.type(DataType.Organization), Filter.type(Testing.DocumentType)),
+        Filter.or(
+          Filter.type(Person.Person),
+          Filter.type(Organization.Organization),
+          Filter.type(TestSchema.DocumentType),
+        ),
       )
       .run();
 
@@ -129,8 +130,7 @@ class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
 
     const { message: enhancedMessage } = await processTranscriptMessage({
       input: { message },
-      executor: this.executor,
-      function: extractionAnthropicFn,
+      function: extractionAnthropicFunction,
     });
 
     return enhancedMessage;
@@ -142,7 +142,7 @@ type UseTestTranscriptionQueue = (
   queueId?: Key.ObjectId,
   running?: boolean,
   interval?: number,
-) => Queue<DataType.Message> | undefined;
+) => Queue<Message.Message> | undefined;
 
 /**
  * Test transcriptionqueue.
@@ -155,7 +155,7 @@ export const useTestTranscriptionQueue: UseTestTranscriptionQueue = (
 ) => {
   // TODO(dmaretskyi): Use space.queues.create() instead.
   const queueDxn = useMemo(() => (space ? createQueueDXN(space.id, queueId) : undefined), [space, queueId]);
-  const queue = useQueue<DataType.Message>(queueDxn);
+  const queue = useQueue<Message.Message>(queueDxn);
   const builder = useMemo(() => new MessageBuilder(space), [space]);
 
   useEffect(() => {
@@ -165,7 +165,7 @@ export const useTestTranscriptionQueue: UseTestTranscriptionQueue = (
 
     const i = setInterval(() => {
       void builder.createMessage(Math.ceil(Math.random() * 3)).then(async (message) => {
-        await queue.append([Obj.make(DataType.Message, message)]);
+        await queue.append([Obj.make(Message.Message, message)]);
       });
     }, interval);
     return () => clearInterval(i);
@@ -186,7 +186,7 @@ export const useTestTranscriptionQueueWithEntityExtraction: UseTestTranscription
 ) => {
   // TODO(dmaretskyi): Use space.queues.create() instead.
   const queueDxn = useMemo(() => (space ? createQueueDXN(space.id, queueId) : undefined), [space, queueId]);
-  const queue = useQueue<DataType.Message>(queueDxn);
+  const queue = useQueue<Message.Message>(queueDxn);
   const [builder] = useState(() => new EntityExtractionMessageBuilder());
 
   useEffect(() => {
@@ -203,7 +203,7 @@ export const useTestTranscriptionQueueWithEntityExtraction: UseTestTranscription
       ctx,
       async () => {
         const message = await builder.createMessage();
-        void queue.append([Obj.make(DataType.Message, message)]);
+        void queue.append([Obj.make(Message.Message, message)]);
       },
       interval,
     );

package/src/transcriber/media-stream-recorder.ts

@@ -10,7 +10,7 @@ import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 import { trace } from '@dxos/tracing';
 
-import { type WavConfig, type AudioChunk, type AudioRecorder } from './audio-recorder';
+import { type AudioChunk, type AudioRecorder, type WavConfig } from './audio-recorder';
 
 let initializingPromise: Promise<void> | undefined;
 
@@ -19,7 +19,7 @@ const initializeExtendableMediaRecorder = async () => {
   await register(await connect());
 };
 
-export type MediaStreamRecorderParams = {
+export type MediaStreamRecorderProps = {
   mediaStreamTrack: MediaStreamTrack;
   config: {
     interval: number;
@@ -37,12 +37,12 @@ export class MediaStreamRecorder implements AudioRecorder {
   /**
    * Default MediaRecorder implementation do not support wav encoding.
    */
-  private readonly _config: MediaStreamRecorderParams['config'];
+  private readonly _config: MediaStreamRecorderProps['config'];
   private _mediaRecorder?: IMediaRecorder = undefined;
   private _header?: Uint8Array = undefined;
   private _onChunk?: (chunk: AudioChunk) => void;
 
-  constructor({ mediaStreamTrack, config }: MediaStreamRecorderParams) {
+  constructor({ mediaStreamTrack, config }: MediaStreamRecorderProps) {
     this._mediaStreamTrack = mediaStreamTrack;
     this._config = config;
   }

package/src/transcriber/transcriber.browser.test.ts

@@ -9,9 +9,9 @@ import { WaveFile } from 'wavefile';
 
 import { Trigger } from '@dxos/async';
 import { log } from '@dxos/log';
-import { type DataType } from '@dxos/schema';
 import { openAndClose } from '@dxos/test-utils';
-import { trace, TRACE_PROCESSOR } from '@dxos/tracing';
+import { TRACE_PROCESSOR, trace } from '@dxos/tracing';
+import { type ContentBlock } from '@dxos/types';
 
 import { type AudioChunk, type AudioRecorder, Transcriber } from '../transcriber';
 import { mergeFloat64Arrays } from '../util';
@@ -93,7 +93,7 @@ describe.skip('Transcriber', () => {
   });
 
   test.skip('transcription of audio recording', { timeout: 10_000 }, async () => {
-    const trigger = new Trigger<DataType.MessageBlock.Transcription[]>({ autoReset: true });
+    const trigger = new Trigger<ContentBlock.Transcript[]>({ autoReset: true });
     const recorder = new MockAudioRecorder({
       buffer: await readFile('test.wav'),
       chunkDuration: 3_000,
@@ -127,7 +127,7 @@ describe.skip('Transcriber', () => {
   });
 
   test.skip('transcription of audio recording with overlapping chunks', { timeout: 20_000 }, async () => {
-    const trigger = new Trigger<DataType.MessageBlock.Transcription[]>({ autoReset: true });
+    const trigger = new Trigger<ContentBlock.Transcript[]>({ autoReset: true });
     const recorder = new MockAudioRecorder({
       buffer: await readFile('test.wav'),
       chunkDuration: 3_000,
@@ -212,7 +212,7 @@ class MockAudioRecorder implements AudioRecorder {
 
   @trace.span()
   async start(): Promise<void> {
-    log.info('start mock audio recorder', { wavParams: this.wav.fmt });
+    log.info('start mock audio recorder', { wavProps: this.wav.fmt });
     const now = Date.now();
     const samples = this.wav.getSamples();
     const sampleRate = this.wavConfig.sampleRate;