@dxos/plugin-transcription 0.8.4-main.fd6878d → 0.9.0

package/src/stories/TranscriptionStory.tsx

@@ -0,0 +1,101 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import React, {
+  type ChangeEvent,
+  type Dispatch,
+  type RefObject,
+  type SetStateAction,
+  useCallback,
+  useRef,
+} from 'react';
+
+import { IconButton, ScrollContainer, Toolbar } from '@dxos/react-ui';
+import { type Message } from '@dxos/types';
+
+import { Transcription } from '../components';
+import { type SerializationModel } from '../model';
+
+export type TranscriptionStoryProps = {
+  audioRef?: RefObject<HTMLAudioElement | null>;
+  model: SerializationModel<Message.Message>;
+  running: boolean;
+  disabled?: boolean;
+  uploadAccept?: string;
+  onRunningChange: Dispatch<SetStateAction<boolean>>;
+  onUpload?: (file: File) => void;
+};
+
+/**
+ * Story wrapper that renders a transcript with playback controls, an optional audio file
+ * upload affordance, and a toolbar slot for additional controls.
+ *
+ * @param onUpload - Callback fired when the user picks a local audio file via the upload button.
+ * @param uploadAccept - HTML `accept` filter for the hidden file input. Defaults to `audio/*`.
+ */
+export const TranscriptionStory = ({
+  audioRef,
+  model,
+  running,
+  disabled,
+  uploadAccept = 'audio/*',
+  onRunningChange,
+  onUpload,
+}: TranscriptionStoryProps) => {
+  const fileInputRef = useRef<HTMLInputElement>(null);
+
+  const handleFileChange = useCallback(
+    (event: ChangeEvent<HTMLInputElement>) => {
+      const file = event.target.files?.[0];
+      if (file && onUpload) {
+        onUpload(file);
+      }
+
+      // Reset so the same file can be re-selected.
+      event.target.value = '';
+    },
+    [onUpload],
+  );
+
+  return (
+    <>
+      {audioRef && <audio ref={audioRef} autoPlay />}
+      <Toolbar.Root>
+        <IconButton
+          iconOnly
+          disabled={disabled}
+          icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
+          label={running ? 'Pause' : 'Play'}
+          onClick={() => onRunningChange((running) => !running)}
+        />
+        {onUpload && (
+          <>
+            <input
+              ref={fileInputRef}
+              type='file'
+              accept={uploadAccept}
+              disabled={disabled}
+              className='hidden'
+              onChange={handleFileChange}
+            />
+            <IconButton
+              iconOnly
+              disabled={disabled}
+              icon='ph--upload--regular'
+              label='Upload audio'
+              onClick={() => fileInputRef.current?.click()}
+            />
+          </>
+        )}
+      </Toolbar.Root>
+      <ScrollContainer.Root pin>
+        <ScrollContainer.Content>
+          <ScrollContainer.Viewport>
+            <Transcription model={model} />
+          </ScrollContainer.Viewport>
+        </ScrollContainer.Content>
+      </ScrollContainer.Root>
+    </>
+  );
+};

package/src/stories/common.ts

@@ -0,0 +1,233 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+// Helpers shared between LiveTranscription / FileTranscription stories.
+// Each "live" story differs only in its *audio source* (mic vs file); everything else —
+// the local in-memory message buffer, the transcriber lifecycle, and the plugin-manager
+// decorator stack — is identical and lives here.
+
+import { type Decorator } from '@storybook/react-vite';
+import * as Effect from 'effect/Effect';
+import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
+
+import { withPluginManager } from '@dxos/app-framework/testing';
+import { AppActivationEvents } from '@dxos/app-toolkit';
+import { scheduleTask } from '@dxos/async';
+import { Context } from '@dxos/context';
+import { Obj } from '@dxos/echo';
+import { SpeakingMonitor } from '@dxos/plugin-calls';
+import { ClientPlugin, initializeIdentity } from '@dxos/plugin-client/testing';
+import { PreviewPlugin } from '@dxos/plugin-preview/testing';
+import { StorybookPlugin, corePlugins } from '@dxos/plugin-testing';
+import { IndexKind } from '@dxos/react-client/echo';
+import { withLayout } from '@dxos/react-ui/testing';
+import { Message, Organization, Person } from '@dxos/types';
+import { seedTestData } from '@dxos/types/testing';
+
+import { useFeedModelAdapter, useTranscriber } from '#hooks';
+import { TestItem } from '#testing';
+import { type TranscriptionCapabilities } from '#types';
+
+import { type Transcriber, type TranscriberProps } from '../transcriber';
+import { TranscriptionPlugin } from '../TranscriptionPlugin';
+import { renderByline } from '../util';
+
+export const useIsSpeaking = (track?: MediaStreamTrack) => {
+  const [isSpeaking, setIsSpeaking] = useState(false);
+  const speakingMonitor = useMemo(() => {
+    if (!track) {
+      return;
+    }
+
+    return new SpeakingMonitor(track);
+  }, [track]);
+
+  useEffect(() => {
+    if (!speakingMonitor) {
+      return;
+    }
+
+    const ctx = new Context();
+    scheduleTask(ctx, async () => {
+      speakingMonitor.speakingChanged.on(ctx, () => setIsSpeaking(speakingMonitor.isSpeaking));
+      await speakingMonitor.open();
+      ctx.onDispose(() => speakingMonitor.close());
+    });
+
+    return () => {
+      void ctx.dispose();
+    };
+  }, [speakingMonitor]);
+
+  return isSpeaking;
+};
+
+//
+// Message model.
+//
+
+/**
+ * In-memory message buffer + model adapter for storybook use (production wires up a real
+ * space-backed `Feed`). Returns the renderable `model` plus an `appendMessage` callback.
+ */
+export const useStoryMessageModel = () => {
+  const [messages, setMessages] = useState<Message.Message[]>([]);
+  const appendMessage = useCallback((message: Message.Message) => setMessages((prev) => [...prev, message]), []);
+  const model = useFeedModelAdapter(renderByline([]), messages);
+  return { model, appendMessage };
+};
+
+/**
+ * Wraps blocks in a `Message.Message` with the given sender and appends it to the local buffer.
+ * Provided as a convenience for stories that don't need entity-extraction or custom enrichment.
+ */
+export const useStoryAppendSegments = (
+  appendMessage: (message: Message.Message) => void,
+  sender: Message.Message['sender'] = {},
+): TranscriberProps['onSegments'] =>
+  useCallback<TranscriberProps['onSegments']>(
+    async (blocks) => {
+      appendMessage(
+        Obj.make(Message.Message, {
+          created: new Date().toISOString(),
+          sender,
+          blocks,
+        }),
+      );
+    },
+    [appendMessage, sender],
+  );
+
+//
+// Transcriber lifecycle.
+//
+
+export type UseStoryTranscriberOptions = Pick<
+  TranscriptionCapabilities.TranscriberProviderProps,
+  'transcriberConfig' | 'recorderConfig' | 'onSegments'
+> & {
+  audioStreamTrack?: MediaStreamTrack;
+  /** Whether the user has the toolbar toggle on. */
+  running: boolean;
+  /** Voice-activity flag — chunks recording only fires while `true`. */
+  isSpeaking: boolean;
+};
+
+/**
+ * Drives the transcriber's open/close + chunk-recording lifecycle off `running`,
+ * the open flag, and an `isSpeaking` voice-activity gate. Returns the transcriber instance
+ * (`undefined` until the capability resolves and an audio track is available).
+ */
+export const useStoryTranscriber = ({
+  audioStreamTrack,
+  running,
+  isSpeaking,
+  transcriberConfig,
+  recorderConfig,
+  onSegments,
+}: UseStoryTranscriberOptions): Transcriber | undefined => {
+  const transcriber = useTranscriber({
+    audioStreamTrack,
+    onSegments,
+    transcriberConfig,
+    recorderConfig,
+  });
+
+  // Open the transcriber when it becomes available; close on teardown.
+  // `tokenRef` is a per-effect identity so a late `close()` completion from a stale
+  // transcriber instance can't clobber `isOpen` after a newer instance has opened.
+  const [isOpen, setIsOpen] = useState(false);
+  const tokenRef = useRef<symbol | null>(null);
+  useEffect(() => {
+    if (!transcriber) {
+      return;
+    }
+    const token = Symbol();
+    tokenRef.current = token;
+    void transcriber.open().then(
+      () => tokenRef.current === token && setIsOpen(true),
+      () => tokenRef.current === token && setIsOpen(false),
+    );
+    return () => {
+      void transcriber.close().then(() => {
+        if (tokenRef.current === token) {
+          setIsOpen(false);
+        }
+      });
+    };
+  }, [transcriber]);
+
+  // Start/stop chunk recording based on running + open + isSpeaking.
+  useEffect(() => {
+    if (!transcriber || !isOpen) {
+      return;
+    }
+    if (running && isSpeaking) {
+      transcriber.startChunksRecording();
+    } else {
+      transcriber.stopChunksRecording();
+    }
+  }, [transcriber, isOpen, running, isSpeaking]);
+
+  return transcriber;
+};
+
+//
+// Plugin manager decorator.
+//
+
+export type StoryDecoratorsOptions = {
+  /**
+   * Enable schema/graph/vector indexes after the client boots; needed for entity-extraction
+   * queries over seeded `Person` / `Organization` objects.
+   * @default false
+   */
+  enableVectorIndex?: boolean;
+};
+
+/**
+ * Standard plugin-manager decorator for transcription stories: core plugins,
+ * `ClientPlugin` with `Person`/`Organization`/`TestItem` schemas + seeded test data, plus
+ * the `PreviewPlugin` and `TranscriptionPlugin`. Toggle `enableVectorIndex` for entity-extraction.
+ */
+export const createStoryDecorators = ({ enableVectorIndex = false }: StoryDecoratorsOptions = {}): Decorator[] => [
+  withLayout({ layout: 'column' }),
+  withPluginManager({
+    plugins: [
+      ...corePlugins(),
+      StorybookPlugin({}),
+      ClientPlugin({
+        types: [TestItem, Person.Person, Organization.Organization],
+        onClientInitialized: ({ client }) =>
+          Effect.gen(function* () {
+            const { personalSpace } = yield* initializeIdentity(client);
+            if (enableVectorIndex) {
+              yield* enableQueryIndexes(client.services.services);
+            }
+            yield* Effect.promise(() => seedTestData(personalSpace));
+          }),
+      }),
+      PreviewPlugin(),
+      TranscriptionPlugin(),
+    ],
+    fireEvents: [AppActivationEvents.SetupAppGraph],
+  }),
+];
+
+// TODO(mykola): Make API easier to use.
+// TODO(mykola): Delete after enabling vector indexing by default.
+const enableQueryIndexes = (services: { QueryService?: any }) =>
+  Effect.gen(function* () {
+    yield* Effect.promise(() =>
+      services.QueryService!.setConfig({
+        enabled: true,
+        indexes: [
+          { kind: IndexKind.Kind.SCHEMA_MATCH },
+          { kind: IndexKind.Kind.GRAPH },
+          { kind: IndexKind.Kind.VECTOR },
+        ],
+      }),
+    );
+    yield* Effect.promise(() => services.QueryService!.reindex());
+  });

package/src/testing/testing.ts

@@ -2,22 +2,22 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Schema } from 'effect';
+import * as Effect from 'effect/Effect';
+import * as Schema from 'effect/Schema';
 import { useEffect, useMemo, useState } from 'react';
 
-import { extractionAnthropicFn, processTranscriptMessage } from '@dxos/assistant/extraction';
+import { extractionAnthropicFunction, processTranscriptMessage } from '@dxos/assistant/extraction';
 import { scheduleTaskInterval } from '@dxos/async';
-import { Filter, type Queue } from '@dxos/client/echo';
+import { createFeedServiceLayer, type Space } from '@dxos/client/echo';
 import { Context } from '@dxos/context';
-import { type Key, Obj, Ref, Type } from '@dxos/echo';
-import { createQueueDXN } from '@dxos/echo-schema';
-import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
+import { DXN, Feed, Filter, Obj, Ref, Type } from '@dxos/echo';
+import { EffectEx } from '@dxos/effect';
 import { IdentityDid } from '@dxos/keys';
 import { log } from '@dxos/log';
-import { faker } from '@dxos/random';
-import { type Space, useQueue } from '@dxos/react-client/echo';
-import { DataType } from '@dxos/schema';
-import { Testing, seedTestData } from '@dxos/schema/testing';
+import { random } from '@dxos/random';
+import { TestSchema } from '@dxos/schema/testing';
+import { type ContentBlock, Message, Organization, Person } from '@dxos/types';
+import { seedTestData } from '@dxos/types/testing';
 
 // TODO(burdon): Reconcile with plugin-markdown. Move to @dxos/schema/testing.
 export const TestItem = Schema.Struct({
@@ -29,16 +29,11 @@ export const TestItem = Schema.Struct({
     title: 'Description',
     description: 'Product description',
   }),
-}).pipe(
-  Type.Obj({
-    typename: 'dxos.org/type/Test',
-    version: '0.1.0',
-  }),
-);
+}).pipe(Type.makeObject(DXN.make('org.dxos.type.test', '0.1.0')));
 
 // TODO(wittjosiah): Make builder generic and reuse for all message types.
 abstract class AbstractMessageBuilder {
-  abstract createMessage(numSegments?: number): Promise<DataType.Message>;
+  abstract createMessage(numSegments?: number): Promise<Message.Message>;
 }
 
 /**
@@ -49,8 +44,8 @@ export class MessageBuilder extends AbstractMessageBuilder {
 
   users = Array.from({ length: 5 }, () => ({
     identityDid: IdentityDid.random().toString(),
-    name: faker.person.fullName(),
-    email: faker.internet.email(),
+    name: random.person.fullName(),
+    email: random.internet.email(),
   }));
 
   start = new Date(Date.now() - 24 * 60 * 60 * 10_000);
@@ -59,22 +54,27 @@ export class MessageBuilder extends AbstractMessageBuilder {
     super();
   }
 
-  override async createMessage(numSegments = 1): Promise<DataType.Message> {
-    return Obj.make(DataType.Message, {
+  override async createMessage(numSegments = 1): Promise<Message.Message> {
+    return Obj.make(Message.Message, {
       created: this.next().toISOString(),
-      sender: faker.helpers.arrayElement(this.users),
+      sender: random.helpers.arrayElement(this.users),
       blocks: Array.from({ length: numSegments }).map(() => this.createBlock()),
     });
   }
 
-  createBlock(): DataType.MessageBlock.Transcript {
-    let text = faker.lorem.paragraph();
+  createBlock(): ContentBlock.Transcript {
+    let text = random.lorem.paragraph();
     if (this._space) {
-      const label = faker.commerce.productName();
-      const obj = this._space.db.add(Obj.make(TestItem, { title: label, description: faker.lorem.paragraph() }));
-      const dxn = Ref.make(obj).dxn.toString();
+      const label = random.commerce.productName();
+      const obj = this._space.db.add(
+        Obj.make(TestItem, {
+          title: label,
+          description: random.lorem.paragraph(),
+        }),
+      );
+      const dxn = Ref.make(obj).uri;
       const words = text.split(' ');
-      words.splice(Math.floor(Math.random() * words.length), 0, `[${label}][${dxn}]`);
+      words.splice(Math.floor(Math.random() * words.length), 0, `[${label}](${dxn})`);
       text = words.join(' ');
     }
 
@@ -93,17 +93,9 @@ export class MessageBuilder extends AbstractMessageBuilder {
 
 // TODO(burdon): Reconcile with BlockBuilder.
 class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
-  executor = new FunctionExecutor(
-    new ServiceContainer().setServices({
-      // ai: {
-      //   client: this.AiService,
-      // },
-    }),
-  );
-
   space: Space | undefined;
   currentMessage: number = 0;
-  transcriptMessages: DataType.Message[] = [];
+  transcriptMessages: Message.Message[] = [];
 
   async connect(space: Space): Promise<void> {
     this.space = space;
@@ -111,14 +103,18 @@ class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
     this.transcriptMessages = transcriptMessages;
   }
 
-  override async createMessage(): Promise<DataType.Message> {
+  override async createMessage(): Promise<Message.Message> {
     if (!this.space) {
       throw new Error('Space not connected');
     }
 
-    const { objects } = await this.space.db
+    const objects = await this.space.db
       .query(
-        Filter.or(Filter.type(DataType.Person), Filter.type(DataType.Organization), Filter.type(Testing.DocumentType)),
+        Filter.or(
+          Filter.type(Person.Person),
+          Filter.type(Organization.Organization),
+          Filter.type(TestSchema.DocumentType),
+        ),
       )
       .run();
 
@@ -129,8 +125,7 @@ class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
 
     const { message: enhancedMessage } = await processTranscriptMessage({
       input: { message },
-      executor: this.executor,
-      function: extractionAnthropicFn,
+      function: extractionAnthropicFunction,
     });
 
     return enhancedMessage;
@@ -139,71 +134,64 @@ class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
 
 type UseTestTranscriptionQueue = (
   space: Space | undefined,
-  queueId?: Key.ObjectId,
   running?: boolean,
   interval?: number,
-) => Queue<DataType.Message> | undefined;
+) => Feed.Feed | undefined;
 
 /**
- * Test transcriptionqueue.
+ * Test transcription feed.
  */
 export const useTestTranscriptionQueue: UseTestTranscriptionQueue = (
   space: Space | undefined,
-  queueId?: Key.ObjectId,
   running = true,
   interval = 1_000,
 ) => {
-  // TODO(dmaretskyi): Use space.queues.create() instead.
-  const queueDxn = useMemo(() => (space ? createQueueDXN(space.id, queueId) : undefined), [space, queueId]);
-  const queue = useQueue<DataType.Message>(queueDxn);
+  const feed = useMemo(() => (space ? space.db.add(Feed.make({ name: 'transcription' })) : undefined), [space]);
   const builder = useMemo(() => new MessageBuilder(space), [space]);
 
   useEffect(() => {
-    if (!queue || !running) {
+    if (!space || !feed || !running) {
       return;
     }
+    const feedServiceLayer = createFeedServiceLayer(space.queues);
 
     const i = setInterval(() => {
       void builder.createMessage(Math.ceil(Math.random() * 3)).then(async (message) => {
-        await queue.append([Obj.make(DataType.Message, message)]);
+        await Feed.append(feed, [message]).pipe(Effect.provide(feedServiceLayer), EffectEx.runAndForwardErrors);
       });
     }, interval);
     return () => clearInterval(i);
-  }, [queue, running, interval]);
+  }, [space, feed, running, interval]);
 
-  return queue;
+  return feed;
 };
 
 /**
- * Test transcription queue.
+ * Test transcription feed with entity extraction.
  */
 // TODO(burdon): Reconcile with useTestTranscriptionQueue.
 export const useTestTranscriptionQueueWithEntityExtraction: UseTestTranscriptionQueue = (
   space: Space | undefined,
-  queueId?: Key.ObjectId,
   running = true,
   interval = 1_000,
 ) => {
-  // TODO(dmaretskyi): Use space.queues.create() instead.
-  const queueDxn = useMemo(() => (space ? createQueueDXN(space.id, queueId) : undefined), [space, queueId]);
-  const queue = useQueue<DataType.Message>(queueDxn);
+  const feed = useMemo(() => (space ? space.db.add(Feed.make({ name: 'transcription' })) : undefined), [space]);
   const [builder] = useState(() => new EntityExtractionMessageBuilder());
 
   useEffect(() => {
-    if (!queue || !running) {
+    if (!space || !feed || !running) {
       return;
     }
 
-    if (space) {
-      void builder.connect(space);
-    }
+    void builder.connect(space);
 
+    const feedServiceLayer = createFeedServiceLayer(space.queues);
     const ctx = new Context();
     scheduleTaskInterval(
       ctx,
       async () => {
         const message = await builder.createMessage();
-        void queue.append([Obj.make(DataType.Message, message)]);
+        await Feed.append(feed, [message]).pipe(Effect.provide(feedServiceLayer), EffectEx.runAndForwardErrors);
       },
       interval,
     );
@@ -211,7 +199,7 @@ export const useTestTranscriptionQueueWithEntityExtraction: UseTestTranscription
     return () => {
       void ctx.dispose();
     };
-  }, [space, queue, running, interval]);
+  }, [space, feed, running, interval]);
 
-  return queue;
+  return feed;
 };

package/src/transcriber/index.ts

@@ -2,7 +2,7 @@
 // Copyright 2025 DXOS.org
 //
 
-export type * from './audio-recorder';
+export * from './audio-recorder';
 export * from './media-stream-recorder';
 export * from './transcriber';
 export * from './transcription-manager';

package/src/transcriber/media-stream-recorder.ts

@@ -19,7 +19,7 @@ const initializeExtendableMediaRecorder = async () => {
   await register(await connect());
 };
 
-export type MediaStreamRecorderParams = {
+export type MediaStreamRecorderProps = {
   mediaStreamTrack: MediaStreamTrack;
   config: {
     interval: number;
@@ -37,12 +37,12 @@ export class MediaStreamRecorder implements AudioRecorder {
   /**
    * Default MediaRecorder implementation do not support wav encoding.
    */
-  private readonly _config: MediaStreamRecorderParams['config'];
+  private readonly _config: MediaStreamRecorderProps['config'];
   private _mediaRecorder?: IMediaRecorder = undefined;
   private _header?: Uint8Array = undefined;
   private _onChunk?: (chunk: AudioChunk) => void;
 
-  constructor({ mediaStreamTrack, config }: MediaStreamRecorderParams) {
+  constructor({ mediaStreamTrack, config }: MediaStreamRecorderProps) {
     this._mediaStreamTrack = mediaStreamTrack;
     this._config = config;
   }

package/src/transcriber/transcriber.browser.test.ts

@@ -4,15 +4,14 @@
 
 import fs from 'node:fs';
 import path from 'node:path';
-
 import { describe, expect, test } from 'vitest';
 import { WaveFile } from 'wavefile';
 
 import { Trigger } from '@dxos/async';
 import { log } from '@dxos/log';
-import { type DataType } from '@dxos/schema';
 import { openAndClose } from '@dxos/test-utils';
 import { TRACE_PROCESSOR, trace } from '@dxos/tracing';
+import { type ContentBlock } from '@dxos/types';
 
 import { type AudioChunk, type AudioRecorder, Transcriber } from '../transcriber';
 import { mergeFloat64Arrays } from '../util';
@@ -94,7 +93,7 @@ describe.skip('Transcriber', () => {
   });
 
   test.skip('transcription of audio recording', { timeout: 10_000 }, async () => {
-    const trigger = new Trigger<DataType.MessageBlock.Transcript[]>({ autoReset: true });
+    const trigger = new Trigger<ContentBlock.Transcript[]>({ autoReset: true });
     const recorder = new MockAudioRecorder({
       buffer: await readFile('test.wav'),
       chunkDuration: 3_000,
@@ -128,7 +127,7 @@ describe.skip('Transcriber', () => {
   });
 
   test.skip('transcription of audio recording with overlapping chunks', { timeout: 20_000 }, async () => {
-    const trigger = new Trigger<DataType.MessageBlock.Transcript[]>({ autoReset: true });
+    const trigger = new Trigger<ContentBlock.Transcript[]>({ autoReset: true });
     const recorder = new MockAudioRecorder({
       buffer: await readFile('test.wav'),
       chunkDuration: 3_000,
@@ -213,7 +212,7 @@ class MockAudioRecorder implements AudioRecorder {
 
   @trace.span()
   async start(): Promise<void> {
-    log.info('start mock audio recorder', { wavParams: this.wav.fmt });
+    log.info('start mock audio recorder', { wavProps: this.wav.fmt });
     const now = Date.now();
     const samples = this.wav.getSamples();
     const sampleRate = this.wavConfig.sampleRate;