@dxos/plugin-transcription 0.8.4-main.dedc0f3 → 0.8.4-main.dfabb4ec29

package/src/capabilities/transcriber.ts

@@ -2,12 +2,14 @@
 // Copyright 2025 DXOS.org
 //
 
-import { type PluginContext, contributes } from '@dxos/app-framework';
+import * as Effect from 'effect/Effect';
+
+import { Capabilities, Capability } from '@dxos/app-framework';
 import { ClientCapabilities } from '@dxos/plugin-client';
 
-import { MediaStreamRecorder, Transcriber, TranscriptionManager } from '../transcriber';
+import { TranscriptionCapabilities } from '#types';
 
-import { TranscriptionCapabilities } from './capabilities';
+import { MediaStreamRecorder, Transcriber, TranscriptionManager } from '../transcriber';
 
 // TODO(burdon): Move to config?
 
@@ -30,48 +32,57 @@ const TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
 /**
  * Records audio while user is speaking and transcribes it after user is done speaking.
  */
-export default (context: PluginContext) => {
-  const getTranscriber: TranscriptionCapabilities.GetTranscriber = ({
-    audioStreamTrack,
-    onSegments,
-    transcriberConfig,
-    recorderConfig,
-  }) => {
-    // Initialize audio transcription.
-    return new Transcriber({
-      config: {
-        transcribeAfterChunksAmount: TRANSCRIBE_AFTER_CHUNKS_AMOUNT,
-        prefixBufferChunksAmount: PREFIXED_CHUNKS_AMOUNT,
-        ...transcriberConfig,
-      },
-      recorder: new MediaStreamRecorder({
-        mediaStreamTrack: audioStreamTrack,
+export default Capability.makeModule(
+  Effect.fnUntraced(function* () {
+    // Get context for lazy capability access in callbacks.
+    const capabilities = yield* Capability.Service;
+    const registry = yield* Capability.get(Capabilities.AtomRegistry);
+
+    const getTranscriber: TranscriptionCapabilities.GetTranscriber = ({
+      audioStreamTrack,
+      onSegments,
+      transcriberConfig,
+      recorderConfig,
+      transcribe,
+    }) => {
+      // Initialize audio transcription.
+      return new Transcriber({
         config: {
-          interval: RECORD_INTERVAL,
-          ...recorderConfig,
+          transcribeAfterChunksAmount: TRANSCRIBE_AFTER_CHUNKS_AMOUNT,
+          prefixBufferChunksAmount: PREFIXED_CHUNKS_AMOUNT,
+          ...transcriberConfig,
         },
-      }),
-      onSegments,
-    });
-  };
+        recorder: new MediaStreamRecorder({
+          mediaStreamTrack: audioStreamTrack,
+          config: {
+            interval: RECORD_INTERVAL,
+            ...recorderConfig,
+          },
+        }),
+        onSegments,
+        transcribe,
+      });
+    };
 
-  const getTranscriptionManager: TranscriptionCapabilities.GetTranscriptionManager = ({ messageEnricher }) => {
-    const client = context.getCapability(ClientCapabilities.Client);
-    const transcriptionManager = new TranscriptionManager({
-      edgeClient: client.edge,
-      messageEnricher,
-    });
+    const getTranscriptionManager: TranscriptionCapabilities.GetTranscriptionManager = ({ messageEnricher }) => {
+      const client = capabilities.get(ClientCapabilities.Client);
+      const transcriptionManager = new TranscriptionManager({
+        edgeClient: client.edge.http,
+        messageEnricher,
+        registry,
+      });
 
-    const identity = client.halo.identity.get();
-    if (identity) {
-      transcriptionManager.setIdentityDid(identity.did);
-    }
+      const identity = client.halo.identity.get();
+      if (identity) {
+        transcriptionManager.setIdentityDid(identity.did);
+      }
 
-    return transcriptionManager;
-  };
+      return transcriptionManager;
+    };
 
-  return [
-    contributes(TranscriptionCapabilities.Transcriber, getTranscriber),
-    contributes(TranscriptionCapabilities.TranscriptionManager, getTranscriptionManager),
-  ];
-};
+    return [
+      Capability.contributes(TranscriptionCapabilities.Transcriber, getTranscriber),
+      Capability.contributes(TranscriptionCapabilities.TranscriptionManager, getTranscriptionManager),
+    ];
+  }),
+);

package/src/components/{Transcript/Transcript.stories.tsx → Transcription/Transcription.stories.tsx}

@@ -2,60 +2,56 @@
 // Copyright 2025 DXOS.org
 //
 
-import '@dxos-theme';
-
 import { type Meta, type StoryObj } from '@storybook/react-vite';
+import * as Effect from 'effect/Effect';
+import type * as Types from 'effect/Types';
 import React, { type FC, useEffect, useMemo, useState } from 'react';
 
-import { IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
 import { withPluginManager } from '@dxos/app-framework/testing';
 import { Key } from '@dxos/echo';
-import { ClientPlugin } from '@dxos/plugin-client';
-import { PreviewPlugin } from '@dxos/plugin-preview';
-import { SpacePlugin } from '@dxos/plugin-space';
-import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
-import { ThemePlugin } from '@dxos/plugin-theme';
-import { faker } from '@dxos/random';
-import { useMembers, useSpace } from '@dxos/react-client/echo';
+import { ClientPlugin } from '@dxos/plugin-client/plugin';
+import { initializeIdentity } from '@dxos/plugin-client/testing';
+import { PreviewPlugin } from '@dxos/plugin-preview/testing';
+import { StorybookPlugin, corePlugins } from '@dxos/plugin-testing';
+import { random } from '@dxos/random';
+import { useMembers, useSpaces } from '@dxos/react-client/echo';
 import { IconButton, Toolbar } from '@dxos/react-ui';
-import { SyntaxHighlighter } from '@dxos/react-ui-syntax-highlighter';
-import { defaultTx } from '@dxos/react-ui-theme';
-import { DataType } from '@dxos/schema';
-import { Testing } from '@dxos/schema/testing';
-import { withLayout } from '@dxos/storybook-utils';
+import { JsonHighlighter } from '@dxos/react-ui-syntax-highlighter';
+import { withLayout } from '@dxos/react-ui/testing';
+import { TestSchema } from '@dxos/schema/testing';
+import { type ContentBlock, type Message, Organization, Person } from '@dxos/types';
 
-import { useQueueModelAdapter } from '../../hooks';
-import { SerializationModel } from '../../model';
+import { useQueueModelAdapter } from '#hooks';
 import {
   MessageBuilder,
   TestItem,
   useTestTranscriptionQueue,
   useTestTranscriptionQueueWithEntityExtraction,
-} from '../../testing';
-import { translations } from '../../translations';
+} from '#testing';
+import { translations } from '#translations';
 
-import { TranscriptView, type TranscriptViewProps, renderByline } from './Transcript';
+import { SerializationModel } from '../../model';
+import { renderByline } from '../../util';
+import { Transcription, type TranscriptionProps } from './Transcription';
 
-faker.seed(1);
+random.seed(1);
 
 /**
  * Story wrapper with test controls.
  */
 const TranscriptContainer: FC<
-  TranscriptViewProps & {
+  TranscriptionProps & {
     running: boolean;
     onRunningChange: (running: boolean) => void;
     onReset?: () => void;
   }
-> = ({ space, model, running, onRunningChange, onReset }) => {
+> = ({ model, running, onRunningChange, onReset }) => {
   return (
     <div className='grid grid-rows-[1fr_40px] grow divide-y divide-separator'>
-      <TranscriptView space={space} model={model} />
+      <Transcription model={model} />
       <div className='grid grid-cols-[1fr_16rem] overflow-hidden'>
         <div className='flex items-center'>
-          <SyntaxHighlighter language='json' className='text-sm'>
-            {JSON.stringify(model.toJSON())}
-          </SyntaxHighlighter>
+          <JsonHighlighter data={model.toJSON()} indent={0} classNames='text-sm' />
         </div>
         <Toolbar.Root classNames='justify-end'>
           <IconButton
@@ -70,20 +66,24 @@ const TranscriptContainer: FC<
   );
 };
 
-type StoryProps = { messages?: DataType.Message[] } & Pick<TranscriptViewProps, 'ignoreAttention' | 'attendableId'>;
+type DefaultStoryProps = { messages?: Message.Message[] } & Pick<
+  TranscriptionProps,
+  'ignoreAttention' | 'attendableId'
+>;
 
 /**
  * Basic story mutates array of messages.
  */
-const BasicStory = ({ messages: initialMessages = [], ...props }: StoryProps) => {
+const BasicStory = ({ messages: initialMessages = [], ...props }: DefaultStoryProps) => {
   const [reset, setReset] = useState({});
   const builder = useMemo(() => new MessageBuilder(), []);
   const model = useMemo(
-    () => new SerializationModel<DataType.Message>(renderByline([]), initialMessages),
+    () => new SerializationModel<Message.Message>(renderByline([]), initialMessages),
     [initialMessages, reset],
   );
   const [running, setRunning] = useState(true);
-  const [currentMessage, setCurrentMessage] = useState<DataType.Message | null>(null);
+  // Use mutable type for local message building (not ECHO-managed).
+  const [currentMessage, setCurrentMessage] = useState<Types.Mutable<Message.Message> | null>(null);
   useEffect(() => {
     if (!running) {
       return;
@@ -104,7 +104,7 @@ const BasicStory = ({ messages: initialMessages = [], ...props }: StoryProps) =>
         return;
       }
 
-      currentMessage.blocks.push(builder.createBlock());
+      (currentMessage.blocks as ContentBlock.Any[]).push(builder.createBlock());
       model.updateChunk(currentMessage);
     }, 3_000);
 
@@ -136,34 +136,27 @@ const QueueStory = ({
   queueId,
   onReset,
   ...props
-}: StoryProps & { queueId: Key.ObjectId; onReset: () => void }) => {
+}: DefaultStoryProps & { queueId: Key.ObjectId; onReset: () => void }) => {
   const [running, setRunning] = useState(true);
-  const space = useSpace();
-  const members = useMembers(space?.key).map((member) => member.identity);
+  const [space] = useSpaces();
+  const members = useMembers(space?.id).map((member) => member.identity);
   const queue = useTestTranscriptionQueue(space, queueId, running, 2_000);
   const model = useQueueModelAdapter(renderByline(members), queue, initialMessages);
 
   return (
-    <TranscriptContainer
-      space={space}
-      model={model}
-      running={running}
-      onRunningChange={setRunning}
-      onReset={onReset}
-      {...props}
-    />
+    <TranscriptContainer model={model} running={running} onRunningChange={setRunning} onReset={onReset} {...props} />
   );
 };
 
 // TODO(burdon): Reconcile with QueueStory.
 const EntityExtractionQueueStory = () => {
   const [running, setRunning] = useState(true);
-  const space = useSpace();
+  const [space] = useSpaces();
   const members = useMembers(space?.key).map((member) => member.identity);
   const queue = useTestTranscriptionQueueWithEntityExtraction(space, undefined, running, 2_000);
   const model = useQueueModelAdapter(renderByline(members), queue, []);
 
-  return <TranscriptContainer space={space} model={model} running={running} onRunningChange={setRunning} />;
+  return <TranscriptContainer model={model} running={running} onRunningChange={setRunning} />;
 };
 
 /**
@@ -180,39 +173,49 @@ const QueueStoryWrapper = () => {
 };
 
 const meta = {
-  title: 'plugins/plugin-transcription/Transcript',
+  title: 'plugins/plugin-transcription/components/Transcription',
   decorators: [
+    withLayout({ layout: 'fullscreen' }),
     withPluginManager({
       plugins: [
-        ThemePlugin({ tx: defaultTx }),
-        StorybookLayoutPlugin(),
+        ...corePlugins(),
+        StorybookPlugin({}),
         ClientPlugin({
-          types: [TestItem, Testing.DocumentType, DataType.Person, DataType.Organization],
-          onClientInitialized: async ({ client }) => {
-            await client.halo.createIdentity();
-          },
+          types: [TestItem, TestSchema.DocumentType, Person.Person, Organization.Organization],
+          onClientInitialized: ({ client }) =>
+            Effect.gen(function* () {
+              yield* initializeIdentity(client);
+            }),
         }),
+
         PreviewPlugin(),
-        SpacePlugin(),
-        SettingsPlugin(),
-        IntentPlugin(),
       ],
     }),
-    withLayout({ fullscreen: true }),
   ],
   parameters: {
+    layout: 'fullscreen',
     translations,
   },
 } satisfies Meta;
 
 export default meta;
 
-export const Default: StoryObj<typeof BasicStory> = {
-  render: BasicStory,
+const DefaultStory = (props: DefaultStoryProps) => {
+  const [messages, setMessages] = useState<Message.Message[]>([]);
+  useEffect(() => {
+    void Promise.all(Array.from({ length: 10 }, () => MessageBuilder.singleton.createMessage())).then(setMessages);
+  }, []);
+  if (messages.length === 0) {
+    return <div>Loading messages...</div>;
+  }
+  return <BasicStory {...props} messages={messages} />;
+};
+
+export const Default: StoryObj<typeof DefaultStory> = {
+  render: DefaultStory,
   args: {
     ignoreAttention: true,
     attendableId: 'story',
-    messages: await Promise.all(Array.from({ length: 10 }, () => MessageBuilder.singleton.createMessage())),
   },
 };
 

package/src/components/Transcription/Transcription.tsx

@@ -0,0 +1,64 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import React from 'react';
+
+import { useThemeContext } from '@dxos/react-ui';
+import { useTextEditor } from '@dxos/react-ui-editor';
+import { type Message, type Transcript } from '@dxos/types';
+import {
+  autoScroll,
+  createBasicExtensions,
+  createMarkdownExtensions,
+  createThemeExtensions,
+  decorateMarkdown,
+  documentSlots,
+  preview,
+} from '@dxos/ui-editor';
+import { composable, composableProps } from '@dxos/ui-theme';
+import { isTruthy } from '@dxos/util';
+
+import { type SerializationModel } from '../../model';
+import { renderByline } from '../../util';
+import { transcription } from './transcription-extension';
+
+export { renderByline };
+
+export type TranscriptionProps = {
+  transcript?: Transcript.Transcript;
+  model: SerializationModel<Message.Message>;
+  // TODO(burdon): Are these still in use?
+  attendableId?: string;
+  ignoreAttention?: boolean;
+};
+
+/**
+ * Transcript component implemented using the editor.
+ */
+export const Transcription = composable<HTMLDivElement, TranscriptionProps>(
+  ({ transcript: object, model, children, ...props }, forwardedRef) => {
+    const { themeMode } = useThemeContext();
+    const { parentRef } = useTextEditor(() => {
+      return {
+        extensions: [
+          createBasicExtensions({ readOnly: true, lineWrapping: true, search: true }),
+          createMarkdownExtensions(),
+          createThemeExtensions({ themeMode, slots: documentSlots }),
+          decorateMarkdown(),
+          preview(),
+          transcription({ model, started: object?.started ? new Date(object.started) : undefined }),
+          autoScroll(),
+        ].filter(isTruthy),
+      };
+    }, [model]);
+
+    return (
+      <div
+        {...composableProps(props, { classNames: 'dx-container' })}
+        ref={parentRef}
+        data-popover-collision-boundary={true /* TODO(thure): Make this a constant and document it. */}
+      />
+    );
+  },
+);

package/src/components/{Transcript → Transcription}/index.ts

@@ -2,4 +2,4 @@
 // Copyright 2025 DXOS.org
 //
 
-export * from './Transcript';
+export * from './Transcription';

package/src/components/{Transcript/transcript-extension.ts → Transcription/transcription-extension.ts}

@@ -8,22 +8,22 @@ import { format } from 'date-fns/format';
 import { intervalToDuration } from 'date-fns/intervalToDuration';
 
 import { type CleanupFn, addEventListener, combine } from '@dxos/async';
-import { type DataType } from '@dxos/schema';
+import { type Message } from '@dxos/types';
 
 import { DocumentAdapter, type SerializationModel } from '../../model';
 
 /**
  * Data structure that maps Chunks queue to lines with transcript state.
  */
-export type TranscriptOptions = {
-  model: SerializationModel<DataType.Message>;
+export type TranscriptionOptions = {
+  model: SerializationModel<Message.Message>;
   started?: Date;
 };
 
 /**
  * Scrolling transcript with timestamps.
  */
-export const transcript = ({ model, started }: TranscriptOptions): Extension => {
+export const transcription = ({ model, started }: TranscriptionOptions): Extension => {
   return [
     // Show timestamps in the gutter.
     gutter({
@@ -131,8 +131,12 @@ export const transcript = ({ model, started }: TranscriptOptions): Extension =>
     ),
 
     EditorView.theme({
+      '.cm-gutters': {
+        backgroundColor: 'var(--color-base-surface)',
+      },
       '.cm-timestamp-gutter': {
         width: '6rem',
+        paddingLeft: '0.5rem',
         paddingRight: '1rem',
       },
       '.cm-timestamp-gutter > .cm-gutterElement > div': {
@@ -156,13 +160,13 @@ class TimestampMarker extends GutterMarker {
     super();
   }
 
-  override eq(other: this): boolean {
-    return other._timestamp === this._timestamp;
+  override eq(other: this) {
+    return this._timestamp === other._timestamp;
   }
 
-  override toDOM(view: EditorView): HTMLDivElement {
+  override toDOM(view: EditorView) {
     const el = document.createElement('div');
-    el.className = 'text-sm text-subdued hover:bg-hoverSurface cursor-pointer';
+    el.className = 'text-sm text-subdued hover:bg-hover-surface cursor-pointer';
     el.textContent = formatTimestamp(this._timestamp, this._started);
     // TODO(burdon): Click to bookmark or copy hyperlink.
     el.onclick = () => {
@@ -176,7 +180,7 @@ class TimestampMarker extends GutterMarker {
   }
 }
 
-const getStartTime = (started?: Date, message?: DataType.Message): Date | undefined => {
+const getStartTime = (started?: Date, message?: Message.Message): Date | undefined => {
   if (started) {
     return started;
   }

package/src/components/index.ts

@@ -2,9 +2,4 @@
 // Copyright 2025 DXOS.org
 //
 
-import { lazy } from 'react';
-
-export * from './Transcript';
-export * from './TranscriptContainer';
-
-export const TranscriptContainer = lazy(() => import('./TranscriptContainer'));
+export * from './Transcription';

package/src/containers/TranscriptionContainer/TranscriptionContainer.tsx

@@ -0,0 +1,35 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import React from 'react';
+
+import { type AppSurface } from '@dxos/app-toolkit/ui';
+import { Feed, Obj } from '@dxos/echo';
+import { useMembers, useQueue } from '@dxos/react-client/echo';
+import { Panel } from '@dxos/react-ui';
+import { type Message, type Transcript } from '@dxos/types';
+
+import { Transcription } from '#components';
+import { useQueueModelAdapter } from '#hooks';
+
+import { renderByline } from '../../util';
+
+export type TranscriptionContainerProps = AppSurface.ObjectArticleProps<Transcript.Transcript>;
+
+export const TranscriptionContainer = ({ role, subject: transcript, attendableId }: TranscriptionContainerProps) => {
+  const db = Obj.getDatabase(transcript);
+  const members = useMembers(db?.spaceId).map((member) => member.identity);
+  const feed = transcript.feed.target;
+  const queueDxn = feed ? Feed.getQueueDxn(feed) : undefined;
+  const queue = useQueue<Message.Message>(queueDxn, { pollInterval: 1_000 });
+  const model = useQueueModelAdapter(renderByline(members), queue);
+
+  return (
+    <Panel.Root role={role}>
+      <Panel.Content asChild>
+        <Transcription attendableId={attendableId} model={model} transcript={transcript} />
+      </Panel.Content>
+    </Panel.Root>
+  );
+};

package/src/containers/TranscriptionContainer/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export { TranscriptionContainer as default } from './TranscriptionContainer';

package/src/containers/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { type ComponentType, lazy } from 'react';
+
+export const TranscriptionContainer: ComponentType<any> = lazy(() => import('./TranscriptionContainer'));

package/src/hooks/useAudioFile.ts

@@ -20,58 +20,63 @@ export const useAudioFile = (audioUrl: string, constraints?: MediaTrackConstrain
     const ctx = new Context();
 
     scheduleTask(ctx, async () => {
-      const response = await fetch(audioUrl);
-      const blob = await response.blob();
-      const objectUrl = URL.createObjectURL(blob);
-      ctx.onDispose(() => {
-        URL.revokeObjectURL(objectUrl);
-      });
+      // TODO(wittjosiah): Fetch to external url fails in headless storybook test.
+      try {
+        const response = await fetch(audioUrl);
+        const blob = await response.blob();
+        const objectUrl = URL.createObjectURL(blob);
+        ctx.onDispose(() => {
+          URL.revokeObjectURL(objectUrl);
+        });
 
-      const audio = new Audio();
-      audio.src = objectUrl;
+        const audio = new Audio();
+        audio.src = objectUrl;
 
-      await new Promise<void>((resolve, reject) => {
-        audio.addEventListener(
-          'error',
-          (err) => {
-            log.error('error', { err });
-            reject(err);
-          },
-          { once: true },
-        );
-        audio.addEventListener(
-          'canplay',
-          async () => {
-            resolve();
-          },
-          { once: true },
-        );
-        audio.load();
-      });
+        await new Promise<void>((resolve, reject) => {
+          audio.addEventListener(
+            'error',
+            (err) => {
+              log.error('error', { err });
+              reject(err);
+            },
+            { once: true },
+          );
+          audio.addEventListener(
+            'canplay',
+            async () => {
+              resolve();
+            },
+            { once: true },
+          );
+          audio.load();
+        });
 
-      const audioCtx = new AudioContext();
+        const audioCtx = new AudioContext();
 
-      // Resume AudioContext if it's suspended.
-      if (audioCtx.state === 'suspended') {
-        await audioCtx.resume();
-      }
+        // Resume AudioContext if it's suspended.
+        if (audioCtx.state === 'suspended') {
+          await audioCtx.resume();
+        }
 
-      const destination = audioCtx.createMediaStreamDestination();
-      destination.channelCount = 1;
+        const destination = audioCtx.createMediaStreamDestination();
+        destination.channelCount = 1;
 
-      const source = audioCtx.createMediaElementSource(audio);
-      source.connect(destination);
-      const track = destination.stream.getAudioTracks()[0];
-      if (constraints) {
-        await track.applyConstraints(constraints).catch((err) => log.catch(err));
-      }
+        const source = audioCtx.createMediaElementSource(audio);
+        source.connect(destination);
+        const track = destination.stream.getAudioTracks()[0];
+        if (constraints) {
+          await track.applyConstraints(constraints).catch((err) => log.catch(err));
+        }
 
-      // Also connect to speakers so audio is audible.
-      setStream({
-        audio,
-        stream: destination.stream,
-        track: destination.stream.getAudioTracks()[0],
-      });
+        // Also connect to speakers so audio is audible.
+        setStream({
+          audio,
+          stream: destination.stream,
+          track: destination.stream.getAudioTracks()[0],
+        });
+      } catch (err) {
+        log.catch(err);
+      }
     });
 
     return () => {

package/src/hooks/useQueueModelAdapter.ts

@@ -12,7 +12,7 @@ import { type ChunkRenderer, SerializationModel } from '../model';
 /**
  * Model adapter for a queue.
  */
-export const useQueueModelAdapter = <T extends Obj.Any>(
+export const useQueueModelAdapter = <T extends Obj.Unknown>(
   renderer: ChunkRenderer<T>,
   queue: Queue<T> | undefined,
   initialChunks: T[] = [],