@dxos/plugin-transcription 0.8.3 → 0.8.4-main.03d5cd7b56

package/src/capabilities/transcriber.ts

@@ -2,10 +2,13 @@
 // Copyright 2025 DXOS.org
 //
 
-import { contributes, type PluginContext } from '@dxos/app-framework';
+import * as Effect from 'effect/Effect';
+
+import { Capabilities, Capability } from '@dxos/app-framework';
 import { ClientCapabilities } from '@dxos/plugin-client';
 
-import { TranscriptionCapabilities } from './capabilities';
+import { TranscriptionCapabilities } from '#types';
+
 import { MediaStreamRecorder, Transcriber, TranscriptionManager } from '../transcriber';
 
 // TODO(burdon): Move to config?
@@ -29,48 +32,57 @@ const TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
 /**
  * Records audio while user is speaking and transcribes it after user is done speaking.
  */
-export default (context: PluginContext) => {
-  const getTranscriber: TranscriptionCapabilities.GetTranscriber = ({
-    audioStreamTrack,
-    onSegments,
-    transcriberConfig,
-    recorderConfig,
-  }) => {
-    // Initialize audio transcription.
-    return new Transcriber({
-      config: {
-        transcribeAfterChunksAmount: TRANSCRIBE_AFTER_CHUNKS_AMOUNT,
-        prefixBufferChunksAmount: PREFIXED_CHUNKS_AMOUNT,
-        ...transcriberConfig,
-      },
-      recorder: new MediaStreamRecorder({
-        mediaStreamTrack: audioStreamTrack,
+export default Capability.makeModule(
+  Effect.fnUntraced(function* () {
+    // Get context for lazy capability access in callbacks.
+    const capabilities = yield* Capability.Service;
+    const registry = yield* Capability.get(Capabilities.AtomRegistry);
+
+    const getTranscriber: TranscriptionCapabilities.GetTranscriber = ({
+      audioStreamTrack,
+      onSegments,
+      transcriberConfig,
+      recorderConfig,
+      transcribe,
+    }) => {
+      // Initialize audio transcription.
+      return new Transcriber({
         config: {
-          interval: RECORD_INTERVAL,
-          ...recorderConfig,
+          transcribeAfterChunksAmount: TRANSCRIBE_AFTER_CHUNKS_AMOUNT,
+          prefixBufferChunksAmount: PREFIXED_CHUNKS_AMOUNT,
+          ...transcriberConfig,
         },
-      }),
-      onSegments,
-    });
-  };
+        recorder: new MediaStreamRecorder({
+          mediaStreamTrack: audioStreamTrack,
+          config: {
+            interval: RECORD_INTERVAL,
+            ...recorderConfig,
+          },
+        }),
+        onSegments,
+        transcribe,
+      });
+    };
 
-  const getTranscriptionManager: TranscriptionCapabilities.GetTranscriptionManager = ({ messageEnricher }) => {
-    const client = context.getCapability(ClientCapabilities.Client);
-    const transcriptionManager = new TranscriptionManager({
-      edgeClient: client.edge,
-      messageEnricher,
-    });
+    const getTranscriptionManager: TranscriptionCapabilities.GetTranscriptionManager = ({ messageEnricher }) => {
+      const client = capabilities.get(ClientCapabilities.Client);
+      const transcriptionManager = new TranscriptionManager({
+        edgeClient: client.edge.http,
+        messageEnricher,
+        registry,
+      });
 
-    const identity = client.halo.identity.get();
-    if (identity) {
-      transcriptionManager.setIdentityDid(identity.did);
-    }
+      const identity = client.halo.identity.get();
+      if (identity) {
+        transcriptionManager.setIdentityDid(identity.did);
+      }
 
-    return transcriptionManager;
-  };
+      return transcriptionManager;
+    };
 
-  return [
-    contributes(TranscriptionCapabilities.Transcriber, getTranscriber),
-    contributes(TranscriptionCapabilities.TranscriptionManager, getTranscriptionManager),
-  ];
-};
+    return [
+      Capability.contributes(TranscriptionCapabilities.Transcriber, getTranscriber),
+      Capability.contributes(TranscriptionCapabilities.TranscriptionManager, getTranscriptionManager),
+    ];
+  }),
+);

package/src/components/{Transcript/Transcript.stories.tsx → Transcription/Transcription.stories.tsx}

@@ -2,59 +2,56 @@
 // Copyright 2025 DXOS.org
 //
 
-import '@dxos-theme';
-
-import { type Meta, type StoryObj } from '@storybook/react';
+import { type Meta, type StoryObj } from '@storybook/react-vite';
+import * as Effect from 'effect/Effect';
+import type * as Types from 'effect/Types';
 import React, { type FC, useEffect, useMemo, useState } from 'react';
 
-import { IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
 import { withPluginManager } from '@dxos/app-framework/testing';
 import { Key } from '@dxos/echo';
-import { ClientPlugin } from '@dxos/plugin-client';
-import { PreviewPlugin } from '@dxos/plugin-preview';
-import { SpacePlugin } from '@dxos/plugin-space';
-import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
-import { ThemePlugin } from '@dxos/plugin-theme';
-import { faker } from '@dxos/random';
-import { useMembers, useSpace } from '@dxos/react-client/echo';
+import { ClientPlugin } from '@dxos/plugin-client/plugin';
+import { initializeIdentity } from '@dxos/plugin-client/testing';
+import { PreviewPlugin } from '@dxos/plugin-preview/testing';
+import { StorybookPlugin, corePlugins } from '@dxos/plugin-testing';
+import { random } from '@dxos/random';
+import { useMembers, useSpaces } from '@dxos/react-client/echo';
 import { IconButton, Toolbar } from '@dxos/react-ui';
-import { SyntaxHighlighter } from '@dxos/react-ui-syntax-highlighter';
-import { defaultTx } from '@dxos/react-ui-theme';
-import { DataType } from '@dxos/schema';
-import { Testing } from '@dxos/schema/testing';
-import { withLayout } from '@dxos/storybook-utils';
-
-import { renderMarkdown, Transcript, type TranscriptProps } from './Transcript';
-import { useQueueModelAdapter } from '../../hooks';
-import { SerializationModel } from '../../model';
+import { JsonHighlighter } from '@dxos/react-ui-syntax-highlighter';
+import { withLayout } from '@dxos/react-ui/testing';
+import { TestSchema } from '@dxos/schema/testing';
+import { type ContentBlock, type Message, Organization, Person } from '@dxos/types';
+
+import { useQueueModelAdapter } from '#hooks';
 import {
   MessageBuilder,
   TestItem,
   useTestTranscriptionQueue,
   useTestTranscriptionQueueWithEntityExtraction,
-} from '../../testing';
-import translations from '../../translations';
+} from '#testing';
+import { translations } from '#translations';
 
-faker.seed(1);
+import { SerializationModel } from '../../model';
+import { renderByline } from '../../util';
+import { Transcription, type TranscriptionProps } from './Transcription';
+
+random.seed(1);
 
 /**
  * Story wrapper with test controls.
  */
 const TranscriptContainer: FC<
-  TranscriptProps & {
+  TranscriptionProps & {
     running: boolean;
     onRunningChange: (running: boolean) => void;
     onReset?: () => void;
   }
-> = ({ space, model, running, onRunningChange, onReset }) => {
+> = ({ model, running, onRunningChange, onReset }) => {
   return (
     <div className='grid grid-rows-[1fr_40px] grow divide-y divide-separator'>
-      <Transcript space={space} model={model} />
+      <Transcription model={model} />
       <div className='grid grid-cols-[1fr_16rem] overflow-hidden'>
         <div className='flex items-center'>
-          <SyntaxHighlighter language='json' className='text-sm'>
-            {JSON.stringify(model.toJSON())}
-          </SyntaxHighlighter>
+          <JsonHighlighter data={model.toJSON()} indent={0} classNames='text-sm' />
         </div>
         <Toolbar.Root classNames='justify-end'>
           <IconButton
@@ -69,20 +66,24 @@ const TranscriptContainer: FC<
   );
 };
 
-type StoryProps = { messages?: DataType.Message[] } & Pick<TranscriptProps, 'ignoreAttention' | 'attendableId'>;
+type DefaultStoryProps = { messages?: Message.Message[] } & Pick<
+  TranscriptionProps,
+  'ignoreAttention' | 'attendableId'
+>;
 
 /**
  * Basic story mutates array of messages.
  */
-const BasicStory = ({ messages: initialMessages = [], ...props }: StoryProps) => {
+const BasicStory = ({ messages: initialMessages = [], ...props }: DefaultStoryProps) => {
   const [reset, setReset] = useState({});
   const builder = useMemo(() => new MessageBuilder(), []);
   const model = useMemo(
-    () => new SerializationModel<DataType.Message>(renderMarkdown([]), initialMessages),
+    () => new SerializationModel<Message.Message>(renderByline([]), initialMessages),
     [initialMessages, reset],
   );
   const [running, setRunning] = useState(true);
-  const [currentMessage, setCurrentMessage] = useState<DataType.Message | null>(null);
+  // Use mutable type for local message building (not ECHO-managed).
+  const [currentMessage, setCurrentMessage] = useState<Types.Mutable<Message.Message> | null>(null);
   useEffect(() => {
     if (!running) {
       return;
@@ -103,7 +104,7 @@ const BasicStory = ({ messages: initialMessages = [], ...props }: StoryProps) =>
         return;
       }
 
-      currentMessage.blocks.push(builder.createBlock());
+      (currentMessage.blocks as ContentBlock.Any[]).push(builder.createBlock());
       model.updateChunk(currentMessage);
     }, 3_000);
 
@@ -135,34 +136,27 @@ const QueueStory = ({
   queueId,
   onReset,
   ...props
-}: StoryProps & { queueId: Key.ObjectId; onReset: () => void }) => {
+}: DefaultStoryProps & { queueId: Key.ObjectId; onReset: () => void }) => {
   const [running, setRunning] = useState(true);
-  const space = useSpace();
-  const members = useMembers(space?.key).map((member) => member.identity);
+  const [space] = useSpaces();
+  const members = useMembers(space?.id).map((member) => member.identity);
   const queue = useTestTranscriptionQueue(space, queueId, running, 2_000);
-  const model = useQueueModelAdapter(renderMarkdown(members), queue, initialMessages);
+  const model = useQueueModelAdapter(renderByline(members), queue, initialMessages);
 
   return (
-    <TranscriptContainer
-      space={space}
-      model={model}
-      running={running}
-      onRunningChange={setRunning}
-      onReset={onReset}
-      {...props}
-    />
+    <TranscriptContainer model={model} running={running} onRunningChange={setRunning} onReset={onReset} {...props} />
   );
 };
 
 // TODO(burdon): Reconcile with QueueStory.
 const EntityExtractionQueueStory = () => {
   const [running, setRunning] = useState(true);
-  const space = useSpace();
+  const [space] = useSpaces();
   const members = useMembers(space?.key).map((member) => member.identity);
   const queue = useTestTranscriptionQueueWithEntityExtraction(space, undefined, running, 2_000);
-  const model = useQueueModelAdapter(renderMarkdown(members), queue, []);
+  const model = useQueueModelAdapter(renderByline(members), queue, []);
 
-  return <TranscriptContainer space={space} model={model} running={running} onRunningChange={setRunning} />;
+  return <TranscriptContainer model={model} running={running} onRunningChange={setRunning} />;
 };
 
 /**
@@ -178,46 +172,54 @@ const QueueStoryWrapper = () => {
   return <QueueStory key={key} queueId={queueId} onReset={handleReset} />;
 };
 
-const meta: Meta<typeof QueueStory> = {
-  title: 'plugins/plugin-transcription/Transcript',
+const meta = {
+  title: 'plugins/plugin-transcription/components/Transcription',
   decorators: [
+    withLayout({ layout: 'fullscreen' }),
     withPluginManager({
       plugins: [
-        ThemePlugin({ tx: defaultTx }),
-        StorybookLayoutPlugin(),
+        ...corePlugins(),
+        StorybookPlugin({}),
         ClientPlugin({
-          types: [TestItem, Testing.DocumentType, DataType.Person, DataType.Organization],
-          onClientInitialized: async (_, client) => {
-            await client.halo.createIdentity();
-          },
+          types: [TestItem, TestSchema.DocumentType, Person.Person, Organization.Organization],
+          onClientInitialized: ({ client }) =>
+            Effect.gen(function* () {
+              yield* initializeIdentity(client);
+            }),
         }),
+
         PreviewPlugin(),
-        SpacePlugin(),
-        SettingsPlugin(),
-        IntentPlugin(),
       ],
     }),
-    withLayout({ fullscreen: true }),
   ],
   parameters: {
+    layout: 'fullscreen',
     translations,
   },
-};
+} satisfies Meta;
 
 export default meta;
 
-type Story = StoryObj<typeof QueueStory>;
+const DefaultStory = (props: DefaultStoryProps) => {
+  const [messages, setMessages] = useState<Message.Message[]>([]);
+  useEffect(() => {
+    void Promise.all(Array.from({ length: 10 }, () => MessageBuilder.singleton.createMessage())).then(setMessages);
+  }, []);
+  if (messages.length === 0) {
+    return <div>Loading messages...</div>;
+  }
+  return <BasicStory {...props} messages={messages} />;
+};
 
-export const Default: Story = {
-  render: BasicStory,
+export const Default: StoryObj<typeof DefaultStory> = {
+  render: DefaultStory,
   args: {
     ignoreAttention: true,
     attendableId: 'story',
-    messages: await Promise.all(Array.from({ length: 10 }, () => MessageBuilder.singleton.createMessage())),
   },
 };
 
-export const Empty: Story = {
+export const Empty: StoryObj<typeof BasicStory> = {
   render: BasicStory,
   args: {
     ignoreAttention: true,
@@ -225,7 +227,7 @@ export const Empty: Story = {
   },
 };
 
-export const WithQueue: Story = {
+export const WithQueue: StoryObj<typeof QueueStoryWrapper> = {
   render: QueueStoryWrapper,
   args: {
     ignoreAttention: true,
@@ -233,10 +235,13 @@ export const WithQueue: Story = {
   },
 };
 
-export const WithEntityExtractionQueue: Story = {
-  render: EntityExtractionQueueStory,
-  args: {
-    ignoreAttention: true,
-    attendableId: 'story',
-  },
-};
+// NOTE: We are running out of free quota on hugging face entity extraction.
+// TODO(mykola): Fix AI service in entity extraction function.
+// TODO(mykola): Fix hugging face quota issues.
+// export const WithEntityExtractionQueue: StoryObj<typeof EntityExtractionQueueStory> = {
+//   render: EntityExtractionQueueStory,
+//   args: {
+//     ignoreAttention: true,
+//     attendableId: 'story',
+//   },
+// };

package/src/components/Transcription/Transcription.tsx

@@ -0,0 +1,64 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import React from 'react';
+
+import { useThemeContext } from '@dxos/react-ui';
+import { useTextEditor } from '@dxos/react-ui-editor';
+import { type Message, type Transcript } from '@dxos/types';
+import {
+  autoScroll,
+  createBasicExtensions,
+  createMarkdownExtensions,
+  createThemeExtensions,
+  decorateMarkdown,
+  documentSlots,
+  preview,
+} from '@dxos/ui-editor';
+import { composable, composableProps } from '@dxos/ui-theme';
+import { isTruthy } from '@dxos/util';
+
+import { type SerializationModel } from '../../model';
+import { renderByline } from '../../util';
+import { transcription } from './transcription-extension';
+
+export { renderByline };
+
+export type TranscriptionProps = {
+  transcript?: Transcript.Transcript;
+  model: SerializationModel<Message.Message>;
+  // TODO(burdon): Are these still in use?
+  attendableId?: string;
+  ignoreAttention?: boolean;
+};
+
+/**
+ * Transcript component implemented using the editor.
+ */
+export const Transcription = composable<HTMLDivElement, TranscriptionProps>(
+  ({ transcript: object, model, children, ...props }, forwardedRef) => {
+    const { themeMode } = useThemeContext();
+    const { parentRef } = useTextEditor(() => {
+      return {
+        extensions: [
+          createBasicExtensions({ readOnly: true, lineWrapping: true, search: true }),
+          createMarkdownExtensions(),
+          createThemeExtensions({ themeMode, slots: documentSlots }),
+          decorateMarkdown(),
+          preview(),
+          transcription({ model, started: object?.started ? new Date(object.started) : undefined }),
+          autoScroll(),
+        ].filter(isTruthy),
+      };
+    }, [model]);
+
+    return (
+      <div
+        {...composableProps(props, { classNames: 'dx-container' })}
+        ref={parentRef}
+        data-popover-collision-boundary={true /* TODO(thure): Make this a constant and document it. */}
+      />
+    );
+  },
+);

package/src/components/{Transcript → Transcription}/index.ts

@@ -2,4 +2,4 @@
 // Copyright 2025 DXOS.org
 //
 
-export * from './Transcript';
+export * from './Transcription';

package/src/components/{Transcript/transcript-extension.ts → Transcription/transcription-extension.ts}

@@ -8,24 +8,22 @@ import { format } from 'date-fns/format';
 import { intervalToDuration } from 'date-fns/intervalToDuration';
 
 import { type CleanupFn, addEventListener, combine } from '@dxos/async';
-import { type RenderCallback } from '@dxos/react-ui-editor';
-import { type DataType } from '@dxos/schema';
+import { type Message } from '@dxos/types';
 
 import { DocumentAdapter, type SerializationModel } from '../../model';
 
 /**
  * Data structure that maps Chunks queue to lines with transcript state.
  */
-export type TranscriptOptions = {
-  model: SerializationModel<DataType.Message>;
+export type TranscriptionOptions = {
+  model: SerializationModel<Message.Message>;
   started?: Date;
-  renderButton?: RenderCallback<{ onClick: () => void }>;
 };
 
 /**
  * Scrolling transcript with timestamps.
  */
-export const transcript = ({ model, started, renderButton }: TranscriptOptions): Extension => {
+export const transcription = ({ model, started }: TranscriptionOptions): Extension => {
   return [
     // Show timestamps in the gutter.
     gutter({
@@ -38,7 +36,7 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
           let line = view.state.doc.lineAt(from);
           while (line.from <= to) {
             const block = model.getChunkAtLine(line.number)?.blocks[0];
-            const timestamp = block?.type === 'transcription' && block.started;
+            const timestamp = block?._tag === 'transcript' && block.started;
             if (timestamp) {
               builder.add(line.from, line.from, new TimestampMarker(line.number, new Date(timestamp), start));
             }
@@ -90,22 +88,6 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
             });
           };
 
-          // Scroll button.
-          if (renderButton) {
-            this._controls = document.createElement('div');
-            this._controls.classList.add('cm-controls', 'transition-opacity', 'duration-300', 'opacity-0');
-            view.dom.appendChild(this._controls);
-            renderButton(
-              this._controls,
-              {
-                onClick: () => {
-                  scrollToBottom(false);
-                },
-              },
-              view,
-            );
-          }
-
           // Event listeners.
           this._cleanup = combine(
             addEventListener(view.scrollDOM, 'scroll', () => {
@@ -131,7 +113,7 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
           );
         }
 
-        update(update: ViewUpdate) {
+        update(_update: ViewUpdate) {
           // Initial sync.
           if (!this._initialized) {
             this._initialized = true;
@@ -149,27 +131,12 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
     ),
 
     EditorView.theme({
-      '.cm-scroller': {
-        overflowY: 'scroll',
-      },
-      '.cm-hide-scrollbar': {
-        scrollbarWidth: 'none',
-        '-ms-overflow-style': 'none',
-      },
-      '.cm-hide-scrollbar::-webkit-scrollbar': {
-        display: 'none',
-      },
-      '.cm-controls': {
-        position: 'absolute',
-        bottom: '0.5rem',
-        right: '0.5rem',
-        zIndex: 1000,
-      },
-      '.cm-line': {
-        paddingRight: '1rem',
+      '.cm-gutters': {
+        backgroundColor: 'var(--color-base-surface)',
       },
       '.cm-timestamp-gutter': {
         width: '6rem',
+        paddingLeft: '0.5rem',
         paddingRight: '1rem',
       },
       '.cm-timestamp-gutter > .cm-gutterElement > div': {
@@ -193,13 +160,13 @@ class TimestampMarker extends GutterMarker {
     super();
   }
 
-  override eq(other: this): boolean {
-    return other._timestamp === this._timestamp;
+  override eq(other: this) {
+    return this._timestamp === other._timestamp;
   }
 
-  override toDOM(view: EditorView): HTMLDivElement {
+  override toDOM(view: EditorView) {
     const el = document.createElement('div');
-    el.className = 'text-sm text-subdued hover:bg-hoverSurface cursor-pointer';
+    el.className = 'text-sm text-subdued hover:bg-hover-surface cursor-pointer';
     el.textContent = formatTimestamp(this._timestamp, this._started);
     // TODO(burdon): Click to bookmark or copy hyperlink.
     el.onclick = () => {
@@ -213,12 +180,12 @@ class TimestampMarker extends GutterMarker {
   }
 }
 
-const getStartTime = (started?: Date, message?: DataType.Message): Date | undefined => {
+const getStartTime = (started?: Date, message?: Message.Message): Date | undefined => {
   if (started) {
     return started;
   }
 
-  if (message?.blocks[0]?.type === 'transcription' && message.blocks[0].started) {
+  if (message?.blocks[0]?._tag === 'transcript' && message.blocks[0].started) {
     return new Date(message.blocks[0].started);
   }
 

package/src/components/index.ts

@@ -2,9 +2,4 @@
 // Copyright 2025 DXOS.org
 //
 
-import { lazy } from 'react';
-
-export * from './Transcript';
-export * from './TranscriptContainer';
-
-export const TranscriptContainer = lazy(() => import('./TranscriptContainer'));
+export * from './Transcription';

package/src/containers/TranscriptionContainer/TranscriptionContainer.tsx

@@ -0,0 +1,35 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import React from 'react';
+
+import { type AppSurface } from '@dxos/app-toolkit/ui';
+import { Feed, Obj } from '@dxos/echo';
+import { useMembers, useQueue } from '@dxos/react-client/echo';
+import { Panel } from '@dxos/react-ui';
+import { type Message, type Transcript } from '@dxos/types';
+
+import { Transcription } from '#components';
+import { useQueueModelAdapter } from '#hooks';
+
+import { renderByline } from '../../util';
+
+export type TranscriptionContainerProps = AppSurface.ObjectArticleProps<Transcript.Transcript>;
+
+export const TranscriptionContainer = ({ role, subject: transcript, attendableId }: TranscriptionContainerProps) => {
+  const db = Obj.getDatabase(transcript);
+  const members = useMembers(db?.spaceId).map((member) => member.identity);
+  const feed = transcript.feed.target;
+  const queueDxn = feed ? Feed.getQueueDxn(feed) : undefined;
+  const queue = useQueue<Message.Message>(queueDxn, { pollInterval: 1_000 });
+  const model = useQueueModelAdapter(renderByline(members), queue);
+
+  return (
+    <Panel.Root role={role}>
+      <Panel.Content asChild>
+        <Transcription attendableId={attendableId} model={model} transcript={transcript} />
+      </Panel.Content>
+    </Panel.Root>
+  );
+};

package/src/containers/TranscriptionContainer/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export { TranscriptionContainer as default } from './TranscriptionContainer';

package/src/containers/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { type ComponentType, lazy } from 'react';
+
+export const TranscriptionContainer: ComponentType<any> = lazy(() => import('./TranscriptionContainer'));