@dxos/plugin-transcription 0.8.3 → 0.8.4-main.1068cf700f

package/src/components/Transcript/Transcript.stories.tsx

@@ -2,29 +2,24 @@
 // Copyright 2025 DXOS.org
 //
 
-import '@dxos-theme';
-
-import { type Meta, type StoryObj } from '@storybook/react';
+import { type Meta, type StoryObj } from '@storybook/react-vite';
+import * as Effect from 'effect/Effect';
+import type * as Types from 'effect/Types';
 import React, { type FC, useEffect, useMemo, useState } from 'react';
 
-import { IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
 import { withPluginManager } from '@dxos/app-framework/testing';
 import { Key } from '@dxos/echo';
 import { ClientPlugin } from '@dxos/plugin-client';
 import { PreviewPlugin } from '@dxos/plugin-preview';
-import { SpacePlugin } from '@dxos/plugin-space';
-import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
-import { ThemePlugin } from '@dxos/plugin-theme';
+import { StorybookPlugin, corePlugins } from '@dxos/plugin-testing';
 import { faker } from '@dxos/random';
 import { useMembers, useSpace } from '@dxos/react-client/echo';
 import { IconButton, Toolbar } from '@dxos/react-ui';
+import { withTheme } from '@dxos/react-ui/testing';
 import { SyntaxHighlighter } from '@dxos/react-ui-syntax-highlighter';
-import { defaultTx } from '@dxos/react-ui-theme';
-import { DataType } from '@dxos/schema';
-import { Testing } from '@dxos/schema/testing';
-import { withLayout } from '@dxos/storybook-utils';
+import { TestSchema } from '@dxos/schema/testing';
+import { type ContentBlock, type Message, Organization, Person } from '@dxos/types';
 
-import { renderMarkdown, Transcript, type TranscriptProps } from './Transcript';
 import { useQueueModelAdapter } from '../../hooks';
 import { SerializationModel } from '../../model';
 import {
@@ -33,7 +28,10 @@ import {
   useTestTranscriptionQueue,
   useTestTranscriptionQueueWithEntityExtraction,
 } from '../../testing';
-import translations from '../../translations';
+import { translations } from '../../translations';
+import { renderByline } from '../../util';
+
+import { TranscriptView, type TranscriptViewProps } from './Transcript';
 
 faker.seed(1);
 
@@ -41,15 +39,15 @@ faker.seed(1);
  * Story wrapper with test controls.
  */
 const TranscriptContainer: FC<
-  TranscriptProps & {
+  TranscriptViewProps & {
     running: boolean;
     onRunningChange: (running: boolean) => void;
     onReset?: () => void;
   }
-> = ({ space, model, running, onRunningChange, onReset }) => {
+> = ({ model, running, onRunningChange, onReset }) => {
   return (
     <div className='grid grid-rows-[1fr_40px] grow divide-y divide-separator'>
-      <Transcript space={space} model={model} />
+      <TranscriptView model={model} />
       <div className='grid grid-cols-[1fr_16rem] overflow-hidden'>
         <div className='flex items-center'>
           <SyntaxHighlighter language='json' className='text-sm'>
@@ -69,7 +67,7 @@ const TranscriptContainer: FC<
   );
 };
 
-type StoryProps = { messages?: DataType.Message[] } & Pick<TranscriptProps, 'ignoreAttention' | 'attendableId'>;
+type StoryProps = { messages?: Message.Message[] } & Pick<TranscriptViewProps, 'ignoreAttention' | 'attendableId'>;
 
 /**
  * Basic story mutates array of messages.
@@ -78,11 +76,12 @@ const BasicStory = ({ messages: initialMessages = [], ...props }: StoryProps) =>
   const [reset, setReset] = useState({});
   const builder = useMemo(() => new MessageBuilder(), []);
   const model = useMemo(
-    () => new SerializationModel<DataType.Message>(renderMarkdown([]), initialMessages),
+    () => new SerializationModel<Message.Message>(renderByline([]), initialMessages),
     [initialMessages, reset],
   );
   const [running, setRunning] = useState(true);
-  const [currentMessage, setCurrentMessage] = useState<DataType.Message | null>(null);
+  // Use mutable type for local message building (not ECHO-managed).
+  const [currentMessage, setCurrentMessage] = useState<Types.Mutable<Message.Message> | null>(null);
   useEffect(() => {
     if (!running) {
       return;
@@ -103,7 +102,7 @@ const BasicStory = ({ messages: initialMessages = [], ...props }: StoryProps) =>
         return;
       }
 
-      currentMessage.blocks.push(builder.createBlock());
+      (currentMessage.blocks as ContentBlock.Any[]).push(builder.createBlock());
       model.updateChunk(currentMessage);
     }, 3_000);
 
@@ -138,19 +137,12 @@ const QueueStory = ({
 }: StoryProps & { queueId: Key.ObjectId; onReset: () => void }) => {
   const [running, setRunning] = useState(true);
   const space = useSpace();
-  const members = useMembers(space?.key).map((member) => member.identity);
+  const members = useMembers(space?.id).map((member) => member.identity);
   const queue = useTestTranscriptionQueue(space, queueId, running, 2_000);
-  const model = useQueueModelAdapter(renderMarkdown(members), queue, initialMessages);
+  const model = useQueueModelAdapter(renderByline(members), queue, initialMessages);
 
   return (
-    <TranscriptContainer
-      space={space}
-      model={model}
-      running={running}
-      onRunningChange={setRunning}
-      onReset={onReset}
-      {...props}
-    />
+    <TranscriptContainer model={model} running={running} onRunningChange={setRunning} onReset={onReset} {...props} />
   );
 };
 
@@ -160,9 +152,9 @@ const EntityExtractionQueueStory = () => {
   const space = useSpace();
   const members = useMembers(space?.key).map((member) => member.identity);
   const queue = useTestTranscriptionQueueWithEntityExtraction(space, undefined, running, 2_000);
-  const model = useQueueModelAdapter(renderMarkdown(members), queue, []);
+  const model = useQueueModelAdapter(renderByline(members), queue, []);
 
-  return <TranscriptContainer space={space} model={model} running={running} onRunningChange={setRunning} />;
+  return <TranscriptContainer model={model} running={running} onRunningChange={setRunning} />;
 };
 
 /**
@@ -178,46 +170,55 @@ const QueueStoryWrapper = () => {
   return <QueueStory key={key} queueId={queueId} onReset={handleReset} />;
 };
 
-const meta: Meta<typeof QueueStory> = {
+const meta = {
   title: 'plugins/plugin-transcription/Transcript',
   decorators: [
+    withTheme(),
     withPluginManager({
       plugins: [
-        ThemePlugin({ tx: defaultTx }),
-        StorybookLayoutPlugin(),
+        ...corePlugins(),
+        StorybookPlugin({}),
         ClientPlugin({
-          types: [TestItem, Testing.DocumentType, DataType.Person, DataType.Organization],
-          onClientInitialized: async (_, client) => {
-            await client.halo.createIdentity();
-          },
+          types: [TestItem, TestSchema.DocumentType, Person.Person, Organization.Organization],
+          onClientInitialized: ({ client }) =>
+            Effect.gen(function* () {
+              yield* Effect.promise(() => client.halo.createIdentity());
+            }),
         }),
+
         PreviewPlugin(),
-        SpacePlugin(),
-        SettingsPlugin(),
-        IntentPlugin(),
       ],
     }),
-    withLayout({ fullscreen: true }),
   ],
   parameters: {
+    layout: 'fullscreen',
     translations,
   },
-};
+} satisfies Meta;
 
 export default meta;
 
-type Story = StoryObj<typeof QueueStory>;
+// TODO(wittjosiah): ECHO objects don't work when passed via Storybook args.
+const DefaultStory = (props: StoryProps) => {
+  const [messages, setMessages] = useState<Message.Message[]>([]);
+  useEffect(() => {
+    void Promise.all(Array.from({ length: 10 }, () => MessageBuilder.singleton.createMessage())).then(setMessages);
+  }, []);
+  if (messages.length === 0) {
+    return <div>Loading messages...</div>;
+  }
+  return <BasicStory {...props} messages={messages} />;
+};
 
-export const Default: Story = {
-  render: BasicStory,
+export const Default: StoryObj<typeof DefaultStory> = {
+  render: DefaultStory,
   args: {
     ignoreAttention: true,
     attendableId: 'story',
-    messages: await Promise.all(Array.from({ length: 10 }, () => MessageBuilder.singleton.createMessage())),
   },
 };
 
-export const Empty: Story = {
+export const Empty: StoryObj<typeof BasicStory> = {
   render: BasicStory,
   args: {
     ignoreAttention: true,
@@ -225,7 +226,7 @@ export const Empty: Story = {
   },
 };
 
-export const WithQueue: Story = {
+export const WithQueue: StoryObj<typeof QueueStoryWrapper> = {
   render: QueueStoryWrapper,
   args: {
     ignoreAttention: true,
@@ -233,10 +234,13 @@ export const WithQueue: Story = {
   },
 };
 
-export const WithEntityExtractionQueue: Story = {
-  render: EntityExtractionQueueStory,
-  args: {
-    ignoreAttention: true,
-    attendableId: 'story',
-  },
-};
+// NOTE: We are running out of free quota on hugging face entity extraction.
+// TODO(mykola): Fix AI service in entity extraction function.
+// TODO(mykola): Fix hugging face quota issues.
+// export const WithEntityExtractionQueue: StoryObj<typeof EntityExtractionQueueStory> = {
+//   render: EntityExtractionQueueStory,
+//   args: {
+//     ignoreAttention: true,
+//     attendableId: 'story',
+//   },
+// };

package/src/components/Transcript/Transcript.tsx

@@ -4,57 +4,32 @@
 
 import React from 'react';
 
-import { type Space } from '@dxos/react-client/echo';
-import { type Identity } from '@dxos/react-client/halo';
-import { IconButton, type ThemedClassName, useThemeContext } from '@dxos/react-ui';
+import { type ThemedClassName, useThemeContext } from '@dxos/react-ui';
+import { useTextEditor } from '@dxos/react-ui-editor';
+import { type Message, type Transcript } from '@dxos/types';
 import {
+  autoScroll,
   createBasicExtensions,
   createMarkdownExtensions,
-  createRenderer,
   createThemeExtensions,
   decorateMarkdown,
-  editorWidth,
+  editorSlots,
   preview,
-  useTextEditor,
-} from '@dxos/react-ui-editor';
-import { mx } from '@dxos/react-ui-theme';
-import { type DataType } from '@dxos/schema';
-import { isNotFalsy } from '@dxos/util';
+} from '@dxos/ui-editor';
+import { mx } from '@dxos/ui-theme';
+import { isTruthy } from '@dxos/util';
 
-import { transcript } from './transcript-extension';
 import { type SerializationModel } from '../../model';
-import { type TranscriptType } from '../../types';
+import { renderByline } from '../../util';
 
-export const renderMarkdown =
-  (identities: Identity[]) =>
-  (message: DataType.Message, index: number, debug = false): string[] => {
-    if (message.sender.role === 'assistant') {
-      // Start/stop block.
-      return [message.blocks.find((block) => block.type === 'transcription')?.text ?? '', ''];
-    }
+import { transcript } from './transcript-extension';
 
-    // TODO(burdon): Use link/reference markup for users (with popover).
-    // TODO(burdon): Color and avatar.
-    const identity = identities.find((identity) => identity.did === message.sender.identityDid);
-    const name =
-      identity?.profile?.displayName ??
-      message.sender.contact?.target?.fullName ??
-      message.sender.name ??
-      message.sender.email ??
-      message.sender.identityDid;
-    const blocks = message.blocks.filter((block) => block.type === 'transcription');
-    return [
-      `###### ${name}` + (debug ? ` [${index}]:${message.id}` : ''),
-      blocks.map((block) => block.text.trim()).join(' '),
-      '',
-    ];
-  };
+export { renderByline };
 
-export type TranscriptProps = ThemedClassName<{
-  transcript?: TranscriptType;
-  space?: Space;
-  model: SerializationModel<DataType.Message>;
-  // TODO(wittjosiah): Move to container.
+export type TranscriptViewProps = ThemedClassName<{
+  transcript?: Transcript.Transcript;
+  model: SerializationModel<Message.Message>;
+  // TODO(burdon): Are these still in use?
   attendableId?: string;
   ignoreAttention?: boolean;
 }>;
@@ -62,39 +37,30 @@ export type TranscriptProps = ThemedClassName<{
 /**
  * Transcript component implemented using the editor.
  */
-export const Transcript = ({
-  classNames,
-  transcript: object,
-  space,
-  model,
-  attendableId,
-  ignoreAttention,
-}: TranscriptProps) => {
+export const TranscriptView = ({ classNames, transcript: object, model }: TranscriptViewProps) => {
   const { themeMode } = useThemeContext();
   const { parentRef } = useTextEditor(() => {
     return {
       extensions: [
-        createBasicExtensions({ readOnly: true, lineWrapping: true }),
-        createMarkdownExtensions({ themeMode }),
-        createThemeExtensions({ themeMode }),
+        createBasicExtensions({ readOnly: true, lineWrapping: true, search: true }),
+        createMarkdownExtensions(),
+        createThemeExtensions({ themeMode, slots: editorSlots }),
         decorateMarkdown(),
-        space && preview(),
+        preview(),
         transcript({
           model,
           started: object?.started ? new Date(object.started) : undefined,
-          renderButton: createRenderer(({ onClick }) => (
-            <IconButton icon='ph--arrow-line-down--regular' iconOnly label='Scroll to bottom' onClick={onClick} />
-          )),
         }),
-      ].filter(isNotFalsy),
+        autoScroll(),
+      ].filter(isTruthy),
     };
-  }, [space, model]);
+  }, [model]);
 
   return (
     <div
       ref={parentRef}
-      className={mx('flex grow overflow-hidden', editorWidth, classNames)}
-      data-popover-collision-boundary={true}
+      className={mx('flex grow overflow-hidden container-max-width', classNames)}
+      data-popover-collision-boundary={true /* TODO(thure): Make this a constant and document it. */}
     />
   );
 };

package/src/components/Transcript/deprecated/OldTranscript.tsx

@@ -3,25 +3,25 @@
 //
 
 import { intervalToDuration } from 'date-fns/intervalToDuration';
-import { Schema } from 'effect';
+import * as Schema from 'effect/Schema';
 import { yieldOrContinue } from 'main-thread-scheduling';
-import React, { type FC, useCallback, useEffect, useMemo, useRef, useState, type WheelEvent } from 'react';
+import React, { type FC, type WheelEvent, useCallback, useEffect, useMemo, useRef, useState } from 'react';
 import { type OnResizeCallback, useResizeDetector } from 'react-resize-detector';
 
 import { Type } from '@dxos/echo';
 import { IconButton, type ThemedClassName, useTranslation } from '@dxos/react-ui';
 import { useAttention } from '@dxos/react-ui-attention';
 import {
+  type DxGridAxisMeta,
   type DxGridCellValue,
   type DxGridElement,
-  type DxGridAxisMeta,
   type DxGridPlaneCells,
   Grid,
   toPlaneCellIndex,
 } from '@dxos/react-ui-grid';
-import { mx } from '@dxos/react-ui-theme';
+import { mx } from '@dxos/ui-theme';
 
-import { TRANSCRIPTION_PLUGIN } from '../../../meta';
+import { meta } from '../../../meta';
 
 /**
  * Transcription fragment.
@@ -43,7 +43,7 @@ const TranscriptBlock = Schema.Struct({
   authorHue: Schema.optional(Schema.String), // TOOD(burdon): Remove.
   segments: Schema.Array(TranscriptSegment),
 }).pipe(
-  Type.Obj({
+  Type.object({
     typename: 'dxos.org/type/TranscriptBlock',
     version: '0.1.0',
   }),
@@ -115,7 +115,7 @@ const measureRows = async (
   return result;
 };
 
-export type TranscriptProps = ThemedClassName<{
+export type TranscriptViewProps = ThemedClassName<{
   blocks?: TranscriptBlock[];
   attendableId?: string;
   ignoreAttention?: boolean;
@@ -124,8 +124,8 @@ export type TranscriptProps = ThemedClassName<{
 /**
  * @@deprecated
  */
-export const Transcript: FC<TranscriptProps> = ({ classNames, blocks, attendableId, ignoreAttention }) => {
-  const { t } = useTranslation(TRANSCRIPTION_PLUGIN);
+export const Transcript: FC<TranscriptViewProps> = ({ classNames, blocks, attendableId, ignoreAttention }) => {
+  const { t } = useTranslation(meta.id);
   const [dxGrid, setDxGrid] = useState<DxGridElement | null>(null);
   const [rows, setRows] = useState<DxGridAxisMeta | undefined>(undefined);
   const [columns, setColumns] = useState<DxGridAxisMeta | undefined>(undefined);
@@ -144,7 +144,7 @@ export const Transcript: FC<TranscriptProps> = ({ classNames, blocks, attendable
     [hasAttention, ignoreAttention],
   );
 
-  const abortControllerRef = useRef<AbortController>();
+  const abortControllerRef = useRef<AbortController>(null);
 
   const handleResize = useCallback(
     async ({ entry }: { entry: { target: HTMLDivElement } | null }) => {
@@ -242,7 +242,7 @@ export const Transcript: FC<TranscriptProps> = ({ classNames, blocks, attendable
           ref={setDxGrid}
         />
       </Grid.Root>
-      <div role='none' {...{ inert: '' }} aria-hidden className={measureClasses} ref={measureRef} />
+      <div role='none' {...{ inert: true }} aria-hidden className={measureClasses} ref={measureRef} />
       <IconButton
         icon='ph--arrow-line-down--regular'
         iconOnly

package/src/components/Transcript/transcript-extension.ts

@@ -8,8 +8,7 @@ import { format } from 'date-fns/format';
 import { intervalToDuration } from 'date-fns/intervalToDuration';
 
 import { type CleanupFn, addEventListener, combine } from '@dxos/async';
-import { type RenderCallback } from '@dxos/react-ui-editor';
-import { type DataType } from '@dxos/schema';
+import { type Message } from '@dxos/types';
 
 import { DocumentAdapter, type SerializationModel } from '../../model';
 
@@ -17,15 +16,14 @@ import { DocumentAdapter, type SerializationModel } from '../../model';
  * Data structure that maps Chunks queue to lines with transcript state.
  */
 export type TranscriptOptions = {
-  model: SerializationModel<DataType.Message>;
+  model: SerializationModel<Message.Message>;
   started?: Date;
-  renderButton?: RenderCallback<{ onClick: () => void }>;
 };
 
 /**
  * Scrolling transcript with timestamps.
  */
-export const transcript = ({ model, started, renderButton }: TranscriptOptions): Extension => {
+export const transcript = ({ model, started }: TranscriptOptions): Extension => {
   return [
     // Show timestamps in the gutter.
     gutter({
@@ -38,7 +36,7 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
           let line = view.state.doc.lineAt(from);
           while (line.from <= to) {
             const block = model.getChunkAtLine(line.number)?.blocks[0];
-            const timestamp = block?.type === 'transcription' && block.started;
+            const timestamp = block?._tag === 'transcript' && block.started;
             if (timestamp) {
               builder.add(line.from, line.from, new TimestampMarker(line.number, new Date(timestamp), start));
             }
@@ -90,22 +88,6 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
             });
           };
 
-          // Scroll button.
-          if (renderButton) {
-            this._controls = document.createElement('div');
-            this._controls.classList.add('cm-controls', 'transition-opacity', 'duration-300', 'opacity-0');
-            view.dom.appendChild(this._controls);
-            renderButton(
-              this._controls,
-              {
-                onClick: () => {
-                  scrollToBottom(false);
-                },
-              },
-              view,
-            );
-          }
-
           // Event listeners.
           this._cleanup = combine(
             addEventListener(view.scrollDOM, 'scroll', () => {
@@ -131,7 +113,7 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
           );
         }
 
-        update(update: ViewUpdate) {
+        update(_update: ViewUpdate) {
           // Initial sync.
           if (!this._initialized) {
             this._initialized = true;
@@ -149,27 +131,12 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
     ),
 
     EditorView.theme({
-      '.cm-scroller': {
-        overflowY: 'scroll',
-      },
-      '.cm-hide-scrollbar': {
-        scrollbarWidth: 'none',
-        '-ms-overflow-style': 'none',
-      },
-      '.cm-hide-scrollbar::-webkit-scrollbar': {
-        display: 'none',
-      },
-      '.cm-controls': {
-        position: 'absolute',
-        bottom: '0.5rem',
-        right: '0.5rem',
-        zIndex: 1000,
-      },
-      '.cm-line': {
-        paddingRight: '1rem',
+      '.cm-gutters': {
+        backgroundColor: 'var(--dx-baseSurface)',
       },
       '.cm-timestamp-gutter': {
         width: '6rem',
+        paddingLeft: '0.5rem',
         paddingRight: '1rem',
       },
       '.cm-timestamp-gutter > .cm-gutterElement > div': {
@@ -193,11 +160,11 @@ class TimestampMarker extends GutterMarker {
     super();
   }
 
-  override eq(other: this): boolean {
-    return other._timestamp === this._timestamp;
+  override eq(other: this) {
+    return this._timestamp === other._timestamp;
   }
 
-  override toDOM(view: EditorView): HTMLDivElement {
+  override toDOM(view: EditorView) {
     const el = document.createElement('div');
     el.className = 'text-sm text-subdued hover:bg-hoverSurface cursor-pointer';
     el.textContent = formatTimestamp(this._timestamp, this._started);
@@ -213,12 +180,12 @@ class TimestampMarker extends GutterMarker {
   }
 }
 
-const getStartTime = (started?: Date, message?: DataType.Message): Date | undefined => {
+const getStartTime = (started?: Date, message?: Message.Message): Date | undefined => {
   if (started) {
     return started;
   }
 
-  if (message?.blocks[0]?.type === 'transcription' && message.blocks[0].started) {
+  if (message?.blocks[0]?._tag === 'transcript' && message.blocks[0].started) {
     return new Date(message.blocks[0].started);
   }
 

package/src/components/TranscriptContainer.tsx

@@ -2,28 +2,32 @@
 // Copyright 2024 DXOS.org
 //
 
-import React, { type FC } from 'react';
+import React from 'react';
 
-import { fullyQualifiedId, getSpace } from '@dxos/client/echo';
+import { type SurfaceComponentProps } from '@dxos/app-toolkit/ui';
+import { Obj } from '@dxos/echo';
 import { useMembers, useQueue } from '@dxos/react-client/echo';
-import { StackItem } from '@dxos/react-ui-stack';
-import { type DataType } from '@dxos/schema';
+import { Layout } from '@dxos/react-ui';
+import { type Message, type Transcript } from '@dxos/types';
 
-import { Transcript, renderMarkdown } from './Transcript';
 import { useQueueModelAdapter } from '../hooks';
-import { type TranscriptType } from '../types';
+import { renderByline } from '../util';
 
-export const TranscriptionContainer: FC<{ role: string; transcript: TranscriptType }> = ({ role, transcript }) => {
-  const attendableId = fullyQualifiedId(transcript);
-  const space = getSpace(transcript);
-  const members = useMembers(space?.key).map((member) => member.identity);
-  const queue = useQueue<DataType.Message>(transcript.queue.dxn, { pollInterval: 1_000 });
-  const model = useQueueModelAdapter(renderMarkdown(members), queue);
+import { TranscriptView } from './Transcript';
+
+export type TranscriptionContainerProps = SurfaceComponentProps<Transcript.Transcript>;
+
+export const TranscriptionContainer = ({ role, subject: transcript }: TranscriptionContainerProps) => {
+  const attendableId = Obj.getDXN(transcript).toString();
+  const db = Obj.getDatabase(transcript);
+  const members = useMembers(db?.spaceId).map((member) => member.identity);
+  const queue = useQueue<Message.Message>(transcript.queue.dxn, { pollInterval: 1_000 });
+  const model = useQueueModelAdapter(renderByline(members), queue);
 
   return (
-    <StackItem.Content role={role} classNames='container-max-width pli-2'>
-      <Transcript attendableId={attendableId} space={space} model={model} transcript={transcript} />
-    </StackItem.Content>
+    <Layout.Main role={role}>
+      <TranscriptView attendableId={attendableId} model={model} transcript={transcript} />
+    </Layout.Main>
   );
 };