@dxos/plugin-transcription 0.8.4-main.28f8d3d → 0.8.4-main.2c6827d

package/src/components/Transcript/Transcript.tsx

@@ -6,29 +6,29 @@ import React from 'react';
 
 import { type Space } from '@dxos/react-client/echo';
 import { type Identity } from '@dxos/react-client/halo';
-import { IconButton, type ThemedClassName, useThemeContext } from '@dxos/react-ui';
+import { type ThemedClassName, useThemeContext } from '@dxos/react-ui';
 import {
+  autoScroll,
   createBasicExtensions,
   createMarkdownExtensions,
-  createRenderer,
   createThemeExtensions,
   decorateMarkdown,
-  editorWidth,
+  editorSlots,
   preview,
   useTextEditor,
 } from '@dxos/react-ui-editor';
 import { mx } from '@dxos/react-ui-theme';
-import { type DataType } from '@dxos/schema';
-import { isNotFalsy } from '@dxos/util';
+import { type Message, type Transcript } from '@dxos/types';
+import { isTruthy } from '@dxos/util';
 
 import { type SerializationModel } from '../../model';
-import { type TranscriptType } from '../../types';
 
 import { transcript } from './transcript-extension';
 
-export const renderMarkdown =
+// TODO(thure): Move this?
+export const renderByline =
   (identities: Identity[]) =>
-  (message: DataType.Message, index: number, debug = false): string[] => {
+  (message: Message.Message, index: number, debug = false): string[] => {
     if (message.sender.role === 'assistant') {
       // Start/stop block.
       return [message.blocks.find((block) => block._tag === 'transcript')?.text ?? '', ''];
@@ -45,17 +45,18 @@ export const renderMarkdown =
       message.sender.identityDid;
     const blocks = message.blocks.filter((block) => block._tag === 'transcript');
     return [
+      // TODO(thure): Use an XML tag with the bits needed here.
       `###### ${name}` + (debug ? ` [${index}]:${message.id}` : ''),
       blocks.map((block) => block.text.trim()).join(' '),
       '',
     ];
   };
 
-export type TranscriptProps = ThemedClassName<{
-  transcript?: TranscriptType;
+export type TranscriptViewProps = ThemedClassName<{
   space?: Space;
-  model: SerializationModel<DataType.Message>;
-  // TODO(wittjosiah): Move to container.
+  transcript?: Transcript.Transcript;
+  model: SerializationModel<Message.Message>;
+  // TODO(burdon): Are these still in use?
   attendableId?: string;
   ignoreAttention?: boolean;
 }>;
@@ -63,39 +64,30 @@ export type TranscriptProps = ThemedClassName<{
 /**
  * Transcript component implemented using the editor.
  */
-export const Transcript = ({
-  classNames,
-  transcript: object,
-  space,
-  model,
-  attendableId,
-  ignoreAttention,
-}: TranscriptProps) => {
+export const TranscriptView = ({ classNames, space, transcript: object, model }: TranscriptViewProps) => {
   const { themeMode } = useThemeContext();
   const { parentRef } = useTextEditor(() => {
     return {
       extensions: [
         createBasicExtensions({ readOnly: true, lineWrapping: true, search: true }),
-        createMarkdownExtensions({ themeMode }),
-        createThemeExtensions({ themeMode }),
+        createMarkdownExtensions(),
+        createThemeExtensions({ themeMode, slots: editorSlots }),
         decorateMarkdown(),
-        space && preview(),
+        preview(),
         transcript({
           model,
           started: object?.started ? new Date(object.started) : undefined,
-          renderButton: createRenderer(({ onClick }) => (
-            <IconButton icon='ph--arrow-line-down--regular' iconOnly label='Scroll to bottom' onClick={onClick} />
-          )),
         }),
-      ].filter(isNotFalsy),
+        autoScroll(),
+      ].filter(isTruthy),
     };
   }, [space, model]);
 
   return (
     <div
       ref={parentRef}
-      className={mx('flex grow overflow-hidden', editorWidth, classNames)}
-      data-popover-collision-boundary={true}
+      className={mx('flex grow overflow-hidden container-max-width', classNames)}
+      data-popover-collision-boundary={true /* TODO(thure): Make this a constant and document it. */}
     />
   );
 };

package/src/components/Transcript/deprecated/OldTranscript.tsx

@@ -3,7 +3,7 @@
 //
 
 import { intervalToDuration } from 'date-fns/intervalToDuration';
-import { Schema } from 'effect';
+import * as Schema from 'effect/Schema';
 import { yieldOrContinue } from 'main-thread-scheduling';
 import React, { type FC, type WheelEvent, useCallback, useEffect, useMemo, useRef, useState } from 'react';
 import { type OnResizeCallback, useResizeDetector } from 'react-resize-detector';
@@ -21,7 +21,7 @@ import {
 } from '@dxos/react-ui-grid';
 import { mx } from '@dxos/react-ui-theme';
 
-import { TRANSCRIPTION_PLUGIN } from '../../../meta';
+import { meta } from '../../../meta';
 
 /**
  * Transcription fragment.
@@ -115,7 +115,7 @@ const measureRows = async (
   return result;
 };
 
-export type TranscriptProps = ThemedClassName<{
+export type TranscriptViewProps = ThemedClassName<{
   blocks?: TranscriptBlock[];
   attendableId?: string;
   ignoreAttention?: boolean;
@@ -124,8 +124,8 @@ export type TranscriptProps = ThemedClassName<{
 /**
  * @@deprecated
  */
-export const Transcript: FC<TranscriptProps> = ({ classNames, blocks, attendableId, ignoreAttention }) => {
-  const { t } = useTranslation(TRANSCRIPTION_PLUGIN);
+export const Transcript: FC<TranscriptViewProps> = ({ classNames, blocks, attendableId, ignoreAttention }) => {
+  const { t } = useTranslation(meta.id);
   const [dxGrid, setDxGrid] = useState<DxGridElement | null>(null);
   const [rows, setRows] = useState<DxGridAxisMeta | undefined>(undefined);
   const [columns, setColumns] = useState<DxGridAxisMeta | undefined>(undefined);
@@ -144,7 +144,7 @@ export const Transcript: FC<TranscriptProps> = ({ classNames, blocks, attendable
     [hasAttention, ignoreAttention],
   );
 
-  const abortControllerRef = useRef<AbortController>();
+  const abortControllerRef = useRef<AbortController>(null);
 
   const handleResize = useCallback(
     async ({ entry }: { entry: { target: HTMLDivElement } | null }) => {
@@ -242,7 +242,7 @@ export const Transcript: FC<TranscriptProps> = ({ classNames, blocks, attendable
           ref={setDxGrid}
         />
       </Grid.Root>
-      <div role='none' {...{ inert: '' }} aria-hidden className={measureClasses} ref={measureRef} />
+      <div role='none' {...{ inert: true }} aria-hidden className={measureClasses} ref={measureRef} />
       <IconButton
         icon='ph--arrow-line-down--regular'
         iconOnly

package/src/components/Transcript/transcript-extension.ts

@@ -8,8 +8,7 @@ import { format } from 'date-fns/format';
 import { intervalToDuration } from 'date-fns/intervalToDuration';
 
 import { type CleanupFn, addEventListener, combine } from '@dxos/async';
-import { type RenderCallback } from '@dxos/react-ui-editor';
-import { type DataType } from '@dxos/schema';
+import { type Message } from '@dxos/types';
 
 import { DocumentAdapter, type SerializationModel } from '../../model';
 
@@ -17,15 +16,14 @@ import { DocumentAdapter, type SerializationModel } from '../../model';
  * Data structure that maps Chunks queue to lines with transcript state.
  */
 export type TranscriptOptions = {
-  model: SerializationModel<DataType.Message>;
+  model: SerializationModel<Message.Message>;
   started?: Date;
-  renderButton?: RenderCallback<{ onClick: () => void }>;
 };
 
 /**
  * Scrolling transcript with timestamps.
  */
-export const transcript = ({ model, started, renderButton }: TranscriptOptions): Extension => {
+export const transcript = ({ model, started }: TranscriptOptions): Extension => {
   return [
     // Show timestamps in the gutter.
     gutter({
@@ -90,22 +88,6 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
             });
           };
 
-          // Scroll button.
-          if (renderButton) {
-            this._controls = document.createElement('div');
-            this._controls.classList.add('cm-controls', 'transition-opacity', 'duration-300', 'opacity-0');
-            view.dom.appendChild(this._controls);
-            renderButton(
-              this._controls,
-              {
-                onClick: () => {
-                  scrollToBottom(false);
-                },
-              },
-              view,
-            );
-          }
-
           // Event listeners.
           this._cleanup = combine(
             addEventListener(view.scrollDOM, 'scroll', () => {
@@ -131,7 +113,7 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
           );
         }
 
-        update(update: ViewUpdate) {
+        update(_update: ViewUpdate) {
           // Initial sync.
           if (!this._initialized) {
             this._initialized = true;
@@ -149,27 +131,12 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
     ),
 
     EditorView.theme({
-      '.cm-scroller': {
-        overflowY: 'scroll',
-      },
-      '.cm-hide-scrollbar': {
-        scrollbarWidth: 'none',
-        '-ms-overflow-style': 'none',
-      },
-      '.cm-hide-scrollbar::-webkit-scrollbar': {
-        display: 'none',
-      },
-      '.cm-controls': {
-        position: 'absolute',
-        bottom: '0.5rem',
-        right: '0.5rem',
-        zIndex: 1000,
-      },
-      '.cm-line': {
-        paddingRight: '1rem',
+      '.cm-gutters': {
+        backgroundColor: 'var(--dx-baseSurface)',
       },
       '.cm-timestamp-gutter': {
         width: '6rem',
+        paddingLeft: '0.5rem',
         paddingRight: '1rem',
       },
       '.cm-timestamp-gutter > .cm-gutterElement > div': {
@@ -193,11 +160,11 @@ class TimestampMarker extends GutterMarker {
     super();
   }
 
-  override eq(other: this): boolean {
-    return other._timestamp === this._timestamp;
+  override eq(other: this) {
+    return this._timestamp === other._timestamp;
   }
 
-  override toDOM(view: EditorView): HTMLDivElement {
+  override toDOM(view: EditorView) {
     const el = document.createElement('div');
     el.className = 'text-sm text-subdued hover:bg-hoverSurface cursor-pointer';
     el.textContent = formatTimestamp(this._timestamp, this._started);
@@ -213,7 +180,7 @@ class TimestampMarker extends GutterMarker {
   }
 }
 
-const getStartTime = (started?: Date, message?: DataType.Message): Date | undefined => {
+const getStartTime = (started?: Date, message?: Message.Message): Date | undefined => {
   if (started) {
     return started;
   }

package/src/components/TranscriptContainer.tsx

@@ -4,31 +4,31 @@
 
 import React from 'react';
 
-import { fullyQualifiedId, getSpace } from '@dxos/client/echo';
+import { getSpace } from '@dxos/client/echo';
+import { Obj } from '@dxos/echo';
 import { useMembers, useQueue } from '@dxos/react-client/echo';
 import { StackItem } from '@dxos/react-ui-stack';
-import { type DataType } from '@dxos/schema';
+import { type Message, type Transcript } from '@dxos/types';
 
 import { useQueueModelAdapter } from '../hooks';
-import { type TranscriptType } from '../types';
 
-import { Transcript, renderMarkdown } from './Transcript';
+import { TranscriptView, renderByline } from './Transcript';
 
 export type TranscriptionContainerProps = {
   role: string;
-  transcript: TranscriptType;
+  transcript: Transcript.Transcript;
 };
 
 export const TranscriptionContainer = ({ transcript }: TranscriptionContainerProps) => {
-  const attendableId = fullyQualifiedId(transcript);
+  const attendableId = Obj.getDXN(transcript).toString();
   const space = getSpace(transcript);
   const members = useMembers(space?.key).map((member) => member.identity);
-  const queue = useQueue<DataType.Message>(transcript.queue.dxn, { pollInterval: 1_000 });
-  const model = useQueueModelAdapter(renderMarkdown(members), queue);
+  const queue = useQueue<Message.Message>(transcript.queue.dxn, { pollInterval: 1_000 });
+  const model = useQueueModelAdapter(renderByline(members), queue);
 
   return (
-    <StackItem.Content classNames='container-max-width pli-2'>
-      <Transcript attendableId={attendableId} space={space} model={model} transcript={transcript} />
+    <StackItem.Content>
+      <TranscriptView attendableId={attendableId} space={space} model={model} transcript={transcript} />
     </StackItem.Content>
   );
 };

package/src/components/stories/FileTranscription.stories.tsx

@@ -2,8 +2,6 @@
 // Copyright 2025 DXOS.org
 //
 
-import '@dxos-theme';
-
 import { type Meta, type StoryObj } from '@storybook/react-vite';
 import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
 
@@ -12,26 +10,27 @@ import { withPluginManager } from '@dxos/app-framework/testing';
 import { scheduleTask } from '@dxos/async';
 import { Context } from '@dxos/context';
 import { Obj } from '@dxos/echo';
+import { createQueueDXN } from '@dxos/echo/internal';
 import { MemoryQueue } from '@dxos/echo-db';
-import { createQueueDXN } from '@dxos/echo-schema';
-import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
+import { FunctionExecutor, ServiceContainer } from '@dxos/functions-runtime';
 import { log } from '@dxos/log';
 import { ClientPlugin } from '@dxos/plugin-client';
 import { PreviewPlugin } from '@dxos/plugin-preview';
 import { SpacePlugin } from '@dxos/plugin-space';
 import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
 import { ThemePlugin } from '@dxos/plugin-theme';
+import { withLayout, withTheme } from '@dxos/react-ui/testing';
 import { defaultTx } from '@dxos/react-ui-theme';
-import { DataType } from '@dxos/schema';
-import { Testing, seedTestData } from '@dxos/schema/testing';
-import { withLayout, withTheme } from '@dxos/storybook-utils';
+import { Testing } from '@dxos/schema/testing';
+import { type Actor, Message, Organization, Person } from '@dxos/types';
+import { seedTestData } from '@dxos/types/testing';
 
 import { useAudioFile, useQueueModelAdapter, useTranscriber } from '../../hooks';
 import { MessageNormalizer, getActorId } from '../../segments-normalization';
 import { TestItem } from '../../testing';
 import { type MediaStreamRecorderParams, type TranscriberParams } from '../../transcriber';
 import { TranscriptionPlugin } from '../../TranscriptionPlugin';
-import { renderMarkdown } from '../Transcript';
+import { renderByline } from '../Transcript';
 
 import { TranscriptionStory } from './TranscriptionStory';
 import { useIsSpeaking } from './useIsSpeaking';
@@ -52,7 +51,7 @@ const AudioFile = ({
   audioConstraints?: MediaTrackConstraints;
 }) => {
   const [running, setRunning] = useState(false);
-  const actor: DataType.Actor = useMemo(() => ({ name: 'You' }), []);
+  const actor: Actor.Actor = useMemo(() => ({ name: 'You' }), []);
 
   // Audio.
   const { audio, track, stream } = useAudioFile(audioUrl, audioConstraints);
@@ -68,7 +67,6 @@ const AudioFile = ({
 
   useEffect(() => {
     if (!audio) {
-      log.warn('no audio');
       return;
     }
 
@@ -82,13 +80,13 @@ const AudioFile = ({
   // Transcriber.
   // TODO(dmaretskyi): Use space.queues.create() instead.
   const queueDxn = useMemo(() => createQueueDXN(), []);
-  const queue = useMemo(() => new MemoryQueue<DataType.Message>(queueDxn), [queueDxn]);
+  const queue = useMemo(() => new MemoryQueue<Message.Message>(queueDxn), [queueDxn]);
 
-  const model = useQueueModelAdapter(renderMarkdown([]), queue);
+  const model = useQueueModelAdapter(renderByline([]), queue);
   const handleSegments = useCallback<TranscriberParams['onSegments']>(
     async (blocks) => {
       void queue?.append([
-        Obj.make(DataType.Message, {
+        Obj.make(Message.Message, {
           created: new Date().toISOString(),
           sender: actor,
           blocks,
@@ -147,10 +145,10 @@ const AudioFile = ({
   }, [normalizer]);
 
   const manageChunkRecording = () => {
-    if (running && isSpeaking) {
+    if (running && isSpeaking && transcriber) {
       log.info('starting transcription');
       transcriber?.startChunksRecording();
-    } else if (!isSpeaking || !running) {
+    } else if ((!isSpeaking || !running) && transcriber) {
       log.info('stopping transcription');
       transcriber?.stopChunksRecording();
     }
@@ -176,18 +174,26 @@ const AudioFile = ({
     manageChunkRecording();
   }, [isSpeaking, stream]);
 
-  return <TranscriptionStory model={model} running={running} onRunningChange={setRunning} audioRef={ref} />;
+  return (
+    <TranscriptionStory
+      disabled={!stream}
+      model={model}
+      running={running}
+      onRunningChange={setRunning}
+      audioRef={ref}
+    />
+  );
 };
 
-const meta: Meta<typeof AudioFile> = {
+const meta = {
   title: 'plugins/plugin-transcription/FileTranscription',
   decorators: [
+    withTheme,
+    withLayout({ container: 'column' }),
     withPluginManager({
       plugins: [
-        ThemePlugin({ tx: defaultTx }),
-        StorybookLayoutPlugin(),
         ClientPlugin({
-          types: [TestItem, DataType.Person, DataType.Organization, Testing.DocumentType],
+          types: [TestItem, Person.Person, Organization.Organization, Testing.DocumentType],
           onClientInitialized: async ({ client }) => {
             await client.halo.createIdentity();
             await client.spaces.waitUntilReady();
@@ -195,21 +201,25 @@ const meta: Meta<typeof AudioFile> = {
             await seedTestData(client.spaces.default);
           },
         }),
-        SpacePlugin(),
+        SpacePlugin({}),
+        IntentPlugin(),
+
+        // UI
+        ThemePlugin({ tx: defaultTx }),
         SettingsPlugin(),
         PreviewPlugin(),
-        IntentPlugin(),
         TranscriptionPlugin(),
+        StorybookLayoutPlugin({}),
       ],
       fireEvents: [Events.SetupAppGraph],
     }),
-    withTheme,
-    withLayout({ fullscreen: true, classNames: 'justify-center' }),
   ],
-};
+} satisfies Meta;
 
 export default meta;
 
+type Story = StoryObj<typeof meta>;
+
 const TRANSCRIBER_CONFIG = {
   transcribeAfterChunksAmount: 100,
   prefixBufferChunksAmount: 50,
@@ -232,15 +242,16 @@ export const Default: StoryObj<typeof AudioFile> = {
   },
 };
 
-export const WithSentenceNormalization: StoryObj<typeof AudioFile> = {
-  render: AudioFile,
-  args: {
-    detectSpeaking: true,
-    normalizeSentences: true,
-    // https://learnenglish.britishcouncil.org/general-english/audio-zone/living-london
-    audioUrl: 'https://dxos.network/audio-london.m4a',
-    // textUrl: 'https://dxos.network/audio-london.txt',
-    transcriberConfig: TRANSCRIBER_CONFIG,
-    recorderConfig: RECORDER_CONFIG,
-  },
-};
+// TODO(mykola): Fix sentence normalization.
+// export const WithSentenceNormalization: StoryObj<typeof AudioFile> = {
+//   render: AudioFile,
+//   args: {
+//     detectSpeaking: true,
+//     normalizeSentences: true,
+//     // https://learnenglish.britishcouncil.org/general-english/audio-zone/living-london
+//     audioUrl: 'https://dxos.network/audio-london.m4a',
+//     // textUrl: 'https://dxos.network/audio-london.txt',
+//     transcriberConfig: TRANSCRIBER_CONFIG,
+//     recorderConfig: RECORDER_CONFIG,
+//   },
+// };

package/src/components/stories/KeyWordRecognition.stories.tsx

@@ -2,22 +2,24 @@
 // Copyright 2025 DXOS.org
 //
 
-import '@dxos-theme';
-
 import { type Meta, type StoryObj } from '@storybook/react-vite';
 import React, { useEffect, useMemo, useState } from 'react';
 
 import { log } from '@dxos/log';
 import { Toolbar } from '@dxos/react-ui';
+import { withLayout, withTheme } from '@dxos/react-ui/testing';
 import { mx } from '@dxos/react-ui-theme';
-import { withLayout, withTheme } from '@dxos/storybook-utils';
 
 type Word = {
   text: string;
   matched: boolean;
 };
 
-const SpeechRecognition = ({ keywords }: { keywords: string[] }) => {
+type StoryProps = {
+  keywords: string[];
+};
+
+const DefaultStory = ({ keywords }: StoryProps) => {
   const [running, setRunning] = useState(false);
   const [matchingWords, setMatchingWords] = useState<Word[]>(keywords.map((word) => ({ text: word, matched: false })));
 
@@ -84,7 +86,7 @@ const SpeechRecognition = ({ keywords }: { keywords: string[] }) => {
   }, [running, recognition]);
 
   return (
-    <div className={mx('flex flex-col gap-2 w-[30rem]')}>
+    <div className={mx('flex flex-col gap-2 is-[30rem]')}>
       <Toolbar.Root>
         <Toolbar.IconButton
           iconOnly
@@ -99,7 +101,7 @@ const SpeechRecognition = ({ keywords }: { keywords: string[] }) => {
           <span
             key={index}
             className={mx(
-              'px-3 py-1 m-2 rounded border',
+              'pli-3 plb-1 m-2 rounded border',
               word.matched ? 'bg-red-200 text-red-800 border-red-300' : 'bg-gray-200 text-gray-600 border-gray-300',
             )}
           >
@@ -113,15 +115,17 @@ const SpeechRecognition = ({ keywords }: { keywords: string[] }) => {
   );
 };
 
-const meta: Meta<typeof SpeechRecognition> = {
+const meta = {
   title: 'plugins/plugin-transcription/KeyWordDetection',
-  decorators: [withTheme, withLayout({ fullscreen: true, classNames: 'justify-center' })],
-};
+  render: DefaultStory,
+  decorators: [withTheme, withLayout({ container: 'column' })],
+} satisfies Meta<typeof DefaultStory>;
 
 export default meta;
 
-export const SpeechRecognitionAPI: StoryObj<typeof SpeechRecognition> = {
-  render: SpeechRecognition,
+type Story = StoryObj<typeof meta>;
+
+export const SpeechRecognitionAPI: Story = {
   args: {
     keywords: ['kai', 'computer', 'hello'],
   },