@dxos/plugin-transcription 0.8.2-main.12df754 → 0.8.2-main.36232bc

package/src/components/Transcript/Transcript.stories.tsx

@@ -5,33 +5,35 @@
 import '@dxos-theme';
 
 import { type Meta, type StoryObj } from '@storybook/react';
-import React, { useEffect, useMemo, useState, type FC } from 'react';
+import React, { type FC, useEffect, useMemo, useState } from 'react';
 
 import { IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
 import { withPluginManager } from '@dxos/app-framework/testing';
+import { ObjectId } from '@dxos/echo-schema';
 import { ClientPlugin } from '@dxos/plugin-client';
 import { PreviewPlugin } from '@dxos/plugin-preview';
 import { SpacePlugin } from '@dxos/plugin-space';
 import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
 import { ThemePlugin } from '@dxos/plugin-theme';
 import { faker } from '@dxos/random';
-import { useSpace } from '@dxos/react-client/echo';
+import { useMembers, useSpace } from '@dxos/react-client/echo';
 import { IconButton, Toolbar } from '@dxos/react-ui';
 import { SyntaxHighlighter } from '@dxos/react-ui-syntax-highlighter';
 import { defaultTx } from '@dxos/react-ui-theme';
+import { Contact, Organization, type MessageType } from '@dxos/schema';
+import { Testing } from '@dxos/schema/testing';
 import { withLayout } from '@dxos/storybook-utils';
 
 import { renderMarkdown, Transcript, type TranscriptProps } from './Transcript';
+import { useQueueModelAdapter } from '../../hooks';
+import { SerializationModel } from '../../model';
 import {
-  BlockBuilder,
+  MessageBuilder,
   TestItem,
   useTestTranscriptionQueue,
   useTestTranscriptionQueueWithEntityExtraction,
-} from './testings';
-import { useQueueModelAdapter } from '../../hooks';
-import { BlockModel } from '../../model';
+} from '../../testing';
 import translations from '../../translations';
-import { type TranscriptBlock } from '../../types';
 
 faker.seed(1);
 
@@ -67,45 +69,49 @@ const TranscriptContainer: FC<
   );
 };
 
-type StoryProps = { blocks?: TranscriptBlock[] } & Pick<TranscriptProps, 'ignoreAttention' | 'attendableId'>;
+type StoryProps = { messages?: MessageType[] } & Pick<TranscriptProps, 'ignoreAttention' | 'attendableId'>;
 
 /**
- * Basic story mutates array of blocks.
+ * Basic story mutates array of messages.
  */
-const BasicStory = ({ blocks: initialBlocks = [], ...props }: StoryProps) => {
+const BasicStory = ({ messages: initialMessages = [], ...props }: StoryProps) => {
   const [reset, setReset] = useState({});
-  const builder = useMemo(() => new BlockBuilder(), []);
-  const model = useMemo(() => new BlockModel<TranscriptBlock>(renderMarkdown, initialBlocks), [initialBlocks, reset]);
+  const builder = useMemo(() => new MessageBuilder(), []);
+  const model = useMemo(
+    () => new SerializationModel<MessageType>(renderMarkdown([]), initialMessages),
+    [initialMessages, reset],
+  );
   const [running, setRunning] = useState(true);
-  const [currentBlock, setCurrentBlock] = useState<TranscriptBlock | null>(null);
+  const [currentMessage, setCurrentMessage] = useState<MessageType | null>(null);
   useEffect(() => {
     if (!running) {
       return;
     }
 
-    if (!currentBlock) {
-      const block = builder.createBlock();
-      model.appendBlock(block);
-      setCurrentBlock(block);
+    if (!currentMessage) {
+      void builder.createMessage().then((message) => {
+        model.appendChunk(message);
+        setCurrentMessage(message);
+      });
       return;
     }
 
     const i = setInterval(() => {
-      if (currentBlock?.segments.length && currentBlock.segments.length >= 3) {
-        setCurrentBlock(null);
+      if (currentMessage?.blocks.length && currentMessage.blocks.length >= 3) {
+        setCurrentMessage(null);
         clearInterval(i);
         return;
       }
 
-      currentBlock.segments.push(builder.createSegment());
-      model.updateBlock(currentBlock);
+      currentMessage.blocks.push(builder.createBlock());
+      model.updateChunk(currentMessage);
     }, 3_000);
 
     return () => clearInterval(i);
-  }, [model, currentBlock, running]);
+  }, [model, currentMessage, running]);
 
   const handleReset = () => {
-    setCurrentBlock(null);
+    setCurrentMessage(null);
     setRunning(false);
     setReset({});
   };
@@ -124,24 +130,54 @@ const BasicStory = ({ blocks: initialBlocks = [], ...props }: StoryProps) => {
 /**
  * Queue story mutates queue with model adapter.
  */
-const QueueStory = ({ blocks: initialBlocks = [], ...props }: StoryProps) => {
+const QueueStory = ({
+  messages: initialMessages = [],
+  queueId,
+  onReset,
+  ...props
+}: StoryProps & { queueId: ObjectId; onReset: () => void }) => {
   const [running, setRunning] = useState(true);
   const space = useSpace();
-  const queue = useTestTranscriptionQueue(space, running, 2_000);
-  const model = useQueueModelAdapter(renderMarkdown, queue, initialBlocks);
+  const members = useMembers(space?.key).map((member) => member.identity);
+  const queue = useTestTranscriptionQueue(space, queueId, running, 2_000);
+  const model = useQueueModelAdapter(renderMarkdown(members), queue, initialMessages);
 
-  return <TranscriptContainer space={space} model={model} running={running} onRunningChange={setRunning} {...props} />;
+  return (
+    <TranscriptContainer
+      space={space}
+      model={model}
+      running={running}
+      onRunningChange={setRunning}
+      onReset={onReset}
+      {...props}
+    />
+  );
 };
 
+// TODO(burdon): Reconcile with QueueStory.
 const EntityExtractionQueueStory = () => {
   const [running, setRunning] = useState(true);
   const space = useSpace();
-  const queue = useTestTranscriptionQueueWithEntityExtraction(space, running, 2_000);
-  const model = useQueueModelAdapter(renderMarkdown, queue, []);
+  const members = useMembers(space?.key).map((member) => member.identity);
+  const queue = useTestTranscriptionQueueWithEntityExtraction(space, undefined, running, 2_000);
+  const model = useQueueModelAdapter(renderMarkdown(members), queue, []);
 
   return <TranscriptContainer space={space} model={model} running={running} onRunningChange={setRunning} />;
 };
 
+/**
+ * Wrapper remounts on refresh to reload queue.
+ */
+const QueueStoryWrapper = () => {
+  const [queueId] = useState(ObjectId.random());
+  const [key, setKey] = useState(ObjectId.random().toString());
+  const handleReset = () => {
+    setKey(ObjectId.random().toString());
+  };
+
+  return <QueueStory key={key} queueId={queueId} onReset={handleReset} />;
+};
+
 const meta: Meta<typeof QueueStory> = {
   title: 'plugins/plugin-transcription/Transcript',
   decorators: [
@@ -150,7 +186,7 @@ const meta: Meta<typeof QueueStory> = {
         ThemePlugin({ tx: defaultTx }),
         StorybookLayoutPlugin(),
         ClientPlugin({
-          types: [TestItem],
+          types: [TestItem, Testing.DocumentType, Contact, Organization],
           onClientInitialized: async (_, client) => {
             await client.halo.createIdentity();
           },
@@ -170,14 +206,14 @@ const meta: Meta<typeof QueueStory> = {
 
 export default meta;
 
-type Story = StoryObj<StoryProps>;
+type Story = StoryObj<typeof QueueStory>;
 
 export const Default: Story = {
   render: BasicStory,
   args: {
     ignoreAttention: true,
     attendableId: 'story',
-    blocks: Array.from({ length: 10 }, () => BlockBuilder.singleton.createBlock()),
+    messages: await Promise.all(Array.from({ length: 10 }, () => MessageBuilder.singleton.createMessage())),
   },
 };
 
@@ -190,7 +226,7 @@ export const Empty: Story = {
 };
 
 export const WithQueue: Story = {
-  render: QueueStory,
+  render: QueueStoryWrapper,
   args: {
     ignoreAttention: true,
     attendableId: 'story',
@@ -200,7 +236,7 @@ export const WithQueue: Story = {
 export const WithEntityExtractionQueue: Story = {
   render: EntityExtractionQueueStory,
   args: {
-    // ignoreAttention: true,
-    // attendableId: 'story',
+    ignoreAttention: true,
+    attendableId: 'story',
   },
 };

package/src/components/Transcript/Transcript.tsx

@@ -4,10 +4,8 @@
 
 import React from 'react';
 
-import { DXN } from '@dxos/keys';
-import { log } from '@dxos/log';
-import { resolveRef, useClient } from '@dxos/react-client';
 import { type Space } from '@dxos/react-client/echo';
+import { type Identity } from '@dxos/react-client/halo';
 import { IconButton, type ThemedClassName, useThemeContext } from '@dxos/react-ui';
 import {
   createBasicExtensions,
@@ -20,26 +18,42 @@ import {
   useTextEditor,
 } from '@dxos/react-ui-editor';
 import { mx } from '@dxos/react-ui-theme';
+import { type MessageType } from '@dxos/schema';
 import { isNotFalsy } from '@dxos/util';
 
 import { transcript } from './transcript-extension';
-import { type BlockModel } from '../../model';
-import { type TranscriptType, type TranscriptBlock } from '../../types';
+import { type SerializationModel } from '../../model';
+import { type TranscriptType } from '../../types';
 
-export const renderMarkdown = (block: TranscriptBlock, debug = false): string[] => {
-  // TODO(burdon): Use link/reference markup for users (with popover).
-  // TODO(burdon): Color and avatar.
-  return [
-    `###### ${block.authorName}` + (debug ? ` (${block.id})` : ''),
-    block.segments.map((segment) => segment.text.trim()).join(' '),
-    '',
-  ];
-};
+export const renderMarkdown =
+  (identities: Identity[]) =>
+  (message: MessageType, index: number, debug = false): string[] => {
+    if (message.sender.role === 'assistant') {
+      // Start/stop block.
+      return [message.blocks.find((block) => block.type === 'transcription')?.text ?? '', ''];
+    }
+
+    // TODO(burdon): Use link/reference markup for users (with popover).
+    // TODO(burdon): Color and avatar.
+    const identity = identities.find((identity) => identity.did === message.sender.identityDid);
+    const name =
+      identity?.profile?.displayName ??
+      message.sender.contact?.target?.fullName ??
+      message.sender.name ??
+      message.sender.email ??
+      message.sender.identityDid;
+    const blocks = message.blocks.filter((block) => block.type === 'transcription');
+    return [
+      `###### ${name}` + (debug ? ` [${index}]:${message.id}` : ''),
+      blocks.map((block) => block.text.trim()).join(' '),
+      '',
+    ];
+  };
 
 export type TranscriptProps = ThemedClassName<{
   transcript?: TranscriptType;
   space?: Space;
-  model: BlockModel<TranscriptBlock>;
+  model: SerializationModel<MessageType>;
   // TODO(wittjosiah): Move to container.
   attendableId?: string;
   ignoreAttention?: boolean;
@@ -56,7 +70,6 @@ export const Transcript = ({
   attendableId,
   ignoreAttention,
 }: TranscriptProps) => {
-  const client = useClient();
   const { themeMode } = useThemeContext();
   const { parentRef } = useTextEditor(() => {
     return {
@@ -65,19 +78,7 @@ export const Transcript = ({
         createMarkdownExtensions({ themeMode }),
         createThemeExtensions({ themeMode }),
         decorateMarkdown(),
-        space &&
-          preview({
-            onLookup: async ({ label, ref }) => {
-              log.info('onLookup', { label, ref });
-              const dxn = DXN.parse(ref);
-              if (!dxn) {
-                return null;
-              }
-
-              const object = await resolveRef(client, dxn, space);
-              return { label, object };
-            },
-          }),
+        space && preview(),
         transcript({
           model,
           started: object?.started ? new Date(object.started) : undefined,

package/src/components/Transcript/deprecated/{Transcript.tsx → OldTranscript.tsx}

@@ -7,6 +7,7 @@ import { yieldOrContinue } from 'main-thread-scheduling';
 import React, { type FC, useCallback, useEffect, useMemo, useRef, useState, type WheelEvent } from 'react';
 import { type OnResizeCallback, useResizeDetector } from 'react-resize-detector';
 
+import { EchoObject, S } from '@dxos/echo-schema';
 import { IconButton, type ThemedClassName, useTranslation } from '@dxos/react-ui';
 import { useAttention } from '@dxos/react-ui-attention';
 import {
@@ -20,7 +21,29 @@ import {
 import { mx } from '@dxos/react-ui-theme';
 
 import { TRANSCRIPTION_PLUGIN } from '../../../meta';
-import { type TranscriptBlock } from '../../../types';
+
+/**
+ * Transcription fragment.
+ */
+const TranscriptSegment = S.Struct({
+  // TODO(burdon): TS from service is not Unix TS (x1000).
+  started: S.String,
+  text: S.String,
+});
+
+type TranscriptSegment = S.Schema.Type<typeof TranscriptSegment>;
+
+/**
+ * Transcription block (from single speaker).
+ */
+const TranscriptBlock = S.Struct({
+  id: S.String,
+  authorName: S.optional(S.String), // TODO(burdon): Replace with identityDid.
+  authorHue: S.optional(S.String), // TOOD(burdon): Remove.
+  segments: S.Array(TranscriptSegment),
+}).pipe(EchoObject({ typename: 'dxos.org/type/TranscriptBlock', version: '0.1.0' }));
+
+type TranscriptBlock = S.Schema.Type<typeof TranscriptBlock>;
 
 // TODO(burdon): Actions (e.g., mark, summarize, translate, label, delete).
 

package/src/components/Transcript/transcript-extension.ts

@@ -9,15 +9,15 @@ import { intervalToDuration } from 'date-fns/intervalToDuration';
 
 import { type CleanupFn, addEventListener, combine } from '@dxos/async';
 import { type RenderCallback } from '@dxos/react-ui-editor';
+import { type MessageType } from '@dxos/schema';
 
-import { DocumentAdapter, type BlockModel } from '../../model';
-import { type TranscriptBlock } from '../../types';
+import { DocumentAdapter, type SerializationModel } from '../../model';
 
 /**
- * Data structure that maps Blocks queue to lines with transcript state.
+ * Data structure that maps Chunks queue to lines with transcript state.
  */
 export type TranscriptOptions = {
-  model: BlockModel<TranscriptBlock>;
+  model: SerializationModel<MessageType>;
   started?: Date;
   renderButton?: RenderCallback<{ onClick: () => void }>;
 };
@@ -32,12 +32,13 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
       class: 'cm-timestamp-gutter',
       lineMarkerChange: (update) => update.docChanged || update.viewportChanged,
       markers: (view) => {
-        const start = getStartTime(started, model.blocks[0]);
+        const start = getStartTime(started, model.chunks[0]);
         const builder = new RangeSetBuilder<GutterMarker>();
         for (const { from, to } of view.visibleRanges) {
           let line = view.state.doc.lineAt(from);
           while (line.from <= to) {
-            const timestamp = model.getBlockAtLine(line.number)?.segments[0]?.started;
+            const block = model.getChunkAtLine(line.number)?.blocks[0];
+            const timestamp = block?.type === 'transcription' && block.started;
             if (timestamp) {
               builder.add(line.from, line.from, new TimestampMarker(line.number, new Date(timestamp), start));
             }
@@ -69,13 +70,15 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
           let isAutoScrolling = false;
           let hasScrolled = false;
 
-          const scrollToBottom = () => {
-            scroller.style.scrollBehavior = 'smooth';
+          let timeout: NodeJS.Timeout | undefined;
+          const scrollToBottom = (smooth = false) => {
+            scroller.style.scrollBehavior = smooth ? 'smooth' : '';
 
             // Temporarily hide scrollbar to prevent flicker.
             scroller.classList.add('cm-hide-scrollbar');
             isAutoScrolling = true;
-            setTimeout(() => {
+            clearTimeout(timeout);
+            timeout = setTimeout(() => {
               this._controls?.classList.add('opacity-0');
               scroller.classList.remove('cm-hide-scrollbar');
               isAutoScrolling = false;
@@ -96,7 +99,7 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
               this._controls,
               {
                 onClick: () => {
-                  scrollToBottom();
+                  scrollToBottom(false);
                 },
               },
               view,
@@ -122,7 +125,7 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
 
               // Scroll.
               if (autoScroll) {
-                scrollToBottom();
+                scrollToBottom(true);
               }
             }),
           );
@@ -210,24 +213,24 @@ class TimestampMarker extends GutterMarker {
   }
 }
 
-const getStartTime = (started?: Date, block?: TranscriptBlock): Date | undefined => {
+const getStartTime = (started?: Date, message?: MessageType): Date | undefined => {
   if (started) {
     return started;
   }
 
-  if (block?.segments[0]?.started) {
-    return new Date(block.segments[0].started);
+  if (message?.blocks[0]?.type === 'transcription' && message.blocks[0].started) {
+    return new Date(message.blocks[0].started);
   }
 
   return undefined;
 };
 
 const formatTimestamp = (timestamp: Date, relative?: Date) => {
-  if (!relative) {
-    return format(timestamp, 'HH:mm:ss');
-  } else {
+  if (relative) {
     const pad = (n = 0) => String(n).padStart(2, '0');
     const { hours, minutes, seconds } = intervalToDuration({ start: relative, end: timestamp });
     return `${pad(hours)}:${pad(minutes)}:${pad(seconds)}`;
+  } else {
+    return format(timestamp, 'HH:mm:ss');
   }
 };

package/src/components/TranscriptContainer.tsx

@@ -5,18 +5,20 @@
 import React, { type FC } from 'react';
 
 import { fullyQualifiedId, getSpace } from '@dxos/client/echo';
-import { useQueue } from '@dxos/react-client/echo';
+import { useMembers, useQueue } from '@dxos/react-client/echo';
 import { StackItem } from '@dxos/react-ui-stack';
+import { type MessageType } from '@dxos/schema';
 
 import { Transcript, renderMarkdown } from './Transcript';
 import { useQueueModelAdapter } from '../hooks';
-import { type TranscriptBlock, type TranscriptType } from '../types';
+import { type TranscriptType } from '../types';
 
 export const TranscriptionContainer: FC<{ role: string; transcript: TranscriptType }> = ({ role, transcript }) => {
   const attendableId = fullyQualifiedId(transcript);
   const space = getSpace(transcript);
-  const queue = useQueue<TranscriptBlock>(transcript.queue.dxn, { pollInterval: 1_000 });
-  const model = useQueueModelAdapter(renderMarkdown, queue);
+  const members = useMembers(space?.key).map((member) => member.identity);
+  const queue = useQueue<MessageType>(transcript.queue.dxn, { pollInterval: 1_000 });
+  const model = useQueueModelAdapter(renderMarkdown(members), queue);
 
   return (
     <StackItem.Content role={role} classNames='container-max-width pli-2'>

package/src/components/TranscriptionSettings/TranscriptionSettings.tsx

@@ -0,0 +1,26 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import React from 'react';
+
+import { Input, useTranslation } from '@dxos/react-ui';
+import { DeprecatedFormContainer, DeprecatedFormInput } from '@dxos/react-ui-form';
+
+import { TRANSCRIPTION_PLUGIN } from '../../meta';
+import { type TranscriptionSettingsProps } from '../../types';
+
+export const TranscriptionSettings = ({ settings }: { settings: TranscriptionSettingsProps }) => {
+  const { t } = useTranslation(TRANSCRIPTION_PLUGIN);
+
+  return (
+    <DeprecatedFormContainer>
+      <DeprecatedFormInput label={t('settings entity extraction label')}>
+        <Input.Switch
+          checked={!!settings.entityExtraction}
+          onCheckedChange={(checked) => (settings.entityExtraction = checked)}
+        />
+      </DeprecatedFormInput>
+    </DeprecatedFormContainer>
+  );
+};

package/src/components/TranscriptionSettings/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './TranscriptionSettings';

package/src/components/index.ts

@@ -6,5 +6,6 @@ import { lazy } from 'react';
 
 export * from './Transcript';
 export * from './TranscriptContainer';
+export * from './TranscriptionSettings';
 
 export const TranscriptContainer = lazy(() => import('./TranscriptContainer'));