@dxos/plugin-transcription 0.8.2-main.fbd8ed0 → 0.8.2-staging.7ac8446

package/src/components/Transcript/Transcript.stories.tsx

@@ -4,200 +4,60 @@
 
 import '@dxos-theme';
 
-import { type Meta, type StoryObj } from '@storybook/react';
-import React, { type FC, useEffect, useMemo, useState } from 'react';
+import { type StoryObj, type Meta } from '@storybook/react';
+import React, { useEffect, useState } from 'react';
 
-import { IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
-import { withPluginManager } from '@dxos/app-framework/testing';
-import { ObjectId } from '@dxos/echo-schema';
-import { ClientPlugin } from '@dxos/plugin-client';
-import { PreviewPlugin } from '@dxos/plugin-preview';
-import { SpacePlugin } from '@dxos/plugin-space';
-import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
-import { ThemePlugin } from '@dxos/plugin-theme';
 import { faker } from '@dxos/random';
-import { useMembers, useSpace } from '@dxos/react-client/echo';
-import { IconButton, Toolbar } from '@dxos/react-ui';
-import { SyntaxHighlighter } from '@dxos/react-ui-syntax-highlighter';
-import { defaultTx } from '@dxos/react-ui-theme';
-import { DataType } from '@dxos/schema';
-import { Testing } from '@dxos/schema/testing';
-import { withLayout } from '@dxos/storybook-utils';
+import { hues } from '@dxos/react-ui-theme';
+import { withTheme, withLayout } from '@dxos/storybook-utils';
 
-import { renderMarkdown, Transcript, type TranscriptProps } from './Transcript';
-import { useQueueModelAdapter } from '../../hooks';
-import { SerializationModel } from '../../model';
-import {
-  MessageBuilder,
-  TestItem,
-  useTestTranscriptionQueue,
-  useTestTranscriptionQueueWithEntityExtraction,
-} from '../../testing';
+import { Transcript } from './Transcript';
 import translations from '../../translations';
 
-faker.seed(1);
-
-/**
- * Story wrapper with test controls.
- */
-const TranscriptContainer: FC<
-  TranscriptProps & {
-    running: boolean;
-    onRunningChange: (running: boolean) => void;
-    onReset?: () => void;
-  }
-> = ({ space, model, running, onRunningChange, onReset }) => {
-  return (
-    <div className='grid grid-rows-[1fr_40px] grow divide-y divide-separator'>
-      <Transcript space={space} model={model} />
-      <div className='grid grid-cols-[1fr_16rem] overflow-hidden'>
-        <div className='flex items-center'>
-          <SyntaxHighlighter language='json' className='text-sm'>
-            {JSON.stringify(model.toJSON())}
-          </SyntaxHighlighter>
-        </div>
-        <Toolbar.Root classNames='justify-end'>
-          <IconButton
-            icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
-            label={running ? 'Pause' : 'Start'}
-            onClick={() => onRunningChange(!running)}
-          />
-          {onReset && <IconButton icon='ph--x--regular' label='Reset' onClick={onReset} />}
-        </Toolbar.Root>
-      </div>
-    </div>
-  );
+let start = new Date(Date.now() - 24 * 60 * 60 * 10_000);
+const next = () => {
+  start = new Date(start.getTime() + Math.random() * 10_000);
+  return start;
 };
 
-type StoryProps = { messages?: DataType.Message[] } & Pick<TranscriptProps, 'ignoreAttention' | 'attendableId'>;
-
-/**
- * Basic story mutates array of messages.
- */
-const BasicStory = ({ messages: initialMessages = [], ...props }: StoryProps) => {
-  const [reset, setReset] = useState({});
-  const builder = useMemo(() => new MessageBuilder(), []);
-  const model = useMemo(
-    () => new SerializationModel<DataType.Message>(renderMarkdown([]), initialMessages),
-    [initialMessages, reset],
-  );
-  const [running, setRunning] = useState(true);
-  const [currentMessage, setCurrentMessage] = useState<DataType.Message | null>(null);
-  useEffect(() => {
-    if (!running) {
-      return;
-    }
-
-    if (!currentMessage) {
-      void builder.createMessage().then((message) => {
-        model.appendChunk(message);
-        setCurrentMessage(message);
-      });
-      return;
-    }
-
-    const i = setInterval(() => {
-      if (currentMessage?.blocks.length && currentMessage.blocks.length >= 3) {
-        setCurrentMessage(null);
-        clearInterval(i);
-        return;
-      }
-
-      currentMessage.blocks.push(builder.createBlock());
-      model.updateChunk(currentMessage);
-    }, 3_000);
-
-    return () => clearInterval(i);
-  }, [model, currentMessage, running]);
-
-  const handleReset = () => {
-    setCurrentMessage(null);
-    setRunning(false);
-    setReset({});
+const users = Array.from({ length: 5 }, () => ({
+  authorName: faker.person.fullName(),
+  authorHue: faker.helpers.arrayElement(hues),
+}));
+
+const createBlock = () => {
+  const author = faker.helpers.arrayElement(users);
+  return {
+    id: faker.string.uuid(),
+    ...author,
+    segments: Array.from({ length: 1 + Math.floor(Math.random() * 2) }, () => ({
+      started: next(),
+      text: faker.lorem.paragraph(),
+    })),
   };
-
-  return (
-    <TranscriptContainer
-      model={model}
-      running={running}
-      onRunningChange={setRunning}
-      onReset={handleReset}
-      {...props}
-    />
-  );
 };
 
-/**
- * Queue story mutates queue with model adapter.
- */
-const QueueStory = ({
-  messages: initialMessages = [],
-  queueId,
-  onReset,
-  ...props
-}: StoryProps & { queueId: ObjectId; onReset: () => void }) => {
-  const [running, setRunning] = useState(true);
-  const space = useSpace();
-  const members = useMembers(space?.key).map((member) => member.identity);
-  const queue = useTestTranscriptionQueue(space, queueId, running, 2_000);
-  const model = useQueueModelAdapter(renderMarkdown(members), queue, initialMessages);
-
-  return (
-    <TranscriptContainer
-      space={space}
-      model={model}
-      running={running}
-      onRunningChange={setRunning}
-      onReset={onReset}
-      {...props}
-    />
-  );
-};
-
-// TODO(burdon): Reconcile with QueueStory.
-const EntityExtractionQueueStory = () => {
-  const [running, setRunning] = useState(true);
-  const space = useSpace();
-  const members = useMembers(space?.key).map((member) => member.identity);
-  const queue = useTestTranscriptionQueueWithEntityExtraction(space, undefined, running, 2_000);
-  const model = useQueueModelAdapter(renderMarkdown(members), queue, []);
-
-  return <TranscriptContainer space={space} model={model} running={running} onRunningChange={setRunning} />;
-};
-
-/**
- * Wrapper remounts on refresh to reload queue.
- */
-const QueueStoryWrapper = () => {
-  const [queueId] = useState(ObjectId.random());
-  const [key, setKey] = useState(ObjectId.random().toString());
-  const handleReset = () => {
-    setKey(ObjectId.random().toString());
-  };
-
-  return <QueueStory key={key} queueId={queueId} onReset={handleReset} />;
-};
-
-const meta: Meta<typeof QueueStory> = {
+const meta: Meta<typeof Transcript> = {
   title: 'plugins/plugin-transcription/Transcript',
+  component: Transcript,
+  render: ({ blocks: initialBlocks = [], ...args }) => {
+    const [blocks, setBlocks] = useState(initialBlocks);
+    useEffect(() => {
+      const i = setInterval(() => {
+        setBlocks((blocks) => [...blocks, createBlock()]);
+      }, 1_000);
+
+      return () => clearInterval(i);
+    }, []);
+
+    return <Transcript {...args} blocks={blocks} />;
+  },
   decorators: [
-    withPluginManager({
-      plugins: [
-        ThemePlugin({ tx: defaultTx }),
-        StorybookLayoutPlugin(),
-        ClientPlugin({
-          types: [TestItem, Testing.DocumentType, DataType.Person, DataType.Organization],
-          onClientInitialized: async (_, client) => {
-            await client.halo.createIdentity();
-          },
-        }),
-        PreviewPlugin(),
-        SpacePlugin(),
-        SettingsPlugin(),
-        IntentPlugin(),
-      ],
+    withTheme,
+    withLayout({
+      tooltips: true,
+      fullscreen: true,
     }),
-    withLayout({ fullscreen: true }),
   ],
   parameters: {
     translations,
@@ -206,35 +66,17 @@ const meta: Meta<typeof QueueStory> = {
 
 export default meta;
 
-type Story = StoryObj<typeof QueueStory>;
+type Story = StoryObj<typeof Transcript>;
 
 export const Default: Story = {
-  render: BasicStory,
   args: {
     ignoreAttention: true,
     attendableId: 'story',
-    messages: await Promise.all(Array.from({ length: 10 }, () => MessageBuilder.singleton.createMessage())),
+    blocks: Array.from({ length: 10 }, createBlock),
   },
 };
 
 export const Empty: Story = {
-  render: BasicStory,
-  args: {
-    ignoreAttention: true,
-    attendableId: 'story',
-  },
-};
-
-export const WithQueue: Story = {
-  render: QueueStoryWrapper,
-  args: {
-    ignoreAttention: true,
-    attendableId: 'story',
-  },
-};
-
-export const WithEntityExtractionQueue: Story = {
-  render: EntityExtractionQueueStory,
   args: {
     ignoreAttention: true,
     attendableId: 'story',

package/src/components/Transcript/Transcript.tsx

@@ -1,100 +1,225 @@
 //
-// Copyright 2025 DXOS.org
+// Copyright 2023 DXOS.org
 //
 
-import React from 'react';
+import { intervalToDuration } from 'date-fns/intervalToDuration';
+import { yieldOrContinue } from 'main-thread-scheduling';
+import React, { type FC, useCallback, useEffect, useMemo, useRef, useState, type WheelEvent } from 'react';
+import { type OnResizeCallback, useResizeDetector } from 'react-resize-detector';
 
-import { type Space } from '@dxos/react-client/echo';
-import { type Identity } from '@dxos/react-client/halo';
-import { IconButton, type ThemedClassName, useThemeContext } from '@dxos/react-ui';
+import { IconButton, type ThemedClassName, useTranslation } from '@dxos/react-ui';
+import { useAttention } from '@dxos/react-ui-attention';
 import {
-  createBasicExtensions,
-  createMarkdownExtensions,
-  createRenderer,
-  createThemeExtensions,
-  decorateMarkdown,
-  editorWidth,
-  preview,
-  useTextEditor,
-} from '@dxos/react-ui-editor';
+  type DxGridCellValue,
+  type DxGridElement,
+  type DxGridAxisMeta,
+  type DxGridPlaneCells,
+  Grid,
+  toPlaneCellIndex,
+} from '@dxos/react-ui-grid';
 import { mx } from '@dxos/react-ui-theme';
-import { type DataType } from '@dxos/schema';
-import { isNotFalsy } from '@dxos/util';
-
-import { transcript } from './transcript-extension';
-import { type SerializationModel } from '../../model';
-import { type TranscriptType } from '../../types';
-
-export const renderMarkdown =
-  (identities: Identity[]) =>
-  (message: DataType.Message, index: number, debug = false): string[] => {
-    if (message.sender.role === 'assistant') {
-      // Start/stop block.
-      return [message.blocks.find((block) => block.type === 'transcription')?.text ?? '', ''];
-    }
 
-    // TODO(burdon): Use link/reference markup for users (with popover).
-    // TODO(burdon): Color and avatar.
-    const identity = identities.find((identity) => identity.did === message.sender.identityDid);
-    const name =
-      identity?.profile?.displayName ??
-      message.sender.contact?.target?.fullName ??
-      message.sender.name ??
-      message.sender.email ??
-      message.sender.identityDid;
-    const blocks = message.blocks.filter((block) => block.type === 'transcription');
-    return [
-      `###### ${name}` + (debug ? ` [${index}]:${message.id}` : ''),
-      blocks.map((block) => block.text.trim()).join(' '),
-      '',
-    ];
-  };
+import { TRANSCRIPTION_PLUGIN } from '../../meta';
+import { type TranscriptBlock } from '../../types';
+
+// TODO(burdon): Actions (e.g., mark, summarize, translate, label, delete).
+
+const lineHeight = 20;
+const cellSpacing = 8 + 2;
+const timestampColumnWidth = 68;
+
+const authorClasses = 'font-medium text-base leading-[20px]';
+const timestampClasses = 'text-xs leading-[20px] text-description pie-0 tabular-nums';
+const segmentTextClasses = 'text-sm whitespace-normal hyphens-auto';
+const measureClasses = mx(
+  // NOTE(thure): The `inline-start` value must equal `timestampColumnWidth` plus grid’s gap (1px)
+  'absolute inline-start-[69px] inline-end-0 invisible z-[-1] border',
+  'pli-[--dx-grid-cell-padding-inline] plb-[--dx-grid-cell-padding-block] leading-[20px]',
+  segmentTextClasses,
+);
+
+const rowDefault = {
+  grid: { size: lineHeight + cellSpacing },
+};
+
+type QueueRows = [number, number][];
+
+const mapTranscriptQueue = (blocks?: TranscriptBlock[]): QueueRows => {
+  if (!blocks || !Array.isArray(blocks) || blocks.length === 0) {
+    return [];
+  } else {
+    return blocks.flatMap((block, blockIndex) => {
+      return [
+        [blockIndex, -1],
+        ...block.segments.map((segment, segmentIndex) => {
+          return [blockIndex, segmentIndex] as [number, number];
+        }),
+      ];
+    });
+  }
+};
+
+const pad = (n = 0) => String(n).padStart(2, '0');
+
+const formatTimestamp = (start: Date, end: Date) => {
+  const { hours, minutes, seconds } = intervalToDuration({ start, end });
+  return `${pad(hours)}:${pad(minutes)}:${pad(seconds)}`;
+};
+
+const measureRows = async (
+  host: HTMLDivElement,
+  blocks: TranscriptBlock[],
+  queueMap: QueueRows,
+  signal: AbortSignal,
+) => {
+  const result: DxGridAxisMeta = { grid: {} };
+  for (let row = 0; row < queueMap.length; row++) {
+    await yieldOrContinue('smooth', signal);
+    const [blockIndex, segmentIndex] = queueMap[row];
+    if (segmentIndex < 0) {
+      result.grid[row] = { size: lineHeight + cellSpacing };
+    } else {
+      host.textContent = blocks[blockIndex]!.segments[segmentIndex]!.text;
+      result.grid[row] = { size: host.offsetHeight };
+    }
+  }
+  return result;
+};
 
 export type TranscriptProps = ThemedClassName<{
-  transcript?: TranscriptType;
-  space?: Space;
-  model: SerializationModel<DataType.Message>;
-  // TODO(wittjosiah): Move to container.
+  blocks?: TranscriptBlock[];
   attendableId?: string;
   ignoreAttention?: boolean;
 }>;
 
-/**
- * Transcript component implemented using the editor.
- */
-export const Transcript = ({
-  classNames,
-  transcript: object,
-  space,
-  model,
-  attendableId,
-  ignoreAttention,
-}: TranscriptProps) => {
-  const { themeMode } = useThemeContext();
-  const { parentRef } = useTextEditor(() => {
-    return {
-      extensions: [
-        createBasicExtensions({ readOnly: true, lineWrapping: true }),
-        createMarkdownExtensions({ themeMode }),
-        createThemeExtensions({ themeMode }),
-        decorateMarkdown(),
-        space && preview(),
-        transcript({
-          model,
-          started: object?.started ? new Date(object.started) : undefined,
-          renderButton: createRenderer(({ onClick }) => (
-            <IconButton icon='ph--arrow-line-down--regular' iconOnly label='Scroll to bottom' onClick={onClick} />
-          )),
-        }),
-      ].filter(isNotFalsy),
-    };
-  }, [space, model]);
+export const Transcript: FC<TranscriptProps> = ({ classNames, blocks, attendableId, ignoreAttention }) => {
+  const { t } = useTranslation(TRANSCRIPTION_PLUGIN);
+  const { hasAttention } = useAttention(attendableId);
+  const [dxGrid, setDxGrid] = useState<DxGridElement | null>(null);
+  const [rows, setRows] = useState<DxGridAxisMeta | undefined>(undefined);
+  const [columns, setColumns] = useState<DxGridAxisMeta | undefined>(undefined);
+  const [autoScroll, setAutoScroll] = useState(true);
+
+  const queueMap = useMemo(() => mapTranscriptQueue(blocks), [blocks]);
+
+  const handleWheel = useCallback(
+    (event: WheelEvent) => {
+      setAutoScroll(false);
+      if (!ignoreAttention && !hasAttention) {
+        event.stopPropagation();
+      }
+    },
+    [hasAttention, ignoreAttention],
+  );
+
+  const abortControllerRef = useRef<AbortController>();
+
+  const handleResize = useCallback(
+    async ({ entry }: { entry: { target: HTMLDivElement } | null }) => {
+      if (entry?.target && Array.isArray(blocks) && blocks[0] && queueMap) {
+        abortControllerRef.current?.abort();
+        abortControllerRef.current = new AbortController();
+        setColumns({ grid: { 0: { size: timestampColumnWidth }, 1: { size: entry.target.offsetWidth } } });
+        return measureRows(entry.target, blocks, queueMap, abortControllerRef.current.signal)
+          .then(setRows)
+          .catch(() => {
+            // Aborted mid-measurement by new size.
+          });
+      }
+    },
+    [blocks, queueMap],
+  );
+
+  const { width, ref: measureRef } = useResizeDetector({
+    onResize: handleResize as OnResizeCallback,
+    refreshOptions: { leading: true },
+  });
+
+  useEffect(() => {
+    if (queueMap.length !== Object.keys(rows?.grid ?? {}).length) {
+      void handleResize({ entry: { target: measureRef.current } }).then(() => {
+        if (autoScroll) {
+          // TODO(thure): Implement a deterministic way to do this when `rows` has fully settled and grid has a new `maxPosBlock`.
+          setTimeout(() => dxGrid?.scrollToEndRow(), 50);
+        }
+      });
+    }
+  }, [blocks, queueMap, width, autoScroll]);
+
+  useEffect(() => {
+    if (dxGrid && Array.isArray(blocks) && blocks[0]) {
+      const transcriptStart = blocks[0]!.segments[0]!.started;
+      dxGrid.getCells = (range, plane) => {
+        switch (plane) {
+          case 'grid': {
+            const cells: DxGridPlaneCells = {};
+            for (let row = range.start.row; row <= range.end.row && row < queueMap.length; row++) {
+              const [blockIndex, segmentIndex] = queueMap[row];
+
+              cells[toPlaneCellIndex({ col: 0, row })] = {
+                readonly: true,
+                value:
+                  segmentIndex < 0
+                    ? formatTimestamp(transcriptStart, blocks[blockIndex]!.segments[Math.max(0, segmentIndex)]!.started)
+                    : '',
+                className: timestampClasses,
+              } satisfies DxGridCellValue;
+
+              cells[toPlaneCellIndex({ col: 1, row })] = (
+                segmentIndex < 0
+                  ? {
+                      readonly: true,
+                      accessoryHtml: `<span data-hue="${blocks[blockIndex]!.authorHue}" class="dx-text-hue">${blocks[blockIndex]!.authorName}</span>`,
+                      className: authorClasses,
+                    }
+                  : {
+                      readonly: true,
+                      value: blocks[blockIndex]!.segments[segmentIndex]!.text,
+                      className: segmentTextClasses,
+                    }
+              ) satisfies DxGridCellValue;
+            }
+            return cells;
+          }
+          default:
+            return {};
+        }
+      };
+    }
+  }, [dxGrid, blocks]);
+
+  const handleScrollToEnd = useCallback(() => {
+    setAutoScroll(true);
+    dxGrid?.scrollToEndRow();
+  }, [dxGrid, autoScroll]);
 
   return (
-    <div
-      ref={parentRef}
-      className={mx('flex grow overflow-hidden', editorWidth, classNames)}
-      data-popover-collision-boundary={true}
-    />
+    <div role='none' className={mx('relative min-bs-0', classNames)}>
+      <Grid.Root id={`${attendableId}--transcript`}>
+        <Grid.Content
+          limitColumns={2}
+          limitRows={queueMap.length}
+          columns={columns}
+          rows={rows}
+          rowDefault={rowDefault}
+          onWheel={handleWheel}
+          className='[--dx-grid-base:var(--dx-baseSurface)] [--dx-grid-lines:var(--dx-baseSurface)] [&_.dx-grid]:min-bs-0 [&_.dx-grid]:min-is-0 [&_.dx-grid]:select-auto'
+          ref={setDxGrid}
+        />
+      </Grid.Root>
+      <div role='none' {...{ inert: '' }} aria-hidden className={measureClasses} ref={measureRef} />
+      <IconButton
+        icon='ph--arrow-line-down--regular'
+        iconOnly
+        label={t('scroll to end label')}
+        tooltipSide='left'
+        data-state={autoScroll ? 'invisible' : 'visible'}
+        classNames={[
+          'absolute inline-end-2 block-end-2 opacity-0 pointer-events-none',
+          'data-[state="visible"]:pointer-events-auto data-[state="visible"]:opacity-100 transition-opacity',
+        ]}
+        onClick={handleScrollToEnd}
+      />
+    </div>
   );
 };

package/src/components/TranscriptContainer.tsx

@@ -4,25 +4,20 @@
 
 import React, { type FC } from 'react';
 
-import { fullyQualifiedId, getSpace } from '@dxos/client/echo';
-import { useMembers, useQueue } from '@dxos/react-client/echo';
+import { fullyQualifiedId } from '@dxos/client/echo';
+import { useQueue } from '@dxos/react-client/echo';
 import { StackItem } from '@dxos/react-ui-stack';
-import { type DataType } from '@dxos/schema';
 
-import { Transcript, renderMarkdown } from './Transcript';
-import { useQueueModelAdapter } from '../hooks';
-import { type TranscriptType } from '../types';
+import { Transcript } from './Transcript';
+import { type TranscriptBlock, type TranscriptType } from '../types';
 
 export const TranscriptionContainer: FC<{ role: string; transcript: TranscriptType }> = ({ role, transcript }) => {
   const attendableId = fullyQualifiedId(transcript);
-  const space = getSpace(transcript);
-  const members = useMembers(space?.key).map((member) => member.identity);
-  const queue = useQueue<DataType.Message>(transcript.queue.dxn, { pollInterval: 1_000 });
-  const model = useQueueModelAdapter(renderMarkdown(members), queue);
+  const queue = useQueue<TranscriptBlock>(transcript.queue.dxn, { pollInterval: 1_000 });
 
   return (
     <StackItem.Content role={role} classNames='container-max-width pli-2'>
-      <Transcript attendableId={attendableId} space={space} model={model} transcript={transcript} />
+      <Transcript attendableId={attendableId} blocks={queue?.items} />
     </StackItem.Content>
   );
 };

package/src/components/index.ts

@@ -6,6 +6,5 @@ import { lazy } from 'react';
 
 export * from './Transcript';
 export * from './TranscriptContainer';
-export * from './TranscriptionSettings';
 
 export const TranscriptContainer = lazy(() => import('./TranscriptContainer'));