@dxos/plugin-transcription 0.8.1 → 0.8.2-main.12df754

package/src/capabilities/intent-resolver.ts

@@ -3,18 +3,18 @@
 //
 
 import { Capabilities, contributes, createResolver, type PluginsContext } from '@dxos/app-framework';
-import { create, refFromDXN } from '@dxos/live-object';
-import { type SpaceId } from '@dxos/react-client/echo';
+import { live, refFromDXN } from '@dxos/live-object';
+import { type SpaceId, randomQueueDxn } from '@dxos/react-client/echo';
 
 import { TranscriptionAction, TranscriptType } from '../types';
-import { getTimeStr, randomQueueDxn } from '../util';
+import { getTimeStr } from '../util';
 
 export default (context: PluginsContext) =>
   contributes(Capabilities.IntentResolver, [
     createResolver({
       intent: TranscriptionAction.Create,
       resolve: ({ name, spaceId }) => {
-        const transcript = create(TranscriptType, {
+        const transcript = live(TranscriptType, {
           queue: refFromDXN(randomQueueDxn(spaceId as SpaceId)),
           name: name ?? `Transcript ${getTimeStr(Date.now())}`,
         });

package/src/capabilities/meeting-transcription-state.ts

@@ -3,12 +3,12 @@
 //
 
 import { contributes } from '@dxos/app-framework';
-import { create } from '@dxos/live-object';
+import { live } from '@dxos/live-object';
 
 import { TranscriptionCapabilities } from './capabilities';
 
 export default () => {
-  const state = create<TranscriptionCapabilities.MeetingTranscriptionState>({
+  const state = live<TranscriptionCapabilities.MeetingTranscriptionState>({
     enabled: false,
   });
 

package/src/capabilities/transcriber.ts

@@ -29,12 +29,13 @@ const TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
  * Records audio while user is speaking and transcribes it after user is done speaking.
  */
 export default () => {
-  const getTranscriber: TranscriptionCapabilities.GetTranscriber = ({ audioStreamTrack, onSegments }) => {
+  const getTranscriber: TranscriptionCapabilities.GetTranscriber = ({ audioStreamTrack, onSegments, config }) => {
     // Initialize audio transcription.
     return new Transcriber({
       config: {
         transcribeAfterChunksAmount: TRANSCRIBE_AFTER_CHUNKS_AMOUNT,
         prefixBufferChunksAmount: PREFIXED_CHUNKS_AMOUNT,
+        ...config,
       },
       recorder: new MediaStreamRecorder({
         mediaStreamTrack: audioStreamTrack,

package/src/components/Transcript/Transcript.stories.tsx

@@ -4,60 +4,164 @@
 
 import '@dxos-theme';
 
-import { type StoryObj, type Meta } from '@storybook/react';
-import React, { useEffect, useState } from 'react';
+import { type Meta, type StoryObj } from '@storybook/react';
+import React, { useEffect, useMemo, useState, type FC } from 'react';
 
+import { IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
+import { withPluginManager } from '@dxos/app-framework/testing';
+import { ClientPlugin } from '@dxos/plugin-client';
+import { PreviewPlugin } from '@dxos/plugin-preview';
+import { SpacePlugin } from '@dxos/plugin-space';
+import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
+import { ThemePlugin } from '@dxos/plugin-theme';
 import { faker } from '@dxos/random';
-import { hues } from '@dxos/react-ui-theme';
-import { withTheme, withLayout } from '@dxos/storybook-utils';
+import { useSpace } from '@dxos/react-client/echo';
+import { IconButton, Toolbar } from '@dxos/react-ui';
+import { SyntaxHighlighter } from '@dxos/react-ui-syntax-highlighter';
+import { defaultTx } from '@dxos/react-ui-theme';
+import { withLayout } from '@dxos/storybook-utils';
 
-import { Transcript } from './Transcript';
+import { renderMarkdown, Transcript, type TranscriptProps } from './Transcript';
+import {
+  BlockBuilder,
+  TestItem,
+  useTestTranscriptionQueue,
+  useTestTranscriptionQueueWithEntityExtraction,
+} from './testings';
+import { useQueueModelAdapter } from '../../hooks';
+import { BlockModel } from '../../model';
 import translations from '../../translations';
+import { type TranscriptBlock } from '../../types';
 
-let start = new Date(Date.now() - 24 * 60 * 60 * 10_000);
-const next = () => {
-  start = new Date(start.getTime() + Math.random() * 10_000);
-  return start;
+faker.seed(1);
+
+/**
+ * Story wrapper with test controls.
+ */
+const TranscriptContainer: FC<
+  TranscriptProps & {
+    running: boolean;
+    onRunningChange: (running: boolean) => void;
+    onReset?: () => void;
+  }
+> = ({ space, model, running, onRunningChange, onReset }) => {
+  return (
+    <div className='grid grid-rows-[1fr_40px] grow divide-y divide-separator'>
+      <Transcript space={space} model={model} />
+      <div className='grid grid-cols-[1fr_16rem] overflow-hidden'>
+        <div className='flex items-center'>
+          <SyntaxHighlighter language='json' className='text-sm'>
+            {JSON.stringify(model.toJSON())}
+          </SyntaxHighlighter>
+        </div>
+        <Toolbar.Root classNames='justify-end'>
+          <IconButton
+            icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
+            label={running ? 'Pause' : 'Start'}
+            onClick={() => onRunningChange(!running)}
+          />
+          {onReset && <IconButton icon='ph--x--regular' label='Reset' onClick={onReset} />}
+        </Toolbar.Root>
+      </div>
+    </div>
+  );
 };
 
-const users = Array.from({ length: 5 }, () => ({
-  authorName: faker.person.fullName(),
-  authorHue: faker.helpers.arrayElement(hues),
-}));
-
-const createBlock = () => {
-  const author = faker.helpers.arrayElement(users);
-  return {
-    id: faker.string.uuid(),
-    ...author,
-    segments: Array.from({ length: 1 + Math.floor(Math.random() * 2) }, () => ({
-      started: next(),
-      text: faker.lorem.paragraph(),
-    })),
+type StoryProps = { blocks?: TranscriptBlock[] } & Pick<TranscriptProps, 'ignoreAttention' | 'attendableId'>;
+
+/**
+ * Basic story mutates array of blocks.
+ */
+const BasicStory = ({ blocks: initialBlocks = [], ...props }: StoryProps) => {
+  const [reset, setReset] = useState({});
+  const builder = useMemo(() => new BlockBuilder(), []);
+  const model = useMemo(() => new BlockModel<TranscriptBlock>(renderMarkdown, initialBlocks), [initialBlocks, reset]);
+  const [running, setRunning] = useState(true);
+  const [currentBlock, setCurrentBlock] = useState<TranscriptBlock | null>(null);
+  useEffect(() => {
+    if (!running) {
+      return;
+    }
+
+    if (!currentBlock) {
+      const block = builder.createBlock();
+      model.appendBlock(block);
+      setCurrentBlock(block);
+      return;
+    }
+
+    const i = setInterval(() => {
+      if (currentBlock?.segments.length && currentBlock.segments.length >= 3) {
+        setCurrentBlock(null);
+        clearInterval(i);
+        return;
+      }
+
+      currentBlock.segments.push(builder.createSegment());
+      model.updateBlock(currentBlock);
+    }, 3_000);
+
+    return () => clearInterval(i);
+  }, [model, currentBlock, running]);
+
+  const handleReset = () => {
+    setCurrentBlock(null);
+    setRunning(false);
+    setReset({});
   };
+
+  return (
+    <TranscriptContainer
+      model={model}
+      running={running}
+      onRunningChange={setRunning}
+      onReset={handleReset}
+      {...props}
+    />
+  );
+};
+
+/**
+ * Queue story mutates queue with model adapter.
+ */
+const QueueStory = ({ blocks: initialBlocks = [], ...props }: StoryProps) => {
+  const [running, setRunning] = useState(true);
+  const space = useSpace();
+  const queue = useTestTranscriptionQueue(space, running, 2_000);
+  const model = useQueueModelAdapter(renderMarkdown, queue, initialBlocks);
+
+  return <TranscriptContainer space={space} model={model} running={running} onRunningChange={setRunning} {...props} />;
 };
 
-const meta: Meta<typeof Transcript> = {
+const EntityExtractionQueueStory = () => {
+  const [running, setRunning] = useState(true);
+  const space = useSpace();
+  const queue = useTestTranscriptionQueueWithEntityExtraction(space, running, 2_000);
+  const model = useQueueModelAdapter(renderMarkdown, queue, []);
+
+  return <TranscriptContainer space={space} model={model} running={running} onRunningChange={setRunning} />;
+};
+
+const meta: Meta<typeof QueueStory> = {
   title: 'plugins/plugin-transcription/Transcript',
-  component: Transcript,
-  render: ({ blocks: initialBlocks = [], ...args }) => {
-    const [blocks, setBlocks] = useState(initialBlocks);
-    useEffect(() => {
-      const i = setInterval(() => {
-        setBlocks((blocks) => [...blocks, createBlock()]);
-      }, 1_000);
-
-      return () => clearInterval(i);
-    }, []);
-
-    return <Transcript {...args} blocks={blocks} />;
-  },
   decorators: [
-    withTheme,
-    withLayout({
-      tooltips: true,
-      fullscreen: true,
+    withPluginManager({
+      plugins: [
+        ThemePlugin({ tx: defaultTx }),
+        StorybookLayoutPlugin(),
+        ClientPlugin({
+          types: [TestItem],
+          onClientInitialized: async (_, client) => {
+            await client.halo.createIdentity();
+          },
+        }),
+        PreviewPlugin(),
+        SpacePlugin(),
+        SettingsPlugin(),
+        IntentPlugin(),
+      ],
     }),
+    withLayout({ tooltips: true, fullscreen: true }),
   ],
   parameters: {
     translations,
@@ -66,19 +170,37 @@ const meta: Meta<typeof Transcript> = {
 
 export default meta;
 
-type Story = StoryObj<typeof Transcript>;
+type Story = StoryObj<StoryProps>;
 
 export const Default: Story = {
+  render: BasicStory,
   args: {
     ignoreAttention: true,
     attendableId: 'story',
-    blocks: Array.from({ length: 10 }, createBlock),
+    blocks: Array.from({ length: 10 }, () => BlockBuilder.singleton.createBlock()),
   },
 };
 
 export const Empty: Story = {
+  render: BasicStory,
+  args: {
+    ignoreAttention: true,
+    attendableId: 'story',
+  },
+};
+
+export const WithQueue: Story = {
+  render: QueueStory,
   args: {
     ignoreAttention: true,
     attendableId: 'story',
   },
 };
+
+export const WithEntityExtractionQueue: Story = {
+  render: EntityExtractionQueueStory,
+  args: {
+    // ignoreAttention: true,
+    // attendableId: 'story',
+  },
+};

package/src/components/Transcript/Transcript.tsx

@@ -1,225 +1,93 @@
 //
-// Copyright 2023 DXOS.org
+// Copyright 2025 DXOS.org
 //
 
-import { intervalToDuration } from 'date-fns/intervalToDuration';
-import { yieldOrContinue } from 'main-thread-scheduling';
-import React, { type FC, useCallback, useEffect, useMemo, useRef, useState, type WheelEvent } from 'react';
-import { type OnResizeCallback, useResizeDetector } from 'react-resize-detector';
+import React from 'react';
 
-import { IconButton, type ThemedClassName, useTranslation } from '@dxos/react-ui';
-import { useAttention } from '@dxos/react-ui-attention';
+import { DXN } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { resolveRef, useClient } from '@dxos/react-client';
+import { type Space } from '@dxos/react-client/echo';
+import { IconButton, type ThemedClassName, useThemeContext } from '@dxos/react-ui';
 import {
-  type DxGridCellValue,
-  type DxGridElement,
-  type DxGridAxisMeta,
-  type DxGridPlaneCells,
-  Grid,
-  toPlaneCellIndex,
-} from '@dxos/react-ui-grid';
+  createBasicExtensions,
+  createMarkdownExtensions,
+  createRenderer,
+  createThemeExtensions,
+  decorateMarkdown,
+  editorWidth,
+  preview,
+  useTextEditor,
+} from '@dxos/react-ui-editor';
 import { mx } from '@dxos/react-ui-theme';
-
-import { TRANSCRIPTION_PLUGIN } from '../../meta';
-import { type TranscriptBlock } from '../../types';
-
-// TODO(burdon): Actions (e.g., mark, summarize, translate, label, delete).
-
-const lineHeight = 20;
-const cellSpacing = 8 + 2;
-const timestampColumnWidth = 68;
-
-const authorClasses = 'font-medium text-base leading-[20px]';
-const timestampClasses = 'text-xs leading-[20px] text-description pie-0 tabular-nums';
-const segmentTextClasses = 'text-sm whitespace-normal hyphens-auto';
-const measureClasses = mx(
-  // NOTE(thure): The `inline-start` value must equal `timestampColumnWidth` plus grid’s gap (1px)
-  'absolute inline-start-[69px] inline-end-0 invisible z-[-1] border',
-  'pli-[--dx-grid-cell-padding-inline] plb-[--dx-grid-cell-padding-block] leading-[20px]',
-  segmentTextClasses,
-);
-
-const rowDefault = {
-  grid: { size: lineHeight + cellSpacing },
-};
-
-type QueueRows = [number, number][];
-
-const mapTranscriptQueue = (blocks?: TranscriptBlock[]): QueueRows => {
-  if (!blocks || !Array.isArray(blocks) || blocks.length === 0) {
-    return [];
-  } else {
-    return blocks.flatMap((block, blockIndex) => {
-      return [
-        [blockIndex, -1],
-        ...block.segments.map((segment, segmentIndex) => {
-          return [blockIndex, segmentIndex] as [number, number];
-        }),
-      ];
-    });
-  }
-};
-
-const pad = (n = 0) => String(n).padStart(2, '0');
-
-const formatTimestamp = (start: Date, end: Date) => {
-  const { hours, minutes, seconds } = intervalToDuration({ start, end });
-  return `${pad(hours)}:${pad(minutes)}:${pad(seconds)}`;
-};
-
-const measureRows = async (
-  host: HTMLDivElement,
-  blocks: TranscriptBlock[],
-  queueMap: QueueRows,
-  signal: AbortSignal,
-) => {
-  const result: DxGridAxisMeta = { grid: {} };
-  for (let row = 0; row < queueMap.length; row++) {
-    await yieldOrContinue('smooth', signal);
-    const [blockIndex, segmentIndex] = queueMap[row];
-    if (segmentIndex < 0) {
-      result.grid[row] = { size: lineHeight + cellSpacing };
-    } else {
-      host.textContent = blocks[blockIndex]!.segments[segmentIndex]!.text;
-      result.grid[row] = { size: host.offsetHeight };
-    }
-  }
-  return result;
+import { isNotFalsy } from '@dxos/util';
+
+import { transcript } from './transcript-extension';
+import { type BlockModel } from '../../model';
+import { type TranscriptType, type TranscriptBlock } from '../../types';
+
+export const renderMarkdown = (block: TranscriptBlock, debug = false): string[] => {
+  // TODO(burdon): Use link/reference markup for users (with popover).
+  // TODO(burdon): Color and avatar.
+  return [
+    `###### ${block.authorName}` + (debug ? ` (${block.id})` : ''),
+    block.segments.map((segment) => segment.text.trim()).join(' '),
+    '',
+  ];
 };
 
 export type TranscriptProps = ThemedClassName<{
-  blocks?: TranscriptBlock[];
+  transcript?: TranscriptType;
+  space?: Space;
+  model: BlockModel<TranscriptBlock>;
+  // TODO(wittjosiah): Move to container.
   attendableId?: string;
   ignoreAttention?: boolean;
 }>;
 
-export const Transcript: FC<TranscriptProps> = ({ classNames, blocks, attendableId, ignoreAttention }) => {
-  const { t } = useTranslation(TRANSCRIPTION_PLUGIN);
-  const { hasAttention } = useAttention(attendableId);
-  const [dxGrid, setDxGrid] = useState<DxGridElement | null>(null);
-  const [rows, setRows] = useState<DxGridAxisMeta | undefined>(undefined);
-  const [columns, setColumns] = useState<DxGridAxisMeta | undefined>(undefined);
-  const [autoScroll, setAutoScroll] = useState(true);
-
-  const queueMap = useMemo(() => mapTranscriptQueue(blocks), [blocks]);
-
-  const handleWheel = useCallback(
-    (event: WheelEvent) => {
-      setAutoScroll(false);
-      if (!ignoreAttention && !hasAttention) {
-        event.stopPropagation();
-      }
-    },
-    [hasAttention, ignoreAttention],
-  );
-
-  const abortControllerRef = useRef<AbortController>();
-
-  const handleResize = useCallback(
-    async ({ entry }: { entry: { target: HTMLDivElement } | null }) => {
-      if (entry?.target && Array.isArray(blocks) && blocks[0] && queueMap) {
-        abortControllerRef.current?.abort();
-        abortControllerRef.current = new AbortController();
-        setColumns({ grid: { 0: { size: timestampColumnWidth }, 1: { size: entry.target.offsetWidth } } });
-        return measureRows(entry.target, blocks, queueMap, abortControllerRef.current.signal)
-          .then(setRows)
-          .catch(() => {
-            // Aborted mid-measurement by new size.
-          });
-      }
-    },
-    [blocks, queueMap],
-  );
-
-  const { width, ref: measureRef } = useResizeDetector({
-    onResize: handleResize as OnResizeCallback,
-    refreshOptions: { leading: true },
-  });
-
-  useEffect(() => {
-    if (queueMap.length !== Object.keys(rows?.grid ?? {}).length) {
-      void handleResize({ entry: { target: measureRef.current } }).then(() => {
-        if (autoScroll) {
-          // TODO(thure): Implement a deterministic way to do this when `rows` has fully settled and grid has a new `maxPosBlock`.
-          setTimeout(() => dxGrid?.scrollToEndRow(), 50);
-        }
-      });
-    }
-  }, [blocks, queueMap, width, autoScroll]);
-
-  useEffect(() => {
-    if (dxGrid && Array.isArray(blocks) && blocks[0]) {
-      const transcriptStart = blocks[0]!.segments[0]!.started;
-      dxGrid.getCells = (range, plane) => {
-        switch (plane) {
-          case 'grid': {
-            const cells: DxGridPlaneCells = {};
-            for (let row = range.start.row; row <= range.end.row && row < queueMap.length; row++) {
-              const [blockIndex, segmentIndex] = queueMap[row];
-
-              cells[toPlaneCellIndex({ col: 0, row })] = {
-                readonly: true,
-                value:
-                  segmentIndex < 0
-                    ? formatTimestamp(transcriptStart, blocks[blockIndex]!.segments[Math.max(0, segmentIndex)]!.started)
-                    : '',
-                className: timestampClasses,
-              } satisfies DxGridCellValue;
-
-              cells[toPlaneCellIndex({ col: 1, row })] = (
-                segmentIndex < 0
-                  ? {
-                      readonly: true,
-                      accessoryHtml: `<span data-hue="${blocks[blockIndex]!.authorHue}" class="dx-text-hue">${blocks[blockIndex]!.authorName}</span>`,
-                      className: authorClasses,
-                    }
-                  : {
-                      readonly: true,
-                      value: blocks[blockIndex]!.segments[segmentIndex]!.text,
-                      className: segmentTextClasses,
-                    }
-              ) satisfies DxGridCellValue;
-            }
-            return cells;
-          }
-          default:
-            return {};
-        }
-      };
-    }
-  }, [dxGrid, blocks]);
-
-  const handleScrollToEnd = useCallback(() => {
-    setAutoScroll(true);
-    dxGrid?.scrollToEndRow();
-  }, [dxGrid, autoScroll]);
+/**
+ * Transcript component implemented using the editor.
+ */
+export const Transcript = ({
+  classNames,
+  transcript: object,
+  space,
+  model,
+  attendableId,
+  ignoreAttention,
+}: TranscriptProps) => {
+  const client = useClient();
+  const { themeMode } = useThemeContext();
+  const { parentRef } = useTextEditor(() => {
+    return {
+      extensions: [
+        createBasicExtensions({ readOnly: true, lineWrapping: true }),
+        createMarkdownExtensions({ themeMode }),
+        createThemeExtensions({ themeMode }),
+        decorateMarkdown(),
+        space &&
+          preview({
+            onLookup: async ({ label, ref }) => {
+              log.info('onLookup', { label, ref });
+              const dxn = DXN.parse(ref);
+              if (!dxn) {
+                return null;
+              }
+
+              const object = await resolveRef(client, dxn, space);
+              return { label, object };
+            },
+          }),
+        transcript({
+          model,
+          started: object?.started ? new Date(object.started) : undefined,
+          renderButton: createRenderer(({ onClick }) => (
+            <IconButton icon='ph--arrow-line-down--regular' iconOnly label='Scroll to bottom' onClick={onClick} />
+          )),
+        }),
+      ].filter(isNotFalsy),
+    };
+  }, [space, model]);
 
-  return (
-    <div role='none' className={mx('relative min-bs-0', classNames)}>
-      <Grid.Root id={`${attendableId}--transcript`}>
-        <Grid.Content
-          limitColumns={2}
-          limitRows={queueMap.length}
-          columns={columns}
-          rows={rows}
-          rowDefault={rowDefault}
-          onWheel={handleWheel}
-          className='[--dx-grid-base:var(--dx-baseSurface)] [--dx-grid-lines:var(--dx-baseSurface)] [&_.dx-grid]:min-bs-0 [&_.dx-grid]:min-is-0 [&_.dx-grid]:select-auto'
-          ref={setDxGrid}
-        />
-      </Grid.Root>
-      <div role='none' {...{ inert: '' }} aria-hidden className={measureClasses} ref={measureRef} />
-      <IconButton
-        icon='ph--arrow-line-down--regular'
-        iconOnly
-        label={t('scroll to end label')}
-        tooltipSide='left'
-        data-state={autoScroll ? 'invisible' : 'visible'}
-        classNames={[
-          'absolute inline-end-2 block-end-2 opacity-0 pointer-events-none',
-          'data-[state="visible"]:pointer-events-auto data-[state="visible"]:opacity-100 transition-opacity',
-        ]}
-        onClick={handleScrollToEnd}
-      />
-    </div>
-  );
+  return <div ref={parentRef} className={mx('flex grow overflow-hidden', editorWidth, classNames)} />;
 };