@dxos/plugin-transcription 0.8.1 → 0.8.2-main.10c050d

package/src/components/Transcript/Transcript.tsx

@@ -1,225 +1,100 @@
 //
-// Copyright 2023 DXOS.org
+// Copyright 2025 DXOS.org
 //
 
-import { intervalToDuration } from 'date-fns/intervalToDuration';
-import { yieldOrContinue } from 'main-thread-scheduling';
-import React, { type FC, useCallback, useEffect, useMemo, useRef, useState, type WheelEvent } from 'react';
-import { type OnResizeCallback, useResizeDetector } from 'react-resize-detector';
+import React from 'react';
 
-import { IconButton, type ThemedClassName, useTranslation } from '@dxos/react-ui';
-import { useAttention } from '@dxos/react-ui-attention';
+import { type Space } from '@dxos/react-client/echo';
+import { type Identity } from '@dxos/react-client/halo';
+import { IconButton, type ThemedClassName, useThemeContext } from '@dxos/react-ui';
 import {
-  type DxGridCellValue,
-  type DxGridElement,
-  type DxGridAxisMeta,
-  type DxGridPlaneCells,
-  Grid,
-  toPlaneCellIndex,
-} from '@dxos/react-ui-grid';
+  createBasicExtensions,
+  createMarkdownExtensions,
+  createRenderer,
+  createThemeExtensions,
+  decorateMarkdown,
+  editorWidth,
+  preview,
+  useTextEditor,
+} from '@dxos/react-ui-editor';
 import { mx } from '@dxos/react-ui-theme';
-
-import { TRANSCRIPTION_PLUGIN } from '../../meta';
-import { type TranscriptBlock } from '../../types';
-
-// TODO(burdon): Actions (e.g., mark, summarize, translate, label, delete).
-
-const lineHeight = 20;
-const cellSpacing = 8 + 2;
-const timestampColumnWidth = 68;
-
-const authorClasses = 'font-medium text-base leading-[20px]';
-const timestampClasses = 'text-xs leading-[20px] text-description pie-0 tabular-nums';
-const segmentTextClasses = 'text-sm whitespace-normal hyphens-auto';
-const measureClasses = mx(
-  // NOTE(thure): The `inline-start` value must equal `timestampColumnWidth` plus grid’s gap (1px)
-  'absolute inline-start-[69px] inline-end-0 invisible z-[-1] border',
-  'pli-[--dx-grid-cell-padding-inline] plb-[--dx-grid-cell-padding-block] leading-[20px]',
-  segmentTextClasses,
-);
-
-const rowDefault = {
-  grid: { size: lineHeight + cellSpacing },
-};
-
-type QueueRows = [number, number][];
-
-const mapTranscriptQueue = (blocks?: TranscriptBlock[]): QueueRows => {
-  if (!blocks || !Array.isArray(blocks) || blocks.length === 0) {
-    return [];
-  } else {
-    return blocks.flatMap((block, blockIndex) => {
-      return [
-        [blockIndex, -1],
-        ...block.segments.map((segment, segmentIndex) => {
-          return [blockIndex, segmentIndex] as [number, number];
-        }),
-      ];
-    });
-  }
-};
-
-const pad = (n = 0) => String(n).padStart(2, '0');
-
-const formatTimestamp = (start: Date, end: Date) => {
-  const { hours, minutes, seconds } = intervalToDuration({ start, end });
-  return `${pad(hours)}:${pad(minutes)}:${pad(seconds)}`;
-};
-
-const measureRows = async (
-  host: HTMLDivElement,
-  blocks: TranscriptBlock[],
-  queueMap: QueueRows,
-  signal: AbortSignal,
-) => {
-  const result: DxGridAxisMeta = { grid: {} };
-  for (let row = 0; row < queueMap.length; row++) {
-    await yieldOrContinue('smooth', signal);
-    const [blockIndex, segmentIndex] = queueMap[row];
-    if (segmentIndex < 0) {
-      result.grid[row] = { size: lineHeight + cellSpacing };
-    } else {
-      host.textContent = blocks[blockIndex]!.segments[segmentIndex]!.text;
-      result.grid[row] = { size: host.offsetHeight };
+import { type DataType } from '@dxos/schema';
+import { isNotFalsy } from '@dxos/util';
+
+import { transcript } from './transcript-extension';
+import { type SerializationModel } from '../../model';
+import { type TranscriptType } from '../../types';
+
+export const renderMarkdown =
+  (identities: Identity[]) =>
+  (message: DataType.Message, index: number, debug = false): string[] => {
+    if (message.sender.role === 'assistant') {
+      // Start/stop block.
+      return [message.blocks.find((block) => block.type === 'transcription')?.text ?? '', ''];
     }
-  }
-  return result;
-};
+
+    // TODO(burdon): Use link/reference markup for users (with popover).
+    // TODO(burdon): Color and avatar.
+    const identity = identities.find((identity) => identity.did === message.sender.identityDid);
+    const name =
+      identity?.profile?.displayName ??
+      message.sender.contact?.target?.fullName ??
+      message.sender.name ??
+      message.sender.email ??
+      message.sender.identityDid;
+    const blocks = message.blocks.filter((block) => block.type === 'transcription');
+    return [
+      `###### ${name}` + (debug ? ` [${index}]:${message.id}` : ''),
+      blocks.map((block) => block.text.trim()).join(' '),
+      '',
+    ];
+  };
 
 export type TranscriptProps = ThemedClassName<{
-  blocks?: TranscriptBlock[];
+  transcript?: TranscriptType;
+  space?: Space;
+  model: SerializationModel<DataType.Message>;
+  // TODO(wittjosiah): Move to container.
   attendableId?: string;
   ignoreAttention?: boolean;
 }>;
 
-export const Transcript: FC<TranscriptProps> = ({ classNames, blocks, attendableId, ignoreAttention }) => {
-  const { t } = useTranslation(TRANSCRIPTION_PLUGIN);
-  const { hasAttention } = useAttention(attendableId);
-  const [dxGrid, setDxGrid] = useState<DxGridElement | null>(null);
-  const [rows, setRows] = useState<DxGridAxisMeta | undefined>(undefined);
-  const [columns, setColumns] = useState<DxGridAxisMeta | undefined>(undefined);
-  const [autoScroll, setAutoScroll] = useState(true);
-
-  const queueMap = useMemo(() => mapTranscriptQueue(blocks), [blocks]);
-
-  const handleWheel = useCallback(
-    (event: WheelEvent) => {
-      setAutoScroll(false);
-      if (!ignoreAttention && !hasAttention) {
-        event.stopPropagation();
-      }
-    },
-    [hasAttention, ignoreAttention],
-  );
-
-  const abortControllerRef = useRef<AbortController>();
-
-  const handleResize = useCallback(
-    async ({ entry }: { entry: { target: HTMLDivElement } | null }) => {
-      if (entry?.target && Array.isArray(blocks) && blocks[0] && queueMap) {
-        abortControllerRef.current?.abort();
-        abortControllerRef.current = new AbortController();
-        setColumns({ grid: { 0: { size: timestampColumnWidth }, 1: { size: entry.target.offsetWidth } } });
-        return measureRows(entry.target, blocks, queueMap, abortControllerRef.current.signal)
-          .then(setRows)
-          .catch(() => {
-            // Aborted mid-measurement by new size.
-          });
-      }
-    },
-    [blocks, queueMap],
-  );
-
-  const { width, ref: measureRef } = useResizeDetector({
-    onResize: handleResize as OnResizeCallback,
-    refreshOptions: { leading: true },
-  });
-
-  useEffect(() => {
-    if (queueMap.length !== Object.keys(rows?.grid ?? {}).length) {
-      void handleResize({ entry: { target: measureRef.current } }).then(() => {
-        if (autoScroll) {
-          // TODO(thure): Implement a deterministic way to do this when `rows` has fully settled and grid has a new `maxPosBlock`.
-          setTimeout(() => dxGrid?.scrollToEndRow(), 50);
-        }
-      });
-    }
-  }, [blocks, queueMap, width, autoScroll]);
-
-  useEffect(() => {
-    if (dxGrid && Array.isArray(blocks) && blocks[0]) {
-      const transcriptStart = blocks[0]!.segments[0]!.started;
-      dxGrid.getCells = (range, plane) => {
-        switch (plane) {
-          case 'grid': {
-            const cells: DxGridPlaneCells = {};
-            for (let row = range.start.row; row <= range.end.row && row < queueMap.length; row++) {
-              const [blockIndex, segmentIndex] = queueMap[row];
-
-              cells[toPlaneCellIndex({ col: 0, row })] = {
-                readonly: true,
-                value:
-                  segmentIndex < 0
-                    ? formatTimestamp(transcriptStart, blocks[blockIndex]!.segments[Math.max(0, segmentIndex)]!.started)
-                    : '',
-                className: timestampClasses,
-              } satisfies DxGridCellValue;
-
-              cells[toPlaneCellIndex({ col: 1, row })] = (
-                segmentIndex < 0
-                  ? {
-                      readonly: true,
-                      accessoryHtml: `<span data-hue="${blocks[blockIndex]!.authorHue}" class="dx-text-hue">${blocks[blockIndex]!.authorName}</span>`,
-                      className: authorClasses,
-                    }
-                  : {
-                      readonly: true,
-                      value: blocks[blockIndex]!.segments[segmentIndex]!.text,
-                      className: segmentTextClasses,
-                    }
-              ) satisfies DxGridCellValue;
-            }
-            return cells;
-          }
-          default:
-            return {};
-        }
-      };
-    }
-  }, [dxGrid, blocks]);
-
-  const handleScrollToEnd = useCallback(() => {
-    setAutoScroll(true);
-    dxGrid?.scrollToEndRow();
-  }, [dxGrid, autoScroll]);
+/**
+ * Transcript component implemented using the editor.
+ */
+export const Transcript = ({
+  classNames,
+  transcript: object,
+  space,
+  model,
+  attendableId,
+  ignoreAttention,
+}: TranscriptProps) => {
+  const { themeMode } = useThemeContext();
+  const { parentRef } = useTextEditor(() => {
+    return {
+      extensions: [
+        createBasicExtensions({ readOnly: true, lineWrapping: true }),
+        createMarkdownExtensions({ themeMode }),
+        createThemeExtensions({ themeMode }),
+        decorateMarkdown(),
+        space && preview(),
+        transcript({
+          model,
+          started: object?.started ? new Date(object.started) : undefined,
+          renderButton: createRenderer(({ onClick }) => (
+            <IconButton icon='ph--arrow-line-down--regular' iconOnly label='Scroll to bottom' onClick={onClick} />
+          )),
+        }),
+      ].filter(isNotFalsy),
+    };
+  }, [space, model]);
 
   return (
-    <div role='none' className={mx('relative min-bs-0', classNames)}>
-      <Grid.Root id={`${attendableId}--transcript`}>
-        <Grid.Content
-          limitColumns={2}
-          limitRows={queueMap.length}
-          columns={columns}
-          rows={rows}
-          rowDefault={rowDefault}
-          onWheel={handleWheel}
-          className='[--dx-grid-base:var(--dx-baseSurface)] [--dx-grid-lines:var(--dx-baseSurface)] [&_.dx-grid]:min-bs-0 [&_.dx-grid]:min-is-0 [&_.dx-grid]:select-auto'
-          ref={setDxGrid}
-        />
-      </Grid.Root>
-      <div role='none' {...{ inert: '' }} aria-hidden className={measureClasses} ref={measureRef} />
-      <IconButton
-        icon='ph--arrow-line-down--regular'
-        iconOnly
-        label={t('scroll to end label')}
-        tooltipSide='left'
-        data-state={autoScroll ? 'invisible' : 'visible'}
-        classNames={[
-          'absolute inline-end-2 block-end-2 opacity-0 pointer-events-none',
-          'data-[state="visible"]:pointer-events-auto data-[state="visible"]:opacity-100 transition-opacity',
-        ]}
-        onClick={handleScrollToEnd}
-      />
-    </div>
+    <div
+      ref={parentRef}
+      className={mx('flex grow overflow-hidden', editorWidth, classNames)}
+      data-popover-collision-boundary={true}
+    />
   );
 };

package/src/components/Transcript/deprecated/OldTranscript.tsx

@@ -0,0 +1,260 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import { intervalToDuration } from 'date-fns/intervalToDuration';
+import { Schema } from 'effect';
+import { yieldOrContinue } from 'main-thread-scheduling';
+import React, { type FC, useCallback, useEffect, useMemo, useRef, useState, type WheelEvent } from 'react';
+import { type OnResizeCallback, useResizeDetector } from 'react-resize-detector';
+
+import { Type } from '@dxos/echo';
+import { IconButton, type ThemedClassName, useTranslation } from '@dxos/react-ui';
+import { useAttention } from '@dxos/react-ui-attention';
+import {
+  type DxGridCellValue,
+  type DxGridElement,
+  type DxGridAxisMeta,
+  type DxGridPlaneCells,
+  Grid,
+  toPlaneCellIndex,
+} from '@dxos/react-ui-grid';
+import { mx } from '@dxos/react-ui-theme';
+
+import { TRANSCRIPTION_PLUGIN } from '../../../meta';
+
+/**
+ * Transcription fragment.
+ */
+const TranscriptSegment = Schema.Struct({
+  // TODO(burdon): TS from service is not Unix TS (x1000).
+  started: Schema.String,
+  text: Schema.String,
+});
+
+type TranscriptSegment = Schema.Schema.Type<typeof TranscriptSegment>;
+
+/**
+ * Transcription block (from single speaker).
+ */
+const TranscriptBlock = Schema.Struct({
+  id: Schema.String,
+  authorName: Schema.optional(Schema.String), // TODO(burdon): Replace with identityDid.
+  authorHue: Schema.optional(Schema.String), // TOOD(burdon): Remove.
+  segments: Schema.Array(TranscriptSegment),
+}).pipe(
+  Type.Obj({
+    typename: 'dxos.org/type/TranscriptBlock',
+    version: '0.1.0',
+  }),
+);
+
+type TranscriptBlock = Schema.Schema.Type<typeof TranscriptBlock>;
+
+// TODO(burdon): Actions (e.g., mark, summarize, translate, label, delete).
+
+const lineHeight = 20;
+const cellSpacing = 8 + 2;
+const timestampColumnWidth = 68;
+
+const authorClasses = 'font-medium text-base leading-[20px]';
+const timestampClasses = 'text-xs leading-[20px] text-description pie-0 tabular-nums';
+const segmentTextClasses = 'text-sm whitespace-normal hyphens-auto';
+const measureClasses = mx(
+  // NOTE(thure): The `inline-start` value must equal `timestampColumnWidth` plus grid’s gap (1px)
+  'absolute inline-start-[69px] inline-end-0 invisible z-[-1] border',
+  'pli-[--dx-grid-cell-padding-inline] plb-[--dx-grid-cell-padding-block] leading-[20px]',
+  segmentTextClasses,
+);
+
+const rowDefault = {
+  grid: { size: lineHeight + cellSpacing },
+};
+
+type QueueRows = [number, number][];
+
+const mapTranscriptQueue = (blocks?: TranscriptBlock[]): QueueRows => {
+  if (!blocks || !Array.isArray(blocks) || blocks.length === 0) {
+    return [];
+  } else {
+    return blocks.flatMap((block, blockIndex) => {
+      return [
+        [blockIndex, -1],
+        ...block.segments.map((segment, segmentIndex) => {
+          return [blockIndex, segmentIndex] as [number, number];
+        }),
+      ];
+    });
+  }
+};
+
+const pad = (n = 0) => String(n).padStart(2, '0');
+
+const formatTimestamp = (start: Date, end: Date) => {
+  const { hours, minutes, seconds } = intervalToDuration({ start, end });
+  return `${pad(hours)}:${pad(minutes)}:${pad(seconds)}`;
+};
+
+const measureRows = async (
+  host: HTMLDivElement,
+  blocks: TranscriptBlock[],
+  queueMap: QueueRows,
+  signal: AbortSignal,
+) => {
+  const result: DxGridAxisMeta = { grid: {} };
+  for (let row = 0; row < queueMap.length; row++) {
+    await yieldOrContinue('smooth', signal);
+    const [blockIndex, segmentIndex] = queueMap[row];
+    if (segmentIndex < 0) {
+      result.grid[row] = { size: lineHeight + cellSpacing };
+    } else {
+      host.textContent = blocks[blockIndex]!.segments[segmentIndex]!.text;
+      result.grid[row] = { size: host.offsetHeight };
+    }
+  }
+  return result;
+};
+
+export type TranscriptProps = ThemedClassName<{
+  blocks?: TranscriptBlock[];
+  attendableId?: string;
+  ignoreAttention?: boolean;
+}>;
+
+/**
+ * @@deprecated
+ */
+export const Transcript: FC<TranscriptProps> = ({ classNames, blocks, attendableId, ignoreAttention }) => {
+  const { t } = useTranslation(TRANSCRIPTION_PLUGIN);
+  const [dxGrid, setDxGrid] = useState<DxGridElement | null>(null);
+  const [rows, setRows] = useState<DxGridAxisMeta | undefined>(undefined);
+  const [columns, setColumns] = useState<DxGridAxisMeta | undefined>(undefined);
+
+  const queueMap = useMemo(() => mapTranscriptQueue(blocks), [blocks]);
+
+  const { hasAttention } = useAttention(attendableId);
+  const [autoScroll, setAutoScroll] = useState(true);
+  const handleWheel = useCallback(
+    (event: WheelEvent) => {
+      setAutoScroll(false);
+      if (!ignoreAttention && !hasAttention) {
+        event.stopPropagation();
+      }
+    },
+    [hasAttention, ignoreAttention],
+  );
+
+  const abortControllerRef = useRef<AbortController>();
+
+  const handleResize = useCallback(
+    async ({ entry }: { entry: { target: HTMLDivElement } | null }) => {
+      if (entry?.target && Array.isArray(blocks) && blocks[0] && queueMap) {
+        abortControllerRef.current?.abort();
+        abortControllerRef.current = new AbortController();
+        setColumns({ grid: { 0: { size: timestampColumnWidth }, 1: { size: entry.target.offsetWidth } } });
+        return measureRows(entry.target, blocks, queueMap, abortControllerRef.current.signal)
+          .then(setRows)
+          .catch(() => {
+            // Aborted mid-measurement by new size.
+          });
+      }
+    },
+    [blocks, queueMap],
+  );
+
+  const { width, ref: measureRef } = useResizeDetector({
+    onResize: handleResize as OnResizeCallback,
+    refreshOptions: { leading: true },
+  });
+
+  useEffect(() => {
+    if (queueMap.length !== Object.keys(rows?.grid ?? {}).length) {
+      void handleResize({ entry: { target: measureRef.current } }).then(() => {
+        if (autoScroll) {
+          // TODO(thure): Implement a deterministic way to do this when `rows` has fully settled and grid has a new `maxPosBlock`.
+          setTimeout(() => dxGrid?.scrollToEndRow(), 50);
+        }
+      });
+    }
+  }, [blocks, queueMap, width, autoScroll]);
+
+  useEffect(() => {
+    if (dxGrid && Array.isArray(blocks) && blocks[0]) {
+      const transcriptStart = blocks[0]!.segments[0]!.started;
+      dxGrid.getCells = (range, plane) => {
+        switch (plane) {
+          case 'grid': {
+            const cells: DxGridPlaneCells = {};
+            for (let row = range.start.row; row <= range.end.row && row < queueMap.length; row++) {
+              const [blockIndex, segmentIndex] = queueMap[row];
+
+              cells[toPlaneCellIndex({ col: 0, row })] = {
+                readonly: true,
+                value:
+                  segmentIndex < 0
+                    ? formatTimestamp(
+                        new Date(transcriptStart),
+                        new Date(blocks[blockIndex]!.segments[Math.max(0, segmentIndex)]!.started),
+                      )
+                    : '',
+                className: timestampClasses,
+              } satisfies DxGridCellValue;
+
+              cells[toPlaneCellIndex({ col: 1, row })] = (
+                segmentIndex < 0
+                  ? {
+                      readonly: true,
+                      accessoryHtml: `<span data-hue="${blocks[blockIndex]!.authorHue}" class="dx-text-hue">${blocks[blockIndex]!.authorName}</span>`,
+                      className: authorClasses,
+                    }
+                  : {
+                      readonly: true,
+                      value: blocks[blockIndex]!.segments[segmentIndex]!.text,
+                      className: segmentTextClasses,
+                    }
+              ) satisfies DxGridCellValue;
+            }
+            return cells;
+          }
+          default:
+            return {};
+        }
+      };
+    }
+  }, [dxGrid, blocks]);
+
+  const handleScrollToEnd = useCallback(() => {
+    setAutoScroll(true);
+    dxGrid?.scrollToEndRow();
+  }, [dxGrid, autoScroll]);
+
+  return (
+    <div role='none' className={mx('relative min-bs-0', classNames)}>
+      <Grid.Root id={`${attendableId}--transcript`}>
+        <Grid.Content
+          limitColumns={2}
+          limitRows={queueMap.length}
+          columns={columns}
+          rows={rows}
+          rowDefault={rowDefault}
+          onWheel={handleWheel}
+          className='[--dx-grid-base:var(--dx-baseSurface)] [--dx-grid-lines:var(--dx-baseSurface)] [&_.dx-grid]:min-bs-0 [&_.dx-grid]:min-is-0 [&_.dx-grid]:select-auto'
+          ref={setDxGrid}
+        />
+      </Grid.Root>
+      <div role='none' {...{ inert: '' }} aria-hidden className={measureClasses} ref={measureRef} />
+      <IconButton
+        icon='ph--arrow-line-down--regular'
+        iconOnly
+        label={t('scroll to end label')}
+        tooltipSide='left'
+        data-state={autoScroll ? 'invisible' : 'visible'}
+        classNames={[
+          'absolute inline-end-2 block-end-2 opacity-0 pointer-events-none',
+          'data-[state="visible"]:pointer-events-auto data-[state="visible"]:opacity-100 transition-opacity',
+        ]}
+        onClick={handleScrollToEnd}
+      />
+    </div>
+  );
+};