@dxos/plugin-transcription 0.8.1 → 0.8.2-main.2f9c567

package/src/components/Transcript/deprecated/OldTranscript.tsx

@@ -0,0 +1,260 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import { intervalToDuration } from 'date-fns/intervalToDuration';
+import { Schema } from 'effect';
+import { yieldOrContinue } from 'main-thread-scheduling';
+import React, { type FC, useCallback, useEffect, useMemo, useRef, useState, type WheelEvent } from 'react';
+import { type OnResizeCallback, useResizeDetector } from 'react-resize-detector';
+
+import { Type } from '@dxos/echo';
+import { IconButton, type ThemedClassName, useTranslation } from '@dxos/react-ui';
+import { useAttention } from '@dxos/react-ui-attention';
+import {
+  type DxGridCellValue,
+  type DxGridElement,
+  type DxGridAxisMeta,
+  type DxGridPlaneCells,
+  Grid,
+  toPlaneCellIndex,
+} from '@dxos/react-ui-grid';
+import { mx } from '@dxos/react-ui-theme';
+
+import { TRANSCRIPTION_PLUGIN } from '../../../meta';
+
+/**
+ * Transcription fragment.
+ */
+const TranscriptSegment = Schema.Struct({
+  // TODO(burdon): TS from service is not Unix TS (x1000).
+  started: Schema.String,
+  text: Schema.String,
+});
+
+type TranscriptSegment = Schema.Schema.Type<typeof TranscriptSegment>;
+
+/**
+ * Transcription block (from single speaker).
+ */
+const TranscriptBlock = Schema.Struct({
+  id: Schema.String,
+  authorName: Schema.optional(Schema.String), // TODO(burdon): Replace with identityDid.
+  authorHue: Schema.optional(Schema.String), // TOOD(burdon): Remove.
+  segments: Schema.Array(TranscriptSegment),
+}).pipe(
+  Type.def({
+    typename: 'dxos.org/type/TranscriptBlock',
+    version: '0.1.0',
+  }),
+);
+
+type TranscriptBlock = Schema.Schema.Type<typeof TranscriptBlock>;
+
+// TODO(burdon): Actions (e.g., mark, summarize, translate, label, delete).
+
+const lineHeight = 20;
+const cellSpacing = 8 + 2;
+const timestampColumnWidth = 68;
+
+const authorClasses = 'font-medium text-base leading-[20px]';
+const timestampClasses = 'text-xs leading-[20px] text-description pie-0 tabular-nums';
+const segmentTextClasses = 'text-sm whitespace-normal hyphens-auto';
+const measureClasses = mx(
+  // NOTE(thure): The `inline-start` value must equal `timestampColumnWidth` plus grid’s gap (1px)
+  'absolute inline-start-[69px] inline-end-0 invisible z-[-1] border',
+  'pli-[--dx-grid-cell-padding-inline] plb-[--dx-grid-cell-padding-block] leading-[20px]',
+  segmentTextClasses,
+);
+
+const rowDefault = {
+  grid: { size: lineHeight + cellSpacing },
+};
+
+type QueueRows = [number, number][];
+
+const mapTranscriptQueue = (blocks?: TranscriptBlock[]): QueueRows => {
+  if (!blocks || !Array.isArray(blocks) || blocks.length === 0) {
+    return [];
+  } else {
+    return blocks.flatMap((block, blockIndex) => {
+      return [
+        [blockIndex, -1],
+        ...block.segments.map((segment, segmentIndex) => {
+          return [blockIndex, segmentIndex] as [number, number];
+        }),
+      ];
+    });
+  }
+};
+
+const pad = (n = 0) => String(n).padStart(2, '0');
+
+const formatTimestamp = (start: Date, end: Date) => {
+  const { hours, minutes, seconds } = intervalToDuration({ start, end });
+  return `${pad(hours)}:${pad(minutes)}:${pad(seconds)}`;
+};
+
+const measureRows = async (
+  host: HTMLDivElement,
+  blocks: TranscriptBlock[],
+  queueMap: QueueRows,
+  signal: AbortSignal,
+) => {
+  const result: DxGridAxisMeta = { grid: {} };
+  for (let row = 0; row < queueMap.length; row++) {
+    await yieldOrContinue('smooth', signal);
+    const [blockIndex, segmentIndex] = queueMap[row];
+    if (segmentIndex < 0) {
+      result.grid[row] = { size: lineHeight + cellSpacing };
+    } else {
+      host.textContent = blocks[blockIndex]!.segments[segmentIndex]!.text;
+      result.grid[row] = { size: host.offsetHeight };
+    }
+  }
+  return result;
+};
+
+export type TranscriptProps = ThemedClassName<{
+  blocks?: TranscriptBlock[];
+  attendableId?: string;
+  ignoreAttention?: boolean;
+}>;
+
+/**
+ * @@deprecated
+ */
+export const Transcript: FC<TranscriptProps> = ({ classNames, blocks, attendableId, ignoreAttention }) => {
+  const { t } = useTranslation(TRANSCRIPTION_PLUGIN);
+  const [dxGrid, setDxGrid] = useState<DxGridElement | null>(null);
+  const [rows, setRows] = useState<DxGridAxisMeta | undefined>(undefined);
+  const [columns, setColumns] = useState<DxGridAxisMeta | undefined>(undefined);
+
+  const queueMap = useMemo(() => mapTranscriptQueue(blocks), [blocks]);
+
+  const { hasAttention } = useAttention(attendableId);
+  const [autoScroll, setAutoScroll] = useState(true);
+  const handleWheel = useCallback(
+    (event: WheelEvent) => {
+      setAutoScroll(false);
+      if (!ignoreAttention && !hasAttention) {
+        event.stopPropagation();
+      }
+    },
+    [hasAttention, ignoreAttention],
+  );
+
+  const abortControllerRef = useRef<AbortController>();
+
+  const handleResize = useCallback(
+    async ({ entry }: { entry: { target: HTMLDivElement } | null }) => {
+      if (entry?.target && Array.isArray(blocks) && blocks[0] && queueMap) {
+        abortControllerRef.current?.abort();
+        abortControllerRef.current = new AbortController();
+        setColumns({ grid: { 0: { size: timestampColumnWidth }, 1: { size: entry.target.offsetWidth } } });
+        return measureRows(entry.target, blocks, queueMap, abortControllerRef.current.signal)
+          .then(setRows)
+          .catch(() => {
+            // Aborted mid-measurement by new size.
+          });
+      }
+    },
+    [blocks, queueMap],
+  );
+
+  const { width, ref: measureRef } = useResizeDetector({
+    onResize: handleResize as OnResizeCallback,
+    refreshOptions: { leading: true },
+  });
+
+  useEffect(() => {
+    if (queueMap.length !== Object.keys(rows?.grid ?? {}).length) {
+      void handleResize({ entry: { target: measureRef.current } }).then(() => {
+        if (autoScroll) {
+          // TODO(thure): Implement a deterministic way to do this when `rows` has fully settled and grid has a new `maxPosBlock`.
+          setTimeout(() => dxGrid?.scrollToEndRow(), 50);
+        }
+      });
+    }
+  }, [blocks, queueMap, width, autoScroll]);
+
+  useEffect(() => {
+    if (dxGrid && Array.isArray(blocks) && blocks[0]) {
+      const transcriptStart = blocks[0]!.segments[0]!.started;
+      dxGrid.getCells = (range, plane) => {
+        switch (plane) {
+          case 'grid': {
+            const cells: DxGridPlaneCells = {};
+            for (let row = range.start.row; row <= range.end.row && row < queueMap.length; row++) {
+              const [blockIndex, segmentIndex] = queueMap[row];
+
+              cells[toPlaneCellIndex({ col: 0, row })] = {
+                readonly: true,
+                value:
+                  segmentIndex < 0
+                    ? formatTimestamp(
+                        new Date(transcriptStart),
+                        new Date(blocks[blockIndex]!.segments[Math.max(0, segmentIndex)]!.started),
+                      )
+                    : '',
+                className: timestampClasses,
+              } satisfies DxGridCellValue;
+
+              cells[toPlaneCellIndex({ col: 1, row })] = (
+                segmentIndex < 0
+                  ? {
+                      readonly: true,
+                      accessoryHtml: `<span data-hue="${blocks[blockIndex]!.authorHue}" class="dx-text-hue">${blocks[blockIndex]!.authorName}</span>`,
+                      className: authorClasses,
+                    }
+                  : {
+                      readonly: true,
+                      value: blocks[blockIndex]!.segments[segmentIndex]!.text,
+                      className: segmentTextClasses,
+                    }
+              ) satisfies DxGridCellValue;
+            }
+            return cells;
+          }
+          default:
+            return {};
+        }
+      };
+    }
+  }, [dxGrid, blocks]);
+
+  const handleScrollToEnd = useCallback(() => {
+    setAutoScroll(true);
+    dxGrid?.scrollToEndRow();
+  }, [dxGrid, autoScroll]);
+
+  return (
+    <div role='none' className={mx('relative min-bs-0', classNames)}>
+      <Grid.Root id={`${attendableId}--transcript`}>
+        <Grid.Content
+          limitColumns={2}
+          limitRows={queueMap.length}
+          columns={columns}
+          rows={rows}
+          rowDefault={rowDefault}
+          onWheel={handleWheel}
+          className='[--dx-grid-base:var(--dx-baseSurface)] [--dx-grid-lines:var(--dx-baseSurface)] [&_.dx-grid]:min-bs-0 [&_.dx-grid]:min-is-0 [&_.dx-grid]:select-auto'
+          ref={setDxGrid}
+        />
+      </Grid.Root>
+      <div role='none' {...{ inert: '' }} aria-hidden className={measureClasses} ref={measureRef} />
+      <IconButton
+        icon='ph--arrow-line-down--regular'
+        iconOnly
+        label={t('scroll to end label')}
+        tooltipSide='left'
+        data-state={autoScroll ? 'invisible' : 'visible'}
+        classNames={[
+          'absolute inline-end-2 block-end-2 opacity-0 pointer-events-none',
+          'data-[state="visible"]:pointer-events-auto data-[state="visible"]:opacity-100 transition-opacity',
+        ]}
+        onClick={handleScrollToEnd}
+      />
+    </div>
+  );
+};

package/src/components/Transcript/transcript-extension.ts

@@ -0,0 +1,236 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type Extension, RangeSetBuilder } from '@codemirror/state';
+import { EditorView, GutterMarker, ViewPlugin, type ViewUpdate, gutter } from '@codemirror/view';
+import { format } from 'date-fns/format';
+import { intervalToDuration } from 'date-fns/intervalToDuration';
+
+import { type CleanupFn, addEventListener, combine } from '@dxos/async';
+import { type RenderCallback } from '@dxos/react-ui-editor';
+import { type DataType } from '@dxos/schema';
+
+import { DocumentAdapter, type SerializationModel } from '../../model';
+
+/**
+ * Data structure that maps Chunks queue to lines with transcript state.
+ */
+export type TranscriptOptions = {
+  model: SerializationModel<DataType.Message>;
+  started?: Date;
+  renderButton?: RenderCallback<{ onClick: () => void }>;
+};
+
+/**
+ * Scrolling transcript with timestamps.
+ */
+export const transcript = ({ model, started, renderButton }: TranscriptOptions): Extension => {
+  return [
+    // Show timestamps in the gutter.
+    gutter({
+      class: 'cm-timestamp-gutter',
+      lineMarkerChange: (update) => update.docChanged || update.viewportChanged,
+      markers: (view) => {
+        const start = getStartTime(started, model.chunks[0]);
+        const builder = new RangeSetBuilder<GutterMarker>();
+        for (const { from, to } of view.visibleRanges) {
+          let line = view.state.doc.lineAt(from);
+          while (line.from <= to) {
+            const block = model.getChunkAtLine(line.number)?.blocks[0];
+            const timestamp = block?.type === 'transcription' && block.started;
+            if (timestamp) {
+              builder.add(line.from, line.from, new TimestampMarker(line.number, new Date(timestamp), start));
+            }
+
+            if (line.to + 1 > view.state.doc.length) {
+              break;
+            }
+
+            line = view.state.doc.lineAt(line.to + 1);
+          }
+        }
+
+        return builder.finish();
+      },
+    }),
+
+    // Listen for model updates.
+    ViewPlugin.fromClass(
+      class {
+        private readonly _controls?: HTMLDivElement;
+        private readonly _cleanup: CleanupFn;
+        private readonly _adapter: DocumentAdapter;
+        private _initialized = false;
+
+        constructor(view: EditorView) {
+          this._adapter = new DocumentAdapter(view);
+
+          const scroller = view.scrollDOM;
+          let isAutoScrolling = false;
+          let hasScrolled = false;
+
+          let timeout: NodeJS.Timeout | undefined;
+          const scrollToBottom = (smooth = false) => {
+            scroller.style.scrollBehavior = smooth ? 'smooth' : '';
+
+            // Temporarily hide scrollbar to prevent flicker.
+            scroller.classList.add('cm-hide-scrollbar');
+            isAutoScrolling = true;
+            clearTimeout(timeout);
+            timeout = setTimeout(() => {
+              this._controls?.classList.add('opacity-0');
+              scroller.classList.remove('cm-hide-scrollbar');
+              isAutoScrolling = false;
+            }, 500);
+
+            // Scroll to bottom.
+            view.dispatch({
+              effects: EditorView.scrollIntoView(view.state.doc.length, { y: 'end' }),
+            });
+          };
+
+          // Scroll button.
+          if (renderButton) {
+            this._controls = document.createElement('div');
+            this._controls.classList.add('cm-controls', 'transition-opacity', 'duration-300', 'opacity-0');
+            view.dom.appendChild(this._controls);
+            renderButton(
+              this._controls,
+              {
+                onClick: () => {
+                  scrollToBottom(false);
+                },
+              },
+              view,
+            );
+          }
+
+          // Event listeners.
+          this._cleanup = combine(
+            addEventListener(view.scrollDOM, 'scroll', () => {
+              if (!isAutoScrolling) {
+                hasScrolled = true;
+                this._controls?.classList.remove('opacity-0');
+              }
+            }),
+
+            model.update.on(() => {
+              // Check if clamped to bottom.
+              const autoScroll =
+                scroller.scrollHeight - scroller.scrollTop - scroller.clientHeight === 0 || !hasScrolled;
+
+              // Sync.
+              model.sync(this._adapter);
+
+              // Scroll.
+              if (autoScroll) {
+                scrollToBottom(true);
+              }
+            }),
+          );
+        }
+
+        update(update: ViewUpdate) {
+          // Initial sync.
+          if (!this._initialized) {
+            this._initialized = true;
+            setTimeout(() => {
+              model.sync(this._adapter);
+            });
+          }
+        }
+
+        destroy() {
+          this._controls?.remove();
+          this._cleanup();
+        }
+      },
+    ),
+
+    EditorView.theme({
+      '.cm-scroller': {
+        overflowY: 'scroll',
+      },
+      '.cm-hide-scrollbar': {
+        scrollbarWidth: 'none',
+        '-ms-overflow-style': 'none',
+      },
+      '.cm-hide-scrollbar::-webkit-scrollbar': {
+        display: 'none',
+      },
+      '.cm-controls': {
+        position: 'absolute',
+        bottom: '0.5rem',
+        right: '0.5rem',
+        zIndex: 1000,
+      },
+      '.cm-line': {
+        paddingRight: '1rem',
+      },
+      '.cm-timestamp-gutter': {
+        width: '6rem',
+        paddingRight: '1rem',
+      },
+      '.cm-timestamp-gutter > .cm-gutterElement > div': {
+        display: 'inline-flex',
+        textAlign: 'right',
+        padding: '3px',
+      },
+    }),
+  ];
+};
+
+/**
+ * Gutter marker that displays a timestamp.
+ */
+class TimestampMarker extends GutterMarker {
+  constructor(
+    readonly _line: number,
+    readonly _timestamp: Date,
+    readonly _started?: Date,
+  ) {
+    super();
+  }
+
+  override eq(other: this) {
+    return other._timestamp === this._timestamp;
+  }
+
+  override toDOM(view: EditorView) {
+    const el = document.createElement('div');
+    el.className = 'text-sm text-subdued hover:bg-hoverSurface cursor-pointer';
+    el.textContent = formatTimestamp(this._timestamp, this._started);
+    // TODO(burdon): Click to bookmark or copy hyperlink.
+    el.onclick = () => {
+      const pos = view.state.doc.line(this._line).from;
+      view.dispatch({
+        effects: EditorView.scrollIntoView(pos, { y: 'start' }),
+      });
+    };
+
+    return el;
+  }
+}
+
+const getStartTime = (started?: Date, message?: DataType.Message): Date | undefined => {
+  if (started) {
+    return started;
+  }
+
+  if (message?.blocks[0]?.type === 'transcription' && message.blocks[0].started) {
+    return new Date(message.blocks[0].started);
+  }
+
+  return undefined;
+};
+
+const formatTimestamp = (timestamp: Date, relative?: Date) => {
+  if (relative) {
+    const pad = (n = 0) => String(n).padStart(2, '0');
+    const { hours, minutes, seconds } = intervalToDuration({ start: relative, end: timestamp });
+    return `${pad(hours)}:${pad(minutes)}:${pad(seconds)}`;
+  } else {
+    return format(timestamp, 'HH:mm:ss');
+  }
+};

package/src/components/TranscriptContainer.tsx

@@ -4,20 +4,25 @@
 
 import React, { type FC } from 'react';
 
-import { fullyQualifiedId } from '@dxos/client/echo';
-import { useQueue } from '@dxos/react-client/echo';
+import { fullyQualifiedId, getSpace } from '@dxos/client/echo';
+import { useMembers, useQueue } from '@dxos/react-client/echo';
 import { StackItem } from '@dxos/react-ui-stack';
+import { type DataType } from '@dxos/schema';
 
-import { Transcript } from './Transcript';
-import { type TranscriptBlock, type TranscriptType } from '../types';
+import { Transcript, renderMarkdown } from './Transcript';
+import { useQueueModelAdapter } from '../hooks';
+import { type TranscriptType } from '../types';
 
 export const TranscriptionContainer: FC<{ role: string; transcript: TranscriptType }> = ({ role, transcript }) => {
   const attendableId = fullyQualifiedId(transcript);
-  const queue = useQueue<TranscriptBlock>(transcript.queue.dxn, { pollInterval: 1_000 });
+  const space = getSpace(transcript);
+  const members = useMembers(space?.key).map((member) => member.identity);
+  const queue = useQueue<DataType.Message>(transcript.queue.dxn, { pollInterval: 1_000 });
+  const model = useQueueModelAdapter(renderMarkdown(members), queue);
 
   return (
     <StackItem.Content role={role} classNames='container-max-width pli-2'>
-      <Transcript attendableId={attendableId} blocks={queue?.items} />
+      <Transcript attendableId={attendableId} space={space} model={model} transcript={transcript} />
     </StackItem.Content>
   );
 };

package/src/components/TranscriptionSettings/TranscriptionSettings.tsx

@@ -0,0 +1,26 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import React from 'react';
+
+import { Input, useTranslation } from '@dxos/react-ui';
+import { DeprecatedFormContainer, DeprecatedFormInput } from '@dxos/react-ui-form';
+
+import { TRANSCRIPTION_PLUGIN } from '../../meta';
+import { type TranscriptionSettingsProps } from '../../types';
+
+export const TranscriptionSettings = ({ settings }: { settings: TranscriptionSettingsProps }) => {
+  const { t } = useTranslation(TRANSCRIPTION_PLUGIN);
+
+  return (
+    <DeprecatedFormContainer>
+      <DeprecatedFormInput label={t('settings entity extraction label')}>
+        <Input.Switch
+          checked={!!settings.entityExtraction}
+          onCheckedChange={(checked) => (settings.entityExtraction = checked)}
+        />
+      </DeprecatedFormInput>
+    </DeprecatedFormContainer>
+  );
+};

package/src/components/TranscriptionSettings/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './TranscriptionSettings';

package/src/components/index.ts

@@ -6,5 +6,6 @@ import { lazy } from 'react';
 
 export * from './Transcript';
 export * from './TranscriptContainer';
+export * from './TranscriptionSettings';
 
 export const TranscriptContainer = lazy(() => import('./TranscriptContainer'));