@dxos/plugin-transcription 0.8.1 → 0.8.2-main.12df754

package/src/components/Transcript/deprecated/Transcript.tsx

@@ -0,0 +1,231 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import { intervalToDuration } from 'date-fns/intervalToDuration';
+import { yieldOrContinue } from 'main-thread-scheduling';
+import React, { type FC, useCallback, useEffect, useMemo, useRef, useState, type WheelEvent } from 'react';
+import { type OnResizeCallback, useResizeDetector } from 'react-resize-detector';
+
+import { IconButton, type ThemedClassName, useTranslation } from '@dxos/react-ui';
+import { useAttention } from '@dxos/react-ui-attention';
+import {
+  type DxGridCellValue,
+  type DxGridElement,
+  type DxGridAxisMeta,
+  type DxGridPlaneCells,
+  Grid,
+  toPlaneCellIndex,
+} from '@dxos/react-ui-grid';
+import { mx } from '@dxos/react-ui-theme';
+
+import { TRANSCRIPTION_PLUGIN } from '../../../meta';
+import { type TranscriptBlock } from '../../../types';
+
+// TODO(burdon): Actions (e.g., mark, summarize, translate, label, delete).
+
+const lineHeight = 20;
+const cellSpacing = 8 + 2;
+const timestampColumnWidth = 68;
+
+const authorClasses = 'font-medium text-base leading-[20px]';
+const timestampClasses = 'text-xs leading-[20px] text-description pie-0 tabular-nums';
+const segmentTextClasses = 'text-sm whitespace-normal hyphens-auto';
+const measureClasses = mx(
+  // NOTE(thure): The `inline-start` value must equal `timestampColumnWidth` plus grid’s gap (1px)
+  'absolute inline-start-[69px] inline-end-0 invisible z-[-1] border',
+  'pli-[--dx-grid-cell-padding-inline] plb-[--dx-grid-cell-padding-block] leading-[20px]',
+  segmentTextClasses,
+);
+
+const rowDefault = {
+  grid: { size: lineHeight + cellSpacing },
+};
+
+type QueueRows = [number, number][];
+
+const mapTranscriptQueue = (blocks?: TranscriptBlock[]): QueueRows => {
+  if (!blocks || !Array.isArray(blocks) || blocks.length === 0) {
+    return [];
+  } else {
+    return blocks.flatMap((block, blockIndex) => {
+      return [
+        [blockIndex, -1],
+        ...block.segments.map((segment, segmentIndex) => {
+          return [blockIndex, segmentIndex] as [number, number];
+        }),
+      ];
+    });
+  }
+};
+
+const pad = (n = 0) => String(n).padStart(2, '0');
+
+const formatTimestamp = (start: Date, end: Date) => {
+  const { hours, minutes, seconds } = intervalToDuration({ start, end });
+  return `${pad(hours)}:${pad(minutes)}:${pad(seconds)}`;
+};
+
+const measureRows = async (
+  host: HTMLDivElement,
+  blocks: TranscriptBlock[],
+  queueMap: QueueRows,
+  signal: AbortSignal,
+) => {
+  const result: DxGridAxisMeta = { grid: {} };
+  for (let row = 0; row < queueMap.length; row++) {
+    await yieldOrContinue('smooth', signal);
+    const [blockIndex, segmentIndex] = queueMap[row];
+    if (segmentIndex < 0) {
+      result.grid[row] = { size: lineHeight + cellSpacing };
+    } else {
+      host.textContent = blocks[blockIndex]!.segments[segmentIndex]!.text;
+      result.grid[row] = { size: host.offsetHeight };
+    }
+  }
+  return result;
+};
+
+export type TranscriptProps = ThemedClassName<{
+  blocks?: TranscriptBlock[];
+  attendableId?: string;
+  ignoreAttention?: boolean;
+}>;
+
+/**
+ * @@deprecated
+ */
+export const Transcript: FC<TranscriptProps> = ({ classNames, blocks, attendableId, ignoreAttention }) => {
+  const { t } = useTranslation(TRANSCRIPTION_PLUGIN);
+  const [dxGrid, setDxGrid] = useState<DxGridElement | null>(null);
+  const [rows, setRows] = useState<DxGridAxisMeta | undefined>(undefined);
+  const [columns, setColumns] = useState<DxGridAxisMeta | undefined>(undefined);
+
+  const queueMap = useMemo(() => mapTranscriptQueue(blocks), [blocks]);
+
+  const { hasAttention } = useAttention(attendableId);
+  const [autoScroll, setAutoScroll] = useState(true);
+  const handleWheel = useCallback(
+    (event: WheelEvent) => {
+      setAutoScroll(false);
+      if (!ignoreAttention && !hasAttention) {
+        event.stopPropagation();
+      }
+    },
+    [hasAttention, ignoreAttention],
+  );
+
+  const abortControllerRef = useRef<AbortController>();
+
+  const handleResize = useCallback(
+    async ({ entry }: { entry: { target: HTMLDivElement } | null }) => {
+      if (entry?.target && Array.isArray(blocks) && blocks[0] && queueMap) {
+        abortControllerRef.current?.abort();
+        abortControllerRef.current = new AbortController();
+        setColumns({ grid: { 0: { size: timestampColumnWidth }, 1: { size: entry.target.offsetWidth } } });
+        return measureRows(entry.target, blocks, queueMap, abortControllerRef.current.signal)
+          .then(setRows)
+          .catch(() => {
+            // Aborted mid-measurement by new size.
+          });
+      }
+    },
+    [blocks, queueMap],
+  );
+
+  const { width, ref: measureRef } = useResizeDetector({
+    onResize: handleResize as OnResizeCallback,
+    refreshOptions: { leading: true },
+  });
+
+  useEffect(() => {
+    if (queueMap.length !== Object.keys(rows?.grid ?? {}).length) {
+      void handleResize({ entry: { target: measureRef.current } }).then(() => {
+        if (autoScroll) {
+          // TODO(thure): Implement a deterministic way to do this when `rows` has fully settled and grid has a new `maxPosBlock`.
+          setTimeout(() => dxGrid?.scrollToEndRow(), 50);
+        }
+      });
+    }
+  }, [blocks, queueMap, width, autoScroll]);
+
+  useEffect(() => {
+    if (dxGrid && Array.isArray(blocks) && blocks[0]) {
+      const transcriptStart = blocks[0]!.segments[0]!.started;
+      dxGrid.getCells = (range, plane) => {
+        switch (plane) {
+          case 'grid': {
+            const cells: DxGridPlaneCells = {};
+            for (let row = range.start.row; row <= range.end.row && row < queueMap.length; row++) {
+              const [blockIndex, segmentIndex] = queueMap[row];
+
+              cells[toPlaneCellIndex({ col: 0, row })] = {
+                readonly: true,
+                value:
+                  segmentIndex < 0
+                    ? formatTimestamp(
+                        new Date(transcriptStart),
+                        new Date(blocks[blockIndex]!.segments[Math.max(0, segmentIndex)]!.started),
+                      )
+                    : '',
+                className: timestampClasses,
+              } satisfies DxGridCellValue;
+
+              cells[toPlaneCellIndex({ col: 1, row })] = (
+                segmentIndex < 0
+                  ? {
+                      readonly: true,
+                      accessoryHtml: `<span data-hue="${blocks[blockIndex]!.authorHue}" class="dx-text-hue">${blocks[blockIndex]!.authorName}</span>`,
+                      className: authorClasses,
+                    }
+                  : {
+                      readonly: true,
+                      value: blocks[blockIndex]!.segments[segmentIndex]!.text,
+                      className: segmentTextClasses,
+                    }
+              ) satisfies DxGridCellValue;
+            }
+            return cells;
+          }
+          default:
+            return {};
+        }
+      };
+    }
+  }, [dxGrid, blocks]);
+
+  const handleScrollToEnd = useCallback(() => {
+    setAutoScroll(true);
+    dxGrid?.scrollToEndRow();
+  }, [dxGrid, autoScroll]);
+
+  return (
+    <div role='none' className={mx('relative min-bs-0', classNames)}>
+      <Grid.Root id={`${attendableId}--transcript`}>
+        <Grid.Content
+          limitColumns={2}
+          limitRows={queueMap.length}
+          columns={columns}
+          rows={rows}
+          rowDefault={rowDefault}
+          onWheel={handleWheel}
+          className='[--dx-grid-base:var(--dx-baseSurface)] [--dx-grid-lines:var(--dx-baseSurface)] [&_.dx-grid]:min-bs-0 [&_.dx-grid]:min-is-0 [&_.dx-grid]:select-auto'
+          ref={setDxGrid}
+        />
+      </Grid.Root>
+      <div role='none' {...{ inert: '' }} aria-hidden className={measureClasses} ref={measureRef} />
+      <IconButton
+        icon='ph--arrow-line-down--regular'
+        iconOnly
+        label={t('scroll to end label')}
+        tooltipSide='left'
+        data-state={autoScroll ? 'invisible' : 'visible'}
+        classNames={[
+          'absolute inline-end-2 block-end-2 opacity-0 pointer-events-none',
+          'data-[state="visible"]:pointer-events-auto data-[state="visible"]:opacity-100 transition-opacity',
+        ]}
+        onClick={handleScrollToEnd}
+      />
+    </div>
+  );
+};

package/src/components/Transcript/testings.ts

@@ -0,0 +1,182 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { useEffect, useMemo, useState } from 'react';
+
+import { AIServiceEdgeClient } from '@dxos/assistant';
+import { AI_SERVICE_ENDPOINT } from '@dxos/assistant/testing';
+import { scheduleTaskInterval } from '@dxos/async';
+import { Context } from '@dxos/context';
+import { AST, create, EchoObject, ObjectId, S } from '@dxos/echo-schema';
+import { DXN, QueueSubspaceTags } from '@dxos/keys';
+import { faker } from '@dxos/random';
+import { live, makeRef, useQueue, type Space } from '@dxos/react-client/echo';
+import { SpaceId } from '@dxos/react-client/echo';
+import { hues } from '@dxos/react-ui-theme';
+import { ContactType } from '@dxos/schema';
+
+import { processTranscriptBlock } from '../../entity-extraction';
+import * as TestData from '../../testing/test-data';
+import { TranscriptBlock } from '../../types';
+
+// TODO(burdon): Reconcile with plugin-markdown. Move to schema-testing.
+export const TestItem = S.Struct({
+  title: S.String.annotations({
+    [AST.TitleAnnotationId]: 'Title',
+    [AST.DescriptionAnnotationId]: 'Product title',
+  }),
+  description: S.String.annotations({
+    [AST.TitleAnnotationId]: 'Description',
+    [AST.DescriptionAnnotationId]: 'Product description',
+  }),
+}).pipe(EchoObject({ typename: 'dxos.org/type/Test', version: '0.1.0' }));
+
+/**
+ * Generator of transcript blocks.
+ */
+export class BlockBuilder {
+  static readonly singleton = new BlockBuilder();
+
+  users = Array.from({ length: 5 }, () => ({
+    authorName: faker.person.fullName(),
+    authorHue: faker.helpers.arrayElement(hues),
+  }));
+
+  start = new Date(Date.now() - 24 * 60 * 60 * 10_000);
+
+  constructor(private readonly _space?: Space) {}
+
+  createBlock(numSegments = 1): TranscriptBlock {
+    return {
+      id: ObjectId.random().toString(),
+      ...faker.helpers.arrayElement(this.users),
+      segments: Array.from({ length: numSegments }).map(() => this.createSegment()),
+    };
+  }
+
+  createSegment() {
+    let text = faker.lorem.paragraph();
+    if (this._space) {
+      const label = faker.commerce.productName();
+      const obj = this._space.db.add(live(TestItem, { title: label, description: faker.lorem.paragraph() }));
+      const dxn = makeRef(obj).dxn.toString();
+      const words = text.split(' ');
+      words.splice(Math.floor(Math.random() * words.length), 0, `[${label}][${dxn}]`);
+      text = words.join(' ');
+    }
+
+    return {
+      started: this.next().toISOString(),
+      text,
+    };
+  }
+
+  next() {
+    this.start = new Date(this.start.getTime() + Math.random() * 10_000);
+    return this.start;
+  }
+}
+
+/**
+ * Test transcriptionqueue.
+ */
+export const useTestTranscriptionQueue = (space: Space | undefined, running = true, interval = 1_000) => {
+  const queueDxn = useMemo(
+    () => (space ? new DXN(DXN.kind.QUEUE, [QueueSubspaceTags.DATA, space.id, ObjectId.random()]) : undefined),
+    [space],
+  );
+  const queue = useQueue<TranscriptBlock>(queueDxn);
+
+  const builder = useMemo(() => new BlockBuilder(space), [space]);
+  useEffect(() => {
+    if (!queue || !running) {
+      return;
+    }
+
+    const i = setInterval(() => {
+      queue.append([create(TranscriptBlock, builder.createBlock(Math.ceil(Math.random() * 3)))]);
+    }, interval);
+    return () => clearInterval(i);
+  }, [queue, running, interval]);
+
+  return queue;
+};
+
+class EntityExtractionBlockBuilder {
+  aiService = new AIServiceEdgeClient({
+    endpoint: AI_SERVICE_ENDPOINT.REMOTE,
+  });
+
+  currentBlock: number = 0;
+
+  seedData(space: Space) {
+    space.db.graph.schemaRegistry.addSchema([ContactType]);
+    // for (const document of TestData.documents) {
+    //   const obj = space.db.add(live(Document, document));
+    //   const dxn = makeRef(obj).dxn.toString();
+    //   document.dxn = dxn;
+    // }
+
+    for (const contact of Object.values(TestData.contacts)) {
+      space.db.add(contact);
+    }
+  }
+
+  async createBlock(): Promise<TranscriptBlock> {
+    const block = TestData.transcriptBlocks[this.currentBlock];
+    this.currentBlock++;
+    this.currentBlock = this.currentBlock % TestData.transcriptBlocks.length;
+
+    const { block: enhancedBlock } = await processTranscriptBlock({
+      block,
+      aiService: this.aiService,
+      context: {
+        objects: [...TestData.documents, ...Object.values(TestData.contacts)],
+      },
+    });
+    return enhancedBlock;
+  }
+}
+
+const randomQueueDXN = (spaceId = SpaceId.random()) =>
+  new DXN(DXN.kind.QUEUE, [QueueSubspaceTags.DATA, spaceId, ObjectId.random()]);
+
+/**
+ * Test transcription queue.
+ */
+export const useTestTranscriptionQueueWithEntityExtraction = (
+  space: Space | undefined,
+  running = true,
+  interval = 1_000,
+) => {
+  const queueDxn = useMemo(() => (space ? randomQueueDXN(space.id) : undefined), [space]);
+  const queue = useQueue<TranscriptBlock>(queueDxn);
+  const [builder] = useState(() => new EntityExtractionBlockBuilder());
+
+  useEffect(() => {
+    if (!queue || !running) {
+      return;
+    }
+
+    if (space) {
+      builder.seedData(space);
+    }
+
+    const ctx = new Context();
+    scheduleTaskInterval(
+      ctx,
+      async () => {
+        const block = await builder.createBlock();
+        queue.append([create(TranscriptBlock, block)]);
+      },
+      interval,
+    );
+
+    return () => {
+      void ctx.dispose();
+    };
+  }, [space, queue, running, interval]);
+
+  return queue;
+};

package/src/components/Transcript/transcript-extension.ts

@@ -0,0 +1,233 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type Extension, RangeSetBuilder } from '@codemirror/state';
+import { EditorView, GutterMarker, ViewPlugin, type ViewUpdate, gutter } from '@codemirror/view';
+import { format } from 'date-fns/format';
+import { intervalToDuration } from 'date-fns/intervalToDuration';
+
+import { type CleanupFn, addEventListener, combine } from '@dxos/async';
+import { type RenderCallback } from '@dxos/react-ui-editor';
+
+import { DocumentAdapter, type BlockModel } from '../../model';
+import { type TranscriptBlock } from '../../types';
+
+/**
+ * Data structure that maps Blocks queue to lines with transcript state.
+ */
+export type TranscriptOptions = {
+  model: BlockModel<TranscriptBlock>;
+  started?: Date;
+  renderButton?: RenderCallback<{ onClick: () => void }>;
+};
+
+/**
+ * Scrolling transcript with timestamps.
+ */
+export const transcript = ({ model, started, renderButton }: TranscriptOptions): Extension => {
+  return [
+    // Show timestamps in the gutter.
+    gutter({
+      class: 'cm-timestamp-gutter',
+      lineMarkerChange: (update) => update.docChanged || update.viewportChanged,
+      markers: (view) => {
+        const start = getStartTime(started, model.blocks[0]);
+        const builder = new RangeSetBuilder<GutterMarker>();
+        for (const { from, to } of view.visibleRanges) {
+          let line = view.state.doc.lineAt(from);
+          while (line.from <= to) {
+            const timestamp = model.getBlockAtLine(line.number)?.segments[0]?.started;
+            if (timestamp) {
+              builder.add(line.from, line.from, new TimestampMarker(line.number, new Date(timestamp), start));
+            }
+
+            if (line.to + 1 > view.state.doc.length) {
+              break;
+            }
+
+            line = view.state.doc.lineAt(line.to + 1);
+          }
+        }
+
+        return builder.finish();
+      },
+    }),
+
+    // Listen for model updates.
+    ViewPlugin.fromClass(
+      class {
+        private readonly _controls?: HTMLDivElement;
+        private readonly _cleanup: CleanupFn;
+        private readonly _adapter: DocumentAdapter;
+        private _initialized = false;
+
+        constructor(view: EditorView) {
+          this._adapter = new DocumentAdapter(view);
+
+          const scroller = view.scrollDOM;
+          let isAutoScrolling = false;
+          let hasScrolled = false;
+
+          const scrollToBottom = () => {
+            scroller.style.scrollBehavior = 'smooth';
+
+            // Temporarily hide scrollbar to prevent flicker.
+            scroller.classList.add('cm-hide-scrollbar');
+            isAutoScrolling = true;
+            setTimeout(() => {
+              this._controls?.classList.add('opacity-0');
+              scroller.classList.remove('cm-hide-scrollbar');
+              isAutoScrolling = false;
+            }, 500);
+
+            // Scroll to bottom.
+            view.dispatch({
+              effects: EditorView.scrollIntoView(view.state.doc.length, { y: 'end' }),
+            });
+          };
+
+          // Scroll button.
+          if (renderButton) {
+            this._controls = document.createElement('div');
+            this._controls.classList.add('cm-controls', 'transition-opacity', 'duration-300', 'opacity-0');
+            view.dom.appendChild(this._controls);
+            renderButton(
+              this._controls,
+              {
+                onClick: () => {
+                  scrollToBottom();
+                },
+              },
+              view,
+            );
+          }
+
+          // Event listeners.
+          this._cleanup = combine(
+            addEventListener(view.scrollDOM, 'scroll', () => {
+              if (!isAutoScrolling) {
+                hasScrolled = true;
+                this._controls?.classList.remove('opacity-0');
+              }
+            }),
+
+            model.update.on(() => {
+              // Check if clamped to bottom.
+              const autoScroll =
+                scroller.scrollHeight - scroller.scrollTop - scroller.clientHeight === 0 || !hasScrolled;
+
+              // Sync.
+              model.sync(this._adapter);
+
+              // Scroll.
+              if (autoScroll) {
+                scrollToBottom();
+              }
+            }),
+          );
+        }
+
+        update(update: ViewUpdate) {
+          // Initial sync.
+          if (!this._initialized) {
+            this._initialized = true;
+            setTimeout(() => {
+              model.sync(this._adapter);
+            });
+          }
+        }
+
+        destroy() {
+          this._controls?.remove();
+          this._cleanup();
+        }
+      },
+    ),
+
+    EditorView.theme({
+      '.cm-scroller': {
+        overflowY: 'scroll',
+      },
+      '.cm-hide-scrollbar': {
+        scrollbarWidth: 'none',
+        '-ms-overflow-style': 'none',
+      },
+      '.cm-hide-scrollbar::-webkit-scrollbar': {
+        display: 'none',
+      },
+      '.cm-controls': {
+        position: 'absolute',
+        bottom: '0.5rem',
+        right: '0.5rem',
+        zIndex: 1000,
+      },
+      '.cm-line': {
+        paddingRight: '1rem',
+      },
+      '.cm-timestamp-gutter': {
+        width: '6rem',
+        paddingRight: '1rem',
+      },
+      '.cm-timestamp-gutter > .cm-gutterElement > div': {
+        display: 'inline-flex',
+        textAlign: 'right',
+        padding: '3px',
+      },
+    }),
+  ];
+};
+
+/**
+ * Gutter marker that displays a timestamp.
+ */
+class TimestampMarker extends GutterMarker {
+  constructor(
+    readonly _line: number,
+    readonly _timestamp: Date,
+    readonly _started?: Date,
+  ) {
+    super();
+  }
+
+  override eq(other: this) {
+    return other._timestamp === this._timestamp;
+  }
+
+  override toDOM(view: EditorView) {
+    const el = document.createElement('div');
+    el.className = 'text-sm text-subdued hover:bg-hoverSurface cursor-pointer';
+    el.textContent = formatTimestamp(this._timestamp, this._started);
+    // TODO(burdon): Click to bookmark or copy hyperlink.
+    el.onclick = () => {
+      const pos = view.state.doc.line(this._line).from;
+      view.dispatch({
+        effects: EditorView.scrollIntoView(pos, { y: 'start' }),
+      });
+    };
+
+    return el;
+  }
+}
+
+const getStartTime = (started?: Date, block?: TranscriptBlock): Date | undefined => {
+  if (started) {
+    return started;
+  }
+
+  if (block?.segments[0]?.started) {
+    return new Date(block.segments[0].started);
+  }
+
+  return undefined;
+};
+
+const formatTimestamp = (timestamp: Date, relative?: Date) => {
+  if (!relative) {
+    return format(timestamp, 'HH:mm:ss');
+  } else {
+    const pad = (n = 0) => String(n).padStart(2, '0');
+    const { hours, minutes, seconds } = intervalToDuration({ start: relative, end: timestamp });
+    return `${pad(hours)}:${pad(minutes)}:${pad(seconds)}`;
+  }
+};

package/src/components/TranscriptContainer.tsx

@@ -4,20 +4,23 @@
 
 import React, { type FC } from 'react';
 
-import { fullyQualifiedId } from '@dxos/client/echo';
+import { fullyQualifiedId, getSpace } from '@dxos/client/echo';
 import { useQueue } from '@dxos/react-client/echo';
 import { StackItem } from '@dxos/react-ui-stack';
 
-import { Transcript } from './Transcript';
+import { Transcript, renderMarkdown } from './Transcript';
+import { useQueueModelAdapter } from '../hooks';
 import { type TranscriptBlock, type TranscriptType } from '../types';
 
 export const TranscriptionContainer: FC<{ role: string; transcript: TranscriptType }> = ({ role, transcript }) => {
   const attendableId = fullyQualifiedId(transcript);
+  const space = getSpace(transcript);
   const queue = useQueue<TranscriptBlock>(transcript.queue.dxn, { pollInterval: 1_000 });
+  const model = useQueueModelAdapter(renderMarkdown, queue);
 
   return (
     <StackItem.Content role={role} classNames='container-max-width pli-2'>
-      <Transcript attendableId={attendableId} blocks={queue?.items} />
+      <Transcript attendableId={attendableId} space={space} model={model} transcript={transcript} />
     </StackItem.Content>
   );
 };