@dxos/plugin-transcription 0.8.3 → 0.8.4-main.1c7ec43d41

package/src/components/Transcript/deprecated/OldTranscript.tsx

@@ -1,260 +0,0 @@
-//
-// Copyright 2023 DXOS.org
-//
-
-import { intervalToDuration } from 'date-fns/intervalToDuration';
-import { Schema } from 'effect';
-import { yieldOrContinue } from 'main-thread-scheduling';
-import React, { type FC, useCallback, useEffect, useMemo, useRef, useState, type WheelEvent } from 'react';
-import { type OnResizeCallback, useResizeDetector } from 'react-resize-detector';
-
-import { Type } from '@dxos/echo';
-import { IconButton, type ThemedClassName, useTranslation } from '@dxos/react-ui';
-import { useAttention } from '@dxos/react-ui-attention';
-import {
-  type DxGridCellValue,
-  type DxGridElement,
-  type DxGridAxisMeta,
-  type DxGridPlaneCells,
-  Grid,
-  toPlaneCellIndex,
-} from '@dxos/react-ui-grid';
-import { mx } from '@dxos/react-ui-theme';
-
-import { TRANSCRIPTION_PLUGIN } from '../../../meta';
-
-/**
- * Transcription fragment.
- */
-const TranscriptSegment = Schema.Struct({
-  // TODO(burdon): TS from service is not Unix TS (x1000).
-  started: Schema.String,
-  text: Schema.String,
-});
-
-type TranscriptSegment = Schema.Schema.Type<typeof TranscriptSegment>;
-
-/**
- * Transcription block (from single speaker).
- */
-const TranscriptBlock = Schema.Struct({
-  id: Schema.String,
-  authorName: Schema.optional(Schema.String), // TODO(burdon): Replace with identityDid.
-  authorHue: Schema.optional(Schema.String), // TOOD(burdon): Remove.
-  segments: Schema.Array(TranscriptSegment),
-}).pipe(
-  Type.Obj({
-    typename: 'dxos.org/type/TranscriptBlock',
-    version: '0.1.0',
-  }),
-);
-
-type TranscriptBlock = Schema.Schema.Type<typeof TranscriptBlock>;
-
-// TODO(burdon): Actions (e.g., mark, summarize, translate, label, delete).
-
-const lineHeight = 20;
-const cellSpacing = 8 + 2;
-const timestampColumnWidth = 68;
-
-const authorClasses = 'font-medium text-base leading-[20px]';
-const timestampClasses = 'text-xs leading-[20px] text-description pie-0 tabular-nums';
-const segmentTextClasses = 'text-sm whitespace-normal hyphens-auto';
-const measureClasses = mx(
-  // NOTE(thure): The `inline-start` value must equal `timestampColumnWidth` plus grid’s gap (1px)
-  'absolute inline-start-[69px] inline-end-0 invisible z-[-1] border',
-  'pli-[--dx-grid-cell-padding-inline] plb-[--dx-grid-cell-padding-block] leading-[20px]',
-  segmentTextClasses,
-);
-
-const rowDefault = {
-  grid: { size: lineHeight + cellSpacing },
-};
-
-type QueueRows = [number, number][];
-
-const mapTranscriptQueue = (blocks?: TranscriptBlock[]): QueueRows => {
-  if (!blocks || !Array.isArray(blocks) || blocks.length === 0) {
-    return [];
-  } else {
-    return blocks.flatMap((block, blockIndex) => {
-      return [
-        [blockIndex, -1],
-        ...block.segments.map((segment, segmentIndex) => {
-          return [blockIndex, segmentIndex] as [number, number];
-        }),
-      ];
-    });
-  }
-};
-
-const pad = (n = 0) => String(n).padStart(2, '0');
-
-const formatTimestamp = (start: Date, end: Date) => {
-  const { hours, minutes, seconds } = intervalToDuration({ start, end });
-  return `${pad(hours)}:${pad(minutes)}:${pad(seconds)}`;
-};
-
-const measureRows = async (
-  host: HTMLDivElement,
-  blocks: TranscriptBlock[],
-  queueMap: QueueRows,
-  signal: AbortSignal,
-) => {
-  const result: DxGridAxisMeta = { grid: {} };
-  for (let row = 0; row < queueMap.length; row++) {
-    await yieldOrContinue('smooth', signal);
-    const [blockIndex, segmentIndex] = queueMap[row];
-    if (segmentIndex < 0) {
-      result.grid[row] = { size: lineHeight + cellSpacing };
-    } else {
-      host.textContent = blocks[blockIndex]!.segments[segmentIndex]!.text;
-      result.grid[row] = { size: host.offsetHeight };
-    }
-  }
-  return result;
-};
-
-export type TranscriptProps = ThemedClassName<{
-  blocks?: TranscriptBlock[];
-  attendableId?: string;
-  ignoreAttention?: boolean;
-}>;
-
-/**
- * @@deprecated
- */
-export const Transcript: FC<TranscriptProps> = ({ classNames, blocks, attendableId, ignoreAttention }) => {
-  const { t } = useTranslation(TRANSCRIPTION_PLUGIN);
-  const [dxGrid, setDxGrid] = useState<DxGridElement | null>(null);
-  const [rows, setRows] = useState<DxGridAxisMeta | undefined>(undefined);
-  const [columns, setColumns] = useState<DxGridAxisMeta | undefined>(undefined);
-
-  const queueMap = useMemo(() => mapTranscriptQueue(blocks), [blocks]);
-
-  const { hasAttention } = useAttention(attendableId);
-  const [autoScroll, setAutoScroll] = useState(true);
-  const handleWheel = useCallback(
-    (event: WheelEvent) => {
-      setAutoScroll(false);
-      if (!ignoreAttention && !hasAttention) {
-        event.stopPropagation();
-      }
-    },
-    [hasAttention, ignoreAttention],
-  );
-
-  const abortControllerRef = useRef<AbortController>();
-
-  const handleResize = useCallback(
-    async ({ entry }: { entry: { target: HTMLDivElement } | null }) => {
-      if (entry?.target && Array.isArray(blocks) && blocks[0] && queueMap) {
-        abortControllerRef.current?.abort();
-        abortControllerRef.current = new AbortController();
-        setColumns({ grid: { 0: { size: timestampColumnWidth }, 1: { size: entry.target.offsetWidth } } });
-        return measureRows(entry.target, blocks, queueMap, abortControllerRef.current.signal)
-          .then(setRows)
-          .catch(() => {
-            // Aborted mid-measurement by new size.
-          });
-      }
-    },
-    [blocks, queueMap],
-  );
-
-  const { width, ref: measureRef } = useResizeDetector({
-    onResize: handleResize as OnResizeCallback,
-    refreshOptions: { leading: true },
-  });
-
-  useEffect(() => {
-    if (queueMap.length !== Object.keys(rows?.grid ?? {}).length) {
-      void handleResize({ entry: { target: measureRef.current } }).then(() => {
-        if (autoScroll) {
-          // TODO(thure): Implement a deterministic way to do this when `rows` has fully settled and grid has a new `maxPosBlock`.
-          setTimeout(() => dxGrid?.scrollToEndRow(), 50);
-        }
-      });
-    }
-  }, [blocks, queueMap, width, autoScroll]);
-
-  useEffect(() => {
-    if (dxGrid && Array.isArray(blocks) && blocks[0]) {
-      const transcriptStart = blocks[0]!.segments[0]!.started;
-      dxGrid.getCells = (range, plane) => {
-        switch (plane) {
-          case 'grid': {
-            const cells: DxGridPlaneCells = {};
-            for (let row = range.start.row; row <= range.end.row && row < queueMap.length; row++) {
-              const [blockIndex, segmentIndex] = queueMap[row];
-
-              cells[toPlaneCellIndex({ col: 0, row })] = {
-                readonly: true,
-                value:
-                  segmentIndex < 0
-                    ? formatTimestamp(
-                        new Date(transcriptStart),
-                        new Date(blocks[blockIndex]!.segments[Math.max(0, segmentIndex)]!.started),
-                      )
-                    : '',
-                className: timestampClasses,
-              } satisfies DxGridCellValue;
-
-              cells[toPlaneCellIndex({ col: 1, row })] = (
-                segmentIndex < 0
-                  ? {
-                      readonly: true,
-                      accessoryHtml: `<span data-hue="${blocks[blockIndex]!.authorHue}" class="dx-text-hue">${blocks[blockIndex]!.authorName}</span>`,
-                      className: authorClasses,
-                    }
-                  : {
-                      readonly: true,
-                      value: blocks[blockIndex]!.segments[segmentIndex]!.text,
-                      className: segmentTextClasses,
-                    }
-              ) satisfies DxGridCellValue;
-            }
-            return cells;
-          }
-          default:
-            return {};
-        }
-      };
-    }
-  }, [dxGrid, blocks]);
-
-  const handleScrollToEnd = useCallback(() => {
-    setAutoScroll(true);
-    dxGrid?.scrollToEndRow();
-  }, [dxGrid, autoScroll]);
-
-  return (
-    <div role='none' className={mx('relative min-bs-0', classNames)}>
-      <Grid.Root id={`${attendableId}--transcript`}>
-        <Grid.Content
-          limitColumns={2}
-          limitRows={queueMap.length}
-          columns={columns}
-          rows={rows}
-          rowDefault={rowDefault}
-          onWheel={handleWheel}
-          className='[--dx-grid-base:var(--dx-baseSurface)] [--dx-grid-lines:var(--dx-baseSurface)] [&_.dx-grid]:min-bs-0 [&_.dx-grid]:min-is-0 [&_.dx-grid]:select-auto'
-          ref={setDxGrid}
-        />
-      </Grid.Root>
-      <div role='none' {...{ inert: '' }} aria-hidden className={measureClasses} ref={measureRef} />
-      <IconButton
-        icon='ph--arrow-line-down--regular'
-        iconOnly
-        label={t('scroll to end label')}
-        tooltipSide='left'
-        data-state={autoScroll ? 'invisible' : 'visible'}
-        classNames={[
-          'absolute inline-end-2 block-end-2 opacity-0 pointer-events-none',
-          'data-[state="visible"]:pointer-events-auto data-[state="visible"]:opacity-100 transition-opacity',
-        ]}
-        onClick={handleScrollToEnd}
-      />
-    </div>
-  );
-};

package/src/components/TranscriptContainer.tsx

@@ -1,30 +0,0 @@
-//
-// Copyright 2024 DXOS.org
-//
-
-import React, { type FC } from 'react';
-
-import { fullyQualifiedId, getSpace } from '@dxos/client/echo';
-import { useMembers, useQueue } from '@dxos/react-client/echo';
-import { StackItem } from '@dxos/react-ui-stack';
-import { type DataType } from '@dxos/schema';
-
-import { Transcript, renderMarkdown } from './Transcript';
-import { useQueueModelAdapter } from '../hooks';
-import { type TranscriptType } from '../types';
-
-export const TranscriptionContainer: FC<{ role: string; transcript: TranscriptType }> = ({ role, transcript }) => {
-  const attendableId = fullyQualifiedId(transcript);
-  const space = getSpace(transcript);
-  const members = useMembers(space?.key).map((member) => member.identity);
-  const queue = useQueue<DataType.Message>(transcript.queue.dxn, { pollInterval: 1_000 });
-  const model = useQueueModelAdapter(renderMarkdown(members), queue);
-
-  return (
-    <StackItem.Content role={role} classNames='container-max-width pli-2'>
-      <Transcript attendableId={attendableId} space={space} model={model} transcript={transcript} />
-    </StackItem.Content>
-  );
-};
-
-export default TranscriptionContainer;

package/src/components/stories/TranscriptionStory.tsx

@@ -1,41 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import '@dxos-theme';
-
-import React, { type Dispatch, type FC, type RefObject, type SetStateAction } from 'react';
-
-import { useSpace } from '@dxos/react-client/echo';
-import { IconButton, Toolbar } from '@dxos/react-ui';
-import { ScrollContainer } from '@dxos/react-ui-components';
-import { type DataType } from '@dxos/schema';
-
-import { type SerializationModel } from '../../model';
-import { Transcript } from '../Transcript';
-
-export const TranscriptionStory: FC<{
-  model: SerializationModel<DataType.Message>;
-  running: boolean;
-  onRunningChange: Dispatch<SetStateAction<boolean>>;
-  audioRef?: RefObject<HTMLAudioElement>;
-}> = ({ model, running, onRunningChange, audioRef }) => {
-  const space = useSpace();
-
-  return (
-    <div className='flex flex-col w-[30rem]'>
-      {audioRef && <audio ref={audioRef} autoPlay />}
-      <Toolbar.Root>
-        <IconButton
-          iconOnly
-          icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
-          label={running ? 'Pause' : 'Play'}
-          onClick={() => onRunningChange((running) => !running)}
-        />
-      </Toolbar.Root>
-      <ScrollContainer>
-        <Transcript space={space} model={model} attendableId='story' />
-      </ScrollContainer>
-    </div>
-  );
-};

package/src/segments-normalization/normalization.ts

@@ -1,105 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { Schema } from 'effect';
-
-import { DEFAULT_EDGE_MODEL, Message } from '@dxos/ai';
-import { AISession } from '@dxos/assistant';
-import { Obj } from '@dxos/echo';
-import { AiService, defineFunction } from '@dxos/functions';
-import { ObjectId } from '@dxos/keys';
-import { log } from '@dxos/log';
-import { DataType } from '@dxos/schema';
-
-const MessageWithRangeId = Schema.extend(
-  DataType.Message,
-  Schema.Struct({
-    // TODO(mykola): Move to meta in DataType.Message. Use `DataType.Message` instead.
-    rangeId: Schema.optional(Schema.Array(Schema.String)).annotations({
-      description: 'The IDs of the messages that contain the sentences.',
-    }),
-  }),
-);
-
-export interface MessageWithRangeId extends Schema.Schema.Type<typeof MessageWithRangeId> {}
-
-export const NormalizationInput = Schema.Struct({
-  // TODO(mykola): Move to meta in DataType.Message. Use `DataType.Message` instead.
-  messages: Schema.Array(MessageWithRangeId).annotations({
-    description: 'Messages to normalize into sentences.',
-  }),
-});
-export interface NormalizationInput extends Schema.Schema.Type<typeof NormalizationInput> {}
-
-export const NormalizationOutput = Schema.Struct({
-  sentences: Schema.Array(MessageWithRangeId.pipe(Schema.mutable)).pipe(Schema.mutable).annotations({
-    description: 'The sentences of the transcript.',
-  }),
-});
-export interface NormalizationOutput extends Schema.Schema.Type<typeof NormalizationOutput> {}
-
-const prompt = `
-You are observing a real-time transcript of a single person speaking.
-The transcription is delivered in chunks of 10 seconds or less. As a result, individual sentences may be split across multiple messages, or multiple sentences may appear within a single message. Additionally, because this is real-time transcription, punctuation and capitalization may be incorrect or missing.
-
-# Task Description:
-  - Your task is to detect and reconstruct broken or incomplete sentences by merging segments into coherent, grammatically correct sentences where appropriate.
-
-# Input Format:
-  - The input is an array of messages that contain transcription blocks.
-  - Each message is a DataType.Message, as described in the output format.
-
-# Output Format:
-  - You have been provided with the tool that defines the output format; make sure to query it.
-  - Do not output anything other than the expected format.
-
-# Segment Handling Rules:
-  - Sort messages by timestamp.
-  - Leave ID of the first message in a sentence.
-  - Each divided sentence should be added to the output as a separate message or block within existing messages, preserving the original order.
-  - If one message contains multiple sentences, you should not split them.
-  - The last sentence may be incomplete; it should still be output as a separate block or message.
-  - Keep track of the original timestamps of the messages and blocks and, use the timestamp of the first message/block of a sentence as the 'started' field of the merged message.
-  - The 'rangeId' field of the merged message should include 'messageId'-s and 'rangeId'-s of all messages that have been used to construct this message.
-
-# Punctuation and Capitalization:
-  - Do not rely on the original punctuation and capitalization—they may be incorrect.
-  - Use logical reasoning to apply appropriate punctuation (e.g., period, comma, question mark, exclamation mark) and capitalization.
-
-# Restrictions:
-  - Do not alter the order of the original messages; maintain the natural flow of speech.
-  - Do not interpret or infer meaning beyond reconstructing sentences.
-  - Do not add or remove any words or phrases.
-`;
-
-export const sentenceNormalization = defineFunction<NormalizationInput, NormalizationOutput>({
-  description: 'Post process of transcription for sentence normalization',
-  inputSchema: NormalizationInput,
-  outputSchema: NormalizationOutput,
-  handler: async ({ data: { messages }, context }) => {
-    log.info('input', { messages });
-    const ai = context.getService(AiService);
-    const session = new AISession({ operationModel: 'configured' });
-
-    const response = await session.runStructured(NormalizationOutput, {
-      generationOptions: { model: DEFAULT_EDGE_MODEL },
-      client: ai.client,
-      tools: [],
-      artifacts: [],
-      history: [
-        Obj.make(Message, {
-          role: 'user',
-          content: messages.map((message) => ({ type: 'text', text: JSON.stringify(message) }) as const),
-        }),
-      ],
-      prompt,
-    });
-
-    response.sentences.forEach((sentence) => {
-      sentence.id = ObjectId.random();
-    });
-    log.info('response', { response });
-    return response;
-  },
-});

package/src/types/schema.ts

@@ -1,41 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { Schema } from 'effect';
-
-import { Queue } from '@dxos/client/echo';
-import { Type } from '@dxos/echo';
-import { TypedObject } from '@dxos/echo-schema';
-
-/**
- * Root transcript object created when the user starts a transcription.
- */
-export const TranscriptSchema = Schema.Struct({
-  // TODO(wittjosiah): Remove?
-  // TODO(burdon): Use Date or string?
-  started: Schema.optional(Schema.String),
-  ended: Schema.optional(Schema.String),
-
-  /**
-   * Queue containing TranscriptBlock objects.
-   */
-  queue: Type.Ref(Queue),
-});
-
-export class TranscriptType extends TypedObject({
-  typename: 'dxos.org/type/Transcript',
-  version: '0.1.0',
-})(TranscriptSchema.fields) {}
-
-// TODO(burdon): Do these need to be kept in sync with EDGE?
-
-/**
- * First message in queue.
- * Contains metadata for the recording and transcript.
- */
-const TranscriptHeader = Schema.Struct({
-  started: Schema.optional(Schema.String),
-});
-
-export type TranscriptHeader = Schema.Schema.Type<typeof TranscriptHeader>;

package/src/types/types.ts

@@ -1,32 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { Schema } from 'effect';
-
-import { SpaceId } from '@dxos/react-client/echo';
-
-import { TranscriptType } from './schema';
-import { TRANSCRIPTION_PLUGIN } from '../meta';
-
-// TODO(burdon): Move to separate proto.
-
-/**
- * Endpoint to the calls service.
- */
-// TODO(burdon): Reconfigure/move.
-export const TRANSCRIPTION_URL = 'https://calls-service.dxos.workers.dev';
-
-export namespace TranscriptionAction {
-  const TRANSCRIPTION_ACTION = `${TRANSCRIPTION_PLUGIN}/action`;
-
-  export class Create extends Schema.TaggedClass<Create>()(`${TRANSCRIPTION_ACTION}/create`, {
-    input: Schema.Struct({
-      name: Schema.optional(Schema.String),
-      spaceId: SpaceId,
-    }),
-    output: Schema.Struct({
-      object: TranscriptType,
-    }),
-  }) {}
-}

package/src/{segments-normalization → normalization}/index.ts

@@ -2,6 +2,6 @@
 // Copyright 2025 DXOS.org
 //
 
-export * from './normalization';
 export * from './message-normalizer';
+export * from './normalization';
 export * from './utils';