@dxos/plugin-transcription 0.8.2-staging.7ac8446 → 0.8.2

package/src/components/Transcript/transcript-extension.ts

@@ -0,0 +1,236 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type Extension, RangeSetBuilder } from '@codemirror/state';
+import { EditorView, GutterMarker, ViewPlugin, type ViewUpdate, gutter } from '@codemirror/view';
+import { format } from 'date-fns/format';
+import { intervalToDuration } from 'date-fns/intervalToDuration';
+
+import { type CleanupFn, addEventListener, combine } from '@dxos/async';
+import { type RenderCallback } from '@dxos/react-ui-editor';
+import { type DataType } from '@dxos/schema';
+
+import { DocumentAdapter, type SerializationModel } from '../../model';
+
+/**
+ * Data structure that maps Chunks queue to lines with transcript state.
+ */
+export type TranscriptOptions = {
+  model: SerializationModel<DataType.Message>;
+  started?: Date;
+  renderButton?: RenderCallback<{ onClick: () => void }>;
+};
+
+/**
+ * Scrolling transcript with timestamps.
+ */
+export const transcript = ({ model, started, renderButton }: TranscriptOptions): Extension => {
+  return [
+    // Show timestamps in the gutter.
+    gutter({
+      class: 'cm-timestamp-gutter',
+      lineMarkerChange: (update) => update.docChanged || update.viewportChanged,
+      markers: (view) => {
+        const start = getStartTime(started, model.chunks[0]);
+        const builder = new RangeSetBuilder<GutterMarker>();
+        for (const { from, to } of view.visibleRanges) {
+          let line = view.state.doc.lineAt(from);
+          while (line.from <= to) {
+            const block = model.getChunkAtLine(line.number)?.blocks[0];
+            const timestamp = block?.type === 'transcription' && block.started;
+            if (timestamp) {
+              builder.add(line.from, line.from, new TimestampMarker(line.number, new Date(timestamp), start));
+            }
+
+            if (line.to + 1 > view.state.doc.length) {
+              break;
+            }
+
+            line = view.state.doc.lineAt(line.to + 1);
+          }
+        }
+
+        return builder.finish();
+      },
+    }),
+
+    // Listen for model updates.
+    ViewPlugin.fromClass(
+      class {
+        private readonly _controls?: HTMLDivElement;
+        private readonly _cleanup: CleanupFn;
+        private readonly _adapter: DocumentAdapter;
+        private _initialized = false;
+
+        constructor(view: EditorView) {
+          this._adapter = new DocumentAdapter(view);
+
+          const scroller = view.scrollDOM;
+          let isAutoScrolling = false;
+          let hasScrolled = false;
+
+          let timeout: NodeJS.Timeout | undefined;
+          const scrollToBottom = (smooth = false) => {
+            scroller.style.scrollBehavior = smooth ? 'smooth' : '';
+
+            // Temporarily hide scrollbar to prevent flicker.
+            scroller.classList.add('cm-hide-scrollbar');
+            isAutoScrolling = true;
+            clearTimeout(timeout);
+            timeout = setTimeout(() => {
+              this._controls?.classList.add('opacity-0');
+              scroller.classList.remove('cm-hide-scrollbar');
+              isAutoScrolling = false;
+            }, 500);
+
+            // Scroll to bottom.
+            view.dispatch({
+              effects: EditorView.scrollIntoView(view.state.doc.length, { y: 'end' }),
+            });
+          };
+
+          // Scroll button.
+          if (renderButton) {
+            this._controls = document.createElement('div');
+            this._controls.classList.add('cm-controls', 'transition-opacity', 'duration-300', 'opacity-0');
+            view.dom.appendChild(this._controls);
+            renderButton(
+              this._controls,
+              {
+                onClick: () => {
+                  scrollToBottom(false);
+                },
+              },
+              view,
+            );
+          }
+
+          // Event listeners.
+          this._cleanup = combine(
+            addEventListener(view.scrollDOM, 'scroll', () => {
+              if (!isAutoScrolling) {
+                hasScrolled = true;
+                this._controls?.classList.remove('opacity-0');
+              }
+            }),
+
+            model.update.on(() => {
+              // Check if clamped to bottom.
+              const autoScroll =
+                scroller.scrollHeight - scroller.scrollTop - scroller.clientHeight === 0 || !hasScrolled;
+
+              // Sync.
+              model.sync(this._adapter);
+
+              // Scroll.
+              if (autoScroll) {
+                scrollToBottom(true);
+              }
+            }),
+          );
+        }
+
+        update(update: ViewUpdate) {
+          // Initial sync.
+          if (!this._initialized) {
+            this._initialized = true;
+            setTimeout(() => {
+              model.sync(this._adapter);
+            });
+          }
+        }
+
+        destroy() {
+          this._controls?.remove();
+          this._cleanup();
+        }
+      },
+    ),
+
+    EditorView.theme({
+      '.cm-scroller': {
+        overflowY: 'scroll',
+      },
+      '.cm-hide-scrollbar': {
+        scrollbarWidth: 'none',
+        '-ms-overflow-style': 'none',
+      },
+      '.cm-hide-scrollbar::-webkit-scrollbar': {
+        display: 'none',
+      },
+      '.cm-controls': {
+        position: 'absolute',
+        bottom: '0.5rem',
+        right: '0.5rem',
+        zIndex: 1000,
+      },
+      '.cm-line': {
+        paddingRight: '1rem',
+      },
+      '.cm-timestamp-gutter': {
+        width: '6rem',
+        paddingRight: '1rem',
+      },
+      '.cm-timestamp-gutter > .cm-gutterElement > div': {
+        display: 'inline-flex',
+        textAlign: 'right',
+        padding: '3px',
+      },
+    }),
+  ];
+};
+
+/**
+ * Gutter marker that displays a timestamp.
+ */
+class TimestampMarker extends GutterMarker {
+  constructor(
+    readonly _line: number,
+    readonly _timestamp: Date,
+    readonly _started?: Date,
+  ) {
+    super();
+  }
+
+  override eq(other: this): boolean {
+    return other._timestamp === this._timestamp;
+  }
+
+  override toDOM(view: EditorView): HTMLDivElement {
+    const el = document.createElement('div');
+    el.className = 'text-sm text-subdued hover:bg-hoverSurface cursor-pointer';
+    el.textContent = formatTimestamp(this._timestamp, this._started);
+    // TODO(burdon): Click to bookmark or copy hyperlink.
+    el.onclick = () => {
+      const pos = view.state.doc.line(this._line).from;
+      view.dispatch({
+        effects: EditorView.scrollIntoView(pos, { y: 'start' }),
+      });
+    };
+
+    return el;
+  }
+}
+
+const getStartTime = (started?: Date, message?: DataType.Message): Date | undefined => {
+  if (started) {
+    return started;
+  }
+
+  if (message?.blocks[0]?.type === 'transcription' && message.blocks[0].started) {
+    return new Date(message.blocks[0].started);
+  }
+
+  return undefined;
+};
+
+const formatTimestamp = (timestamp: Date, relative?: Date) => {
+  if (relative) {
+    const pad = (n = 0) => String(n).padStart(2, '0');
+    const { hours, minutes, seconds } = intervalToDuration({ start: relative, end: timestamp });
+    return `${pad(hours)}:${pad(minutes)}:${pad(seconds)}`;
+  } else {
+    return format(timestamp, 'HH:mm:ss');
+  }
+};

package/src/components/TranscriptContainer.tsx

@@ -4,20 +4,25 @@
 
 import React, { type FC } from 'react';
 
-import { fullyQualifiedId } from '@dxos/client/echo';
-import { useQueue } from '@dxos/react-client/echo';
+import { fullyQualifiedId, getSpace } from '@dxos/client/echo';
+import { useMembers, useQueue } from '@dxos/react-client/echo';
 import { StackItem } from '@dxos/react-ui-stack';
+import { type DataType } from '@dxos/schema';
 
-import { Transcript } from './Transcript';
-import { type TranscriptBlock, type TranscriptType } from '../types';
+import { Transcript, renderMarkdown } from './Transcript';
+import { useQueueModelAdapter } from '../hooks';
+import { type TranscriptType } from '../types';
 
 export const TranscriptionContainer: FC<{ role: string; transcript: TranscriptType }> = ({ role, transcript }) => {
   const attendableId = fullyQualifiedId(transcript);
-  const queue = useQueue<TranscriptBlock>(transcript.queue.dxn, { pollInterval: 1_000 });
+  const space = getSpace(transcript);
+  const members = useMembers(space?.key).map((member) => member.identity);
+  const queue = useQueue<DataType.Message>(transcript.queue.dxn, { pollInterval: 1_000 });
+  const model = useQueueModelAdapter(renderMarkdown(members), queue);
 
   return (
     <StackItem.Content role={role} classNames='container-max-width pli-2'>
-      <Transcript attendableId={attendableId} blocks={queue?.items} />
+      <Transcript attendableId={attendableId} space={space} model={model} transcript={transcript} />
     </StackItem.Content>
   );
 };

package/src/components/stories/FileTranscription.stories.tsx

@@ -0,0 +1,240 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import '@dxos-theme';
+
+import { type Meta, type StoryObj } from '@storybook/react';
+import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
+
+import { AIServiceEdgeClient } from '@dxos/ai';
+import { AI_SERVICE_ENDPOINT } from '@dxos/ai/testing';
+import { Events, IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
+import { withPluginManager } from '@dxos/app-framework/testing';
+import { scheduleTask } from '@dxos/async';
+import { Context } from '@dxos/context';
+import { MemoryQueue } from '@dxos/echo-db';
+import { create, createQueueDxn } from '@dxos/echo-schema';
+import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
+import { log } from '@dxos/log';
+import { ClientPlugin } from '@dxos/plugin-client';
+import { PreviewPlugin } from '@dxos/plugin-preview';
+import { SpacePlugin } from '@dxos/plugin-space';
+import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
+import { ThemePlugin } from '@dxos/plugin-theme';
+import { defaultTx } from '@dxos/react-ui-theme';
+import { DataType } from '@dxos/schema';
+import { seedTestData, Testing } from '@dxos/schema/testing';
+import { withLayout, withTheme } from '@dxos/storybook-utils';
+
+import { TranscriptionStory } from './TranscriptionStory';
+import { useIsSpeaking } from './useIsSpeaking';
+import { TranscriptionPlugin } from '../../TranscriptionPlugin';
+import { useAudioFile, useQueueModelAdapter, useTranscriber } from '../../hooks';
+import { getActorId, MessageNormalizer } from '../../segments-normalization';
+import { TestItem } from '../../testing';
+import { type MediaStreamRecorderParams, type TranscriberParams } from '../../transcriber';
+import { renderMarkdown } from '../Transcript';
+
+const AudioFile = ({
+  detectSpeaking,
+  audioUrl,
+  normalizeSentences,
+  transcriberConfig,
+  recorderConfig,
+  audioConstraints,
+}: {
+  detectSpeaking?: boolean;
+  normalizeSentences?: boolean;
+  audioUrl: string;
+  transcriberConfig?: TranscriberParams['config'];
+  recorderConfig?: MediaStreamRecorderParams['config'];
+  audioConstraints?: MediaTrackConstraints;
+}) => {
+  const [running, setRunning] = useState(false);
+  const actor: DataType.Actor = useMemo(() => ({ name: 'You' }), []);
+
+  // Audio.
+  const { audio, track, stream } = useAudioFile(audioUrl, audioConstraints);
+
+  // Speaking.
+  const isSpeaking = detectSpeaking ? useIsSpeaking(track) : true;
+  const ref = useRef<HTMLAudioElement>(null);
+  useEffect(() => {
+    if (ref.current && stream) {
+      ref.current.srcObject = stream;
+    }
+  }, [stream]);
+
+  useEffect(() => {
+    if (!audio) {
+      log.warn('no audio');
+      return;
+    }
+
+    if (running) {
+      void audio.play();
+    } else {
+      void audio.pause();
+    }
+  }, [audio, running]);
+
+  // Transcriber.
+  const queueDxn = useMemo(() => createQueueDxn(), []);
+  const queue = useMemo(() => new MemoryQueue<DataType.Message>(queueDxn), [queueDxn]);
+
+  const model = useQueueModelAdapter(renderMarkdown([]), queue);
+  const handleSegments = useCallback<TranscriberParams['onSegments']>(
+    async (blocks) => {
+      const message = create(DataType.Message, { sender: actor, created: new Date().toISOString(), blocks });
+      void queue?.append([message]);
+    },
+    [queue],
+  );
+
+  const transcriber = useTranscriber({
+    audioStreamTrack: track,
+    onSegments: handleSegments,
+    transcriberConfig,
+    recorderConfig,
+  });
+
+  // Normalize sentences.
+  const normalizer = useMemo(() => {
+    if (!normalizeSentences) {
+      return;
+    }
+    const executor = new FunctionExecutor(
+      new ServiceContainer().setServices({
+        ai: {
+          client: new AIServiceEdgeClient({
+            endpoint: AI_SERVICE_ENDPOINT.REMOTE,
+            defaultGenerationOptions: {
+              model: '@anthropic/claude-3-5-sonnet-20241022',
+            },
+          }),
+        },
+      }),
+    );
+
+    return new MessageNormalizer({
+      functionExecutor: executor,
+      queue,
+      startingCursor: { actorId: getActorId(actor), timestamp: new Date().toISOString() },
+    });
+  }, [normalizeSentences, queue, actor]);
+
+  useEffect(() => {
+    if (!normalizer) {
+      return;
+    }
+    const ctx = new Context();
+    scheduleTask(ctx, async () => {
+      await normalizer.open();
+      ctx.onDispose(async () => {
+        await normalizer.close();
+      });
+    });
+    return () => {
+      void ctx.dispose();
+    };
+  }, [normalizer]);
+
+  const manageChunkRecording = () => {
+    if (running && isSpeaking) {
+      log.info('starting transcription');
+      transcriber?.startChunksRecording();
+    } else if (!isSpeaking || !running) {
+      log.info('stopping transcription');
+      transcriber?.stopChunksRecording();
+    }
+  };
+
+  useEffect(() => {
+    const ctx = new Context();
+    scheduleTask(ctx, async () => {
+      if (transcriber && running) {
+        await transcriber.open();
+      } else if (!running) {
+        await transcriber?.close();
+      }
+      manageChunkRecording();
+    });
+
+    return () => {
+      void ctx.dispose();
+    };
+  }, [transcriber, running]);
+
+  useEffect(() => {
+    manageChunkRecording();
+  }, [isSpeaking, stream]);
+
+  return <TranscriptionStory model={model} running={running} onRunningChange={setRunning} audioRef={ref} />;
+};
+
+const meta: Meta<typeof AudioFile> = {
+  title: 'plugins/plugin-transcription/FileTranscription',
+  decorators: [
+    withPluginManager({
+      plugins: [
+        ThemePlugin({ tx: defaultTx }),
+        StorybookLayoutPlugin(),
+        ClientPlugin({
+          types: [TestItem, DataType.Person, DataType.Organization, Testing.DocumentType],
+          onClientInitialized: async (_, client) => {
+            await client.halo.createIdentity();
+            await client.spaces.waitUntilReady();
+            await client.spaces.default.waitUntilReady();
+            await seedTestData(client.spaces.default);
+          },
+        }),
+        SpacePlugin(),
+        SettingsPlugin(),
+        PreviewPlugin(),
+        IntentPlugin(),
+        TranscriptionPlugin(),
+      ],
+      fireEvents: [Events.SetupAppGraph],
+    }),
+    withTheme,
+    withLayout({ fullscreen: true, classNames: 'justify-center' }),
+  ],
+};
+
+export default meta;
+
+const TRANSCRIBER_CONFIG = {
+  transcribeAfterChunksAmount: 100,
+  prefixBufferChunksAmount: 50,
+  normalizeSentences: true,
+};
+
+const RECORDER_CONFIG = {
+  interval: 200,
+};
+
+export const Default: StoryObj<typeof AudioFile> = {
+  render: AudioFile,
+  args: {
+    detectSpeaking: true,
+    // https://learnenglish.britishcouncil.org/general-english/audio-zone/living-london
+    audioUrl: 'https://dxos.network/audio-london.m4a',
+    // textUrl: 'https://dxos.network/audio-london.txt',
+    transcriberConfig: TRANSCRIBER_CONFIG,
+    recorderConfig: RECORDER_CONFIG,
+  },
+};
+
+export const WithSentenceNormalization: StoryObj<typeof AudioFile> = {
+  render: AudioFile,
+  args: {
+    detectSpeaking: true,
+    normalizeSentences: true,
+    // https://learnenglish.britishcouncil.org/general-english/audio-zone/living-london
+    audioUrl: 'https://dxos.network/audio-london.m4a',
+    // textUrl: 'https://dxos.network/audio-london.txt',
+    transcriberConfig: TRANSCRIBER_CONFIG,
+    recorderConfig: RECORDER_CONFIG,
+  },
+};

package/src/components/stories/KeyWordRecognition.stories.tsx

@@ -0,0 +1,128 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import '@dxos-theme';
+
+import { type StoryObj, type Meta } from '@storybook/react';
+import React, { useEffect, useMemo, useState } from 'react';
+
+import { log } from '@dxos/log';
+import { Toolbar } from '@dxos/react-ui';
+import { mx } from '@dxos/react-ui-theme';
+import { withLayout, withTheme } from '@dxos/storybook-utils';
+
+type Word = {
+  text: string;
+  matched: boolean;
+};
+
+const SpeechRecognition = ({ keywords }: { keywords: string[] }) => {
+  const [running, setRunning] = useState(false);
+  const [matchingWords, setMatchingWords] = useState<Word[]>(keywords.map((word) => ({ text: word, matched: false })));
+
+  const [transcript, setTranscript] = useState('');
+  const recognition = useMemo(() => {
+    // TODO(mykola): Fix types
+    const SpeechRecognition = (window as any).SpeechRecognition || (window as any).webkitSpeechRecognition;
+    const SpeechGrammarList = (window as any).SpeechGrammarList || (window as any).webkitSpeechGrammarList;
+    if (!SpeechRecognition || !SpeechGrammarList) {
+      log.error('Speech recognition not supported in this browser');
+      return;
+    }
+
+    const recognition = new SpeechRecognition();
+    recognition.continuous = true;
+    recognition.interimResults = true;
+    recognition.lang = 'en-US';
+    recognition.maxAlternatives = 1;
+    // Add this line to enable punctuation
+    recognition.grammars = new SpeechGrammarList();
+
+    // TODO(mykola): Fix types
+    recognition.onresult = (event: any) => {
+      log.info('recognition result', { event });
+      const current = event.resultIndex;
+      const transcript = event.results[current][0].transcript;
+
+      // Remove punctuation and normalize whitespace for comparison
+      const normalizeText = (text: string) =>
+        text
+          .toLowerCase()
+          .replace(/[.,!?]/g, '')
+          .replace(/\s+/g, ' ')
+          .trim();
+
+      const normalizedTranscript = normalizeText(transcript);
+
+      // Check each keyword and update matching words
+      const updatedWords = matchingWords.map((word) => {
+        const normalizedKeyword = normalizeText(word.text);
+        const matched = normalizedTranscript.includes(normalizedKeyword);
+        return { ...word, matched };
+      });
+
+      setMatchingWords(updatedWords);
+      setTranscript(transcript);
+    };
+
+    // TODO(mykola): Fix types
+    recognition.onerror = (event: any) => {
+      log.error('Speech recognition error:', { error: event.error });
+      setRunning(false);
+    };
+
+    return recognition;
+  }, []);
+
+  useEffect(() => {
+    if (running) {
+      recognition?.start();
+    } else {
+      recognition?.stop();
+    }
+  }, [running, recognition]);
+
+  return (
+    <div className={mx('flex flex-col gap-2 w-[30rem]')}>
+      <Toolbar.Root>
+        <Toolbar.IconButton
+          iconOnly
+          icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
+          label={running ? 'Pause' : 'Play'}
+          onClick={() => setRunning((running) => !running)}
+        />
+      </Toolbar.Root>
+
+      <div className='flex flex-wrap'>
+        {matchingWords.map((word, index) => (
+          <span
+            key={index}
+            className={mx(
+              'px-3 py-1 m-2 rounded border',
+              word.matched ? 'bg-red-200 text-red-800 border-red-300' : 'bg-gray-200 text-gray-600 border-gray-300',
+            )}
+          >
+            {word.text}
+          </span>
+        ))}
+      </div>
+
+      <span>{transcript || (running ? 'Listening...' : 'Click play to start speech recognition')}</span>
+    </div>
+  );
+};
+
+const meta: Meta<typeof SpeechRecognition> = {
+  title: 'plugins/plugin-transcription/KeyWordDetection',
+  decorators: [withTheme, withLayout({ fullscreen: true, classNames: 'justify-center' })],
+};
+
+export default meta;
+
+export const SpeechRecognitionAPI: StoryObj<typeof SpeechRecognition> = {
+  render: SpeechRecognition,
+  args: {
+    keywords: ['kai', 'computer', 'hello'],
+  },
+};