@dxos/plugin-transcription 0.7.5-main.b19bfc8

package/src/components/TranscriptContainer.tsx

@@ -0,0 +1,167 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { pipe } from 'effect';
+import React, { useCallback, useMemo, useState, type FC } from 'react';
+
+import { chain, createIntent, LayoutAction, useIntentDispatcher } from '@dxos/app-framework';
+import { Message } from '@dxos/artifact';
+import { AIServiceClientImpl, DEFAULT_LLM_MODEL, MixedStreamParser, type AIServiceClient } from '@dxos/assistant';
+import { create, getSpace, makeRef } from '@dxos/client/echo';
+import { QueueImpl } from '@dxos/echo-db';
+import { createStatic, isInstanceOf } from '@dxos/echo-schema';
+import { invariant } from '@dxos/invariant';
+import { DXN } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { DocumentType } from '@dxos/plugin-markdown/types';
+import { CollectionType, SpaceAction } from '@dxos/plugin-space/types';
+import { useConfig } from '@dxos/react-client';
+import { useEdgeClient, useQueue, type EdgeHttpClient } from '@dxos/react-edge-client';
+import { IconButton, Toolbar, useTranslation } from '@dxos/react-ui';
+import { ScrollContainer } from '@dxos/react-ui-components';
+import { StackItem } from '@dxos/react-ui-stack';
+import { TextType } from '@dxos/schema';
+
+import { Transcript } from './Transcript';
+import { TRANSCRIPTION_PLUGIN } from '../meta';
+import { TranscriptBlock, type TranscriptType } from '../types';
+
+export const TranscriptionContainer: FC<{ transcript: TranscriptType }> = ({ transcript }) => {
+  const { t } = useTranslation(TRANSCRIPTION_PLUGIN);
+  const edge = useEdgeClient();
+  const aiService = useAiServiceClient();
+  const { dispatchPromise: dispatch } = useIntentDispatcher();
+
+  const queue = useQueue<TranscriptBlock>(edge, transcript.queue ? DXN.parse(transcript.queue) : undefined, {
+    pollInterval: 1_000,
+  });
+
+  // TODO(dmaretskyi): Pending state and errors should be handled by the framework!!!
+  const [isSummarizing, setIsSummarizing] = useState(false);
+  const handleSummarize = useCallback(async () => {
+    setIsSummarizing(true);
+    try {
+      const document = await summarizeTranscript(edge, aiService, transcript);
+      const space = getSpace(transcript);
+      const target = space?.properties[CollectionType.typename]?.target;
+      await dispatch(
+        pipe(
+          createIntent(SpaceAction.AddObject, { object: document, target }),
+          chain(LayoutAction.Open, { part: 'main' }),
+        ),
+      );
+    } finally {
+      setIsSummarizing(false);
+    }
+  }, [transcript, edge]);
+
+  // TODO(dmaretskyi): Move action to menu.
+  return (
+    <StackItem.Content toolbar={true}>
+      <StackItem.Heading>
+        <Toolbar.Root classNames='flex gap-2'>
+          <IconButton
+            icon='ph--subtitles--regular'
+            iconOnly
+            size={5}
+            disabled={isSummarizing}
+            onClick={handleSummarize}
+            label={t('summary button')}
+          />
+          {isSummarizing && <div className='text-sm'>{t('summarizing label')}</div>}
+        </Toolbar.Root>
+      </StackItem.Heading>
+      <ScrollContainer>
+        <Transcript blocks={queue?.items} />
+      </ScrollContainer>
+    </StackItem.Content>
+  );
+};
+
+export default TranscriptionContainer;
+
+// TODO(dmaretskyi): Extract to the new file once transcript refactoring has settled.
+const summarizeTranscript = async (
+  edgeClient: EdgeHttpClient,
+  aiService: AIServiceClient,
+  transcript: TranscriptType,
+) => {
+  invariant(transcript.queue, 'No queue found for transcript');
+
+  const queue = new QueueImpl(edgeClient, DXN.parse(transcript.queue));
+  await queue.refresh();
+  const content = queue.items
+    .filter((block) => isInstanceOf(TranscriptBlock, block))
+    .map((block) => `${block.author}: ${block.segments.map((segment) => segment.text).join('\n')}`)
+    .join('\n\n');
+
+  const parser = new MixedStreamParser();
+
+  log.info('summarizing transcript', { blockCount: queue.items.length });
+  const output = await parser.parse(
+    await aiService.generate({
+      model: DEFAULT_LLM_MODEL,
+      systemPrompt: SUMMARIZE_PROMPT,
+      history: [createStatic(Message, { role: 'user', content: [{ type: 'text', text: content }] })],
+    }),
+  );
+
+  log.info('transcript summary', { output });
+  invariant(output[0].content[0].type === 'text', 'Expected text content');
+  const summary = output[0].content[0].text;
+
+  // TODO(dmaretskyi): .started is missing.
+  const name = `Summary ${(transcript.started ?? new Date()).toLocaleString('en-US', {
+    year: 'numeric',
+    month: 'long',
+    day: 'numeric',
+    hour: '2-digit',
+    minute: '2-digit',
+  })}`;
+
+  return create(DocumentType, {
+    name,
+    content: makeRef(create(TextType, { content: summary })),
+    threads: [],
+  });
+};
+
+// TODO(dmaretskyi): Add example to set consistent structure for the summary.
+const SUMMARIZE_PROMPT = `
+  # Goal
+  Create a markdown summary of the transcript provided.
+
+  # Formatting
+  - Format the summary as a markdown document without extra comments like "Here is the summary of the transcript:".
+  - Use markdown formatting for headings and bullet points.
+  - Format the summary as a list of key points and takeaways.
+  - All names of people should be in bold.
+
+  # Note Taking
+  - Correlate items in the summary with the person of origin to build a coherent narrative.
+  - Include short quotes verbatim where appropriate. Especially when concerned with design decisions and problem descriptions.
+
+  # Tasks
+  At the end of the summary include tasks.
+  Extract only the tasks that are:
+  - Directly actionable
+  - Clearly assigned to a person or team (or can easily be inferred)
+  - Strongly implied by the conversation and/or user note (no speculative tasks)
+  - Specific enough that someone reading them would know exactly what to do next
+
+  Format all tasks as markdown checkboxes using the syntax:
+  - [ ] Task description
+
+  Additional information can be included (indented).
+
+  If no actionable tasks are found, omit this tasks section.
+`;
+
+// TODO(dmaretskyi): Extract?
+//  Also this conflicts with plugin automation providing the ai client via a capability. Need to reconcile.
+const useAiServiceClient = (): AIServiceClient => {
+  const config = useConfig();
+  const endpoint = config.values.runtime?.services?.ai?.server ?? 'http://localhost:8788';
+  return useMemo(() => new AIServiceClientImpl({ endpoint }), [endpoint]);
+};

package/src/components/index.ts

@@ -0,0 +1,10 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { lazy } from 'react';
+
+export * from './Transcript';
+export * from './TranscriptContainer';
+
+export const TranscriptContainer = lazy(() => import('./TranscriptContainer'));

package/src/components/transcription.stories.tsx

@@ -0,0 +1,198 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import '@dxos-theme';
+
+import { type StoryObj, type Meta } from '@storybook/react';
+import React, {
+  type Dispatch,
+  type FC,
+  type SetStateAction,
+  useCallback,
+  useEffect,
+  useMemo,
+  useRef,
+  useState,
+} from 'react';
+
+import { createStatic } from '@dxos/echo-schema';
+import { type DXN } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { Config } from '@dxos/react-client';
+import { withClientProvider } from '@dxos/react-client/testing';
+import { useEdgeClient, useQueue } from '@dxos/react-edge-client';
+import { IconButton, Toolbar } from '@dxos/react-ui';
+import { ScrollContainer } from '@dxos/react-ui-components';
+import { withTheme, withLayout } from '@dxos/storybook-utils';
+
+import { Transcript } from './Transcript';
+import { useTranscriber, useAudioFile, useAudioTrack } from '../hooks';
+import { type TranscriberParams } from '../transcriber';
+import { TranscriptBlock } from '../types';
+import { randomQueueDxn } from '../util';
+
+const UX: FC<{
+  playing: boolean;
+  setPlaying: Dispatch<SetStateAction<boolean>>;
+  blocks?: TranscriptBlock[];
+}> = ({ playing, setPlaying, blocks }) => {
+  return (
+    <div className='flex flex-col w-[30rem]'>
+      <Toolbar.Root>
+        <IconButton
+          iconOnly
+          icon={playing ? 'ph--pause--regular' : 'ph--play--regular'}
+          label={playing ? 'Pause' : 'Play'}
+          onClick={() => setPlaying((playing) => !playing)}
+        />
+      </Toolbar.Root>
+      <ScrollContainer>
+        <Transcript blocks={blocks} />
+      </ScrollContainer>
+    </div>
+  );
+};
+
+const Microphone = () => {
+  const [playing, setPlaying] = useState(false);
+
+  // Audio.
+  const track = useAudioTrack(playing);
+
+  // Queue.
+  const queueDxn = useMemo(() => randomQueueDxn(), []);
+  const echoClient = useEdgeClient();
+  const queue = useQueue<TranscriptBlock>(echoClient, queueDxn, { pollInterval: 500 });
+
+  // Transcriber.
+  const handleSegments = useCallback<TranscriberParams['onSegments']>(
+    async (segments) => {
+      const block = createStatic(TranscriptBlock, { segments });
+      queue?.append([block]);
+    },
+    [queue],
+  );
+  const transcriber = useTranscriber({ audioStreamTrack: track, onSegments: handleSegments });
+  useEffect(() => {
+    void transcriber?.open();
+    return () => {
+      void transcriber?.close();
+    };
+  }, [transcriber]);
+
+  // Manage transcription state.
+  useEffect(() => {
+    if (playing && transcriber?.isOpen) {
+      void transcriber?.startChunksRecording();
+    } else if (!playing) {
+      void transcriber?.stopChunksRecording();
+    }
+  }, [transcriber, playing, transcriber?.isOpen]);
+
+  return <UX playing={playing} setPlaying={setPlaying} blocks={queue?.items} />;
+};
+
+const AudioFile = ({ queueDxn, audioUrl }: { queueDxn: DXN; audioUrl: string; transcriptUrl: string }) => {
+  const [playing, setPlaying] = useState(false);
+
+  // Audio.
+  const audioElement = useRef<HTMLAudioElement>(null);
+  const { audio, stream, track } = useAudioFile(audioUrl);
+  useEffect(() => {
+    if (stream && audioElement.current) {
+      audioElement.current.srcObject = stream;
+    }
+  }, [stream, audioElement.current]);
+
+  useEffect(() => {
+    if (!audio) {
+      log.warn('no audio');
+      return;
+    }
+
+    if (playing) {
+      void audio.play();
+    } else {
+      void audio.pause();
+    }
+  }, [audio, playing]);
+
+  // Transcriber.
+  const echoClient = useEdgeClient();
+  const queue = useQueue<TranscriptBlock>(echoClient, queueDxn, { pollInterval: 500 });
+  const handleSegments = useCallback<TranscriberParams['onSegments']>(
+    async (segments) => {
+      const block = createStatic(TranscriptBlock, { author: 'test', segments });
+      queue?.append([block]);
+    },
+    [queue],
+  );
+
+  const transcriber = useTranscriber({
+    audioStreamTrack: track,
+    onSegments: handleSegments,
+  });
+
+  useEffect(() => {
+    if (transcriber && playing) {
+      void transcriber.open();
+    } else if (!playing) {
+      transcriber?.stopChunksRecording();
+      void transcriber?.close();
+    }
+  }, [transcriber, playing]);
+
+  useEffect(() => {
+    if (track?.readyState === 'live' && transcriber?.isOpen) {
+      log.info('starting transcription');
+      transcriber.startChunksRecording();
+    } else if (track?.readyState !== 'live' && transcriber) {
+      log.info('stopping transcription');
+      transcriber.stopChunksRecording();
+    }
+  }, [transcriber, track?.readyState, transcriber?.isOpen]);
+
+  return <UX playing={playing} setPlaying={setPlaying} blocks={queue?.items} />;
+};
+
+const meta: Meta<typeof AudioFile> = {
+  title: 'plugins/plugin-transcription/transcription',
+  decorators: [
+    withClientProvider({
+      config: new Config({
+        runtime: {
+          client: { edgeFeatures: { signaling: true } },
+          services: {
+            edge: { url: 'https://edge.dxos.workers.dev/' },
+            iceProviders: [{ urls: 'https://edge.dxos.workers.dev/ice' }],
+          },
+        },
+      }),
+    }),
+    withTheme,
+    withLayout({
+      tooltips: true,
+      fullscreen: true,
+      classNames: 'justify-center',
+    }),
+  ],
+};
+
+export default meta;
+
+type Story = StoryObj<typeof AudioFile>;
+
+export const Default: Story = {
+  render: Microphone,
+};
+
+export const File: Story = {
+  render: AudioFile,
+  args: {
+    queueDxn: randomQueueDxn(),
+    // https://learnenglish.britishcouncil.org/general-english/audio-zone/living-london
+    transcriptUrl: 'https://dxos.network/audio-london.txt',
+    audioUrl: 'https://dxos.network/audio-london.m4a',
+  },
+};

package/src/hooks/index.ts

@@ -0,0 +1,9 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './useAudioFile';
+export * from './useAudioTrack';
+export * from './useIsSpeaking';
+export * from './useTranscriber';
+export * from './useVoiceInput';

package/src/hooks/useAudioFile.ts

@@ -0,0 +1,62 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { useEffect, useState } from 'react';
+
+import { log } from '@dxos/log';
+
+export type UseAudioState = {
+  audio?: HTMLAudioElement;
+  stream?: MediaStream;
+  track?: MediaStreamTrack;
+};
+
+export const useAudioFile = (audioUrl: string): UseAudioState => {
+  const [{ audio, stream, track }, setStream] = useState<UseAudioState>({});
+  useEffect(() => {
+    const t = setTimeout(async () => {
+      const response = await fetch(audioUrl);
+      const blob = await response.blob();
+      const audio = new Audio();
+      audio.src = URL.createObjectURL(blob);
+      await new Promise<void>((resolve, reject) => {
+        audio.addEventListener(
+          'error',
+          (err) => {
+            log.error('error', { err });
+            reject(err);
+          },
+          { once: true },
+        );
+        audio.addEventListener(
+          'canplay',
+          () => {
+            log.info('starting...');
+            resolve();
+          },
+          { once: true },
+        );
+        audio.load();
+      });
+
+      const ctx = new AudioContext();
+
+      const destination = ctx.createMediaStreamDestination();
+      destination.channelCount = 1;
+
+      const source = ctx.createMediaElementSource(audio);
+      source.connect(destination);
+
+      setStream({
+        audio,
+        stream: destination.stream,
+        track: destination.stream.getAudioTracks()[0],
+      });
+    });
+
+    return () => clearTimeout(t);
+  }, [audioUrl]);
+
+  return { audio, stream, track };
+};

package/src/hooks/useAudioTrack.ts

@@ -0,0 +1,35 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { useEffect, useState } from 'react';
+
+/**
+ *
+ */
+// TODO(burdon): Reconcile with react-ui-sfx and plugin-calls.
+export const useAudioTrack = (active?: boolean): MediaStreamTrack | undefined => {
+  const [track, setTrack] = useState<MediaStreamTrack>();
+  useEffect(() => {
+    if (!active) {
+      track?.stop();
+      setTrack(undefined);
+      return;
+    }
+
+    const initAudio = async () => {
+      const audio = new Audio();
+      audio.srcObject = await navigator.mediaDevices.getUserMedia({ audio: true });
+      const [track] = audio.srcObject.getAudioTracks();
+      if (track) {
+        setTrack(track);
+      }
+    };
+
+    if (active) {
+      void initAudio();
+    }
+  }, [active]);
+
+  return track;
+};

package/src/hooks/useIsSpeaking.ts

@@ -0,0 +1,77 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { useEffect, useRef, useState } from 'react';
+
+import { monitorAudioLevel } from '../util';
+
+export const useIsSpeaking = (mediaStreamTrack?: MediaStreamTrack) => {
+  const [isSpeaking, setIsSpeaking] = useState(false);
+
+  // The audio level is monitored very rapidly, and we don't want
+  // react involved in tracking the state because it causes way
+  // too many re-renders. To work around this, we use the isSpeakingRef
+  // to track the state and then sync it using another effect.
+  const isSpeakingRef = useRef(isSpeaking);
+
+  // this effect syncs the state on a 50ms interval
+  useEffect(() => {
+    if (!mediaStreamTrack) {
+      setIsSpeaking(false);
+      return;
+    }
+
+    isSpeakingRef.current = isSpeaking;
+    const interval = window.setInterval(() => {
+      // state is already in sync do nothing
+      if (isSpeaking === isSpeakingRef.current) {
+        return;
+      }
+
+      // sync state
+      setIsSpeaking(isSpeakingRef.current);
+    }, 50);
+
+    return () => {
+      clearInterval(interval);
+    };
+  }, [isSpeaking, isSpeakingRef, mediaStreamTrack]);
+
+  useEffect(() => {
+    if (!mediaStreamTrack) {
+      return;
+    }
+
+    let timeout = -1;
+    const cleanup = monitorAudioLevel({
+      mediaStreamTrack,
+      onMeasure: (vol) => {
+        // Once the user has been determined to be speaking, we want
+        // to lower the threshold because speech patterns don't always
+        // kick up above 0.05
+        const audioLevelAboveThreshold = vol > (isSpeakingRef.current ? 0.02 : 0.05);
+        if (audioLevelAboveThreshold) {
+          // user is still speaking, clear timeout & reset.
+          clearTimeout(timeout);
+          timeout = -1;
+          // track state.
+          isSpeakingRef.current = true;
+        } else if (timeout === -1) {
+          // user is not speaking and timeout is not set.
+          timeout = window.setTimeout(() => {
+            isSpeakingRef.current = false;
+            // reset timeout
+            timeout = -1;
+          }, 1_000);
+        }
+      },
+    });
+
+    return () => {
+      cleanup();
+    };
+  }, [isSpeaking, mediaStreamTrack]);
+
+  return isSpeaking;
+};

package/src/hooks/useTranscriber.ts

@@ -0,0 +1,62 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { useEffect, useMemo } from 'react';
+
+import { MediaStreamRecorder, Transcriber, type TranscriberParams } from '../transcriber';
+
+// TODO(burdon): Move to config?
+
+/**
+ * Length of the chunk in ms.
+ */
+const RECORD_INTERVAL = 200;
+
+/**
+ * Number of chunks to save before the user starts speaking.
+ */
+const PREFIXED_CHUNKS_AMOUNT = 10;
+
+/**
+ * Number of chunks to transcribe automatically after.
+ * Combined should be mess than 25MB or whisper would fail.
+ */
+const TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
+
+export type UseTranscriberProps = {
+  audioStreamTrack?: MediaStreamTrack;
+  onSegments?: TranscriberParams['onSegments'];
+};
+
+/**
+ * Records audio while user is speaking and transcribes it after user is done speaking.
+ */
+export const useTranscriber = ({ audioStreamTrack, onSegments }: UseTranscriberProps) => {
+  // Initialize audio transcription.
+  const transcriber = useMemo<Transcriber | undefined>(() => {
+    if (!onSegments || !audioStreamTrack) {
+      return undefined;
+    }
+
+    return new Transcriber({
+      config: {
+        transcribeAfterChunksAmount: TRANSCRIBE_AFTER_CHUNKS_AMOUNT,
+        prefixBufferChunksAmount: PREFIXED_CHUNKS_AMOUNT,
+      },
+      recorder: new MediaStreamRecorder({
+        mediaStreamTrack: audioStreamTrack,
+        interval: RECORD_INTERVAL,
+      }),
+      onSegments,
+    });
+  }, [audioStreamTrack, onSegments]);
+
+  useEffect(() => {
+    return () => {
+      void transcriber?.close();
+    };
+  }, [transcriber]);
+
+  return transcriber;
+};

package/src/hooks/useVoiceInput.ts

@@ -0,0 +1,66 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { useState, useEffect, useCallback } from 'react';
+
+import { scheduleMicroTask } from '@dxos/async';
+import { Context } from '@dxos/context';
+import { log } from '@dxos/log';
+import { useSoundEffect } from '@dxos/react-ui-sfx';
+
+import { useAudioTrack } from './useAudioTrack';
+import { useTranscriber } from './useTranscriber';
+import { type TranscriberParams } from '../transcriber';
+
+export type UseVoiceInputProps = {
+  active?: boolean;
+  onUpdate: (text: string) => void;
+};
+
+/**
+ * Hook for voice input.
+ */
+export const useVoiceInput = ({ active, onUpdate }: UseVoiceInputProps) => {
+  const soundStart = useSoundEffect('StartRecording');
+  const soundStop = useSoundEffect('StopRecording');
+
+  const handleSegments = useCallback<TranscriberParams['onSegments']>(async (segments) => {
+    const text = segments.map((str) => str.text.trim().replace(/[^\w\s]/g, '')).join(' ');
+    onUpdate(text);
+  }, []);
+
+  // Audio/transcription.
+  const [recording, setRecording] = useState(false);
+  const track = useAudioTrack(active);
+  const transcriber = useTranscriber({
+    audioStreamTrack: track,
+    onSegments: handleSegments,
+  });
+
+  // Start/stop transcription.
+  useEffect(() => {
+    const ctx = new Context();
+    scheduleMicroTask(ctx, async () => {
+      if (active && transcriber) {
+        await transcriber.open();
+        log.info('starting...');
+        setRecording(true);
+        void soundStart.play();
+        transcriber.startChunksRecording();
+      } else if (!active && transcriber?.isOpen) {
+        transcriber?.stopChunksRecording();
+        await transcriber?.close();
+        log.info('stopped');
+        setRecording(false);
+        void soundStop.play();
+      }
+    });
+
+    return () => {
+      void ctx.dispose();
+    };
+  }, [active, transcriber]);
+
+  return { recording };
+};