@dxos/plugin-transcription 0.8.4-main.fd6878d → 0.9.0

package/src/operations/create.ts

@@ -0,0 +1,24 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+
+import { Operation } from '@dxos/compute';
+import { Feed, Ref } from '@dxos/echo';
+import { Transcript } from '@dxos/types';
+
+import { TranscriptOperation } from '../types';
+
+const handler: Operation.WithHandler<typeof TranscriptOperation.Create> = TranscriptOperation.Create.pipe(
+  Operation.withHandler(
+    Effect.fn(function* ({ space }) {
+      const feed = space.db.add(Feed.make());
+      return {
+        object: Transcript.make(Ref.make(feed)),
+      };
+    }),
+  ),
+);
+
+export default handler;

package/src/operations/index.ts

@@ -0,0 +1,11 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { OperationHandlerSet } from '@dxos/compute';
+
+export const TranscriptionOperationHandlerSet = OperationHandlerSet.lazy(
+  () => import('./create'),
+  () => import('./open'),
+  () => import('./summarize'),
+);

package/src/operations/open.ts

@@ -0,0 +1,28 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+
+import { Operation } from '@dxos/compute';
+import { Database, Feed, Filter } from '@dxos/echo';
+import { Message } from '@dxos/types';
+
+import { TranscriptOperation } from '../types';
+import { renderByline } from '../util';
+
+const handler: Operation.WithHandler<typeof TranscriptOperation.Open> = TranscriptOperation.Open.pipe(
+  Operation.withHandler(
+    Effect.fn(function* ({ transcript }) {
+      const transcriptObj = yield* Database.load(transcript);
+      const feed = yield* Database.load(transcriptObj.feed);
+      const messages = yield* Feed.runQuery(feed, Filter.type(Message.Message));
+      const content = messages
+        .flatMap((message: Message.Message, index: number) => renderByline([])(message, index))
+        .join('\n\n');
+      return { content };
+    }),
+  ),
+);
+
+export default handler;

package/src/operations/summarize.ts

@@ -0,0 +1,102 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Array from 'effect/Array';
+import * as Effect from 'effect/Effect';
+import * as Function from 'effect/Function';
+import * as Layer from 'effect/Layer';
+import * as Option from 'effect/Option';
+
+import { AiService, ConsolePrinter, ToolExecutionService, ToolResolverService } from '@dxos/ai';
+import { AiRequest, GenerationObserver } from '@dxos/assistant';
+import { Trace, Operation } from '@dxos/compute';
+import { Database } from '@dxos/echo';
+import { registryLayerNoop } from '@dxos/echo/testing';
+import { trim } from '@dxos/util';
+
+import { TranscriptOperation } from '../types';
+
+/**
+ * Summarize a transcript of a meeting.
+ */
+const handler: Operation.WithHandler<typeof TranscriptOperation.Summarize> = TranscriptOperation.Summarize.pipe(
+  Operation.withHandler(
+    Effect.fnUntraced(
+      function* ({ transcript, notes }) {
+        const result = yield* new AiRequest.Request({
+          observer: GenerationObserver.fromPrinter(new ConsolePrinter({ tag: 'summarize' })),
+        }).run({
+          prompt: `Transcript: ${transcript}\n\nNotes: ${notes}`,
+          history: [],
+          system: systemPrompt,
+        });
+
+        const summary = Function.pipe(
+          result,
+          Array.findLast((msg) => msg.sender.role === 'assistant' && msg.blocks.some((block) => block._tag === 'text')),
+          Option.flatMap((msg) =>
+            Function.pipe(
+              msg.blocks,
+              Array.findLast((block) => block._tag === 'text'),
+              Option.map((block) => block.text),
+            ),
+          ),
+          Option.getOrThrowWith(() => new Error('No summary found')),
+        );
+
+        return { summary };
+      },
+      Effect.provide(
+        Layer.mergeAll(
+          AiService.model('ai.claude.model.claude-sonnet-4-0'),
+          ToolResolverService.layerEmpty,
+          ToolExecutionService.layerEmpty,
+          Trace.writerLayerNoop,
+          Database.notAvailable,
+          Layer.succeed(Operation.Service, {
+            invoke: () => Effect.die('Not available.'),
+            schedule: () => Effect.die('Not available.'),
+            invokePromise: async () => ({ error: new Error('Not available.') }),
+          } as any),
+          registryLayerNoop,
+        ),
+      ),
+    ),
+  ),
+);
+
+export default handler;
+
+const systemPrompt = trim`
+  You are a helpful assistant that summarizes transcripts of meetings.
+
+  # Goal
+  Create a markdown summary of the meeting transcript with text notes provided.
+  Notes are very important so make sure to include them in the summary if they contain meaningful information.
+
+  # Formatting
+  - Format the summary as a markdown document without extra comments like "Here is the summary of the transcript:".
+  - Use markdown formatting for headings and bullet points.
+  - Format the summary as a list of key points and takeaways.
+  - All names of people should be in bold.
+
+  # Note Taking
+  - Correlate items in the summary with the person of origin to build a coherent narrative.
+  - Include short quotes verbatim where appropriate. Especially when concerned with design decisions and problem descriptions.
+
+  # Tasks
+  At the end of the summary include tasks.
+  Extract only the tasks that are:
+  - Directly actionable.
+  - Clearly assigned to a person or team (or can easily be inferred).
+  - Strongly implied by the conversation and/or user note (no speculative tasks).
+  - Specific enough that someone reading them would know exactly what to do next.
+
+  Format all tasks as markdown checkboxes using the syntax:
+  - [ ] Task description.
+
+  Additional information can be included (indented).
+
+  If no actionable tasks are found, omit this tasks section.
+`;

package/src/plugin.ts

@@ -0,0 +1,11 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Plugin } from '@dxos/app-framework';
+
+import { meta } from './meta';
+
+export const TranscriptionPlugin = Plugin.lazy(meta, () => import('#plugin'));
+
+export { TranscriptionOperationHandlerSet } from './operations';

package/src/stories/FileTranscription.stories.tsx

@@ -0,0 +1,143 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type Meta, type StoryObj } from '@storybook/react-vite';
+import React, { useCallback, useEffect, useRef, useState } from 'react';
+
+import { useAudioFile } from '#hooks';
+
+import { type MediaStreamRecorderProps, type TranscriberProps } from '../transcriber';
+import {
+  useIsSpeaking,
+  createStoryDecorators,
+  useStoryAppendSegments,
+  useStoryMessageModel,
+  useStoryTranscriber,
+} from './common';
+import { TranscriptionStory } from './TranscriptionStory';
+
+const DEFAULT_TRANSCRIBER_CONFIG = {
+  transcribeAfterChunksAmount: 100,
+  prefixBufferChunksAmount: 50,
+  normalizeSentences: true,
+};
+
+const DEFAULT_RECORDER_CONFIG = {
+  interval: 200,
+};
+
+type DefaultStoryProps = {
+  detectSpeaking?: boolean;
+  normalizeSentences?: boolean;
+  audioUrl: string;
+  transcriberConfig?: TranscriberProps['config'];
+  recorderConfig?: MediaStreamRecorderProps['config'];
+  audioConstraints?: MediaTrackConstraints;
+};
+
+const DefaultStory = ({
+  detectSpeaking,
+  audioUrl,
+  normalizeSentences,
+  transcriberConfig = DEFAULT_TRANSCRIBER_CONFIG,
+  recorderConfig = DEFAULT_RECORDER_CONFIG,
+  audioConstraints,
+}: DefaultStoryProps) => {
+  const [running, setRunning] = useState(false);
+
+  // Optional uploaded file overrides the default URL; revoke the previous URL on change/unmount.
+  const [uploadedUrl, setUploadedUrl] = useState<string>();
+  useEffect(() => {
+    return () => {
+      if (uploadedUrl) {
+        URL.revokeObjectURL(uploadedUrl);
+      }
+    };
+  }, [uploadedUrl]);
+
+  const handleUpload = useCallback((file: File) => {
+    setRunning(false);
+    setUploadedUrl(URL.createObjectURL(file));
+  }, []);
+
+  const sourceUrl = uploadedUrl ?? audioUrl;
+  const { audio, track, stream } = useAudioFile(sourceUrl, audioConstraints);
+  const isSpeaking = detectSpeaking ? useIsSpeaking(track) : true;
+
+  // Pipe the decoded stream into the <audio> element so the user hears playback.
+  const ref = useRef<HTMLAudioElement>(null);
+  useEffect(() => {
+    if (ref.current && stream) {
+      ref.current.srcObject = stream;
+    }
+  }, [stream]);
+
+  // Mirror the toolbar's running flag to the underlying HTMLAudioElement.
+  useEffect(() => {
+    if (!audio) {
+      return;
+    }
+
+    if (running) {
+      void audio.play();
+    } else {
+      void audio.pause();
+    }
+  }, [audio, running]);
+
+  const { model, appendMessage } = useStoryMessageModel();
+  const handleSegments = useStoryAppendSegments(appendMessage);
+
+  useStoryTranscriber({
+    audioStreamTrack: track,
+    running,
+    isSpeaking,
+    transcriberConfig,
+    recorderConfig,
+    onSegments: handleSegments,
+  });
+
+  // TODO(burdon): Sentence normalization moved to require a real space-backed Feed + FeedService runtime.
+  //  Re-enable here once the story has access to one.
+  void normalizeSentences;
+
+  return (
+    <TranscriptionStory
+      audioRef={ref}
+      disabled={!stream}
+      model={model}
+      running={running}
+      onRunningChange={setRunning}
+      onUpload={handleUpload}
+    />
+  );
+};
+
+const meta = {
+  title: 'plugins/plugin-transcription/stories/FileTranscription',
+  render: DefaultStory,
+  decorators: createStoryDecorators(),
+} satisfies Meta<DefaultStoryProps>;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+export const Default: Story = {
+  args: {
+    detectSpeaking: true,
+    // https://learnenglish.britishcouncil.org/general-english/audio-zone/living-london
+    audioUrl: 'https://dxos.network/audio-london.m4a',
+    // textUrl: 'https://dxos.network/audio-london.txt',
+  },
+};
+
+// TODO(mykola): Fix sentence normalization.
+// export const WithSentenceNormalization: StoryObj<typeof AudioFile> = {
+//   args: {
+//     detectSpeaking: true,
+//     normalizeSentences: true,
+//     audioUrl: 'https://dxos.network/audio-london.m4a',
+//   },
+// };

package/src/{components/stories → stories}/KeyWordRecognition.stories.tsx

@@ -2,26 +2,27 @@
 // Copyright 2025 DXOS.org
 //
 
-import '@dxos-theme';
-
 import { type Meta, type StoryObj } from '@storybook/react-vite';
 import React, { useEffect, useMemo, useState } from 'react';
 
 import { log } from '@dxos/log';
-import { Toolbar } from '@dxos/react-ui';
-import { mx } from '@dxos/react-ui-theme';
-import { withLayout, withTheme } from '@dxos/storybook-utils';
+import { Panel, Toolbar } from '@dxos/react-ui';
+import { withLayout, withTheme } from '@dxos/react-ui/testing';
+import { mx } from '@dxos/ui-theme';
 
 type Word = {
   text: string;
   matched: boolean;
 };
 
-const SpeechRecognition = ({ keywords }: { keywords: string[] }) => {
+type DefaultStoryProps = {
+  keywords: string[];
+};
+
+const DefaultStory = ({ keywords }: DefaultStoryProps) => {
   const [running, setRunning] = useState(false);
   const [matchingWords, setMatchingWords] = useState<Word[]>(keywords.map((word) => ({ text: word, matched: false })));
 
-  const [transcript, setTranscript] = useState('');
   const recognition = useMemo(() => {
     // TODO(mykola): Fix types
     const SpeechRecognition = (window as any).SpeechRecognition || (window as any).webkitSpeechRecognition;
@@ -36,16 +37,16 @@ const SpeechRecognition = ({ keywords }: { keywords: string[] }) => {
     recognition.interimResults = true;
     recognition.lang = 'en-US';
     recognition.maxAlternatives = 1;
-    // Add this line to enable punctuation
+    // Add this line to enable punctuation.
     recognition.grammars = new SpeechGrammarList();
 
-    // TODO(mykola): Fix types
+    // TODO(mykola): Fix types.
     recognition.onresult = (event: any) => {
       log.info('recognition result', { event });
       const current = event.resultIndex;
       const transcript = event.results[current][0].transcript;
 
-      // Remove punctuation and normalize whitespace for comparison
+      // Remove punctuation and normalize whitespace for comparison.
       const normalizeText = (text: string) =>
         text
           .toLowerCase()
@@ -55,7 +56,7 @@ const SpeechRecognition = ({ keywords }: { keywords: string[] }) => {
 
       const normalizedTranscript = normalizeText(transcript);
 
-      // Check each keyword and update matching words
+      // Check each keyword and update matching words.
       const updatedWords = matchingWords.map((word) => {
         const normalizedKeyword = normalizeText(word.text);
         const matched = normalizedTranscript.includes(normalizedKeyword);
@@ -63,10 +64,9 @@ const SpeechRecognition = ({ keywords }: { keywords: string[] }) => {
       });
 
       setMatchingWords(updatedWords);
-      setTranscript(transcript);
     };
 
-    // TODO(mykola): Fix types
+    // TODO(mykola): Fix types.
     recognition.onerror = (event: any) => {
       log.error('Speech recognition error:', { error: event.error });
       setRunning(false);
@@ -84,45 +84,49 @@ const SpeechRecognition = ({ keywords }: { keywords: string[] }) => {
   }, [running, recognition]);
 
   return (
-    <div className={mx('flex flex-col gap-2 w-[30rem]')}>
-      <Toolbar.Root>
-        <Toolbar.IconButton
-          iconOnly
-          icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
-          label={running ? 'Pause' : 'Play'}
-          onClick={() => setRunning((running) => !running)}
-        />
-      </Toolbar.Root>
-
-      <div className='flex flex-wrap'>
-        {matchingWords.map((word, index) => (
-          <span
-            key={index}
-            className={mx(
-              'px-3 py-1 m-2 rounded border',
-              word.matched ? 'bg-red-200 text-red-800 border-red-300' : 'bg-gray-200 text-gray-600 border-gray-300',
-            )}
-          >
-            {word.text}
-          </span>
-        ))}
-      </div>
-
-      <span>{transcript || (running ? 'Listening...' : 'Click play to start speech recognition')}</span>
-    </div>
+    <Panel.Root>
+      <Panel.Toolbar asChild>
+        <Toolbar.Root>
+          <Toolbar.IconButton
+            iconOnly
+            icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
+            label={running ? 'Pause' : 'Play'}
+            onClick={() => setRunning((running) => !running)}
+          />
+        </Toolbar.Root>
+      </Panel.Toolbar>
+
+      <Panel.Content>
+        <div className='p-2 grid grid-cols-[repeat(auto-fill,minmax(6rem,1fr))] gap-2'>
+          {matchingWords.map((word, index) => (
+            <span
+              key={index}
+              className={mx(
+                'grid place-items-center p-2 aspect-square rounded-sm border border-separator',
+                word.matched && 'border-red-300',
+              )}
+            >
+              {word.text}
+            </span>
+          ))}
+        </div>
+      </Panel.Content>
+    </Panel.Root>
   );
 };
 
-const meta: Meta<typeof SpeechRecognition> = {
-  title: 'plugins/plugin-transcription/KeyWordDetection',
-  decorators: [withTheme, withLayout({ fullscreen: true, classNames: 'justify-center' })],
-};
+const meta = {
+  title: 'plugins/plugin-transcription/stories/KeyWordDetection',
+  render: DefaultStory,
+  decorators: [withTheme(), withLayout({ layout: 'column' })],
+} satisfies Meta<typeof DefaultStory>;
 
 export default meta;
 
-export const SpeechRecognitionAPI: StoryObj<typeof SpeechRecognition> = {
-  render: SpeechRecognition,
+type Story = StoryObj<typeof meta>;
+
+export const SpeechRecognitionAPI: Story = {
   args: {
-    keywords: ['kai', 'computer', 'hello'],
+    keywords: ['kai', 'computer', 'hello', 'dxos'],
   },
 };

package/src/stories/LiveTranscription.stories.tsx

@@ -0,0 +1,153 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type Meta, type StoryObj } from '@storybook/react-vite';
+import React, { useCallback, useMemo, useState } from 'react';
+
+import {
+  type ExtractionFunction,
+  extractionAnthropicFunction,
+  extractionNerFunction,
+  getNer,
+  processTranscriptMessage,
+} from '@dxos/assistant/extraction';
+import { Filter, type Obj } from '@dxos/echo';
+import { invariant } from '@dxos/invariant';
+import { useSpaces } from '@dxos/react-client/echo';
+import { Message, Organization, Person } from '@dxos/types';
+
+import { useAudioTrack } from '#hooks';
+
+import { type MediaStreamRecorderProps, type TranscriberProps } from '../transcriber';
+import { useIsSpeaking, createStoryDecorators, useStoryMessageModel, useStoryTranscriber } from './common';
+import { TranscriptionStory } from './TranscriptionStory';
+
+const DEFAULT_TRANSCRIBER_CONFIG = {
+  transcribeAfterChunksAmount: 50,
+  prefixBufferChunksAmount: 10,
+};
+
+const DEFAULT_RECORDER_CONFIG = {
+  interval: 200,
+};
+
+type DefaultStoryProps = {
+  detectSpeaking?: boolean;
+  transcriberConfig?: TranscriberProps['config'];
+  recorderConfig?: MediaStreamRecorderProps['config'];
+  audioConstraints?: MediaTrackConstraints;
+  entityExtraction?: 'ner' | 'llm';
+};
+
+const DefaultStory = ({
+  detectSpeaking,
+  entityExtraction,
+  transcriberConfig = DEFAULT_TRANSCRIBER_CONFIG,
+  recorderConfig = DEFAULT_RECORDER_CONFIG,
+  audioConstraints,
+}: DefaultStoryProps) => {
+  const [running, setRunning] = useState(false);
+  const track = useAudioTrack(running, audioConstraints);
+  const isSpeaking = detectSpeaking ? useIsSpeaking(track) : true;
+  const { model, appendMessage } = useStoryMessageModel();
+  const [space] = useSpaces();
+
+  // Resolve the entity-extraction function + nearby objects once per `entityExtraction` toggle.
+  const { extractionFunction, objects } = useMemo(() => {
+    if (!space || !entityExtraction) {
+      return {};
+    }
+
+    let extractionFunction: ExtractionFunction | undefined;
+    let objects: Promise<Obj.Unknown[]> | undefined;
+    if (entityExtraction === 'ner') {
+      void getNer(); // Init model loading. Takes time.
+      extractionFunction = extractionNerFunction;
+    } else if (entityExtraction === 'llm') {
+      extractionFunction = extractionAnthropicFunction;
+      objects = space.db.query(Filter.or(Filter.type(Person.Person), Filter.type(Organization.Organization))).run();
+    }
+    return { extractionFunction, objects };
+  }, [entityExtraction, space]);
+
+  // Build a Message from each transcribed batch; optionally enrich via entity extraction.
+  const handleSegments = useCallback<TranscriberProps['onSegments']>(
+    async (blocks) => {
+      const message = Message.make({
+        sender: { name: 'You' },
+        created: new Date().toISOString(),
+        blocks,
+      });
+
+      if (entityExtraction && space) {
+        invariant(extractionFunction, 'extractionFunction is required');
+        const result = await processTranscriptMessage({
+          function: extractionFunction,
+          input: { message, objects: await objects },
+          options: { fallbackToRaw: true, timeout: 30_000 },
+        });
+        appendMessage(result.message);
+      } else {
+        appendMessage(message);
+      }
+    },
+    [appendMessage, space, entityExtraction, extractionFunction, objects],
+  );
+
+  useStoryTranscriber({
+    audioStreamTrack: track,
+    running,
+    isSpeaking,
+    transcriberConfig,
+    recorderConfig,
+    onSegments: handleSegments,
+  });
+
+  return <TranscriptionStory model={model} running={running} onRunningChange={setRunning} />;
+};
+
+const meta = {
+  title: 'plugins/plugin-transcription/stories/LiveTranscription',
+  render: DefaultStory,
+  decorators: createStoryDecorators({ enableVectorIndex: true }),
+} satisfies Meta<typeof DefaultStory>;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+export const Default: Story = {
+  args: {
+    detectSpeaking: false,
+    audioConstraints: {
+      echoCancellation: false,
+      noiseSuppression: false,
+      autoGainControl: false,
+    },
+  },
+};
+
+export const SpeechDetection: Story = {
+  args: {
+    detectSpeaking: true,
+    audioConstraints: {
+      echoCancellation: true,
+      noiseSuppression: true,
+      autoGainControl: true,
+    },
+  },
+};
+
+// TODO(mykola): Fix hugging face quota issues.
+// export const EntityExtraction: Story = {
+//   args: {
+//     detectSpeaking: true,
+//     entityExtraction: 'ner',
+//     audioConstraints: {
+//       echoCancellation: true,
+//       noiseSuppression: true,
+//       autoGainControl: true,
+//     },
+//   },
+// };