@dxos/plugin-transcription 0.8.4-main.28f8d3d → 0.8.4-main.2c6827d

package/src/components/stories/MicrophoneTranscription.stories.tsx

@@ -2,8 +2,6 @@
 // Copyright 2025 DXOS.org
 //
 
-import '@dxos-theme';
-
 import { type Meta, type StoryObj } from '@storybook/react-vite';
 import React, { useCallback, useEffect, useMemo, useState } from 'react';
 
@@ -11,15 +9,15 @@ import { Events, IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
 import { withPluginManager } from '@dxos/app-framework/testing';
 import {
   type ExtractionFunction,
-  extractionAnthropicFn,
-  extractionNerFn,
+  extractionAnthropicFunction,
+  extractionNerFunction,
   getNer,
   processTranscriptMessage,
 } from '@dxos/assistant/extraction';
 import { Filter, Obj, type Type } from '@dxos/echo';
+import { createQueueDXN } from '@dxos/echo/internal';
 import { MemoryQueue } from '@dxos/echo-db';
-import { createQueueDXN } from '@dxos/echo-schema';
-import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
+import { FunctionExecutor, ServiceContainer } from '@dxos/functions-runtime';
 import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 import { ClientPlugin } from '@dxos/plugin-client';
@@ -28,16 +26,17 @@ import { SpacePlugin } from '@dxos/plugin-space';
 import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
 import { ThemePlugin } from '@dxos/plugin-theme';
 import { IndexKind, useSpace } from '@dxos/react-client/echo';
+import { withLayout, withTheme } from '@dxos/react-ui/testing';
 import { defaultTx } from '@dxos/react-ui-theme';
-import { DataType } from '@dxos/schema';
-import { Testing, seedTestData } from '@dxos/schema/testing';
-import { withLayout, withTheme } from '@dxos/storybook-utils';
+import { Testing } from '@dxos/schema/testing';
+import { Message, Organization, Person } from '@dxos/types';
+import { seedTestData } from '@dxos/types/testing';
 
 import { useAudioTrack, useQueueModelAdapter, useTranscriber } from '../../hooks';
 import { TestItem } from '../../testing';
 import { type MediaStreamRecorderParams, type TranscriberParams } from '../../transcriber';
 import { TranscriptionPlugin } from '../../TranscriptionPlugin';
-import { renderMarkdown } from '../Transcript';
+import { renderByline } from '../Transcript';
 
 import { TranscriptionStory } from './TranscriptionStory';
 import { useIsSpeaking } from './useIsSpeaking';
@@ -77,8 +76,8 @@ const DefaultStory = ({
   // Queue.
   // TODO(dmaretskyi): Use space.queues.create() instead.
   const queueDxn = useMemo(() => createQueueDXN(), []);
-  const queue = useMemo(() => new MemoryQueue<DataType.Message>(queueDxn), [queueDxn]);
-  const model = useQueueModelAdapter(renderMarkdown([]), queue);
+  const queue = useMemo(() => new MemoryQueue<Message.Message>(queueDxn), [queueDxn]);
+  const model = useQueueModelAdapter(renderByline([]), queue);
   const space = useSpace();
 
   useEffect(() => {
@@ -101,14 +100,14 @@ const DefaultStory = ({
     if (entityExtraction === 'ner') {
       // Init model loading. Takes time.
       void getNer();
-      extractionFunction = extractionNerFn;
+      extractionFunction = extractionNerFunction;
     } else if (entityExtraction === 'llm') {
-      extractionFunction = extractionAnthropicFn;
+      extractionFunction = extractionAnthropicFunction;
       objects = space.db
         .query(
           Filter.or(
-            Filter.type(DataType.Person),
-            Filter.type(DataType.Organization),
+            Filter.type(Person.Person),
+            Filter.type(Organization.Organization),
             Filter.type(Testing.DocumentType),
           ),
         )
@@ -134,7 +133,7 @@ const DefaultStory = ({
   // Transcriber.
   const handleSegments = useCallback<TranscriberParams['onSegments']>(
     async (blocks) => {
-      const message = Obj.make(DataType.Message, {
+      const message = Obj.make(Message.Message, {
         sender: { name: 'You' },
         created: new Date().toISOString(),
         blocks,
@@ -193,16 +192,16 @@ const DefaultStory = ({
   return <TranscriptionStory model={model} running={running} onRunningChange={setRunning} />;
 };
 
-const meta: Meta<typeof DefaultStory> = {
+const meta = {
   title: 'plugins/plugin-transcription/MicrophoneTranscription',
   render: DefaultStory,
   decorators: [
+    withTheme,
+    withLayout({ container: 'column' }),
     withPluginManager({
       plugins: [
-        ThemePlugin({ tx: defaultTx }),
-        StorybookLayoutPlugin(),
         ClientPlugin({
-          types: [TestItem, DataType.Person, DataType.Organization, Testing.DocumentType],
+          types: [TestItem, Person.Person, Organization.Organization, Testing.DocumentType],
           onClientInitialized: async ({ client }) => {
             await client.halo.createIdentity();
             await client.spaces.waitUntilReady();
@@ -223,22 +222,24 @@ const meta: Meta<typeof DefaultStory> = {
             await seedTestData(client.spaces.default);
           },
         }),
-        SpacePlugin(),
+        SpacePlugin({}),
+        IntentPlugin(),
         SettingsPlugin(),
+
+        // UI
+        ThemePlugin({ tx: defaultTx }),
         PreviewPlugin(),
-        IntentPlugin(),
         TranscriptionPlugin(),
+        StorybookLayoutPlugin({}),
       ],
       fireEvents: [Events.SetupAppGraph],
     }),
-    withTheme,
-    withLayout({ fullscreen: true, classNames: 'justify-center' }),
   ],
-};
+} satisfies Meta<typeof DefaultStory>;
 
 export default meta;
 
-type Story = StoryObj<typeof DefaultStory>;
+type Story = StoryObj<typeof meta>;
 
 export const Default: Story = {
   args: {
@@ -254,19 +255,21 @@ export const Default: Story = {
   },
 };
 
-export const EntityExtraction: Story = {
-  args: {
-    detectSpeaking: true,
-    entityExtraction: 'ner',
-    transcriberConfig: TRANSCRIBER_CONFIG,
-    recorderConfig: RECORDER_CONFIG,
-    audioConstraints: {
-      echoCancellation: true,
-      noiseSuppression: true,
-      autoGainControl: true,
-    },
-  },
-};
+// NOTE: We are running out of free quota on hugging face entity extraction.
+// TODO(mykola): Fix hugging face quota issues.
+// export const EntityExtraction: Story = {
+//   args: {
+//     detectSpeaking: true,
+//     entityExtraction: 'ner',
+//     transcriberConfig: TRANSCRIBER_CONFIG,
+//     recorderConfig: RECORDER_CONFIG,
+//     audioConstraints: {
+//       echoCancellation: true,
+//       noiseSuppression: true,
+//       autoGainControl: true,
+//     },
+//   },
+// };
 
 export const SpeechDetection: Story = {
   args: {

package/src/components/stories/TranscriptionStory.tsx

@@ -7,35 +7,38 @@ import '@dxos-theme';
 import React, { type Dispatch, type FC, type RefObject, type SetStateAction } from 'react';
 
 import { useSpace } from '@dxos/react-client/echo';
-import { IconButton, Toolbar } from '@dxos/react-ui';
-import { ScrollContainer } from '@dxos/react-ui-components';
-import { type DataType } from '@dxos/schema';
+import { IconButton, ScrollContainer, Toolbar } from '@dxos/react-ui';
+import { type Message } from '@dxos/types';
 
 import { type SerializationModel } from '../../model';
-import { Transcript } from '../Transcript';
+import { TranscriptView } from '../Transcript';
 
 export const TranscriptionStory: FC<{
-  model: SerializationModel<DataType.Message>;
+  model: SerializationModel<Message.Message>;
+  disabled?: boolean;
   running: boolean;
   onRunningChange: Dispatch<SetStateAction<boolean>>;
-  audioRef?: RefObject<HTMLAudioElement>;
-}> = ({ model, running, onRunningChange, audioRef }) => {
+  audioRef?: RefObject<HTMLAudioElement | null>;
+}> = ({ model, running, onRunningChange, audioRef, disabled }) => {
   const space = useSpace();
 
   return (
-    <div className='flex flex-col w-[30rem]'>
+    <div className='flex flex-col is-[30rem]'>
       {audioRef && <audio ref={audioRef} autoPlay />}
       <Toolbar.Root>
         <IconButton
           iconOnly
+          disabled={disabled}
           icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
           label={running ? 'Pause' : 'Play'}
           onClick={() => onRunningChange((running) => !running)}
         />
       </Toolbar.Root>
-      <ScrollContainer>
-        <Transcript space={space} model={model} attendableId='story' />
-      </ScrollContainer>
+      <ScrollContainer.Root pin>
+        <ScrollContainer.Viewport>
+          <TranscriptView space={space} model={model} />
+        </ScrollContainer.Viewport>
+      </ScrollContainer.Root>
     </div>
   );
 };

package/src/functions/index.ts

@@ -0,0 +1,6 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export { default as open } from './open';
+export { default as summarize } from './summarize';

package/src/functions/open.ts

@@ -0,0 +1,38 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+import * as Schema from 'effect/Schema';
+
+import { ArtifactId } from '@dxos/assistant';
+import { Obj } from '@dxos/echo';
+import { DatabaseService, QueueService, defineFunction } from '@dxos/functions';
+import { Message, Transcript } from '@dxos/types';
+
+import { renderByline } from '../components';
+
+export default defineFunction({
+  key: 'dxos.org/function/transcription/open',
+  name: 'Open',
+  description: 'Opens and reads the contents of a transcription object.',
+  inputSchema: Schema.Struct({
+    id: ArtifactId.annotations({
+      description: 'The ID of the transcription object.',
+    }),
+  }),
+  outputSchema: Schema.Struct({
+    content: Schema.String,
+  }),
+  handler: Effect.fn(function* ({ data: { id } }) {
+    const transcript = yield* DatabaseService.resolve(ArtifactId.toDXN(id), Transcript.Transcript);
+    const { dxn } = yield* Effect.promise(() => transcript.queue.load());
+    const queue = yield* QueueService.getQueue(dxn);
+    yield* Effect.promise(() => queue?.queryObjects());
+    const content = queue?.objects
+      .filter((message) => Obj.instanceOf(Message.Message, message))
+      .flatMap((message, index) => renderByline([])(message, index))
+      .join('\n\n');
+    return { content };
+  }),
+});

package/src/functions/summarize.ts

@@ -0,0 +1,103 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Array from 'effect/Array';
+import * as Effect from 'effect/Effect';
+import * as Function from 'effect/Function';
+import * as Layer from 'effect/Layer';
+import * as Option from 'effect/Option';
+import * as Schema from 'effect/Schema';
+
+import { AiService, ConsolePrinter, ToolExecutionService, ToolResolverService } from '@dxos/ai';
+import { AiSession, GenerationObserver } from '@dxos/assistant';
+import { TracingService, defineFunction } from '@dxos/functions';
+import { trim } from '@dxos/util';
+
+/**
+ * Summarize a transcript of a meeting.
+ */
+export default defineFunction({
+  key: 'dxos.org/function/transcription/summarize',
+  name: 'Summarize',
+  description: 'Summarize a transcript of a meeting.',
+  inputSchema: Schema.Struct({
+    transcript: Schema.String.annotations({
+      description: 'The transcript of the meeting.',
+    }),
+    notes: Schema.optional(Schema.String).annotations({
+      description: 'Additional notes from the participants.',
+    }),
+  }),
+  outputSchema: Schema.Struct({
+    summary: Schema.String.annotations({
+      description: 'The summary of the transcript.',
+    }),
+  }),
+  handler: Effect.fnUntraced(
+    function* ({ data: { transcript, notes } }) {
+      const result = yield* new AiSession().run({
+        prompt: `Transcript: ${transcript}\n\nNotes: ${notes}`,
+        history: [],
+        system: systemPrompt,
+        observer: GenerationObserver.fromPrinter(new ConsolePrinter({ tag: 'summarize' })),
+      });
+
+      const summary = Function.pipe(
+        result,
+        Array.findLast((msg) => msg.sender.role === 'assistant' && msg.blocks.some((block) => block._tag === 'text')),
+        Option.flatMap((msg) =>
+          Function.pipe(
+            msg.blocks,
+            Array.findLast((block) => block._tag === 'text'),
+            Option.map((block) => block.text),
+          ),
+        ),
+        Option.getOrThrowWith(() => new Error('No summary found')),
+      );
+
+      return { summary };
+    },
+    Effect.provide(
+      Layer.mergeAll(
+        AiService.model('@anthropic/claude-sonnet-4-0'),
+        ToolResolverService.layerEmpty,
+        ToolExecutionService.layerEmpty,
+        TracingService.layerNoop,
+      ),
+    ),
+  ),
+});
+
+const systemPrompt = trim`
+  You are a helpful assistant that summarizes transcripts of meetings.
+
+  # Goal
+  Create a markdown summary of the meeting transcript with text notes provided.
+  Notes are very important so make sure to include them in the summary if they contain meaningful information.
+
+  # Formatting
+  - Format the summary as a markdown document without extra comments like "Here is the summary of the transcript:".
+  - Use markdown formatting for headings and bullet points.
+  - Format the summary as a list of key points and takeaways.
+  - All names of people should be in bold.
+
+  # Note Taking
+  - Correlate items in the summary with the person of origin to build a coherent narrative.
+  - Include short quotes verbatim where appropriate. Especially when concerned with design decisions and problem descriptions.
+
+  # Tasks
+  At the end of the summary include tasks.
+  Extract only the tasks that are:
+  - Directly actionable.
+  - Clearly assigned to a person or team (or can easily be inferred).
+  - Strongly implied by the conversation and/or user note (no speculative tasks).
+  - Specific enough that someone reading them would know exactly what to do next.
+
+  Format all tasks as markdown checkboxes using the syntax:
+  - [ ] Task description.
+
+  Additional information can be included (indented).
+
+  If no actionable tasks are found, omit this tasks section.
+`;

package/src/hooks/useAudioFile.ts

@@ -20,67 +20,63 @@ export const useAudioFile = (audioUrl: string, constraints?: MediaTrackConstrain
     const ctx = new Context();
 
     scheduleTask(ctx, async () => {
-      const response = await fetch(audioUrl);
-      const blob = await response.blob();
-      const objectUrl = URL.createObjectURL(blob);
-      ctx.onDispose(() => {
-        URL.revokeObjectURL(objectUrl);
-      });
+      // TODO(wittjosiah): Fetch to external url fails in headless storybook test.
+      try {
+        const response = await fetch(audioUrl);
+        const blob = await response.blob();
+        const objectUrl = URL.createObjectURL(blob);
+        ctx.onDispose(() => {
+          URL.revokeObjectURL(objectUrl);
+        });
 
-      const audio = new Audio();
-      audio.src = objectUrl;
+        const audio = new Audio();
+        audio.src = objectUrl;
 
-      await new Promise<void>((resolve, reject) => {
-        audio.addEventListener(
-          'error',
-          (err) => {
-            log.error('error', { err });
-            reject(err);
-          },
-          { once: true },
-        );
-        audio.addEventListener(
-          'canplay',
-          async () => {
-            log.info('starting...');
-            try {
-              // Try to play the audio
-              await audio.play();
+        await new Promise<void>((resolve, reject) => {
+          audio.addEventListener(
+            'error',
+            (err) => {
+              log.error('error', { err });
+              reject(err);
+            },
+            { once: true },
+          );
+          audio.addEventListener(
+            'canplay',
+            async () => {
               resolve();
-            } catch (playError) {
-              log.error('Play failed', { playError });
-              // Still resolve as the audio is ready, even if autoplay failed.
-              resolve();
-            }
-          },
-          { once: true },
-        );
-        audio.load();
-      });
+            },
+            { once: true },
+          );
+          audio.load();
+        });
 
-      const audioCtx = new AudioContext();
+        const audioCtx = new AudioContext();
 
-      // Resume AudioContext if it's suspended.
-      if (audioCtx.state === 'suspended') {
-        await audioCtx.resume();
-      }
+        // Resume AudioContext if it's suspended.
+        if (audioCtx.state === 'suspended') {
+          await audioCtx.resume();
+        }
 
-      const destination = audioCtx.createMediaStreamDestination();
-      destination.channelCount = 1;
+        const destination = audioCtx.createMediaStreamDestination();
+        destination.channelCount = 1;
 
-      const source = audioCtx.createMediaElementSource(audio);
-      source.connect(destination);
-      const track = destination.stream.getAudioTracks()[0];
-      if (constraints) {
-        await track.applyConstraints(constraints).catch((err) => log.catch(err));
-      }
+        const source = audioCtx.createMediaElementSource(audio);
+        source.connect(destination);
+        const track = destination.stream.getAudioTracks()[0];
+        if (constraints) {
+          await track.applyConstraints(constraints).catch((err) => log.catch(err));
+        }
 
-      // Also connect to speakers so audio is audible.
-      setStream({
-        audio,
-        stream: destination.stream,
-        track: destination.stream.getAudioTracks()[0],
-      });
+        // Also connect to speakers so audio is audible.
+        setStream({
+          audio,
+          stream: destination.stream,
+          track: destination.stream.getAudioTracks()[0],
+        });
+      } catch (err) {
+        log.catch(err);
+      }
     });
 
     return () => {

package/src/hooks/useTranscriber.ts

@@ -4,7 +4,7 @@
 
 import { useEffect, useMemo } from 'react';
 
-import { useCapabilities } from '@dxos/app-framework';
+import { useCapabilities } from '@dxos/app-framework/react';
 
 import { TranscriptionCapabilities } from '../capabilities';
 import { type Transcriber } from '../transcriber';
@@ -26,7 +26,12 @@ export const useTranscriber = ({
       return undefined;
     }
 
-    return getTranscriber({ audioStreamTrack, recorderConfig, transcriberConfig, onSegments });
+    return getTranscriber({
+      audioStreamTrack,
+      recorderConfig,
+      transcriberConfig,
+      onSegments,
+    });
   }, [getTranscriber, audioStreamTrack, recorderConfig, transcriberConfig, onSegments]);
 
   useEffect(() => {

package/src/meta.ts

@@ -3,14 +3,17 @@
 //
 
 import { type PluginMeta } from '@dxos/app-framework';
-
-export const TRANSCRIPTION_PLUGIN = 'dxos.org/plugin/transcription';
+import { trim } from '@dxos/util';
 
 export const meta: PluginMeta = {
-  id: TRANSCRIPTION_PLUGIN,
+  id: 'dxos.org/plugin/transcription',
   name: 'Transcription',
-  description: 'Voice transcription.',
+  description: trim`
+    Real-time voice-to-text transcription service for capturing spoken content.
+    Convert audio input into searchable text that integrates seamlessly with notes and documents.
+  `,
   icon: 'ph--microphone--regular',
+  iconHue: 'sky',
   source: 'https://github.com/dxos/dxos/tree/main/packages/plugins/plugin-transcription',
   tags: ['labs'],
   screenshots: [],

package/src/model/model.test.ts

@@ -7,12 +7,12 @@ import { describe, test } from 'vitest';
 
 import { Obj } from '@dxos/echo';
 import type { ObjectId } from '@dxos/keys';
-import { DataType } from '@dxos/schema';
+import { Message } from '@dxos/types';
 
 import { type ChunkRenderer, DocumentAdapter, SerializationModel } from './model';
 
-const blockToMarkdown: ChunkRenderer<DataType.Message> = (
-  message: DataType.Message,
+const blockToMarkdown: ChunkRenderer<Message.Message> = (
+  message: Message.Message,
   index: number,
   debug = true,
 ): string[] => {
@@ -27,12 +27,12 @@ const createDate = () => new Date().toISOString();
 
 describe('SerializationModel', () => {
   test('basic', ({ expect }) => {
-    const model = new SerializationModel<DataType.Message>(blockToMarkdown);
+    const model = new SerializationModel<Message.Message>(blockToMarkdown);
     expect(model.chunks.length).to.eq(0);
     expect(model.doc.toString()).to.eq('');
 
     // Create message.
-    const message = Obj.make(DataType.Message, {
+    const message = Obj.make(Message.Message, {
       created: createDate(),
       sender: { name: 'Alice' },
       blocks: [
@@ -62,14 +62,14 @@ describe('SerializationModel', () => {
 
   test('sync - append', ({ expect }) => {
     const view = new EditorView({ extensions: [], doc: '' });
-    const model = new SerializationModel<DataType.Message>(blockToMarkdown);
+    const model = new SerializationModel<Message.Message>(blockToMarkdown);
     const adapter = new DocumentAdapter(view);
     expect(adapter.lineCount()).to.eq(0);
     expect(view.state.doc.toString()).to.eq('');
 
     // Append message.
     {
-      const message = Obj.make(DataType.Message, {
+      const message = Obj.make(Message.Message, {
         created: createDate(),
         sender: { name: 'Alice' },
         blocks: [
@@ -87,7 +87,7 @@ describe('SerializationModel', () => {
 
     // Append message.
     {
-      const message = Obj.make(DataType.Message, {
+      const message = Obj.make(Message.Message, {
         created: createDate(),
         sender: { name: 'Bob' },
         blocks: [
@@ -106,7 +106,7 @@ describe('SerializationModel', () => {
 
   test('sync - append, update, delete', ({ expect }) => {
     const view = new EditorView({ extensions: [], doc: '' });
-    const model = new SerializationModel<DataType.Message>(blockToMarkdown);
+    const model = new SerializationModel<Message.Message>(blockToMarkdown);
     const adapter = new DocumentAdapter(view);
     expect(adapter.lineCount()).to.eq(0);
 
@@ -114,7 +114,7 @@ describe('SerializationModel', () => {
 
     // Append message.
     {
-      const message = Obj.make(DataType.Message, {
+      const message = Obj.make(Message.Message, {
         created: createDate(),
         sender: { name: 'Alice' },
         blocks: [
@@ -147,7 +147,7 @@ describe('SerializationModel', () => {
 
     // Append message.
     {
-      const message = Obj.make(DataType.Message, {
+      const message = Obj.make(Message.Message, {
         created: createDate(),
         sender: { name: 'Bob' },
         blocks: [{ _tag: 'transcript', started: createDate(), text: 'Hello again!' }],

package/src/segments-normalization/message-normalizer.ts

@@ -11,11 +11,11 @@ import { effect } from '@preact/signals-core';
 import { DeferredTask, asyncTimeout } from '@dxos/async';
 import { LifecycleState, Resource } from '@dxos/context';
 import { type Queue } from '@dxos/echo-db';
-import { type FunctionExecutor } from '@dxos/functions';
+import { type FunctionExecutor } from '@dxos/functions-runtime';
 import { log } from '@dxos/log';
-import { type DataType } from '@dxos/schema';
+import { type Message } from '@dxos/types';
 
-import { type MessageWithRangeId, sentenceNormalization } from './normalization';
+import { type MessageWithRangeId, type NormalizationOutput, sentenceNormalization } from './normalization';
 import { getActorId } from './utils';
 
 const PROCESSING_TIMEOUT = 20_000; // ms
@@ -23,7 +23,7 @@ const MAX_RANGE_ID_COUNT = 10;
 
 export type SegmentsNormalizerParams = {
   functionExecutor: FunctionExecutor;
-  queue: Queue<DataType.Message>;
+  queue: Queue<Message.Message>;
   startingCursor: QueueCursor;
 };
 
@@ -36,7 +36,7 @@ export type QueueCursor = {
 // TODO(mykola): .
 export class MessageNormalizer extends Resource {
   private readonly _functionExecutor: FunctionExecutor;
-  private _queue: Queue<DataType.Message>;
+  private _queue: Queue<Message.Message>;
   private _cursor: QueueCursor;
   private _messagesToProcess: MessageWithRangeId[] = [];
   private _normalizationTask?: DeferredTask;
@@ -81,7 +81,7 @@ export class MessageNormalizer extends Resource {
 
     try {
       // TODO(mykola): Executor should support timeout.
-      const response = await asyncTimeout(
+      const response: NormalizationOutput = await asyncTimeout(
         this._functionExecutor.invoke(sentenceNormalization, { messages }),
         PROCESSING_TIMEOUT,
       );