@dxos/plugin-transcription 0.8.3 → 0.8.4-main.1068cf700f

package/src/functions/open.ts

@@ -0,0 +1,37 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+import * as Schema from 'effect/Schema';
+
+import { Database, Obj, Type } from '@dxos/echo';
+import { QueueService, defineFunction } from '@dxos/functions';
+import { Message, Transcript } from '@dxos/types';
+
+import { renderByline } from '../util';
+
+export default defineFunction({
+  key: 'dxos.org/function/transcription/open',
+  name: 'Open',
+  description: 'Opens and reads the contents of a transcription object.',
+  inputSchema: Schema.Struct({
+    transcript: Type.Ref(Transcript.Transcript).annotations({
+      description: 'The ID of the transcription object.',
+    }),
+  }),
+  outputSchema: Schema.Struct({
+    content: Schema.String,
+  }),
+  handler: Effect.fn(function* ({ data: { transcript } }) {
+    const transcriptObj = yield* Database.load(transcript);
+    const { dxn } = yield* Effect.promise(() => transcriptObj.queue.load());
+    const queue = yield* QueueService.getQueue(dxn);
+    yield* Effect.promise(() => queue?.queryObjects());
+    const content = queue?.objects
+      .filter((message) => Obj.instanceOf(Message.Message, message))
+      .flatMap((message, index) => renderByline([])(message, index))
+      .join('\n\n');
+    return { content };
+  }),
+});

package/src/functions/summarize.ts

@@ -0,0 +1,103 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Array from 'effect/Array';
+import * as Effect from 'effect/Effect';
+import * as Function from 'effect/Function';
+import * as Layer from 'effect/Layer';
+import * as Option from 'effect/Option';
+import * as Schema from 'effect/Schema';
+
+import { AiService, ConsolePrinter, ToolExecutionService, ToolResolverService } from '@dxos/ai';
+import { AiSession, GenerationObserver } from '@dxos/assistant';
+import { TracingService, defineFunction } from '@dxos/functions';
+import { trim } from '@dxos/util';
+
+/**
+ * Summarize a transcript of a meeting.
+ */
+export default defineFunction({
+  key: 'dxos.org/function/transcription/summarize',
+  name: 'Summarize',
+  description: 'Summarize a transcript of a meeting.',
+  inputSchema: Schema.Struct({
+    transcript: Schema.String.annotations({
+      description: 'The transcript of the meeting.',
+    }),
+    notes: Schema.optional(Schema.String).annotations({
+      description: 'Additional notes from the participants.',
+    }),
+  }),
+  outputSchema: Schema.Struct({
+    summary: Schema.String.annotations({
+      description: 'The summary of the transcript.',
+    }),
+  }),
+  handler: Effect.fnUntraced(
+    function* ({ data: { transcript, notes } }) {
+      const result = yield* new AiSession().run({
+        prompt: `Transcript: ${transcript}\n\nNotes: ${notes}`,
+        history: [],
+        system: systemPrompt,
+        observer: GenerationObserver.fromPrinter(new ConsolePrinter({ tag: 'summarize' })),
+      });
+
+      const summary = Function.pipe(
+        result,
+        Array.findLast((msg) => msg.sender.role === 'assistant' && msg.blocks.some((block) => block._tag === 'text')),
+        Option.flatMap((msg) =>
+          Function.pipe(
+            msg.blocks,
+            Array.findLast((block) => block._tag === 'text'),
+            Option.map((block) => block.text),
+          ),
+        ),
+        Option.getOrThrowWith(() => new Error('No summary found')),
+      );
+
+      return { summary };
+    },
+    Effect.provide(
+      Layer.mergeAll(
+        AiService.model('@anthropic/claude-sonnet-4-0'),
+        ToolResolverService.layerEmpty,
+        ToolExecutionService.layerEmpty,
+        TracingService.layerNoop,
+      ),
+    ),
+  ),
+});
+
+const systemPrompt = trim`
+  You are a helpful assistant that summarizes transcripts of meetings.
+
+  # Goal
+  Create a markdown summary of the meeting transcript with text notes provided.
+  Notes are very important so make sure to include them in the summary if they contain meaningful information.
+
+  # Formatting
+  - Format the summary as a markdown document without extra comments like "Here is the summary of the transcript:".
+  - Use markdown formatting for headings and bullet points.
+  - Format the summary as a list of key points and takeaways.
+  - All names of people should be in bold.
+
+  # Note Taking
+  - Correlate items in the summary with the person of origin to build a coherent narrative.
+  - Include short quotes verbatim where appropriate. Especially when concerned with design decisions and problem descriptions.
+
+  # Tasks
+  At the end of the summary include tasks.
+  Extract only the tasks that are:
+  - Directly actionable.
+  - Clearly assigned to a person or team (or can easily be inferred).
+  - Strongly implied by the conversation and/or user note (no speculative tasks).
+  - Specific enough that someone reading them would know exactly what to do next.
+
+  Format all tasks as markdown checkboxes using the syntax:
+  - [ ] Task description.
+
+  Additional information can be included (indented).
+
+  If no actionable tasks are found, omit this tasks section.
+`;

package/src/hooks/useAudioFile.ts

@@ -20,67 +20,63 @@ export const useAudioFile = (audioUrl: string, constraints?: MediaTrackConstrain
     const ctx = new Context();
 
     scheduleTask(ctx, async () => {
-      const response = await fetch(audioUrl);
-      const blob = await response.blob();
-      const objectUrl = URL.createObjectURL(blob);
-      ctx.onDispose(() => {
-        URL.revokeObjectURL(objectUrl);
-      });
+      // TODO(wittjosiah): Fetch to external url fails in headless storybook test.
+      try {
+        const response = await fetch(audioUrl);
+        const blob = await response.blob();
+        const objectUrl = URL.createObjectURL(blob);
+        ctx.onDispose(() => {
+          URL.revokeObjectURL(objectUrl);
+        });
 
-      const audio = new Audio();
-      audio.src = objectUrl;
+        const audio = new Audio();
+        audio.src = objectUrl;
 
-      await new Promise<void>((resolve, reject) => {
-        audio.addEventListener(
-          'error',
-          (err) => {
-            log.error('error', { err });
-            reject(err);
-          },
-          { once: true },
-        );
-        audio.addEventListener(
-          'canplay',
-          async () => {
-            log.info('starting...');
-            try {
-              // Try to play the audio
-              await audio.play();
+        await new Promise<void>((resolve, reject) => {
+          audio.addEventListener(
+            'error',
+            (err) => {
+              log.error('error', { err });
+              reject(err);
+            },
+            { once: true },
+          );
+          audio.addEventListener(
+            'canplay',
+            async () => {
               resolve();
-            } catch (playError) {
-              log.error('Play failed', { playError });
-              // Still resolve as the audio is ready, even if autoplay failed.
-              resolve();
-            }
-          },
-          { once: true },
-        );
-        audio.load();
-      });
+            },
+            { once: true },
+          );
+          audio.load();
+        });
 
-      const audioCtx = new AudioContext();
+        const audioCtx = new AudioContext();
 
-      // Resume AudioContext if it's suspended.
-      if (audioCtx.state === 'suspended') {
-        await audioCtx.resume();
-      }
+        // Resume AudioContext if it's suspended.
+        if (audioCtx.state === 'suspended') {
+          await audioCtx.resume();
+        }
 
-      const destination = audioCtx.createMediaStreamDestination();
-      destination.channelCount = 1;
+        const destination = audioCtx.createMediaStreamDestination();
+        destination.channelCount = 1;
 
-      const source = audioCtx.createMediaElementSource(audio);
-      source.connect(destination);
-      const track = destination.stream.getAudioTracks()[0];
-      if (constraints) {
-        await track.applyConstraints(constraints).catch((err) => log.catch(err));
-      }
+        const source = audioCtx.createMediaElementSource(audio);
+        source.connect(destination);
+        const track = destination.stream.getAudioTracks()[0];
+        if (constraints) {
+          await track.applyConstraints(constraints).catch((err) => log.catch(err));
+        }
 
-      // Also connect to speakers so audio is audible.
-      setStream({
-        audio,
-        stream: destination.stream,
-        track: destination.stream.getAudioTracks()[0],
-      });
+        // Also connect to speakers so audio is audible.
+        setStream({
+          audio,
+          stream: destination.stream,
+          track: destination.stream.getAudioTracks()[0],
+        });
+      } catch (err) {
+        log.catch(err);
+      }
     });
 
     return () => {

package/src/hooks/useQueueModelAdapter.ts

@@ -12,7 +12,7 @@ import { type ChunkRenderer, SerializationModel } from '../model';
 /**
  * Model adapter for a queue.
  */
-export const useQueueModelAdapter = <T extends Obj.Any>(
+export const useQueueModelAdapter = <T extends Obj.Unknown>(
   renderer: ChunkRenderer<T>,
   queue: Queue<T> | undefined,
   initialChunks: T[] = [],

package/src/hooks/useTranscriber.ts

@@ -4,30 +4,35 @@
 
 import { useEffect, useMemo } from 'react';
 
-import { useCapabilities } from '@dxos/app-framework';
+import { useCapabilities } from '@dxos/app-framework/ui';
 
-import { TranscriptionCapabilities } from '../capabilities';
 import { type Transcriber } from '../transcriber';
+import { TranscriptionCapabilities } from '../types';
 
 /**
  * Records audio while user is speaking and transcribes it after user is done speaking.
  */
 export const useTranscriber = ({
   audioStreamTrack,
-  onSegments,
-  transcriberConfig,
   recorderConfig,
+  transcriberConfig,
+  onSegments,
 }: Partial<TranscriptionCapabilities.GetTranscriberProps>) => {
   const [getTranscriber] = useCapabilities(TranscriptionCapabilities.Transcriber);
 
   // Initialize audio transcription.
   const transcriber = useMemo<Transcriber | undefined>(() => {
-    if (!onSegments || !audioStreamTrack || !getTranscriber) {
+    if (!getTranscriber || !audioStreamTrack || !onSegments) {
       return undefined;
     }
 
-    return getTranscriber({ audioStreamTrack, onSegments, transcriberConfig, recorderConfig });
-  }, [audioStreamTrack, onSegments, getTranscriber, transcriberConfig, recorderConfig]);
+    return getTranscriber({
+      audioStreamTrack,
+      recorderConfig,
+      transcriberConfig,
+      onSegments,
+    });
+  }, [getTranscriber, audioStreamTrack, recorderConfig, transcriberConfig, onSegments]);
 
   useEffect(() => {
     return () => {

package/src/hooks/useVoiceInput.ts

@@ -2,16 +2,17 @@
 // Copyright 2025 DXOS.org
 //
 
-import { useState, useEffect, useCallback } from 'react';
+import { useCallback, useEffect, useState } from 'react';
 
 import { scheduleMicroTask } from '@dxos/async';
 import { Context } from '@dxos/context';
 import { log } from '@dxos/log';
 import { useSoundEffect } from '@dxos/react-ui-sfx';
 
+import { type TranscriberProps } from '../transcriber';
+
 import { useAudioTrack } from './useAudioTrack';
 import { useTranscriber } from './useTranscriber';
-import { type TranscriberParams } from '../transcriber';
 
 export type UseVoiceInputProps = {
   active?: boolean;
@@ -25,7 +26,7 @@ export const useVoiceInput = ({ active, onUpdate }: UseVoiceInputProps) => {
   const soundStart = useSoundEffect('StartRecording');
   const soundStop = useSoundEffect('StopRecording');
 
-  const handleSegments = useCallback<TranscriberParams['onSegments']>(async (segments) => {
+  const handleSegments = useCallback<TranscriberProps['onSegments']>(async (segments) => {
     const text = segments.map((str) => str.text.trim().replace(/[^\w\s]/g, '')).join(' ');
     onUpdate(text);
   }, []);

package/src/index.ts

@@ -2,7 +2,7 @@
 // Copyright 2025 DXOS.org
 //
 
-export { TranscriptionCapabilities } from './capabilities';
+export { TranscriptionCapabilities } from './types';
 
 export * from './components';
 export * from './hooks';

package/src/meta.ts

@@ -2,15 +2,18 @@
 // Copyright 2023 DXOS.org
 //
 
-import { type PluginMeta } from '@dxos/app-framework';
+import { type Plugin } from '@dxos/app-framework';
+import { trim } from '@dxos/util';
 
-export const TRANSCRIPTION_PLUGIN = 'dxos.org/plugin/transcription';
-
-export const meta: PluginMeta = {
-  id: TRANSCRIPTION_PLUGIN,
+export const meta: Plugin.Meta = {
+  id: 'dxos.org/plugin/transcription',
   name: 'Transcription',
-  description: 'Voice transcription.',
+  description: trim`
+    Real-time voice-to-text transcription service for capturing spoken content.
+    Convert audio input into searchable text that integrates seamlessly with notes and documents.
+  `,
   icon: 'ph--microphone--regular',
+  iconHue: 'sky',
   source: 'https://github.com/dxos/dxos/tree/main/packages/plugins/plugin-transcription',
   tags: ['labs'],
   screenshots: [],

package/src/model/model.test.ts

@@ -7,18 +7,18 @@ import { describe, test } from 'vitest';
 
 import { Obj } from '@dxos/echo';
 import type { ObjectId } from '@dxos/keys';
-import { DataType } from '@dxos/schema';
+import { Message } from '@dxos/types';
 
-import { SerializationModel, DocumentAdapter, type ChunkRenderer } from './model';
+import { type ChunkRenderer, DocumentAdapter, SerializationModel } from './model';
 
-const blockToMarkdown: ChunkRenderer<DataType.Message> = (
-  message: DataType.Message,
+const blockToMarkdown: ChunkRenderer<Message.Message> = (
+  message: Message.Message,
   index: number,
   debug = true,
 ): string[] => {
   return [
     `###### ${message.sender.name}`,
-    ...message.blocks.filter((block) => block.type === 'transcription').map((block) => block.text),
+    ...message.blocks.filter((block) => block._tag === 'transcript').map((block) => block.text),
     '',
   ];
 };
@@ -27,17 +27,17 @@ const createDate = () => new Date().toISOString();
 
 describe('SerializationModel', () => {
   test('basic', ({ expect }) => {
-    const model = new SerializationModel<DataType.Message>(blockToMarkdown);
+    const model = new SerializationModel<Message.Message>(blockToMarkdown);
     expect(model.chunks.length).to.eq(0);
     expect(model.doc.toString()).to.eq('');
 
     // Create message.
-    const message = Obj.make(DataType.Message, {
+    const message = Obj.make(Message.Message, {
       created: createDate(),
       sender: { name: 'Alice' },
       blocks: [
         {
-          type: 'transcription',
+          _tag: 'transcript',
           started: createDate(),
           text: 'Hello world!',
         },
@@ -51,7 +51,9 @@ describe('SerializationModel', () => {
     }
 
     // Update message.
-    message.blocks.push({ type: 'transcription', started: createDate(), text: 'Hello again!' });
+    Obj.change(message, (m) => {
+      m.blocks.push({ _tag: 'transcript', started: createDate(), text: 'Hello again!' });
+    });
     model.updateChunk(message);
     {
       const text = model.doc.toString();
@@ -62,19 +64,19 @@ describe('SerializationModel', () => {
 
   test('sync - append', ({ expect }) => {
     const view = new EditorView({ extensions: [], doc: '' });
-    const model = new SerializationModel<DataType.Message>(blockToMarkdown);
+    const model = new SerializationModel<Message.Message>(blockToMarkdown);
     const adapter = new DocumentAdapter(view);
     expect(adapter.lineCount()).to.eq(0);
     expect(view.state.doc.toString()).to.eq('');
 
     // Append message.
     {
-      const message = Obj.make(DataType.Message, {
+      const message = Obj.make(Message.Message, {
         created: createDate(),
         sender: { name: 'Alice' },
         blocks: [
           {
-            type: 'transcription',
+            _tag: 'transcript',
             started: createDate(),
             text: 'Hello world!',
           },
@@ -87,12 +89,12 @@ describe('SerializationModel', () => {
 
     // Append message.
     {
-      const message = Obj.make(DataType.Message, {
+      const message = Obj.make(Message.Message, {
         created: createDate(),
         sender: { name: 'Bob' },
         blocks: [
           {
-            type: 'transcription',
+            _tag: 'transcript',
             started: createDate(),
             text: 'Hello world!',
           },
@@ -106,7 +108,7 @@ describe('SerializationModel', () => {
 
   test('sync - append, update, delete', ({ expect }) => {
     const view = new EditorView({ extensions: [], doc: '' });
-    const model = new SerializationModel<DataType.Message>(blockToMarkdown);
+    const model = new SerializationModel<Message.Message>(blockToMarkdown);
     const adapter = new DocumentAdapter(view);
     expect(adapter.lineCount()).to.eq(0);
 
@@ -114,12 +116,12 @@ describe('SerializationModel', () => {
 
     // Append message.
     {
-      const message = Obj.make(DataType.Message, {
+      const message = Obj.make(Message.Message, {
         created: createDate(),
         sender: { name: 'Alice' },
         blocks: [
           {
-            type: 'transcription',
+            _tag: 'transcript',
             started: createDate(),
             text: 'Hello world!',
           },
@@ -131,13 +133,17 @@ describe('SerializationModel', () => {
       expect(view.state.doc.toString()).to.eq('###### Alice\nHello world!\n\n');
 
       // Update message (add block).
-      message.blocks.push({ type: 'transcription', started: createDate(), text: 'Hello again!' });
+      Obj.change(message, (m) => {
+        m.blocks.push({ _tag: 'transcript', started: createDate(), text: 'Hello again!' });
+      });
       model.updateChunk(message);
       model.sync(adapter);
       expect(view.state.doc.toString()).to.eq('###### Alice\nHello world!\nHello again!\n\n');
 
       // Update message (remove block).
-      message.blocks.splice(0, 1);
+      Obj.change(message, (m) => {
+        m.blocks.splice(0, 1);
+      });
       model.updateChunk(message);
       model.sync(adapter);
       expect(view.state.doc.toString()).to.eq('###### Alice\nHello again!\n\n');
@@ -147,10 +153,10 @@ describe('SerializationModel', () => {
 
     // Append message.
     {
-      const message = Obj.make(DataType.Message, {
+      const message = Obj.make(Message.Message, {
         created: createDate(),
         sender: { name: 'Bob' },
-        blocks: [{ type: 'transcription', started: createDate(), text: 'Hello again!' }],
+        blocks: [{ _tag: 'transcript', started: createDate(), text: 'Hello again!' }],
       });
       model.appendChunk(message);
       model.sync(adapter);

package/src/segments-normalization/message-normalizer.ts

@@ -6,24 +6,22 @@
 // Copyright 2025 DXOS.org
 //
 
-import { effect } from '@preact/signals-core';
-
-import { asyncTimeout, DeferredTask } from '@dxos/async';
+import { DeferredTask, asyncTimeout } from '@dxos/async';
 import { LifecycleState, Resource } from '@dxos/context';
 import { type Queue } from '@dxos/echo-db';
-import { type FunctionExecutor } from '@dxos/functions';
+import { type FunctionExecutor } from '@dxos/functions-runtime';
 import { log } from '@dxos/log';
-import { type DataType } from '@dxos/schema';
+import { type Message } from '@dxos/types';
 
-import { type MessageWithRangeId, sentenceNormalization } from './normalization';
+import { type MessageWithRangeId, type NormalizationOutput, sentenceNormalization } from './normalization';
 import { getActorId } from './utils';
 
 const PROCESSING_TIMEOUT = 20_000; // ms
 const MAX_RANGE_ID_COUNT = 10;
 
-export type SegmentsNormalizerParams = {
+export type SegmentsNormalizerProps = {
   functionExecutor: FunctionExecutor;
-  queue: Queue<DataType.Message>;
+  queue: Queue<Message.Message>;
   startingCursor: QueueCursor;
 };
 
@@ -36,13 +34,13 @@ export type QueueCursor = {
 // TODO(mykola): .
 export class MessageNormalizer extends Resource {
   private readonly _functionExecutor: FunctionExecutor;
-  private _queue: Queue<DataType.Message>;
+  private _queue: Queue<Message.Message>;
   private _cursor: QueueCursor;
   private _messagesToProcess: MessageWithRangeId[] = [];
   private _normalizationTask?: DeferredTask;
   private _lastProcessedMessageIds?: string[];
 
-  constructor({ functionExecutor, queue, startingCursor }: SegmentsNormalizerParams) {
+  constructor({ functionExecutor, queue, startingCursor }: SegmentsNormalizerProps) {
     super();
     this._functionExecutor = functionExecutor;
     this._queue = queue;
@@ -51,7 +49,8 @@ export class MessageNormalizer extends Resource {
 
   protected override async _open(): Promise<void> {
     this._normalizationTask = new DeferredTask(this._ctx, () => this._processMessages());
-    const unsubscribe = effect(() => {
+
+    const updateMessages = () => {
       if (this._lifecycleState !== LifecycleState.OPEN) {
         return;
       }
@@ -62,8 +61,14 @@ export class MessageNormalizer extends Resource {
       });
 
       this._normalizationTask!.schedule();
-    });
-    this._ctx.onDispose(() => unsubscribe());
+    };
+
+    // Initial update.
+    updateMessages();
+
+    // Subscribe to queue changes.
+    const unsubscribe = this._queue.subscribe(updateMessages);
+    this._ctx.onDispose(unsubscribe);
   }
 
   // Need to unpack strings from blocks from messages run them through the function and then pack them back into blocks into messages.
@@ -81,7 +86,7 @@ export class MessageNormalizer extends Resource {
 
     try {
       // TODO(mykola): Executor should support timeout.
-      const response = await asyncTimeout(
+      const response: NormalizationOutput = await asyncTimeout(
         this._functionExecutor.invoke(sentenceNormalization, { messages }),
         PROCESSING_TIMEOUT,
       );