@dxos/plugin-transcription 0.8.4-main.2e9d522 → 0.8.4-main.406dc2a

package/src/components/stories/TranscriptionStory.tsx

@@ -12,14 +12,15 @@ import { ScrollContainer } from '@dxos/react-ui-components';
 import { type DataType } from '@dxos/schema';
 
 import { type SerializationModel } from '../../model';
-import { Transcript } from '../Transcript';
+import { TranscriptView } from '../Transcript';
 
 export const TranscriptionStory: FC<{
   model: SerializationModel<DataType.Message>;
+  disabled?: boolean;
   running: boolean;
   onRunningChange: Dispatch<SetStateAction<boolean>>;
-  audioRef?: RefObject<HTMLAudioElement>;
-}> = ({ model, running, onRunningChange, audioRef }) => {
+  audioRef?: RefObject<HTMLAudioElement | null>;
+}> = ({ model, running, onRunningChange, audioRef, disabled }) => {
   const space = useSpace();
 
   return (
@@ -28,14 +29,17 @@ export const TranscriptionStory: FC<{
       <Toolbar.Root>
         <IconButton
           iconOnly
+          disabled={disabled}
           icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
           label={running ? 'Pause' : 'Play'}
           onClick={() => onRunningChange((running) => !running)}
         />
       </Toolbar.Root>
-      <ScrollContainer>
-        <Transcript space={space} model={model} attendableId='story' />
-      </ScrollContainer>
+      <ScrollContainer.Root pin>
+        <ScrollContainer.Content>
+          <TranscriptView space={space} model={model} />
+        </ScrollContainer.Content>
+      </ScrollContainer.Root>
     </div>
   );
 };

package/src/functions/index.ts

@@ -0,0 +1,6 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export { default as open } from './open';
+export { default as summarize } from './summarize';

package/src/functions/open.ts

@@ -0,0 +1,39 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+import * as Schema from 'effect/Schema';
+
+import { ArtifactId } from '@dxos/assistant';
+import { Obj } from '@dxos/echo';
+import { DatabaseService, QueueService, defineFunction } from '@dxos/functions';
+import { DataType } from '@dxos/schema';
+
+import { renderByline } from '../components';
+import { Transcript } from '../types';
+
+export default defineFunction({
+  key: 'dxos.org/function/transcription/open',
+  name: 'Open',
+  description: 'Opens and reads the contents of a transcription object.',
+  inputSchema: Schema.Struct({
+    id: ArtifactId.annotations({
+      description: 'The ID of the transcription object.',
+    }),
+  }),
+  outputSchema: Schema.Struct({
+    content: Schema.String,
+  }),
+  handler: Effect.fn(function* ({ data: { id } }) {
+    const transcript = yield* DatabaseService.resolve(ArtifactId.toDXN(id), Transcript.Transcript);
+    const { dxn } = yield* Effect.promise(() => transcript.queue.load());
+    const queue = yield* QueueService.getQueue(dxn);
+    yield* Effect.promise(() => queue?.queryObjects());
+    const content = queue?.objects
+      .filter((message) => Obj.instanceOf(DataType.Message, message))
+      .flatMap((message, index) => renderByline([])(message, index))
+      .join('\n\n');
+    return { content };
+  }),
+});

package/src/functions/summarize.ts

@@ -0,0 +1,103 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Array from 'effect/Array';
+import * as Effect from 'effect/Effect';
+import * as Function from 'effect/Function';
+import * as Layer from 'effect/Layer';
+import * as Option from 'effect/Option';
+import * as Schema from 'effect/Schema';
+
+import { AiService, ConsolePrinter, ToolExecutionService, ToolResolverService } from '@dxos/ai';
+import { AiSession, GenerationObserver } from '@dxos/assistant';
+import { TracingService, defineFunction } from '@dxos/functions';
+import { trim } from '@dxos/util';
+
+/**
+ * Summarize a transcript of a meeting.
+ */
+export default defineFunction({
+  key: 'dxos.org/function/transcription/summarize',
+  name: 'Summarize',
+  description: 'Summarize a transcript of a meeting.',
+  inputSchema: Schema.Struct({
+    transcript: Schema.String.annotations({
+      description: 'The transcript of the meeting.',
+    }),
+    notes: Schema.optional(Schema.String).annotations({
+      description: 'Additional notes from the participants.',
+    }),
+  }),
+  outputSchema: Schema.Struct({
+    summary: Schema.String.annotations({
+      description: 'The summary of the transcript.',
+    }),
+  }),
+  handler: Effect.fnUntraced(
+    function* ({ data: { transcript, notes } }) {
+      const result = yield* new AiSession().run({
+        prompt: `Transcript: ${transcript}\n\nNotes: ${notes}`,
+        history: [],
+        system: systemPrompt,
+        observer: GenerationObserver.fromPrinter(new ConsolePrinter({ tag: 'summarize' })),
+      });
+
+      const summary = Function.pipe(
+        result,
+        Array.findLast((msg) => msg.sender.role === 'assistant' && msg.blocks.some((block) => block._tag === 'text')),
+        Option.flatMap((msg) =>
+          Function.pipe(
+            msg.blocks,
+            Array.findLast((block) => block._tag === 'text'),
+            Option.map((block) => block.text),
+          ),
+        ),
+        Option.getOrThrowWith(() => new Error('No summary found')),
+      );
+
+      return { summary };
+    },
+    Effect.provide(
+      Layer.mergeAll(
+        AiService.model('@anthropic/claude-sonnet-4-0'),
+        ToolResolverService.layerEmpty,
+        ToolExecutionService.layerEmpty,
+        TracingService.layerNoop,
+      ),
+    ),
+  ),
+});
+
+const systemPrompt = trim`
+  You are a helpful assistant that summarizes transcripts of meetings.
+
+  # Goal
+  Create a markdown summary of the meeting transcript with text notes provided.
+  Notes are very important so make sure to include them in the summary if they contain meaningful information.
+
+  # Formatting
+  - Format the summary as a markdown document without extra comments like "Here is the summary of the transcript:".
+  - Use markdown formatting for headings and bullet points.
+  - Format the summary as a list of key points and takeaways.
+  - All names of people should be in bold.
+
+  # Note Taking
+  - Correlate items in the summary with the person of origin to build a coherent narrative.
+  - Include short quotes verbatim where appropriate. Especially when concerned with design decisions and problem descriptions.
+
+  # Tasks
+  At the end of the summary include tasks.
+  Extract only the tasks that are:
+  - Directly actionable.
+  - Clearly assigned to a person or team (or can easily be inferred).
+  - Strongly implied by the conversation and/or user note (no speculative tasks).
+  - Specific enough that someone reading them would know exactly what to do next.
+
+  Format all tasks as markdown checkboxes using the syntax:
+  - [ ] Task description.
+
+  Additional information can be included (indented).
+
+  If no actionable tasks are found, omit this tasks section.
+`;

package/src/hooks/useAudioFile.ts

@@ -20,67 +20,63 @@ export const useAudioFile = (audioUrl: string, constraints?: MediaTrackConstrain
     const ctx = new Context();
 
     scheduleTask(ctx, async () => {
-      const response = await fetch(audioUrl);
-      const blob = await response.blob();
-      const objectUrl = URL.createObjectURL(blob);
-      ctx.onDispose(() => {
-        URL.revokeObjectURL(objectUrl);
-      });
+      // TODO(wittjosiah): Fetch to external url fails in headless storybook test.
+      try {
+        const response = await fetch(audioUrl);
+        const blob = await response.blob();
+        const objectUrl = URL.createObjectURL(blob);
+        ctx.onDispose(() => {
+          URL.revokeObjectURL(objectUrl);
+        });
 
-      const audio = new Audio();
-      audio.src = objectUrl;
+        const audio = new Audio();
+        audio.src = objectUrl;
 
-      await new Promise<void>((resolve, reject) => {
-        audio.addEventListener(
-          'error',
-          (err) => {
-            log.error('error', { err });
-            reject(err);
-          },
-          { once: true },
-        );
-        audio.addEventListener(
-          'canplay',
-          async () => {
-            log.info('starting...');
-            try {
-              // Try to play the audio
-              await audio.play();
+        await new Promise<void>((resolve, reject) => {
+          audio.addEventListener(
+            'error',
+            (err) => {
+              log.error('error', { err });
+              reject(err);
+            },
+            { once: true },
+          );
+          audio.addEventListener(
+            'canplay',
+            async () => {
               resolve();
-            } catch (playError) {
-              log.error('Play failed', { playError });
-              // Still resolve as the audio is ready, even if autoplay failed.
-              resolve();
-            }
-          },
-          { once: true },
-        );
-        audio.load();
-      });
+            },
+            { once: true },
+          );
+          audio.load();
+        });
 
-      const audioCtx = new AudioContext();
+        const audioCtx = new AudioContext();
 
-      // Resume AudioContext if it's suspended.
-      if (audioCtx.state === 'suspended') {
-        await audioCtx.resume();
-      }
+        // Resume AudioContext if it's suspended.
+        if (audioCtx.state === 'suspended') {
+          await audioCtx.resume();
+        }
 
-      const destination = audioCtx.createMediaStreamDestination();
-      destination.channelCount = 1;
+        const destination = audioCtx.createMediaStreamDestination();
+        destination.channelCount = 1;
 
-      const source = audioCtx.createMediaElementSource(audio);
-      source.connect(destination);
-      const track = destination.stream.getAudioTracks()[0];
-      if (constraints) {
-        await track.applyConstraints(constraints).catch((err) => log.catch(err));
-      }
+        const source = audioCtx.createMediaElementSource(audio);
+        source.connect(destination);
+        const track = destination.stream.getAudioTracks()[0];
+        if (constraints) {
+          await track.applyConstraints(constraints).catch((err) => log.catch(err));
+        }
 
-      // Also connect to speakers so audio is audible.
-      setStream({
-        audio,
-        stream: destination.stream,
-        track: destination.stream.getAudioTracks()[0],
-      });
+        // Also connect to speakers so audio is audible.
+        setStream({
+          audio,
+          stream: destination.stream,
+          track: destination.stream.getAudioTracks()[0],
+        });
+      } catch (err) {
+        log.catch(err);
+      }
     });
 
     return () => {

package/src/hooks/useVoiceInput.ts

@@ -2,16 +2,17 @@
 // Copyright 2025 DXOS.org
 //
 
-import { useState, useEffect, useCallback } from 'react';
+import { useCallback, useEffect, useState } from 'react';
 
 import { scheduleMicroTask } from '@dxos/async';
 import { Context } from '@dxos/context';
 import { log } from '@dxos/log';
 import { useSoundEffect } from '@dxos/react-ui-sfx';
 
+import { type TranscriberParams } from '../transcriber';
+
 import { useAudioTrack } from './useAudioTrack';
 import { useTranscriber } from './useTranscriber';
-import { type TranscriberParams } from '../transcriber';
 
 export type UseVoiceInputProps = {
   active?: boolean;

package/src/meta.ts

@@ -4,10 +4,8 @@
 
 import { type PluginMeta } from '@dxos/app-framework';
 
-export const TRANSCRIPTION_PLUGIN = 'dxos.org/plugin/transcription';
-
 export const meta: PluginMeta = {
-  id: TRANSCRIPTION_PLUGIN,
+  id: 'dxos.org/plugin/transcription',
   name: 'Transcription',
   description: 'Voice transcription.',
   icon: 'ph--microphone--regular',

package/src/model/model.test.ts

@@ -9,7 +9,7 @@ import { Obj } from '@dxos/echo';
 import type { ObjectId } from '@dxos/keys';
 import { DataType } from '@dxos/schema';
 
-import { SerializationModel, DocumentAdapter, type ChunkRenderer } from './model';
+import { type ChunkRenderer, DocumentAdapter, SerializationModel } from './model';
 
 const blockToMarkdown: ChunkRenderer<DataType.Message> = (
   message: DataType.Message,

package/src/segments-normalization/message-normalizer.ts

@@ -8,7 +8,7 @@
 
 import { effect } from '@preact/signals-core';
 
-import { asyncTimeout, DeferredTask } from '@dxos/async';
+import { DeferredTask, asyncTimeout } from '@dxos/async';
 import { LifecycleState, Resource } from '@dxos/context';
 import { type Queue } from '@dxos/echo-db';
 import { type FunctionExecutor } from '@dxos/functions';

package/src/segments-normalization/normalization.test.ts

@@ -8,8 +8,8 @@ import { describe, test } from 'vitest';
 import { scheduleTaskInterval } from '@dxos/async';
 import { Context } from '@dxos/context';
 import { Obj } from '@dxos/echo';
+import { createQueueDXN } from '@dxos/echo/internal';
 import { MemoryQueue } from '@dxos/echo-db';
-import { createQueueDXN } from '@dxos/echo-schema';
 import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
 import { log } from '@dxos/log';
 import { DataType } from '@dxos/schema';
@@ -49,14 +49,13 @@ const messages: MessageWithRangeId[] = [
   'in classical physics objects have well-defined properties such as position speed and momentum',
 ].map((string, index) =>
   Obj.make(DataType.Message, {
-    created: new Date(Date.now() + 1000 * index).toISOString(),
+    created: new Date(Date.now() + 1_000 * index).toISOString(),
     sender,
-    blocks: [{ _tag: 'transcript', started: new Date(Date.now() + 1000 * index).toISOString(), text: string }],
+    blocks: [{ _tag: 'transcript', started: new Date(Date.now() + 1_000 * index).toISOString(), text: string }],
   }),
 );
 
-// eslint-disable-next-line no-unused-vars
-const _REMOTE_AI = true;
+// const REMOTE_AI = true;
 
 describe.skip('SentenceNormalization', () => {
   const getExecutor = () => {
@@ -98,6 +97,7 @@ describe.skip('SentenceNormalization', () => {
         buffer = sentences.slice(activeSentenceIndex);
       }
     }
+
     log.info('sentences', {
       originalMessages: JSON.stringify(messages, null, 2),
       sentences: JSON.stringify(sentences, null, 2),

package/src/segments-normalization/normalization.ts

@@ -2,18 +2,19 @@
 // Copyright 2025 DXOS.org
 //
 
-// TODO(burdon): Needs refactoring.
+// ISSUE(burdon): defineFunction
 // @ts-nocheck
 
-import { Schema } from 'effect';
+import * as Schema from 'effect/Schema';
 
-import { DEFAULT_EDGE_MODEL, AiService } from '@dxos/ai';
+import { AiService, DEFAULT_EDGE_MODEL } from '@dxos/ai';
 import { AiSession } from '@dxos/assistant';
 import { Obj } from '@dxos/echo';
 import { defineFunction } from '@dxos/functions';
 import { ObjectId } from '@dxos/keys';
 import { log } from '@dxos/log';
 import { DataType } from '@dxos/schema';
+import { trim } from '@dxos/util';
 
 const MessageWithRangeId = Schema.extend(
   DataType.Message,
@@ -42,40 +43,6 @@ export const NormalizationOutput = Schema.Struct({
 });
 export interface NormalizationOutput extends Schema.Schema.Type<typeof NormalizationOutput> {}
 
-const prompt = `
-You are observing a real-time transcript of a single person speaking.
-The transcription is delivered in chunks of 10 seconds or less. As a result, individual sentences may be split across multiple messages, or multiple sentences may appear within a single message. Additionally, because this is real-time transcription, punctuation and capitalization may be incorrect or missing.
-
-# Task Description:
-  - Your task is to detect and reconstruct broken or incomplete sentences by merging segments into coherent, grammatically correct sentences where appropriate.
-
-# Input Format:
-  - The input is an array of messages that contain transcription blocks.
-  - Each message is a DataType.Message, as described in the output format.
-
-# Output Format:
-  - You have been provided with the tool that defines the output format; make sure to query it.
-  - Do not output anything other than the expected format.
-
-# Segment Handling Rules:
-  - Sort messages by timestamp.
-  - Leave ID of the first message in a sentence.
-  - Each divided sentence should be added to the output as a separate message or block within existing messages, preserving the original order.
-  - If one message contains multiple sentences, you should not split them.
-  - The last sentence may be incomplete; it should still be output as a separate block or message.
-  - Keep track of the original timestamps of the messages and blocks and, use the timestamp of the first message/block of a sentence as the 'started' field of the merged message.
-  - The 'rangeId' field of the merged message should include 'messageId'-s and 'rangeId'-s of all messages that have been used to construct this message.
-
-# Punctuation and Capitalization:
-  - Do not rely on the original punctuation and capitalization—they may be incorrect.
-  - Use logical reasoning to apply appropriate punctuation (e.g., period, comma, question mark, exclamation mark) and capitalization.
-
-# Restrictions:
-  - Do not alter the order of the original messages; maintain the natural flow of speech.
-  - Do not interpret or infer meaning beyond reconstructing sentences.
-  - Do not add or remove any words or phrases.
-`;
-
 export const sentenceNormalization = defineFunction<NormalizationInput, NormalizationOutput>({
   description: 'Post process of transcription for sentence normalization',
   inputSchema: NormalizationInput,
@@ -112,3 +79,37 @@ export const sentenceNormalization = defineFunction<NormalizationInput, Normaliz
     return response;
   },
 });
+
+const prompt = trim`
+  You are observing a real-time transcript of a single person speaking.
+  The transcription is delivered in chunks of 10 seconds or less. As a result, individual sentences may be split across multiple messages, or multiple sentences may appear within a single message. Additionally, because this is real-time transcription, punctuation and capitalization may be incorrect or missing.
+
+  # Task Description:
+    - Your task is to detect and reconstruct broken or incomplete sentences by merging segments into coherent, grammatically correct sentences where appropriate.
+
+  # Input Format:
+    - The input is an array of messages that contain transcription blocks.
+    - Each message is a DataType.Message, as described in the output format.
+
+  # Output Format:
+    - You have been provided with the tool that defines the output format; make sure to query it.
+    - Do not output anything other than the expected format.
+
+  # Segment Handling Rules:
+    - Sort messages by timestamp.
+    - Leave ID of the first message in a sentence.
+    - Each divided sentence should be added to the output as a separate message or block within existing messages, preserving the original order.
+    - If one message contains multiple sentences, you should not split them.
+    - The last sentence may be incomplete; it should still be output as a separate block or message.
+    - Keep track of the original timestamps of the messages and blocks and, use the timestamp of the first message/block of a sentence as the 'started' field of the merged message.
+    - The 'rangeId' field of the merged message should include 'messageId'-s and 'rangeId'-s of all messages that have been used to construct this message.
+
+  # Punctuation and Capitalization:
+    - Do not rely on the original punctuation and capitalization—they may be incorrect.
+    - Use logical reasoning to apply appropriate punctuation (e.g., period, comma, question mark, exclamation mark) and capitalization.
+
+  # Restrictions:
+    - Do not alter the order of the original messages; maintain the natural flow of speech.
+    - Do not interpret or infer meaning beyond reconstructing sentences.
+    - Do not add or remove any words or phrases.
+`;

package/src/testing/testing.ts

@@ -2,20 +2,20 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Schema } from 'effect';
+import * as Schema from 'effect/Schema';
 import { useEffect, useMemo, useState } from 'react';
 
-import { extractionAnthropicFn, processTranscriptMessage } from '@dxos/assistant';
+import { extractionAnthropicFunction, processTranscriptMessage } from '@dxos/assistant/extraction';
 import { scheduleTaskInterval } from '@dxos/async';
 import { Filter, type Queue } from '@dxos/client/echo';
 import { Context } from '@dxos/context';
-import { Obj, Ref, Type, type Key } from '@dxos/echo';
-import { createQueueDXN } from '@dxos/echo-schema';
+import { type Key, Obj, Ref, Type } from '@dxos/echo';
+import { createQueueDXN } from '@dxos/echo/internal';
 import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
 import { IdentityDid } from '@dxos/keys';
 import { log } from '@dxos/log';
 import { faker } from '@dxos/random';
-import { useQueue, type Space } from '@dxos/react-client/echo';
+import { type Space, useQueue } from '@dxos/react-client/echo';
 import { DataType } from '@dxos/schema';
 import { Testing, seedTestData } from '@dxos/schema/testing';
 
@@ -74,7 +74,7 @@ export class MessageBuilder extends AbstractMessageBuilder {
       const obj = this._space.db.add(Obj.make(TestItem, { title: label, description: faker.lorem.paragraph() }));
       const dxn = Ref.make(obj).dxn.toString();
       const words = text.split(' ');
-      words.splice(Math.floor(Math.random() * words.length), 0, `[${label}][${dxn}]`);
+      words.splice(Math.floor(Math.random() * words.length), 0, `[${label}](${dxn})`);
       text = words.join(' ');
     }
 
@@ -130,7 +130,7 @@ class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
     const { message: enhancedMessage } = await processTranscriptMessage({
       input: { message },
       executor: this.executor,
-      function: extractionAnthropicFn,
+      function: extractionAnthropicFunction,
     });
 
     return enhancedMessage;

package/src/transcriber/media-stream-recorder.ts

@@ -10,7 +10,7 @@ import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 import { trace } from '@dxos/tracing';
 
-import { type WavConfig, type AudioChunk, type AudioRecorder } from './audio-recorder';
+import { type AudioChunk, type AudioRecorder, type WavConfig } from './audio-recorder';
 
 let initializingPromise: Promise<void> | undefined;
 

package/src/transcriber/transcriber.browser.test.ts

@@ -4,6 +4,7 @@
 
 import fs from 'node:fs';
 import path from 'node:path';
+
 import { describe, expect, test } from 'vitest';
 import { WaveFile } from 'wavefile';
 
@@ -11,7 +12,7 @@ import { Trigger } from '@dxos/async';
 import { log } from '@dxos/log';
 import { type DataType } from '@dxos/schema';
 import { openAndClose } from '@dxos/test-utils';
-import { trace, TRACE_PROCESSOR } from '@dxos/tracing';
+import { TRACE_PROCESSOR, trace } from '@dxos/tracing';
 
 import { type AudioChunk, type AudioRecorder, Transcriber } from '../transcriber';
 import { mergeFloat64Arrays } from '../util';

package/src/transcriber/transcriber.ts

@@ -10,10 +10,11 @@ import { log } from '@dxos/log';
 import { type DataType } from '@dxos/schema';
 import { trace } from '@dxos/tracing';
 
-import { type AudioRecorder, type AudioChunk } from './audio-recorder';
 import { TRANSCRIPTION_URL } from '../types';
 import { mergeFloat64Arrays } from '../util';
 
+import { type AudioChunk, type AudioRecorder } from './audio-recorder';
+
 type WhisperWord = {
   word: string;