@dxos/plugin-transcription 0.8.4-main.28f8d3d → 0.8.4-main.406dc2a

package/src/functions/index.ts

@@ -0,0 +1,6 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export { default as open } from './open';
+export { default as summarize } from './summarize';

package/src/functions/open.ts

@@ -0,0 +1,39 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+import * as Schema from 'effect/Schema';
+
+import { ArtifactId } from '@dxos/assistant';
+import { Obj } from '@dxos/echo';
+import { DatabaseService, QueueService, defineFunction } from '@dxos/functions';
+import { DataType } from '@dxos/schema';
+
+import { renderByline } from '../components';
+import { Transcript } from '../types';
+
+export default defineFunction({
+  key: 'dxos.org/function/transcription/open',
+  name: 'Open',
+  description: 'Opens and reads the contents of a transcription object.',
+  inputSchema: Schema.Struct({
+    id: ArtifactId.annotations({
+      description: 'The ID of the transcription object.',
+    }),
+  }),
+  outputSchema: Schema.Struct({
+    content: Schema.String,
+  }),
+  handler: Effect.fn(function* ({ data: { id } }) {
+    const transcript = yield* DatabaseService.resolve(ArtifactId.toDXN(id), Transcript.Transcript);
+    const { dxn } = yield* Effect.promise(() => transcript.queue.load());
+    const queue = yield* QueueService.getQueue(dxn);
+    yield* Effect.promise(() => queue?.queryObjects());
+    const content = queue?.objects
+      .filter((message) => Obj.instanceOf(DataType.Message, message))
+      .flatMap((message, index) => renderByline([])(message, index))
+      .join('\n\n');
+    return { content };
+  }),
+});

package/src/functions/summarize.ts

@@ -0,0 +1,103 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Array from 'effect/Array';
+import * as Effect from 'effect/Effect';
+import * as Function from 'effect/Function';
+import * as Layer from 'effect/Layer';
+import * as Option from 'effect/Option';
+import * as Schema from 'effect/Schema';
+
+import { AiService, ConsolePrinter, ToolExecutionService, ToolResolverService } from '@dxos/ai';
+import { AiSession, GenerationObserver } from '@dxos/assistant';
+import { TracingService, defineFunction } from '@dxos/functions';
+import { trim } from '@dxos/util';
+
+/**
+ * Summarize a transcript of a meeting.
+ */
+export default defineFunction({
+  key: 'dxos.org/function/transcription/summarize',
+  name: 'Summarize',
+  description: 'Summarize a transcript of a meeting.',
+  inputSchema: Schema.Struct({
+    transcript: Schema.String.annotations({
+      description: 'The transcript of the meeting.',
+    }),
+    notes: Schema.optional(Schema.String).annotations({
+      description: 'Additional notes from the participants.',
+    }),
+  }),
+  outputSchema: Schema.Struct({
+    summary: Schema.String.annotations({
+      description: 'The summary of the transcript.',
+    }),
+  }),
+  handler: Effect.fnUntraced(
+    function* ({ data: { transcript, notes } }) {
+      const result = yield* new AiSession().run({
+        prompt: `Transcript: ${transcript}\n\nNotes: ${notes}`,
+        history: [],
+        system: systemPrompt,
+        observer: GenerationObserver.fromPrinter(new ConsolePrinter({ tag: 'summarize' })),
+      });
+
+      const summary = Function.pipe(
+        result,
+        Array.findLast((msg) => msg.sender.role === 'assistant' && msg.blocks.some((block) => block._tag === 'text')),
+        Option.flatMap((msg) =>
+          Function.pipe(
+            msg.blocks,
+            Array.findLast((block) => block._tag === 'text'),
+            Option.map((block) => block.text),
+          ),
+        ),
+        Option.getOrThrowWith(() => new Error('No summary found')),
+      );
+
+      return { summary };
+    },
+    Effect.provide(
+      Layer.mergeAll(
+        AiService.model('@anthropic/claude-sonnet-4-0'),
+        ToolResolverService.layerEmpty,
+        ToolExecutionService.layerEmpty,
+        TracingService.layerNoop,
+      ),
+    ),
+  ),
+});
+
+const systemPrompt = trim`
+  You are a helpful assistant that summarizes transcripts of meetings.
+
+  # Goal
+  Create a markdown summary of the meeting transcript with text notes provided.
+  Notes are very important so make sure to include them in the summary if they contain meaningful information.
+
+  # Formatting
+  - Format the summary as a markdown document without extra comments like "Here is the summary of the transcript:".
+  - Use markdown formatting for headings and bullet points.
+  - Format the summary as a list of key points and takeaways.
+  - All names of people should be in bold.
+
+  # Note Taking
+  - Correlate items in the summary with the person of origin to build a coherent narrative.
+  - Include short quotes verbatim where appropriate. Especially when concerned with design decisions and problem descriptions.
+
+  # Tasks
+  At the end of the summary include tasks.
+  Extract only the tasks that are:
+  - Directly actionable.
+  - Clearly assigned to a person or team (or can easily be inferred).
+  - Strongly implied by the conversation and/or user note (no speculative tasks).
+  - Specific enough that someone reading them would know exactly what to do next.
+
+  Format all tasks as markdown checkboxes using the syntax:
+  - [ ] Task description.
+
+  Additional information can be included (indented).
+
+  If no actionable tasks are found, omit this tasks section.
+`;

package/src/hooks/useAudioFile.ts

@@ -20,67 +20,63 @@ export const useAudioFile = (audioUrl: string, constraints?: MediaTrackConstrain
     const ctx = new Context();
 
     scheduleTask(ctx, async () => {
-      const response = await fetch(audioUrl);
-      const blob = await response.blob();
-      const objectUrl = URL.createObjectURL(blob);
-      ctx.onDispose(() => {
-        URL.revokeObjectURL(objectUrl);
-      });
+      // TODO(wittjosiah): Fetch to external url fails in headless storybook test.
+      try {
+        const response = await fetch(audioUrl);
+        const blob = await response.blob();
+        const objectUrl = URL.createObjectURL(blob);
+        ctx.onDispose(() => {
+          URL.revokeObjectURL(objectUrl);
+        });
 
-      const audio = new Audio();
-      audio.src = objectUrl;
+        const audio = new Audio();
+        audio.src = objectUrl;
 
-      await new Promise<void>((resolve, reject) => {
-        audio.addEventListener(
-          'error',
-          (err) => {
-            log.error('error', { err });
-            reject(err);
-          },
-          { once: true },
-        );
-        audio.addEventListener(
-          'canplay',
-          async () => {
-            log.info('starting...');
-            try {
-              // Try to play the audio
-              await audio.play();
+        await new Promise<void>((resolve, reject) => {
+          audio.addEventListener(
+            'error',
+            (err) => {
+              log.error('error', { err });
+              reject(err);
+            },
+            { once: true },
+          );
+          audio.addEventListener(
+            'canplay',
+            async () => {
               resolve();
-            } catch (playError) {
-              log.error('Play failed', { playError });
-              // Still resolve as the audio is ready, even if autoplay failed.
-              resolve();
-            }
-          },
-          { once: true },
-        );
-        audio.load();
-      });
+            },
+            { once: true },
+          );
+          audio.load();
+        });
 
-      const audioCtx = new AudioContext();
+        const audioCtx = new AudioContext();
 
-      // Resume AudioContext if it's suspended.
-      if (audioCtx.state === 'suspended') {
-        await audioCtx.resume();
-      }
+        // Resume AudioContext if it's suspended.
+        if (audioCtx.state === 'suspended') {
+          await audioCtx.resume();
+        }
 
-      const destination = audioCtx.createMediaStreamDestination();
-      destination.channelCount = 1;
+        const destination = audioCtx.createMediaStreamDestination();
+        destination.channelCount = 1;
 
-      const source = audioCtx.createMediaElementSource(audio);
-      source.connect(destination);
-      const track = destination.stream.getAudioTracks()[0];
-      if (constraints) {
-        await track.applyConstraints(constraints).catch((err) => log.catch(err));
-      }
+        const source = audioCtx.createMediaElementSource(audio);
+        source.connect(destination);
+        const track = destination.stream.getAudioTracks()[0];
+        if (constraints) {
+          await track.applyConstraints(constraints).catch((err) => log.catch(err));
+        }
 
-      // Also connect to speakers so audio is audible.
-      setStream({
-        audio,
-        stream: destination.stream,
-        track: destination.stream.getAudioTracks()[0],
-      });
+        // Also connect to speakers so audio is audible.
+        setStream({
+          audio,
+          stream: destination.stream,
+          track: destination.stream.getAudioTracks()[0],
+        });
+      } catch (err) {
+        log.catch(err);
+      }
     });
 
     return () => {

package/src/meta.ts

@@ -4,10 +4,8 @@
 
 import { type PluginMeta } from '@dxos/app-framework';
 
-export const TRANSCRIPTION_PLUGIN = 'dxos.org/plugin/transcription';
-
 export const meta: PluginMeta = {
-  id: TRANSCRIPTION_PLUGIN,
+  id: 'dxos.org/plugin/transcription',
   name: 'Transcription',
   description: 'Voice transcription.',
   icon: 'ph--microphone--regular',

package/src/segments-normalization/normalization.test.ts

@@ -8,8 +8,8 @@ import { describe, test } from 'vitest';
 import { scheduleTaskInterval } from '@dxos/async';
 import { Context } from '@dxos/context';
 import { Obj } from '@dxos/echo';
+import { createQueueDXN } from '@dxos/echo/internal';
 import { MemoryQueue } from '@dxos/echo-db';
-import { createQueueDXN } from '@dxos/echo-schema';
 import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
 import { log } from '@dxos/log';
 import { DataType } from '@dxos/schema';

package/src/segments-normalization/normalization.ts

@@ -5,7 +5,7 @@
 // ISSUE(burdon): defineFunction
 // @ts-nocheck
 
-import { Schema } from 'effect';
+import * as Schema from 'effect/Schema';
 
 import { AiService, DEFAULT_EDGE_MODEL } from '@dxos/ai';
 import { AiSession } from '@dxos/assistant';

package/src/testing/testing.ts

@@ -2,15 +2,15 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Schema } from 'effect';
+import * as Schema from 'effect/Schema';
 import { useEffect, useMemo, useState } from 'react';
 
-import { extractionAnthropicFn, processTranscriptMessage } from '@dxos/assistant/extraction';
+import { extractionAnthropicFunction, processTranscriptMessage } from '@dxos/assistant/extraction';
 import { scheduleTaskInterval } from '@dxos/async';
 import { Filter, type Queue } from '@dxos/client/echo';
 import { Context } from '@dxos/context';
 import { type Key, Obj, Ref, Type } from '@dxos/echo';
-import { createQueueDXN } from '@dxos/echo-schema';
+import { createQueueDXN } from '@dxos/echo/internal';
 import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
 import { IdentityDid } from '@dxos/keys';
 import { log } from '@dxos/log';
@@ -74,7 +74,7 @@ export class MessageBuilder extends AbstractMessageBuilder {
       const obj = this._space.db.add(Obj.make(TestItem, { title: label, description: faker.lorem.paragraph() }));
       const dxn = Ref.make(obj).dxn.toString();
       const words = text.split(' ');
-      words.splice(Math.floor(Math.random() * words.length), 0, `[${label}][${dxn}]`);
+      words.splice(Math.floor(Math.random() * words.length), 0, `[${label}](${dxn})`);
       text = words.join(' ');
     }
 
@@ -130,7 +130,7 @@ class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
     const { message: enhancedMessage } = await processTranscriptMessage({
       input: { message },
       executor: this.executor,
-      function: extractionAnthropicFn,
+      function: extractionAnthropicFunction,
     });
 
     return enhancedMessage;

package/src/transcriber/transcription-manager.ts

@@ -90,19 +90,17 @@ export class TranscriptionManager extends Resource {
     return this;
   }
 
-  /**
-   * Enable or disable transcription.
-   * @param enabled - Whether to enable transcription.
-   */
-  @synchronized
-  async setEnabled(enabled?: boolean): Promise<void> {
+  async setEnabled(enabled: boolean): Promise<void> {
     if (this._enabled.value === enabled) {
       return;
     }
 
     this._enabled.value = enabled ?? false;
-    // TODO(burdon): Why is toggle called here?
-    this.isOpen && (await this._toggleTranscriber());
+    if (enabled) {
+      await this._maybeRestartTranscriber();
+    } else {
+      await this._stopTranscriber();
+    }
   }
 
   @synchronized
@@ -112,44 +110,19 @@ export class TranscriptionManager extends Resource {
     }
 
     this._audioStreamTrack = track;
-    this.isOpen && (await this._toggleTranscriber());
+    await this._maybeRestartTranscriber();
   }
 
   protected override async _open(): Promise<void> {
-    await this._toggleTranscriber();
+    await this._maybeRestartTranscriber();
   }
 
   protected override async _close(): Promise<void> {
-    void this._transcriber?.close();
-  }
-
-  // TODO(burdon): Change this to setEnables (explicit), not toggle.
-  private async _toggleTranscriber(): Promise<void> {
-    await this._maybeReinitTranscriber();
-
-    // Open or close transcriber if transcription is enabled or disabled.
-    if (this._enabled.value) {
-      await this._transcriber?.open();
-      // TODO(burdon): Started and stopped blocks appear twice.
-      const block = Obj.make(DataType.Message, {
-        created: new Date().toISOString(),
-        blocks: [{ _tag: 'transcript', text: 'Started', started: new Date().toISOString() }],
-        sender: { role: 'assistant' },
-      });
-      await this._queue?.append([block]);
-    } else {
-      await this._transcriber?.close();
-      const block = Obj.make(DataType.Message, {
-        created: new Date().toISOString(),
-        blocks: [{ _tag: 'transcript', text: 'Stopped', started: new Date().toISOString() }],
-        sender: { role: 'assistant' },
-      });
-      await this._queue?.append([block]);
-    }
+    await this._stopTranscriber();
   }
 
-  private async _maybeReinitTranscriber(): Promise<void> {
-    if (!this._audioStreamTrack || !this._enabled.value) {
+  private async _maybeRestartTranscriber(): Promise<void> {
+    if (!this._audioStreamTrack || !this._enabled.value || !this.isOpen) {
       return;
     }
 
@@ -172,9 +145,15 @@ export class TranscriptionManager extends Resource {
         recorder: this._mediaRecorder,
         onSegments: (segments) => this._onSegments(segments),
       });
+
+      await this._transcriber?.open();
     }
   }
 
+  private async _stopTranscriber(): Promise<void> {
+    await this._transcriber?.close();
+  }
+
   private async _onSegments(segments: DataType.MessageBlock.Transcript[]): Promise<void> {
     if (!this.isOpen || !this._queue) {
       return;

package/src/types/{schema.ts → Transcript.ts}

@@ -2,16 +2,15 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Schema } from 'effect';
+import * as Schema from 'effect/Schema';
 
 import { Queue } from '@dxos/client/echo';
-import { Type } from '@dxos/echo';
-import { TypedObject } from '@dxos/echo-schema';
+import { type DXN, Obj, Ref, Type } from '@dxos/echo';
 
 /**
  * Root transcript object created when the user starts a transcription.
  */
-export const TranscriptSchema = Schema.Struct({
+export const Transcript = Schema.Struct({
   // TODO(wittjosiah): Remove?
   // TODO(burdon): Use Date or string?
   started: Schema.optional(Schema.String),
@@ -21,12 +20,14 @@ export const TranscriptSchema = Schema.Struct({
    * Queue containing TranscriptBlock objects.
    */
   queue: Type.Ref(Queue),
-});
+}).pipe(
+  Type.Obj({
+    typename: 'dxos.org/type/Transcript',
+    version: '0.1.0',
+  }),
+);
 
-export class TranscriptType extends TypedObject({
-  typename: 'dxos.org/type/Transcript',
-  version: '0.1.0',
-})(TranscriptSchema.fields) {}
+export type Transcript = Schema.Schema.Type<typeof Transcript>;
 
 // TODO(burdon): Do these need to be kept in sync with EDGE?
 
@@ -39,3 +40,5 @@ const TranscriptHeader = Schema.Struct({
 });
 
 export type TranscriptHeader = Schema.Schema.Type<typeof TranscriptHeader>;
+
+export const makeTranscript = (queueDxn: DXN) => Obj.make(Transcript, { queue: Ref.fromDXN(queueDxn) });

package/src/types/TranscriptAction.ts

@@ -0,0 +1,24 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+import { SpaceSchema } from '@dxos/client/echo';
+
+import { meta } from '../meta';
+
+import { Transcript } from './Transcript';
+
+/**
+ * Endpoint to the calls service.
+ */
+export class Create extends Schema.TaggedClass<Create>()(`${meta.id}/action/create`, {
+  input: Schema.Struct({
+    name: Schema.optional(Schema.String),
+    space: SpaceSchema,
+  }),
+  output: Schema.Struct({
+    object: Transcript,
+  }),
+}) {}

package/src/types/index.ts

@@ -2,5 +2,6 @@
 // Copyright 2025 DXOS.org
 //
 
-export * from './schema';
+export * as TranscriptAction from './TranscriptAction';
+export * as Transcript from './Transcript';
 export * from './types';

package/src/types/types.ts

@@ -2,32 +2,8 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Schema } from 'effect';
-
-import { SpaceId } from '@dxos/react-client/echo';
-
-import { TRANSCRIPTION_PLUGIN } from '../meta';
-
-import { TranscriptType } from './schema';
-
-// TODO(burdon): Move to separate proto.
-
 /**
  * Endpoint to the calls service.
  */
 // TODO(burdon): Reconfigure/move.
 export const TRANSCRIPTION_URL = 'https://calls-service.dxos.workers.dev';
-
-export namespace TranscriptionAction {
-  const TRANSCRIPTION_ACTION = `${TRANSCRIPTION_PLUGIN}/action`;
-
-  export class Create extends Schema.TaggedClass<Create>()(`${TRANSCRIPTION_ACTION}/create`, {
-    input: Schema.Struct({
-      name: Schema.optional(Schema.String),
-      spaceId: SpaceId,
-    }),
-    output: Schema.Struct({
-      object: TranscriptType,
-    }),
-  }) {}
-}

package/dist/lib/browser/TranscriptContainer-FH7VAWYA.mjs

@@ -1,11 +0,0 @@
-import {
-  TranscriptContainer_default,
-  TranscriptionContainer
-} from "./chunk-BJWYA2BU.mjs";
-import "./chunk-SREIAQSP.mjs";
-import "./chunk-5WH45KBY.mjs";
-export {
-  TranscriptionContainer,
-  TranscriptContainer_default as default
-};
-//# sourceMappingURL=TranscriptContainer-FH7VAWYA.mjs.map

package/dist/lib/browser/chunk-5WH45KBY.mjs.map

@@ -1,7 +0,0 @@
-{
-  "version": 3,
-  "sources": ["../../../src/meta.ts"],
-  "sourcesContent": ["//\n// Copyright 2023 DXOS.org\n//\n\nimport { type PluginMeta } from '@dxos/app-framework';\n\nexport const TRANSCRIPTION_PLUGIN = 'dxos.org/plugin/transcription';\n\nexport const meta: PluginMeta = {\n  id: TRANSCRIPTION_PLUGIN,\n  name: 'Transcription',\n  description: 'Voice transcription.',\n  icon: 'ph--microphone--regular',\n  source: 'https://github.com/dxos/dxos/tree/main/packages/plugins/plugin-transcription',\n  tags: ['labs'],\n  screenshots: [],\n};\n"],
-  "mappings": ";AAMO,IAAMA,uBAAuB;AAE7B,IAAMC,OAAmB;EAC9BC,IAAIF;EACJG,MAAM;EACNC,aAAa;EACbC,MAAM;EACNC,QAAQ;EACRC,MAAM;IAAC;;EACPC,aAAa,CAAA;AACf;",
-  "names": ["TRANSCRIPTION_PLUGIN", "meta", "id", "name", "description", "icon", "source", "tags", "screenshots"]
-}