@dxos/plugin-transcription 0.8.3 → 0.8.4-main.03d5cd7b56

package/src/transcriber/transcriber.test.ts

@@ -0,0 +1,104 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { describe, test } from 'vitest';
+
+import { type AudioChunk } from './audio-recorder';
+import { alignWhisperSegments, type WhisperSegment } from './transcriber';
+
+const T0 = Date.parse('2026-01-01T00:00:00.000Z');
+
+const chunk = (offsetMs = 0, samples = 16): AudioChunk => ({
+  timestamp: T0 + offsetMs,
+  data: new Float64Array(samples),
+});
+
+const segment = (overrides: Partial<WhisperSegment>): WhisperSegment => ({
+  text: '',
+  start: 0,
+  end: 0,
+  no_speech_prob: 0,
+  words: [],
+  ...overrides,
+});
+
+describe('alignWhisperSegments', () => {
+  test('returns empty when no chunks were transcribed', ({ expect }) => {
+    const result = alignWhisperSegments(
+      [segment({ text: 'x', end: 1, words: [{ word: 'x', start: 0, end: 1 }] })],
+      [],
+      0,
+    );
+    expect(result.transcripts).toEqual([]);
+    expect(result.lastUsedTimestamp).toBeUndefined();
+  });
+
+  test('absolute-timestamps and deduplicates segments against lastUsedTimestamp', ({ expect }) => {
+    const segments: WhisperSegment[] = [
+      segment({
+        text: 'hello world',
+        start: 0,
+        end: 1,
+        words: [
+          { word: 'hello ', start: 0, end: 0.5 },
+          { word: 'world', start: 0.5, end: 1 },
+        ],
+      }),
+    ];
+
+    const first = alignWhisperSegments(segments, [chunk(0)], 0);
+    expect(first.transcripts).toHaveLength(1);
+    expect(first.transcripts[0]._tag).toBe('transcript');
+    expect(first.transcripts[0].text).toBe('hello world');
+    expect(new Date(first.transcripts[0].started).getTime()).toBe(T0);
+    expect(first.lastUsedTimestamp).toBe(T0 + 1_000);
+
+    // Same segments, same chunk start: every word's absolute start is < lastUsedTimestamp -> filtered out.
+    const second = alignWhisperSegments(segments, [chunk(0)], first.lastUsedTimestamp!);
+    expect(second.transcripts).toEqual([]);
+    expect(second.lastUsedTimestamp).toBeUndefined();
+  });
+
+  test('keeps later words when earlier words fall before lastUsedTimestamp', ({ expect }) => {
+    const segments: WhisperSegment[] = [
+      segment({
+        text: 'old new',
+        start: 0,
+        end: 2,
+        words: [
+          { word: 'old ', start: 0, end: 1 },
+          { word: 'new', start: 1.5, end: 2 },
+        ],
+      }),
+    ];
+
+    // First word starts at T0, second at T0+1500 — only the second survives a cursor at T0+1000.
+    const result = alignWhisperSegments(segments, [chunk(0)], T0 + 1_000);
+    expect(result.transcripts).toHaveLength(1);
+    expect(result.transcripts[0].text).toBe('new');
+    expect(new Date(result.transcripts[0].started).getTime()).toBe(T0 + 1_500);
+    expect(result.lastUsedTimestamp).toBe(T0 + 2_000);
+  });
+
+  test('drops segments that end before lastUsedTimestamp', ({ expect }) => {
+    const segments: WhisperSegment[] = [
+      segment({
+        text: 'stale',
+        start: 0,
+        end: 0.5,
+        words: [{ word: 'stale', start: 0, end: 0.5 }],
+      }),
+      segment({
+        text: 'fresh',
+        start: 1,
+        end: 2,
+        words: [{ word: 'fresh', start: 1.5, end: 2 }],
+      }),
+    ];
+
+    const result = alignWhisperSegments(segments, [chunk(0)], T0 + 1_000);
+    expect(result.transcripts).toHaveLength(1);
+    expect(result.transcripts[0].text).toBe('fresh');
+  });
+});

package/src/transcriber/transcriber.ts

@@ -7,14 +7,15 @@ import { WaveFile } from 'wavefile';
 import { DeferredTask, Trigger } from '@dxos/async';
 import { type Context, LifecycleState, Resource } from '@dxos/context';
 import { log } from '@dxos/log';
-import { type DataType } from '@dxos/schema';
 import { trace } from '@dxos/tracing';
+import { type ContentBlock } from '@dxos/types';
+
+import { TRANSCRIPTION_URL } from '#types';
 
-import { type AudioRecorder, type AudioChunk } from './audio-recorder';
-import { TRANSCRIPTION_URL } from '../types';
 import { mergeFloat64Arrays } from '../util';
+import { type AudioChunk, type AudioRecorder } from './audio-recorder';
 
-type WhisperWord = {
+export type WhisperWord = {
   word: string;
 
   /**
@@ -28,7 +29,7 @@ type WhisperWord = {
   end: number;
 };
 
-type WhisperSegment = {
+export type WhisperSegment = {
   text: string;
 
   /**
@@ -60,16 +61,32 @@ export type TranscribeConfig = {
    * This is needed to catch the beginning of the user's speech.
    */
   prefixBufferChunksAmount: number;
+
+  /**
+   * Override the transcription endpoint base URL.
+   * Defaults to the DXOS calls service.
+   */
+  endpoint?: string;
 };
 
-export type TranscriberParams = {
+/**
+ * Function that converts a base64-encoded WAV payload into Whisper segments.
+ * Allows callers to swap in alternative providers or mock the transport.
+ */
+export type TranscribeFn = (audio: string) => Promise<WhisperSegment[]>;
+
+export type TranscriberProps = {
   config: TranscribeConfig;
   recorder: AudioRecorder;
   /**
    * Callback to handle the transcribed segments, after all segment transformers are applied.
    * @param segments - The transcribed segments.
    */
-  onSegments: (segments: DataType.MessageBlock.Transcription[]) => Promise<void>;
+  onSegments: (segments: ContentBlock.Transcript[]) => Promise<void>;
+  /**
+   * Optional override of the transcription transport. When provided, supersedes `config.endpoint`.
+   */
+  transcribe?: TranscribeFn;
 };
 
 /**
@@ -85,16 +102,18 @@ export class Transcriber extends Resource {
 
   private readonly _config: TranscribeConfig;
   private readonly _recorder: AudioRecorder;
-  private readonly _onSegments: TranscriberParams['onSegments'];
+  private readonly _onSegments: TranscriberProps['onSegments'];
+  private readonly _transcribeFn?: TranscribeFn;
 
   private _recording = false;
   private _transcribeTask?: DeferredTask = undefined;
 
-  constructor({ config, recorder, onSegments }: TranscriberParams) {
+  constructor({ config, recorder, onSegments, transcribe }: TranscriberProps) {
     super();
     this._config = config;
     this._recorder = recorder;
     this._onSegments = onSegments;
+    this._transcribeFn = transcribe;
   }
 
   protected override async _open(ctx: Context): Promise<void> {
@@ -191,70 +210,90 @@ export class Transcriber extends Resource {
   @trace.span({ showInBrowserTimeline: true })
   private async _fetchTranscription(audio: string): Promise<WhisperSegment[]> {
     if (audio.length === 0) {
-      this._audioChunks = [];
       throw new Error('No audio to send for transcribing');
     }
 
-    const response = await fetch(`${TRANSCRIPTION_URL}/transcribe`, {
-      method: 'POST',
-      body: JSON.stringify({ audio }),
-      headers: {
-        'Content-Type': 'application/json',
-      },
-    });
-
-    if (!response.ok) {
-      this._audioChunks = [];
-      throw new Error(`Transcription failed: ${response.statusText}`);
+    let segments: unknown;
+    if (this._transcribeFn) {
+      segments = await this._transcribeFn(audio);
+    } else {
+      const endpoint = this._config.endpoint ?? TRANSCRIPTION_URL;
+      const response = await fetch(`${endpoint}/transcribe`, {
+        method: 'POST',
+        body: JSON.stringify({ audio }),
+        headers: {
+          'Content-Type': 'application/json',
+        },
+      });
+
+      if (!response.ok) {
+        throw new Error(`Transcription failed: ${response.statusText}`);
+      }
+
+      ({ segments } = (await response.json()) as { segments?: unknown });
     }
 
-    const { segments } = (await response.json()) as {
-      segments: WhisperSegment[];
-    };
+    if (!Array.isArray(segments)) {
+      throw new Error('Transcription response payload is invalid');
+    }
 
-    log.info('transcription response', {
-      segments: segments.length,
-      string: segments.map((segments) => segments.text).join(' '),
-    });
+    log.info('transcription response', { segments: segments.length });
 
-    return segments;
+    return segments as WhisperSegment[];
   }
 
-  private _alignSegments(
-    segments: WhisperSegment[],
-    originalChunks: AudioChunk[],
-  ): DataType.MessageBlock.Transcription[] {
-    // Absolute zero for all relative timestamps in the segments.
-    const zeroTimestamp = originalChunks.at(0)!.timestamp;
-
-    const filteredSegments = segments
-      // Use segments that end after the last used timestamp.
-      // Segments could overlap, so we need to filter words that starts after the last used timestamp.
-      .filter((segment) => zeroTimestamp + segment.end * 1_000 >= this._lastUsedTimestamp)
-      // Use words that starts after the last used timestamp.
-      .map((segment) => {
-        const words = segment.words.filter((word) => zeroTimestamp + word.start * 1_000 >= this._lastUsedTimestamp);
-        return {
-          ...segment,
-          words,
-          text: words.map((word) => word.word).join(''),
-          start: words.at(0)?.start ?? segment.end,
-        };
-      })
-      .filter((segment) => segment.words.length > 0);
-
-    if (filteredSegments.length === 0) {
-      return [];
+  private _alignSegments(segments: WhisperSegment[], originalChunks: AudioChunk[]): ContentBlock.Transcript[] {
+    const result = alignWhisperSegments(segments, originalChunks, this._lastUsedTimestamp);
+    if (result.lastUsedTimestamp !== undefined) {
+      this._lastUsedTimestamp = result.lastUsedTimestamp;
     }
+    return result.transcripts;
+  }
+}
+
+/**
+ * Pure helper that filters and absolute-timestamps Whisper segments against a chunk timeline.
+ * Exported for testing.
+ *
+ * @param segments - Raw segments returned by the Whisper service for `originalChunks`.
+ * @param originalChunks - Chunks fed into the transcription request (used for the zero timestamp).
+ * @param lastUsedTimestamp - Most recent absolute timestamp already emitted; segments whose end
+ *   falls strictly before this point are deduped out, as are individual words whose start does.
+ */
+export const alignWhisperSegments = (
+  segments: WhisperSegment[],
+  originalChunks: AudioChunk[],
+  lastUsedTimestamp: number,
+): { transcripts: ContentBlock.Transcript[]; lastUsedTimestamp?: number } => {
+  if (originalChunks.length === 0) {
+    return { transcripts: [] };
+  }
 
-    // Update last timestamp.
-    this._lastUsedTimestamp = zeroTimestamp + filteredSegments.at(-1)!.end * 1_000;
+  const zeroTimestamp = originalChunks[0].timestamp;
+
+  const filteredSegments = segments
+    .filter((segment) => zeroTimestamp + segment.end * 1_000 >= lastUsedTimestamp)
+    .map((segment) => {
+      const words = segment.words.filter((word) => zeroTimestamp + word.start * 1_000 >= lastUsedTimestamp);
+      return {
+        ...segment,
+        words,
+        text: words.map((word) => word.word).join(''),
+        start: words.at(0)?.start ?? segment.end,
+      };
+    })
+    .filter((segment) => segment.words.length > 0);
+
+  if (filteredSegments.length === 0) {
+    return { transcripts: [] };
+  }
 
-    // Add absolute timestamp to each segment.
-    return filteredSegments.map((segment) => ({
-      type: 'transcription',
+  return {
+    transcripts: filteredSegments.map((segment) => ({
+      _tag: 'transcript',
       started: new Date(zeroTimestamp + segment.start * 1_000).toISOString(),
       text: segment.text.trim(),
-    }));
-  }
-}
+    })),
+    lastUsedTimestamp: zeroTimestamp + filteredSegments.at(-1)!.end * 1_000,
+  };
+};

package/src/transcriber/transcription-manager.ts

@@ -2,14 +2,14 @@
 // Copyright 2025 DXOS.org
 //
 
-import { signal } from '@preact/signals-core';
+import { Atom, type Registry } from '@effect-atom/atom-react';
 
 import { synchronized } from '@dxos/async';
 import { Resource } from '@dxos/context';
 import { Obj } from '@dxos/echo';
 import { type Queue } from '@dxos/echo-db';
 import { type EdgeHttpClient } from '@dxos/react-edge-client';
-import { DataType } from '@dxos/schema';
+import { type ContentBlock, Message } from '@dxos/types';
 
 import { MediaStreamRecorder } from './media-stream-recorder';
 import { Transcriber } from './transcriber';
@@ -30,10 +30,11 @@ const PREFIXED_CHUNKS_AMOUNT = 10;
  */
 const TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
 
-export type TranscriptMessageEnricher = (message: DataType.Message) => Promise<DataType.Message>;
+export type TranscriptMessageEnricher = (message: Message.Message) => Promise<Message.Message>;
 
 export type TranscriptionManagerOptions = {
   edgeClient: EdgeHttpClient;
+  registry: Registry.Registry;
 
   /**
    * Enrich the message before it is written to the transcription queue.
@@ -47,25 +48,30 @@ export type TranscriptionManagerOptions = {
 export class TranscriptionManager extends Resource {
   private readonly _edgeClient: EdgeHttpClient;
   private readonly _messageEnricher?: TranscriptMessageEnricher;
+  private readonly _registry: Registry.Registry;
   private _audioStreamTrack?: MediaStreamTrack = undefined;
   private _identityDid?: string = undefined;
   private _mediaRecorder?: MediaStreamRecorder = undefined;
   private _transcriber?: Transcriber = undefined;
-  private _queue?: Queue<DataType.Message> = undefined;
-  private _enabled = signal(false);
+  private _queue?: Queue<Message.Message> = undefined;
+  private _enabledAtom = Atom.make(false);
 
   constructor(options: TranscriptionManagerOptions) {
     super();
     this._edgeClient = options.edgeClient;
     this._messageEnricher = options.messageEnricher;
+    this._registry = options.registry;
   }
 
-  /** @reactive */
-  get enabled() {
-    return this._enabled.value;
+  get enabled(): Atom.Atom<boolean> {
+    return this._enabledAtom;
   }
 
-  setQueue(queue: Queue<DataType.Message>): this {
+  getEnabled(): boolean {
+    return this._registry.get(this._enabledAtom);
+  }
+
+  setQueue(queue: Queue<Message.Message>): this {
     this._queue = queue;
     return this;
   }
@@ -78,7 +84,7 @@ export class TranscriptionManager extends Resource {
   }
 
   setRecording(recording?: boolean): this {
-    if (!this.isOpen || !this._enabled.value) {
+    if (!this.isOpen || !this.getEnabled()) {
       return this;
     }
 
@@ -90,19 +96,17 @@ export class TranscriptionManager extends Resource {
     return this;
   }
 
-  /**
-   * Enable or disable transcription.
-   * @param enabled - Whether to enable transcription.
-   */
-  @synchronized
-  async setEnabled(enabled?: boolean): Promise<void> {
-    if (this._enabled.value === enabled) {
+  async setEnabled(enabled: boolean): Promise<void> {
+    if (this.getEnabled() === enabled) {
       return;
     }
 
-    this._enabled.value = enabled ?? false;
-    // TODO(burdon): Why is toggle called here?
-    this.isOpen && (await this._toggleTranscriber());
+    this._registry.set(this._enabledAtom, enabled ?? false);
+    if (enabled) {
+      await this._maybeRestartTranscriber();
+    } else {
+      await this._stopTranscriber();
+    }
   }
 
   @synchronized
@@ -112,44 +116,19 @@ export class TranscriptionManager extends Resource {
     }
 
     this._audioStreamTrack = track;
-    this.isOpen && (await this._toggleTranscriber());
+    await this._maybeRestartTranscriber();
   }
 
   protected override async _open(): Promise<void> {
-    await this._toggleTranscriber();
+    await this._maybeRestartTranscriber();
   }
 
   protected override async _close(): Promise<void> {
-    void this._transcriber?.close();
-  }
-
-  // TODO(burdon): Change this to setEnables (explicit), not toggle.
-  private async _toggleTranscriber(): Promise<void> {
-    await this._maybeReinitTranscriber();
-
-    // Open or close transcriber if transcription is enabled or disabled.
-    if (this._enabled.value) {
-      await this._transcriber?.open();
-      // TODO(burdon): Started and stopped blocks appear twice.
-      const block = Obj.make(DataType.Message, {
-        created: new Date().toISOString(),
-        blocks: [{ type: 'transcription', text: 'Started', started: new Date().toISOString() }],
-        sender: { role: 'assistant' },
-      });
-      await this._queue?.append([block]);
-    } else {
-      await this._transcriber?.close();
-      const block = Obj.make(DataType.Message, {
-        created: new Date().toISOString(),
-        blocks: [{ type: 'transcription', text: 'Stopped', started: new Date().toISOString() }],
-        sender: { role: 'assistant' },
-      });
-      await this._queue?.append([block]);
-    }
+    await this._stopTranscriber();
   }
 
-  private async _maybeReinitTranscriber(): Promise<void> {
-    if (!this._audioStreamTrack || !this._enabled.value) {
+  private async _maybeRestartTranscriber(): Promise<void> {
+    if (!this._audioStreamTrack || !this.getEnabled() || !this.isOpen) {
       return;
     }
 
@@ -172,15 +151,21 @@ export class TranscriptionManager extends Resource {
         recorder: this._mediaRecorder,
         onSegments: (segments) => this._onSegments(segments),
       });
+
+      await this._transcriber?.open();
     }
   }
 
-  private async _onSegments(segments: DataType.MessageBlock.Transcription[]): Promise<void> {
+  private async _stopTranscriber(): Promise<void> {
+    await this._transcriber?.close();
+  }
+
+  private async _onSegments(segments: ContentBlock.Transcript[]): Promise<void> {
     if (!this.isOpen || !this._queue) {
       return;
     }
 
-    let block = Obj.make(DataType.Message, {
+    let block = Obj.make(Message.Message, {
       created: new Date().toISOString(),
       blocks: segments,
       sender: { identityDid: this._identityDid },

package/src/translations.ts

@@ -2,21 +2,23 @@
 // Copyright 2023 DXOS.org
 //
 
-import { TRANSCRIPTION_PLUGIN } from './meta';
+import { type Resource } from '@dxos/react-ui';
 
-export default [
+import { meta } from '#meta';
+
+export const translations = [
   {
     'en-US': {
-      [TRANSCRIPTION_PLUGIN]: {
-        'plugin name': 'Transcription',
-        'transcript companion label': 'Transcript',
+      [meta.id]: {
+        'plugin.name': 'Transcription',
+        'transcript-companion.label': 'Transcript',
 
-        'delete button': 'Delete',
-        'bookmark button': 'Bookmark',
-        'scroll to end label': 'Scroll to latest',
+        'delete.button': 'Delete',
+        'bookmark.button': 'Bookmark',
+        'scroll-to-end.label': 'Scroll to latest',
 
-        'settings entity extraction label': 'Entity extraction',
+        'settings.entity-extraction.label': 'Entity extraction',
       },
     },
   },
-];
+] as const satisfies Resource[];

package/src/types/TranscriptOperation.ts

@@ -0,0 +1,103 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+// @import-as-namespace
+
+import * as Schema from 'effect/Schema';
+
+import { AiService } from '@dxos/ai';
+import { SpaceSchema } from '@dxos/client/echo';
+import { Operation } from '@dxos/compute';
+import { Database, Feed, Ref } from '@dxos/echo';
+import { Message, Transcript } from '@dxos/types';
+
+import { meta } from '#meta';
+
+const TRANSCRIPT_OPERATION = `${meta.id}.operation`;
+
+export const Create = Operation.make({
+  meta: { key: `${TRANSCRIPT_OPERATION}.create`, name: 'Create Transcript' },
+  input: Schema.Struct({
+    name: Schema.optional(Schema.String),
+    space: SpaceSchema,
+  }),
+  output: Schema.Struct({
+    object: Transcript.Transcript,
+  }),
+});
+
+export const MessageWithRangeId = Schema.extend(
+  Message.Message,
+  Schema.Struct({
+    rangeId: Schema.optional(Schema.Array(Schema.String)).annotations({
+      description: 'The IDs of the messages that contain the sentences.',
+    }),
+  }),
+);
+
+export type MessageWithRangeIdType = Schema.Schema.Type<typeof MessageWithRangeId>;
+
+export const Open = Operation.make({
+  meta: {
+    key: 'org.dxos.function.transcription.open',
+    name: 'Open',
+    description: 'Opens and reads the contents of a transcription object.',
+  },
+  input: Schema.Struct({
+    transcript: Ref.Ref(Transcript.Transcript).annotations({
+      description: 'The ID of the transcription object.',
+    }),
+  }),
+  output: Schema.Struct({
+    content: Schema.String,
+  }),
+  services: [Database.Service, Feed.FeedService],
+});
+
+export const Summarize = Operation.make({
+  meta: {
+    key: 'org.dxos.function.transcription.summarize',
+    name: 'Summarize',
+    description: 'Summarize a transcript of a meeting.',
+  },
+  input: Schema.Struct({
+    transcript: Schema.String.annotations({
+      description: 'The transcript of the meeting.',
+    }),
+    notes: Schema.optional(Schema.String).annotations({
+      description: 'Additional notes from the participants.',
+    }),
+  }),
+  output: Schema.Struct({
+    summary: Schema.String.annotations({
+      description: 'The summary of the transcript.',
+    }),
+  }),
+  services: [AiService.AiService],
+});
+
+export const SentenceNormalizationInput = Schema.Struct({
+  messages: Schema.Array(MessageWithRangeId).annotations({
+    description: 'Messages to normalize into sentences.',
+  }),
+});
+
+export type SentenceNormalizationInputType = Schema.Schema.Type<typeof SentenceNormalizationInput>;
+
+export const SentenceNormalizationOutput = Schema.Struct({
+  sentences: Schema.Array(MessageWithRangeId.pipe(Schema.mutable)).pipe(Schema.mutable).annotations({
+    description: 'The sentences of the transcript.',
+  }),
+});
+
+export const SentenceNormalization = Operation.make({
+  meta: {
+    key: 'org.dxos.function.transcription.sentence-normalization',
+    name: 'Sentence Normalization',
+    description: 'Post process of transcription for sentence normalization',
+  },
+  input: SentenceNormalizationInput,
+  output: SentenceNormalizationOutput,
+  services: [],
+});

package/src/types/TranscriptionCapabilities.ts

@@ -0,0 +1,39 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+// @import-as-namespace
+
+import { Capability } from '@dxos/app-framework';
+
+import { meta } from '#meta';
+
+import {
+  type MediaStreamRecorderProps,
+  type Transcriber as TranscriberType,
+  type TranscriberProps,
+  type TranscriptMessageEnricher,
+  type TranscriptionManager as TranscriptionManagerType,
+} from '../transcriber';
+
+export type GetTranscriberProps = {
+  audioStreamTrack: MediaStreamTrack;
+  recorderConfig?: Partial<MediaStreamRecorderProps['config']>;
+  transcriberConfig?: Partial<TranscriberProps['config']>;
+  onSegments: TranscriberProps['onSegments'];
+  transcribe?: TranscriberProps['transcribe'];
+};
+
+export type GetTranscriber = (props: GetTranscriberProps) => TranscriberType;
+
+export type GetTranscriptionManagerProps = {
+  messageEnricher?: TranscriptMessageEnricher;
+};
+
+export type GetTranscriptionManager = (props: GetTranscriptionManagerProps) => TranscriptionManagerType;
+
+export const Transcriber = Capability.make<GetTranscriber>(`${meta.id}.capability.transcriber`);
+
+export const TranscriptionManager = Capability.make<GetTranscriptionManager>(
+  `${meta.id}.capability.transcription-manager`,
+);