@dxos/plugin-transcription 0.8.3 → 0.8.4-main.1c7ec43d41

package/src/testing/testing.ts

@@ -2,24 +2,22 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Schema } from 'effect';
+import * as Schema from 'effect/Schema';
 import { useEffect, useMemo, useState } from 'react';
 
-import { EdgeAiServiceClient } from '@dxos/ai';
-import { AI_SERVICE_ENDPOINT } from '@dxos/ai/testing';
-import { extractionAnthropicFn, processTranscriptMessage } from '@dxos/assistant';
+import { extractionAnthropicFunction, processTranscriptMessage } from '@dxos/assistant/extraction';
 import { scheduleTaskInterval } from '@dxos/async';
 import { Filter, type Queue } from '@dxos/client/echo';
 import { Context } from '@dxos/context';
 import { type Key, Obj, Ref, Type } from '@dxos/echo';
-import { createQueueDXN } from '@dxos/echo-schema';
-import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
+import { createQueueDXN } from '@dxos/echo/internal';
 import { IdentityDid } from '@dxos/keys';
 import { log } from '@dxos/log';
-import { faker } from '@dxos/random';
-import { useQueue, type Space } from '@dxos/react-client/echo';
-import { DataType } from '@dxos/schema';
-import { Testing, seedTestData } from '@dxos/schema/testing';
+import { random } from '@dxos/random';
+import { type Space, useQueue } from '@dxos/react-client/echo';
+import { TestSchema } from '@dxos/schema/testing';
+import { type ContentBlock, Message, Organization, Person } from '@dxos/types';
+import { seedTestData } from '@dxos/types/testing';
 
 // TODO(burdon): Reconcile with plugin-markdown. Move to @dxos/schema/testing.
 export const TestItem = Schema.Struct({
@@ -32,15 +30,15 @@ export const TestItem = Schema.Struct({
     description: 'Product description',
   }),
 }).pipe(
-  Type.Obj({
-    typename: 'dxos.org/type/Test',
+  Type.object({
+    typename: 'org.dxos.type.test',
     version: '0.1.0',
   }),
 );
 
 // TODO(wittjosiah): Make builder generic and reuse for all message types.
 abstract class AbstractMessageBuilder {
-  abstract createMessage(numSegments?: number): Promise<DataType.Message>;
+  abstract createMessage(numSegments?: number): Promise<Message.Message>;
 }
 
 /**
@@ -51,8 +49,8 @@ export class MessageBuilder extends AbstractMessageBuilder {
 
   users = Array.from({ length: 5 }, () => ({
     identityDid: IdentityDid.random().toString(),
-    name: faker.person.fullName(),
-    email: faker.internet.email(),
+    name: random.person.fullName(),
+    email: random.internet.email(),
   }));
 
   start = new Date(Date.now() - 24 * 60 * 60 * 10_000);
@@ -61,27 +59,32 @@ export class MessageBuilder extends AbstractMessageBuilder {
     super();
   }
 
-  override async createMessage(numSegments = 1): Promise<DataType.Message> {
-    return Obj.make(DataType.Message, {
+  override async createMessage(numSegments = 1): Promise<Message.Message> {
+    return Obj.make(Message.Message, {
       created: this.next().toISOString(),
-      sender: faker.helpers.arrayElement(this.users),
+      sender: random.helpers.arrayElement(this.users),
       blocks: Array.from({ length: numSegments }).map(() => this.createBlock()),
     });
   }
 
-  createBlock(): DataType.MessageBlock.Transcription {
-    let text = faker.lorem.paragraph();
+  createBlock(): ContentBlock.Transcript {
+    let text = random.lorem.paragraph();
     if (this._space) {
-      const label = faker.commerce.productName();
-      const obj = this._space.db.add(Obj.make(TestItem, { title: label, description: faker.lorem.paragraph() }));
+      const label = random.commerce.productName();
+      const obj = this._space.db.add(
+        Obj.make(TestItem, {
+          title: label,
+          description: random.lorem.paragraph(),
+        }),
+      );
       const dxn = Ref.make(obj).dxn.toString();
       const words = text.split(' ');
-      words.splice(Math.floor(Math.random() * words.length), 0, `[${label}][${dxn}]`);
+      words.splice(Math.floor(Math.random() * words.length), 0, `[${label}](${dxn})`);
       text = words.join(' ');
     }
 
     return {
-      type: 'transcription',
+      _tag: 'transcript',
       started: this.next().toISOString(),
       text,
     };
@@ -95,15 +98,9 @@ export class MessageBuilder extends AbstractMessageBuilder {
 
 // TODO(burdon): Reconcile with BlockBuilder.
 class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
-  AiService = new EdgeAiServiceClient({
-    endpoint: AI_SERVICE_ENDPOINT.REMOTE,
-  });
-
-  executor = new FunctionExecutor(new ServiceContainer().setServices({ ai: { client: this.AiService } }));
-
   space: Space | undefined;
   currentMessage: number = 0;
-  transcriptMessages: DataType.Message[] = [];
+  transcriptMessages: Message.Message[] = [];
 
   async connect(space: Space): Promise<void> {
     this.space = space;
@@ -111,14 +108,18 @@ class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
     this.transcriptMessages = transcriptMessages;
   }
 
-  override async createMessage(): Promise<DataType.Message> {
+  override async createMessage(): Promise<Message.Message> {
     if (!this.space) {
       throw new Error('Space not connected');
     }
 
-    const { objects } = await this.space.db
+    const objects = await this.space.db
       .query(
-        Filter.or(Filter.type(DataType.Person), Filter.type(DataType.Organization), Filter.type(Testing.DocumentType)),
+        Filter.or(
+          Filter.type(Person.Person),
+          Filter.type(Organization.Organization),
+          Filter.type(TestSchema.DocumentType),
+        ),
       )
       .run();
 
@@ -129,8 +130,7 @@ class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
 
     const { message: enhancedMessage } = await processTranscriptMessage({
       input: { message },
-      executor: this.executor,
-      function: extractionAnthropicFn,
+      function: extractionAnthropicFunction,
     });
 
     return enhancedMessage;
@@ -142,7 +142,7 @@ type UseTestTranscriptionQueue = (
   queueId?: Key.ObjectId,
   running?: boolean,
   interval?: number,
-) => Queue<DataType.Message> | undefined;
+) => Queue<Message.Message> | undefined;
 
 /**
  * Test transcriptionqueue.
@@ -155,7 +155,7 @@ export const useTestTranscriptionQueue: UseTestTranscriptionQueue = (
 ) => {
   // TODO(dmaretskyi): Use space.queues.create() instead.
   const queueDxn = useMemo(() => (space ? createQueueDXN(space.id, queueId) : undefined), [space, queueId]);
-  const queue = useQueue<DataType.Message>(queueDxn);
+  const queue = useQueue<Message.Message>(queueDxn);
   const builder = useMemo(() => new MessageBuilder(space), [space]);
 
   useEffect(() => {
@@ -165,7 +165,7 @@ export const useTestTranscriptionQueue: UseTestTranscriptionQueue = (
 
     const i = setInterval(() => {
       void builder.createMessage(Math.ceil(Math.random() * 3)).then(async (message) => {
-        await queue.append([Obj.make(DataType.Message, message)]);
+        await queue.append([Obj.make(Message.Message, message)]);
       });
     }, interval);
     return () => clearInterval(i);
@@ -186,7 +186,7 @@ export const useTestTranscriptionQueueWithEntityExtraction: UseTestTranscription
 ) => {
   // TODO(dmaretskyi): Use space.queues.create() instead.
   const queueDxn = useMemo(() => (space ? createQueueDXN(space.id, queueId) : undefined), [space, queueId]);
-  const queue = useQueue<DataType.Message>(queueDxn);
+  const queue = useQueue<Message.Message>(queueDxn);
   const [builder] = useState(() => new EntityExtractionMessageBuilder());
 
   useEffect(() => {
@@ -203,7 +203,7 @@ export const useTestTranscriptionQueueWithEntityExtraction: UseTestTranscription
       ctx,
       async () => {
         const message = await builder.createMessage();
-        void queue.append([Obj.make(DataType.Message, message)]);
+        void queue.append([Obj.make(Message.Message, message)]);
       },
       interval,
     );

package/src/transcriber/media-stream-recorder.ts

@@ -10,7 +10,7 @@ import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 import { trace } from '@dxos/tracing';
 
-import { type WavConfig, type AudioChunk, type AudioRecorder } from './audio-recorder';
+import { type AudioChunk, type AudioRecorder, type WavConfig } from './audio-recorder';
 
 let initializingPromise: Promise<void> | undefined;
 
@@ -19,7 +19,7 @@ const initializeExtendableMediaRecorder = async () => {
   await register(await connect());
 };
 
-export type MediaStreamRecorderParams = {
+export type MediaStreamRecorderProps = {
   mediaStreamTrack: MediaStreamTrack;
   config: {
     interval: number;
@@ -37,12 +37,12 @@ export class MediaStreamRecorder implements AudioRecorder {
   /**
    * Default MediaRecorder implementation do not support wav encoding.
    */
-  private readonly _config: MediaStreamRecorderParams['config'];
+  private readonly _config: MediaStreamRecorderProps['config'];
   private _mediaRecorder?: IMediaRecorder = undefined;
   private _header?: Uint8Array = undefined;
   private _onChunk?: (chunk: AudioChunk) => void;
 
-  constructor({ mediaStreamTrack, config }: MediaStreamRecorderParams) {
+  constructor({ mediaStreamTrack, config }: MediaStreamRecorderProps) {
     this._mediaStreamTrack = mediaStreamTrack;
     this._config = config;
   }

package/src/transcriber/transcriber.browser.test.ts

@@ -9,9 +9,9 @@ import { WaveFile } from 'wavefile';
 
 import { Trigger } from '@dxos/async';
 import { log } from '@dxos/log';
-import { type DataType } from '@dxos/schema';
 import { openAndClose } from '@dxos/test-utils';
-import { trace, TRACE_PROCESSOR } from '@dxos/tracing';
+import { TRACE_PROCESSOR, trace } from '@dxos/tracing';
+import { type ContentBlock } from '@dxos/types';
 
 import { type AudioChunk, type AudioRecorder, Transcriber } from '../transcriber';
 import { mergeFloat64Arrays } from '../util';
@@ -93,7 +93,7 @@ describe.skip('Transcriber', () => {
   });
 
   test.skip('transcription of audio recording', { timeout: 10_000 }, async () => {
-    const trigger = new Trigger<DataType.MessageBlock.Transcription[]>({ autoReset: true });
+    const trigger = new Trigger<ContentBlock.Transcript[]>({ autoReset: true });
     const recorder = new MockAudioRecorder({
       buffer: await readFile('test.wav'),
       chunkDuration: 3_000,
@@ -127,7 +127,7 @@ describe.skip('Transcriber', () => {
   });
 
   test.skip('transcription of audio recording with overlapping chunks', { timeout: 20_000 }, async () => {
-    const trigger = new Trigger<DataType.MessageBlock.Transcription[]>({ autoReset: true });
+    const trigger = new Trigger<ContentBlock.Transcript[]>({ autoReset: true });
     const recorder = new MockAudioRecorder({
       buffer: await readFile('test.wav'),
       chunkDuration: 3_000,
@@ -212,7 +212,7 @@ class MockAudioRecorder implements AudioRecorder {
 
   @trace.span()
   async start(): Promise<void> {
-    log.info('start mock audio recorder', { wavParams: this.wav.fmt });
+    log.info('start mock audio recorder', { wavProps: this.wav.fmt });
     const now = Date.now();
     const samples = this.wav.getSamples();
     const sampleRate = this.wavConfig.sampleRate;

package/src/transcriber/transcriber.test.ts

@@ -0,0 +1,104 @@
+//
+// Copyright 2026 DXOS.org
+//
+
+import { describe, test } from 'vitest';
+
+import { type AudioChunk } from './audio-recorder';
+import { alignWhisperSegments, type WhisperSegment } from './transcriber';
+
+const T0 = Date.parse('2026-01-01T00:00:00.000Z');
+
+const chunk = (offsetMs = 0, samples = 16): AudioChunk => ({
+  timestamp: T0 + offsetMs,
+  data: new Float64Array(samples),
+});
+
+const segment = (overrides: Partial<WhisperSegment>): WhisperSegment => ({
+  text: '',
+  start: 0,
+  end: 0,
+  no_speech_prob: 0,
+  words: [],
+  ...overrides,
+});
+
+describe('alignWhisperSegments', () => {
+  test('returns empty when no chunks were transcribed', ({ expect }) => {
+    const result = alignWhisperSegments(
+      [segment({ text: 'x', end: 1, words: [{ word: 'x', start: 0, end: 1 }] })],
+      [],
+      0,
+    );
+    expect(result.transcripts).toEqual([]);
+    expect(result.lastUsedTimestamp).toBeUndefined();
+  });
+
+  test('absolute-timestamps and deduplicates segments against lastUsedTimestamp', ({ expect }) => {
+    const segments: WhisperSegment[] = [
+      segment({
+        text: 'hello world',
+        start: 0,
+        end: 1,
+        words: [
+          { word: 'hello ', start: 0, end: 0.5 },
+          { word: 'world', start: 0.5, end: 1 },
+        ],
+      }),
+    ];
+
+    const first = alignWhisperSegments(segments, [chunk(0)], 0);
+    expect(first.transcripts).toHaveLength(1);
+    expect(first.transcripts[0]._tag).toBe('transcript');
+    expect(first.transcripts[0].text).toBe('hello world');
+    expect(new Date(first.transcripts[0].started).getTime()).toBe(T0);
+    expect(first.lastUsedTimestamp).toBe(T0 + 1_000);
+
+    // Same segments, same chunk start: every word's absolute start is < lastUsedTimestamp -> filtered out.
+    const second = alignWhisperSegments(segments, [chunk(0)], first.lastUsedTimestamp!);
+    expect(second.transcripts).toEqual([]);
+    expect(second.lastUsedTimestamp).toBeUndefined();
+  });
+
+  test('keeps later words when earlier words fall before lastUsedTimestamp', ({ expect }) => {
+    const segments: WhisperSegment[] = [
+      segment({
+        text: 'old new',
+        start: 0,
+        end: 2,
+        words: [
+          { word: 'old ', start: 0, end: 1 },
+          { word: 'new', start: 1.5, end: 2 },
+        ],
+      }),
+    ];
+
+    // First word starts at T0, second at T0+1500 — only the second survives a cursor at T0+1000.
+    const result = alignWhisperSegments(segments, [chunk(0)], T0 + 1_000);
+    expect(result.transcripts).toHaveLength(1);
+    expect(result.transcripts[0].text).toBe('new');
+    expect(new Date(result.transcripts[0].started).getTime()).toBe(T0 + 1_500);
+    expect(result.lastUsedTimestamp).toBe(T0 + 2_000);
+  });
+
+  test('drops segments that end before lastUsedTimestamp', ({ expect }) => {
+    const segments: WhisperSegment[] = [
+      segment({
+        text: 'stale',
+        start: 0,
+        end: 0.5,
+        words: [{ word: 'stale', start: 0, end: 0.5 }],
+      }),
+      segment({
+        text: 'fresh',
+        start: 1,
+        end: 2,
+        words: [{ word: 'fresh', start: 1.5, end: 2 }],
+      }),
+    ];
+
+    const result = alignWhisperSegments(segments, [chunk(0)], T0 + 1_000);
+    expect(result.transcripts).toHaveLength(1);
+    expect(result.transcripts[0].text).toBe('fresh');
+  });
+});

package/src/transcriber/transcriber.ts

@@ -7,14 +7,15 @@ import { WaveFile } from 'wavefile';
 import { DeferredTask, Trigger } from '@dxos/async';
 import { type Context, LifecycleState, Resource } from '@dxos/context';
 import { log } from '@dxos/log';
-import { type DataType } from '@dxos/schema';
 import { trace } from '@dxos/tracing';
+import { type ContentBlock } from '@dxos/types';
+
+import { TRANSCRIPTION_URL } from '#types';
 
-import { type AudioRecorder, type AudioChunk } from './audio-recorder';
-import { TRANSCRIPTION_URL } from '../types';
 import { mergeFloat64Arrays } from '../util';
+import { type AudioChunk, type AudioRecorder } from './audio-recorder';
 
-type WhisperWord = {
+export type WhisperWord = {
   word: string;
 
   /**
@@ -28,7 +29,7 @@ type WhisperWord = {
   end: number;
 };
 
-type WhisperSegment = {
+export type WhisperSegment = {
   text: string;
 
   /**
@@ -60,16 +61,32 @@ export type TranscribeConfig = {
    * This is needed to catch the beginning of the user's speech.
    */
   prefixBufferChunksAmount: number;
+
+  /**
+   * Override the transcription endpoint base URL.
+   * Defaults to the DXOS calls service.
+   */
+  endpoint?: string;
 };
 
-export type TranscriberParams = {
+/**
+ * Function that converts a base64-encoded WAV payload into Whisper segments.
+ * Allows callers to swap in alternative providers or mock the transport.
+ */
+export type TranscribeFn = (audio: string) => Promise<WhisperSegment[]>;
+
+export type TranscriberProps = {
   config: TranscribeConfig;
   recorder: AudioRecorder;
   /**
    * Callback to handle the transcribed segments, after all segment transformers are applied.
    * @param segments - The transcribed segments.
    */
-  onSegments: (segments: DataType.MessageBlock.Transcription[]) => Promise<void>;
+  onSegments: (segments: ContentBlock.Transcript[]) => Promise<void>;
+  /**
+   * Optional override of the transcription transport. When provided, supersedes `config.endpoint`.
+   */
+  transcribe?: TranscribeFn;
 };
 
 /**
@@ -85,16 +102,18 @@ export class Transcriber extends Resource {
 
   private readonly _config: TranscribeConfig;
   private readonly _recorder: AudioRecorder;
-  private readonly _onSegments: TranscriberParams['onSegments'];
+  private readonly _onSegments: TranscriberProps['onSegments'];
+  private readonly _transcribeFn?: TranscribeFn;
 
   private _recording = false;
   private _transcribeTask?: DeferredTask = undefined;
 
-  constructor({ config, recorder, onSegments }: TranscriberParams) {
+  constructor({ config, recorder, onSegments, transcribe }: TranscriberProps) {
     super();
     this._config = config;
     this._recorder = recorder;
     this._onSegments = onSegments;
+    this._transcribeFn = transcribe;
   }
 
   protected override async _open(ctx: Context): Promise<void> {
@@ -191,70 +210,90 @@ export class Transcriber extends Resource {
   @trace.span({ showInBrowserTimeline: true })
   private async _fetchTranscription(audio: string): Promise<WhisperSegment[]> {
     if (audio.length === 0) {
-      this._audioChunks = [];
       throw new Error('No audio to send for transcribing');
     }
 
-    const response = await fetch(`${TRANSCRIPTION_URL}/transcribe`, {
-      method: 'POST',
-      body: JSON.stringify({ audio }),
-      headers: {
-        'Content-Type': 'application/json',
-      },
-    });
-
-    if (!response.ok) {
-      this._audioChunks = [];
-      throw new Error(`Transcription failed: ${response.statusText}`);
+    let segments: unknown;
+    if (this._transcribeFn) {
+      segments = await this._transcribeFn(audio);
+    } else {
+      const endpoint = this._config.endpoint ?? TRANSCRIPTION_URL;
+      const response = await fetch(`${endpoint}/transcribe`, {
+        method: 'POST',
+        body: JSON.stringify({ audio }),
+        headers: {
+          'Content-Type': 'application/json',
+        },
+      });
+
+      if (!response.ok) {
+        throw new Error(`Transcription failed: ${response.statusText}`);
+      }
+
+      ({ segments } = (await response.json()) as { segments?: unknown });
     }
 
-    const { segments } = (await response.json()) as {
-      segments: WhisperSegment[];
-    };
+    if (!Array.isArray(segments)) {
+      throw new Error('Transcription response payload is invalid');
+    }
 
-    log.info('transcription response', {
-      segments: segments.length,
-      string: segments.map((segments) => segments.text).join(' '),
-    });
+    log.info('transcription response', { segments: segments.length });
 
-    return segments;
+    return segments as WhisperSegment[];
   }
 
-  private _alignSegments(
-    segments: WhisperSegment[],
-    originalChunks: AudioChunk[],
-  ): DataType.MessageBlock.Transcription[] {
-    // Absolute zero for all relative timestamps in the segments.
-    const zeroTimestamp = originalChunks.at(0)!.timestamp;
-
-    const filteredSegments = segments
-      // Use segments that end after the last used timestamp.
-      // Segments could overlap, so we need to filter words that starts after the last used timestamp.
-      .filter((segment) => zeroTimestamp + segment.end * 1_000 >= this._lastUsedTimestamp)
-      // Use words that starts after the last used timestamp.
-      .map((segment) => {
-        const words = segment.words.filter((word) => zeroTimestamp + word.start * 1_000 >= this._lastUsedTimestamp);
-        return {
-          ...segment,
-          words,
-          text: words.map((word) => word.word).join(''),
-          start: words.at(0)?.start ?? segment.end,
-        };
-      })
-      .filter((segment) => segment.words.length > 0);
-
-    if (filteredSegments.length === 0) {
-      return [];
+  private _alignSegments(segments: WhisperSegment[], originalChunks: AudioChunk[]): ContentBlock.Transcript[] {
+    const result = alignWhisperSegments(segments, originalChunks, this._lastUsedTimestamp);
+    if (result.lastUsedTimestamp !== undefined) {
+      this._lastUsedTimestamp = result.lastUsedTimestamp;
     }
+    return result.transcripts;
+  }
+}
+
+/**
+ * Pure helper that filters and absolute-timestamps Whisper segments against a chunk timeline.
+ * Exported for testing.
+ *
+ * @param segments - Raw segments returned by the Whisper service for `originalChunks`.
+ * @param originalChunks - Chunks fed into the transcription request (used for the zero timestamp).
+ * @param lastUsedTimestamp - Most recent absolute timestamp already emitted; segments whose end
+ *   falls strictly before this point are deduped out, as are individual words whose start does.
+ */
+export const alignWhisperSegments = (
+  segments: WhisperSegment[],
+  originalChunks: AudioChunk[],
+  lastUsedTimestamp: number,
+): { transcripts: ContentBlock.Transcript[]; lastUsedTimestamp?: number } => {
+  if (originalChunks.length === 0) {
+    return { transcripts: [] };
+  }
 
-    // Update last timestamp.
-    this._lastUsedTimestamp = zeroTimestamp + filteredSegments.at(-1)!.end * 1_000;
+  const zeroTimestamp = originalChunks[0].timestamp;
+
+  const filteredSegments = segments
+    .filter((segment) => zeroTimestamp + segment.end * 1_000 >= lastUsedTimestamp)
+    .map((segment) => {
+      const words = segment.words.filter((word) => zeroTimestamp + word.start * 1_000 >= lastUsedTimestamp);
+      return {
+        ...segment,
+        words,
+        text: words.map((word) => word.word).join(''),
+        start: words.at(0)?.start ?? segment.end,
+      };
+    })
+    .filter((segment) => segment.words.length > 0);
+
+  if (filteredSegments.length === 0) {
+    return { transcripts: [] };
+  }
 
-    // Add absolute timestamp to each segment.
-    return filteredSegments.map((segment) => ({
-      type: 'transcription',
+  return {
+    transcripts: filteredSegments.map((segment) => ({
+      _tag: 'transcript',
       started: new Date(zeroTimestamp + segment.start * 1_000).toISOString(),
       text: segment.text.trim(),
-    }));
-  }
-}
+    })),
+    lastUsedTimestamp: zeroTimestamp + filteredSegments.at(-1)!.end * 1_000,
+  };
+};