@dxos/plugin-transcription 0.7.5-main.b19bfc8

package/src/index.ts

@@ -0,0 +1,11 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './capabilities';
+export * from './components';
+export * from './hooks';
+export * from './meta';
+export * from './transcriber';
+
+export * from './TranscriptionPlugin';

package/src/meta.ts

@@ -0,0 +1,17 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import { type PluginMeta } from '@dxos/app-framework';
+
+export const TRANSCRIPTION_PLUGIN = 'dxos.org/plugin/transcription';
+
+export const meta = {
+  id: TRANSCRIPTION_PLUGIN,
+  name: 'Transcription',
+  description: 'Voice transcription.',
+  icon: 'ph--microphone--regular',
+  source: 'https://github.com/dxos/dxos/tree/main/packages/plugins/experimental/plugin-transcription',
+  tags: ['experimental'],
+  screenshots: [],
+} satisfies PluginMeta;

package/src/sanity.test.ts

@@ -0,0 +1,11 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { describe, expect, test } from 'vitest';
+
+describe('sanity', () => {
+  test('sanity', () => {
+    expect(true).toBe(true);
+  });
+});

package/src/transcriber/audio-recorder.ts

@@ -0,0 +1,56 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export type AudioChunk = {
+  /**
+   * Timestamp of the beginning of the chunk.
+   */
+  timestamp: number;
+
+  /**
+   * 16-bit PCM data.
+   */
+  data: Float64Array;
+};
+
+export type WavConfig = {
+  channels: number;
+  sampleRate: number;
+  bitDepthCode: string;
+};
+
+/**
+ * Records MediaStreamTrack to Uint8Array 16-bit PCM data.
+ * The data is passed to the onChunk callback with the timestamp of the chunk.
+ * All chunks are aligned and sequential.
+ * Time ─────────────────────────────────────────────────►
+ *
+ * Recording:
+ * t1    ┌──────┐
+ * t2    ┌──────┬──────┐
+ * t3    ┌──────┬──────┬──────┐
+ * t4    ┌──────┬──────┬──────┬──────┐
+ *       └──────┴──────┴──────┴──────┘
+ *         1st     2nd     3rd     4th
+ *        chunk   chunk   chunk   chunk
+ *
+ */
+export interface AudioRecorder {
+  wavConfig: WavConfig;
+
+  /**
+   * Sets callback for receiving audio chunks.
+   */
+  setOnChunk(onChunk: (chunk: AudioChunk) => void): void;
+
+  /**
+   * Starts the recorder.
+   */
+  start(): Promise<void>;
+
+  /**
+   * Stops the recorder.
+   */
+  stop(): Promise<void>;
+}

package/src/transcriber/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './audio-recorder';
+export * from './media-stream-recorder';
+export * from './transcriber';

package/src/transcriber/media-stream-recorder.ts

@@ -0,0 +1,109 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type IBlobEvent, type IMediaRecorder, MediaRecorder, register } from 'extendable-media-recorder';
+import { WaveFile } from 'wavefile';
+
+import { synchronized } from '@dxos/async';
+import { invariant } from '@dxos/invariant';
+import { log } from '@dxos/log';
+import { trace } from '@dxos/tracing';
+
+import { type WavConfig, type AudioChunk, type AudioRecorder } from './audio-recorder';
+
+let initializingPromise: Promise<void> | undefined;
+
+const initializeExtendableMediaRecorder = async () => {
+  const { connect } = await import('extendable-media-recorder-wav-encoder');
+  await register(await connect());
+};
+
+/**
+ * Recorder that uses the MediaContext API and AudioNode API to record audio.
+ *
+ * It records MediaStream using https://www.npmjs.com/package/extendable-media-recorder.
+ */
+@trace.resource()
+export class MediaStreamRecorder implements AudioRecorder {
+  private readonly _mediaStreamTrack: MediaStreamTrack;
+  /**
+   * Default MediaRecorder implementation do not support wav encoding.
+   */
+  private readonly _interval: number;
+  private _mediaRecorder?: IMediaRecorder = undefined;
+  private _header?: Uint8Array = undefined;
+  private _onChunk?: (chunk: AudioChunk) => void;
+
+  constructor({ mediaStreamTrack, interval }: { mediaStreamTrack: MediaStreamTrack; interval: number }) {
+    this._mediaStreamTrack = mediaStreamTrack;
+    this._interval = interval;
+  }
+
+  get wavConfig(): WavConfig {
+    const settings = this._mediaStreamTrack.getSettings();
+    return {
+      channels: settings.channelCount ?? 1,
+      sampleRate: settings.sampleRate ?? 16000,
+      bitDepthCode: settings.sampleSize ? String(settings.sampleSize) : '16',
+    };
+  }
+
+  setOnChunk(onChunk: (chunk: AudioChunk) => void): void {
+    this._onChunk = onChunk;
+  }
+
+  async start() {
+    await (initializingPromise ??= initializeExtendableMediaRecorder()).catch((err) =>
+      log.info('initializeExtendableMediaRecorder', { err }),
+    );
+
+    if (!this._mediaRecorder) {
+      const stream = new MediaStream([this._mediaStreamTrack]);
+      this._mediaRecorder = new MediaRecorder(stream, { mimeType: 'audio/wav' });
+    }
+
+    if (this._mediaRecorder.state === 'recording') {
+      return;
+    }
+    invariant(this._onChunk, 'MediaStreamRecorder: onChunk is not set');
+    this._mediaRecorder.ondataavailable = (event) => this._ondataavailable(event);
+    this._mediaRecorder.start(this._interval);
+  }
+
+  async stop() {
+    if (this._mediaRecorder?.state !== 'recording') {
+      return;
+    }
+    this._mediaRecorder.stop();
+  }
+
+  @synchronized
+  private async _ondataavailable(event: IBlobEvent) {
+    const blob = event.data;
+    const uint8Array = new Uint8Array(await blob.arrayBuffer());
+
+    // First chunk from the MediaRecorder has a header.
+    const isHeader =
+      uint8Array[0] === 0x52 && // R
+      uint8Array[1] === 0x49 && // I
+      uint8Array[2] === 0x46 && // F
+      uint8Array[3] === 0x46; // F
+    let wav: WaveFile;
+
+    if (isHeader) {
+      wav = new WaveFile(uint8Array);
+      this._header = uint8Array.slice(0, 44);
+    } else if (this._header) {
+      wav = new WaveFile(new Uint8Array([...this._header, ...uint8Array]));
+    } else {
+      log.warn('MediaStreamRecorder: no header');
+      return;
+    }
+
+    this._onChunk!({
+      timestamp: Date.now(),
+      data: wav.getSamples(),
+    });
+  }
+}

package/src/transcriber/transcriber.test.ts

@@ -0,0 +1,230 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { describe, expect, test } from 'vitest';
+import { WaveFile } from 'wavefile';
+
+import { Trigger } from '@dxos/async';
+import { log } from '@dxos/log';
+import { openAndClose } from '@dxos/test-utils';
+import { trace, TRACE_PROCESSOR } from '@dxos/tracing';
+
+import { type AudioChunk, type AudioRecorder, Transcriber } from '../transcriber';
+import { type TranscriptSegment } from '../types';
+import { mergeFloat64Arrays } from '../util';
+
+// TODO(burdon): ReferenceError: Worker is not defined (Can only run with web workers).
+
+// This is a playground for testing the transcription, requires `calls-service` to be running.
+describe.skip('Transcriber', () => {
+  const DIR_PATH = path.join(__dirname, 'assets');
+
+  const readFile = async (filename: string) => {
+    return fs.readFileSync(path.join(DIR_PATH, filename));
+  };
+
+  const writeOutputFile = async (filename: string, data: Uint8Array) => {
+    const outputPath = path.join(DIR_PATH, 'out', filename);
+    if (!fs.existsSync(path.dirname(outputPath))) {
+      fs.mkdirSync(path.dirname(outputPath), { recursive: true });
+    }
+
+    return fs.writeFileSync(outputPath, data);
+  };
+
+  test.skip('split wav file', async () => {
+    // Read the WAV file
+    const fileBuffer = await readFile('./testing/test.wav');
+    const wav = new WaveFile(fileBuffer);
+
+    // Get PCM samples and format info.
+    const samples = wav.getSamples();
+    const format = wav.fmt as any;
+
+    // Take second half of samples.
+    const halfSamples = samples.slice(Math.floor(samples.length / 2));
+
+    // Create new WAV file with half the samples.
+    const halfWav = new WaveFile();
+    halfWav.fromScratch(1, format.sampleRate!, '16', halfSamples);
+
+    // Save the half-length file
+    await writeOutputFile('half.wav', halfWav.toBuffer());
+  });
+
+  test.skip('merge audio chunks', async () => {
+    const chunks: AudioChunk[] = [];
+    const trigger = new Trigger({ autoReset: true });
+
+    const recorder = new MockAudioRecorder({
+      buffer: await readFile('test.wav'),
+      chunkDuration: 3_000,
+    });
+
+    recorder.setOnChunk((chunk) => {
+      chunks.push(chunk);
+      trigger.wake();
+    });
+
+    await recorder.start();
+
+    const chunksNumber = 3;
+    for (let i = 0; i < chunksNumber; i++) {
+      recorder.emitChunk();
+    }
+
+    while (chunks.length < chunksNumber) {
+      await trigger.wait();
+    }
+
+    const wav = new WaveFile();
+    wav.fromScratch(
+      1, //
+      recorder.wavConfig.sampleRate,
+      '16',
+      mergeFloat64Arrays(chunks.map((chunk) => chunk.data)),
+    );
+    await writeOutputFile('merged.wav', wav.toBuffer());
+
+    await recorder.stop();
+  });
+
+  test.skip('transcription of audio recording', { timeout: 10_000 }, async () => {
+    const trigger = new Trigger<TranscriptSegment[]>({ autoReset: true });
+    const recorder = new MockAudioRecorder({
+      buffer: await readFile('test.wav'),
+      chunkDuration: 3_000,
+    });
+    const transcription = new Transcriber({
+      recorder,
+      onSegments: async (segments) => {
+        log.info('transcription', { segments });
+        trigger.wake(segments);
+      },
+      config: {
+        transcribeAfterChunksAmount: 1000,
+        prefixBufferChunksAmount: 1,
+      },
+    });
+
+    await openAndClose(transcription);
+
+    transcription.startChunksRecording();
+
+    recorder.emitChunk();
+    recorder.emitChunk();
+    recorder.emitChunk();
+
+    transcription.stopChunksRecording();
+
+    // Could fail, not critical.
+    expect(
+      (await trigger.wait()).some((segment) => segment.text.includes('I will tell you some information about myself')),
+    );
+  });
+
+  test.skip('transcription of audio recording with overlapping chunks', { timeout: 20_000 }, async () => {
+    const trigger = new Trigger<TranscriptSegment[]>({ autoReset: true });
+    const recorder = new MockAudioRecorder({
+      buffer: await readFile('test.wav'),
+      chunkDuration: 3_000,
+    });
+    const transcription = new Transcriber({
+      recorder,
+      onSegments: async (segments) => {
+        log.info('transcription', { segments });
+        trigger.wake(segments);
+      },
+      config: {
+        transcribeAfterChunksAmount: 1000,
+        prefixBufferChunksAmount: 1,
+      },
+    });
+
+    await openAndClose(transcription);
+
+    transcription.startChunksRecording();
+    recorder.emitChunk();
+    recorder.emitChunk();
+    transcription.stopChunksRecording();
+    expect((await trigger.wait()).some((segment) => segment.text.includes("Hello, I'm")));
+
+    transcription.startChunksRecording();
+    recorder.emitChunk();
+    recorder.emitChunk();
+    transcription.stopChunksRecording();
+    expect(
+      (await trigger.wait()).some((segment) => segment.text.includes('I will tell you some information about myself')),
+    );
+
+    log.info('Done', { trace: TRACE_PROCESSOR.getDiagnostics() });
+  });
+
+  test.skip('double file', async () => {
+    const fileBuffer = await readFile('test.wav');
+    const wav = new WaveFile(fileBuffer);
+    const samples = wav.getSamples();
+    const newWav = new WaveFile();
+    newWav.fromScratch(1, 24000, '16', mergeFloat64Arrays([samples, samples]));
+    await writeOutputFile('double.wav', newWav.toBuffer());
+  });
+});
+
+@trace.resource()
+class MockAudioRecorder implements AudioRecorder {
+  public chunks: AudioChunk[] = [];
+  public wav: WaveFile;
+  public chunkDuration: number;
+
+  private _onChunk?: (chunk: AudioChunk) => void = undefined;
+
+  constructor({ buffer, chunkDuration }: { buffer: Uint8Array; chunkDuration: number }) {
+    this.wav = new WaveFile(buffer);
+    this.chunkDuration = chunkDuration;
+  }
+
+  get wavConfig() {
+    return {
+      channels: 1,
+      sampleRate: (this.wav.fmt as any).sampleRate,
+      bitDepthCode: '16',
+    };
+  }
+
+  setOnChunk(onChunk: (chunk: AudioChunk) => void) {
+    this._onChunk = onChunk;
+  }
+
+  /**
+   * Emit a chunk in controlled manner for testing purposes.
+   */
+  emitChunk() {
+    const chunk = this.chunks.shift();
+    if (!chunk) {
+      return;
+    }
+
+    this._onChunk!(chunk);
+  }
+
+  @trace.span()
+  async start() {
+    log.info('start mock audio recorder', { wavParams: this.wav.fmt });
+    const now = Date.now();
+    const samples = this.wav.getSamples();
+    const sampleRate = this.wavConfig.sampleRate;
+    const increment = sampleRate * (this.chunkDuration / 1000);
+    for (let i = 0; i < samples.length; i += increment) {
+      const chunk = samples.slice(i, i + increment);
+      this.chunks.push({
+        timestamp: now + (i / increment) * this.chunkDuration,
+        data: chunk,
+      });
+    }
+  }
+
+  async stop() {}
+}

package/src/transcriber/transcriber.ts

@@ -0,0 +1,247 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { WaveFile } from 'wavefile';
+
+import { DeferredTask, Trigger } from '@dxos/async';
+import { type Context, LifecycleState, Resource } from '@dxos/context';
+import { log } from '@dxos/log';
+import { trace } from '@dxos/tracing';
+
+import { type AudioRecorder, type AudioChunk } from './audio-recorder';
+import { TRANSCRIPTION_URL, type TranscriptSegment } from '../types';
+import { mergeFloat64Arrays } from '../util';
+
+type WhisperWord = {
+  word: string;
+
+  /**
+   * Time in seconds from the start of the audio.
+   */
+  start: number;
+
+  /**
+   * Time in seconds from the start of the audio.
+   */
+  end: number;
+};
+
+type WhisperSegment = {
+  text: string;
+
+  /**
+   * Time in seconds from the start of the audio.
+   */
+  start: number;
+
+  /**
+   * Time in seconds from the start of the audio.
+   */
+  end: number;
+
+  /**
+   * Probability of no speech in the segment.
+   */
+  no_speech_prob: number;
+
+  words: WhisperWord[];
+};
+
+export type TranscribeConfig = {
+  /**
+   * Number of chunks to transcribe automatically after.
+   */
+  transcribeAfterChunksAmount: number;
+
+  /**
+   * Number of chunks to save before the user starts speaking.
+   * This is needed to catch the beginning of the user's speech.
+   */
+  prefixBufferChunksAmount: number;
+};
+
+export type TranscriberParams = {
+  config: TranscribeConfig;
+  recorder: AudioRecorder;
+  onSegments: (segments: TranscriptSegment[]) => Promise<void>;
+};
+
+/**
+ * Handles transcription of audio chunks.
+ * If user is not speaking, the last `minChunksAmount` chunks are saved and transcribed.
+ * If user is speaking, the chunks are added to the buffer until the user is done talking.
+ */
+export class Transcriber extends Resource {
+  private _audioChunks: AudioChunk[] = [];
+  private _lastUsedTimestamp = 0;
+
+  private readonly _openTrigger = new Trigger({ autoReset: true });
+
+  private readonly _config: TranscribeConfig;
+  private readonly _recorder: AudioRecorder;
+  private readonly _onSegments: TranscriberParams['onSegments'];
+
+  private _recording = false;
+  private _transcribeTask?: DeferredTask = undefined;
+
+  constructor({ config, recorder, onSegments }: TranscriberParams) {
+    super();
+    this._config = config;
+    this._recorder = recorder;
+    this._onSegments = onSegments;
+  }
+
+  protected override async _open(ctx: Context) {
+    this._recorder.setOnChunk((chunk) => this._saveAudioChunk(chunk));
+    await this._recorder.start();
+    this._transcribeTask = new DeferredTask(ctx, async () => this._transcribe());
+    this._openTrigger.wake();
+  }
+
+  protected override async _close() {
+    this._recording = false;
+    this._transcribeTask = undefined;
+    await this._recorder.stop();
+  }
+
+  startChunksRecording() {
+    if (this._lifecycleState !== LifecycleState.OPEN) {
+      return;
+    }
+    this._recording = true;
+  }
+
+  stopChunksRecording() {
+    if (this._lifecycleState !== LifecycleState.OPEN || !this._recording) {
+      return;
+    }
+    this._recording = false;
+    this._transcribeTask?.schedule();
+  }
+
+  private _saveAudioChunk(chunk: AudioChunk) {
+    log('saving audio chunk', { chunk });
+    this._audioChunks.push(chunk);
+
+    // Clean the buffer if the user is not speaking and the transcription task is not scheduled.
+    if (
+      !this._recording &&
+      (!this._transcribeTask || !this._transcribeTask.scheduled) &&
+      this._audioChunks.length >= this._config.prefixBufferChunksAmount
+    ) {
+      this._audioChunks = this._dropOldChunks();
+    } else if (this._audioChunks.length >= this._config.transcribeAfterChunksAmount && this._recording) {
+      this._transcribeTask?.schedule();
+    }
+  }
+
+  private async _transcribe() {
+    log('transcribing', { chunks: this._audioChunks.length });
+    const chunks = this._audioChunks;
+    this._audioChunks = this._dropOldChunks();
+    if (chunks.length === 0) {
+      return;
+    }
+
+    const audio = await this._mergeAudioChunks(chunks);
+    const segments = await this._fetchTranscription(audio);
+    if (!Array.isArray(segments) || segments.length === 0) {
+      return;
+    }
+
+    const alignedSegments = this._alignSegments(segments, chunks);
+    if (alignedSegments.length > 0) {
+      await this._onSegments(alignedSegments);
+    }
+  }
+
+  private _dropOldChunks() {
+    return this._config.prefixBufferChunksAmount > 0
+      ? this._audioChunks.slice(-this._config.prefixBufferChunksAmount)
+      : [];
+  }
+
+  @trace.span({ showInBrowserTimeline: true })
+  private async _mergeAudioChunks(chunks: AudioChunk[]) {
+    const file = new WaveFile();
+    const wavConfig = this._recorder.wavConfig;
+
+    file.fromScratch(
+      wavConfig.channels,
+      wavConfig.sampleRate,
+      wavConfig.bitDepthCode,
+      mergeFloat64Arrays(chunks.map(({ data }) => data)),
+    );
+
+    return file.toBase64();
+  }
+
+  @trace.span({ showInBrowserTimeline: true })
+  private async _fetchTranscription(audio: string) {
+    if (audio.length === 0) {
+      this._audioChunks = [];
+      throw new Error('No audio to send for transcribing');
+    }
+
+    const response = await fetch(`${TRANSCRIPTION_URL}/transcribe`, {
+      method: 'POST',
+      body: JSON.stringify({ audio }),
+      headers: {
+        'Content-Type': 'application/json',
+      },
+    });
+
+    if (!response.ok) {
+      this._audioChunks = [];
+      throw new Error('Failed to transcribe');
+    }
+
+    const {
+      segments,
+    }: {
+      segments: WhisperSegment[];
+    } = await response.json();
+
+    log.info('transcription response', {
+      segments,
+      string: segments.map((segments) => segments.text).join(' '),
+    });
+
+    return segments;
+  }
+
+  private _alignSegments(segments: WhisperSegment[], originalChunks: AudioChunk[]): TranscriptSegment[] {
+    // Absolute zero for all relative timestamps in the segments.
+    const zeroTimestamp = originalChunks.at(0)!.timestamp;
+
+    const filteredSegments = segments
+      // Use segments that end after the last used timestamp.
+      // Segments could overlap, so we need to filter words that starts after the last used timestamp.
+      .filter((segment) => zeroTimestamp + segment.end * 1_000 >= this._lastUsedTimestamp)
+      // Use words that starts after the last used timestamp.
+      .map((segment) => {
+        const words = segment.words.filter((word) => zeroTimestamp + word.start * 1_000 >= this._lastUsedTimestamp);
+        return {
+          ...segment,
+          words,
+          text: words.map((word) => word.word).join(''),
+          start: words.at(0)?.start ?? segment.end,
+        };
+      })
+      .filter((segment) => segment.words.length > 0);
+
+    if (filteredSegments.length === 0) {
+      return [];
+    }
+
+    // Update last timestamp.
+    this._lastUsedTimestamp = zeroTimestamp + filteredSegments.at(-1)!.end * 1_000;
+
+    // Add absolute timestamp to each segment.
+    return filteredSegments.map((segment) => ({
+      started: new Date(zeroTimestamp + segment.start * 1_000),
+      text: segment.text,
+    }));
+  }
+}