@dxos/plugin-transcription 0.9.1-main.c7dcc2e112 → 0.10.0

package/src/transcriber/transcriber.browser.test.ts

@@ -1,230 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import fs from 'node:fs';
-import path from 'node:path';
-import { describe, expect, test } from 'vitest';
-import { WaveFile } from 'wavefile';
-
-import { Trigger } from '@dxos/async';
-import { log } from '@dxos/log';
-import { openAndClose } from '@dxos/test-utils';
-import { TRACE_PROCESSOR, trace } from '@dxos/tracing';
-import { type ContentBlock } from '@dxos/types';
-
-import { type AudioChunk, type AudioRecorder, Transcriber } from '../transcriber';
-import { mergeFloat64Arrays } from '../util';
-
-// TODO(burdon): ReferenceError: Worker is not defined (Can only run with web workers).
-
-// This is a playground for testing the transcription, requires `calls-service` to be running.
-describe.skip('Transcriber', () => {
-  const DIR_PATH = path.join(__dirname, 'assets');
-
-  const readFile = async (filename: string) => {
-    return fs.readFileSync(path.join(DIR_PATH, filename));
-  };
-
-  const writeOutputFile = async (filename: string, data: Uint8Array) => {
-    const outputPath = path.join(DIR_PATH, 'out', filename);
-    if (!fs.existsSync(path.dirname(outputPath))) {
-      fs.mkdirSync(path.dirname(outputPath), { recursive: true });
-    }
-
-    return fs.writeFileSync(outputPath, data);
-  };
-
-  test.skip('split wav file', async () => {
-    // Read the WAV file
-    const fileBuffer = await readFile('./testing/test.wav');
-    const wav = new WaveFile(fileBuffer);
-
-    // Get PCM samples and format info.
-    const samples = wav.getSamples();
-    const format = wav.fmt as any;
-
-    // Take second half of samples.
-    const halfSamples = samples.slice(Math.floor(samples.length / 2));
-
-    // Create new WAV file with half the samples.
-    const halfWav = new WaveFile();
-    halfWav.fromScratch(1, format.sampleRate!, '16', halfSamples);
-
-    // Save the half-length file
-    await writeOutputFile('half.wav', halfWav.toBuffer());
-  });
-
-  test.skip('merge audio chunks', async () => {
-    const chunks: AudioChunk[] = [];
-    const trigger = new Trigger({ autoReset: true });
-
-    const recorder = new MockAudioRecorder({
-      buffer: await readFile('test.wav'),
-      chunkDuration: 3_000,
-    });
-
-    recorder.setOnChunk((chunk) => {
-      chunks.push(chunk);
-      trigger.wake();
-    });
-
-    await recorder.start();
-
-    const chunksNumber = 3;
-    for (let i = 0; i < chunksNumber; i++) {
-      recorder.emitChunk();
-    }
-
-    while (chunks.length < chunksNumber) {
-      await trigger.wait();
-    }
-
-    const wav = new WaveFile();
-    wav.fromScratch(
-      1, //
-      recorder.wavConfig.sampleRate,
-      '16',
-      mergeFloat64Arrays(chunks.map((chunk) => chunk.data)),
-    );
-    await writeOutputFile('merged.wav', wav.toBuffer());
-
-    await recorder.stop();
-  });
-
-  test.skip('transcription of audio recording', { timeout: 10_000 }, async () => {
-    const trigger = new Trigger<ContentBlock.Transcript[]>({ autoReset: true });
-    const recorder = new MockAudioRecorder({
-      buffer: await readFile('test.wav'),
-      chunkDuration: 3_000,
-    });
-    const transcription = new Transcriber({
-      recorder,
-      onSegments: async (segments) => {
-        log.info('transcription', { segments });
-        trigger.wake(segments);
-      },
-      config: {
-        transcribeAfterChunksAmount: 1000,
-        prefixBufferChunksAmount: 1,
-      },
-    });
-
-    await openAndClose(transcription);
-
-    transcription.startChunksRecording();
-
-    recorder.emitChunk();
-    recorder.emitChunk();
-    recorder.emitChunk();
-
-    transcription.stopChunksRecording();
-
-    // Could fail, not critical.
-    expect(
-      (await trigger.wait()).some((segment) => segment.text.includes('I will tell you some information about myself')),
-    );
-  });
-
-  test.skip('transcription of audio recording with overlapping chunks', { timeout: 20_000 }, async () => {
-    const trigger = new Trigger<ContentBlock.Transcript[]>({ autoReset: true });
-    const recorder = new MockAudioRecorder({
-      buffer: await readFile('test.wav'),
-      chunkDuration: 3_000,
-    });
-    const transcription = new Transcriber({
-      recorder,
-      onSegments: async (segments) => {
-        log.info('transcription', { segments });
-        trigger.wake(segments);
-      },
-      config: {
-        transcribeAfterChunksAmount: 1000,
-        prefixBufferChunksAmount: 1,
-      },
-    });
-
-    await openAndClose(transcription);
-
-    transcription.startChunksRecording();
-    recorder.emitChunk();
-    recorder.emitChunk();
-    transcription.stopChunksRecording();
-    expect((await trigger.wait()).some((segment) => segment.text.includes("Hello, I'm")));
-
-    transcription.startChunksRecording();
-    recorder.emitChunk();
-    recorder.emitChunk();
-    transcription.stopChunksRecording();
-    expect(
-      (await trigger.wait()).some((segment) => segment.text.includes('I will tell you some information about myself')),
-    );
-
-    log.info('Done', { trace: TRACE_PROCESSOR.getDiagnostics() });
-  });
-
-  test.skip('double file', async () => {
-    const fileBuffer = await readFile('test.wav');
-    const wav = new WaveFile(fileBuffer);
-    const samples = wav.getSamples();
-    const newWav = new WaveFile();
-    newWav.fromScratch(1, 24000, '16', mergeFloat64Arrays([samples, samples]));
-    await writeOutputFile('double.wav', newWav.toBuffer());
-  });
-});
-
-@trace.resource()
-class MockAudioRecorder implements AudioRecorder {
-  public chunks: AudioChunk[] = [];
-  public wav: WaveFile;
-  public chunkDuration: number;
-
-  private _onChunk?: (chunk: AudioChunk) => void = undefined;
-
-  constructor({ buffer, chunkDuration }: { buffer: Uint8Array; chunkDuration: number }) {
-    this.wav = new WaveFile(buffer);
-    this.chunkDuration = chunkDuration;
-  }
-
-  get wavConfig() {
-    return {
-      channels: 1,
-      sampleRate: (this.wav.fmt as any).sampleRate,
-      bitDepthCode: '16',
-    };
-  }
-
-  setOnChunk(onChunk: (chunk: AudioChunk) => void): void {
-    this._onChunk = onChunk;
-  }
-
-  /**
-   * Emit a chunk in controlled manner for testing purposes.
-   */
-  emitChunk(): void {
-    const chunk = this.chunks.shift();
-    if (!chunk) {
-      return;
-    }
-
-    this._onChunk!(chunk);
-  }
-
-  @trace.span()
-  async start(): Promise<void> {
-    log.info('start mock audio recorder', { wavProps: this.wav.fmt });
-    const now = Date.now();
-    const samples = this.wav.getSamples();
-    const sampleRate = this.wavConfig.sampleRate;
-    const increment = sampleRate * (this.chunkDuration / 1000);
-    for (let i = 0; i < samples.length; i += increment) {
-      const chunk = samples.slice(i, i + increment);
-      this.chunks.push({
-        timestamp: now + (i / increment) * this.chunkDuration,
-        data: chunk,
-      });
-    }
-  }
-
-  async stop(): Promise<void> {}
-}

package/src/transcriber/transcriber.test.ts

@@ -1,104 +0,0 @@
-//
-// Copyright 2026 DXOS.org
-//
-
-import { describe, test } from 'vitest';
-
-import { type AudioChunk } from './audio-recorder';
-import { alignWhisperSegments, type WhisperSegment } from './transcriber';
-
-const T0 = Date.parse('2026-01-01T00:00:00.000Z');
-
-const chunk = (offsetMs = 0, samples = 16): AudioChunk => ({
-  timestamp: T0 + offsetMs,
-  data: new Float64Array(samples),
-});
-
-const segment = (overrides: Partial<WhisperSegment>): WhisperSegment => ({
-  text: '',
-  start: 0,
-  end: 0,
-  no_speech_prob: 0,
-  words: [],
-  ...overrides,
-});
-
-describe('alignWhisperSegments', () => {
-  test('returns empty when no chunks were transcribed', ({ expect }) => {
-    const result = alignWhisperSegments(
-      [segment({ text: 'x', end: 1, words: [{ word: 'x', start: 0, end: 1 }] })],
-      [],
-      0,
-    );
-    expect(result.transcripts).toEqual([]);
-    expect(result.lastTimestamp).toBeUndefined();
-  });
-
-  test('absolute-timestamps and deduplicates segments against lastTimestamp', ({ expect }) => {
-    const segments: WhisperSegment[] = [
-      segment({
-        text: 'hello world',
-        start: 0,
-        end: 1,
-        words: [
-          { word: 'hello ', start: 0, end: 0.5 },
-          { word: 'world', start: 0.5, end: 1 },
-        ],
-      }),
-    ];
-
-    const first = alignWhisperSegments(segments, [chunk(0)], 0);
-    expect(first.transcripts).toHaveLength(1);
-    expect(first.transcripts[0]._tag).toBe('transcript');
-    expect(first.transcripts[0].text).toBe('hello world');
-    expect(new Date(first.transcripts[0].started).getTime()).toBe(T0);
-    expect(first.lastTimestamp).toBe(T0 + 1_000);
-
-    // Same segments, same chunk start: every word's absolute start is < lastTimestamp -> filtered out.
-    const second = alignWhisperSegments(segments, [chunk(0)], first.lastTimestamp!);
-    expect(second.transcripts).toEqual([]);
-    expect(second.lastTimestamp).toBeUndefined();
-  });
-
-  test('keeps later words when earlier words fall before lastTimestamp', ({ expect }) => {
-    const segments: WhisperSegment[] = [
-      segment({
-        text: 'old new',
-        start: 0,
-        end: 2,
-        words: [
-          { word: 'old ', start: 0, end: 1 },
-          { word: 'new', start: 1.5, end: 2 },
-        ],
-      }),
-    ];
-
-    // First word starts at T0, second at T0+1500 — only the second survives a cursor at T0+1000.
-    const result = alignWhisperSegments(segments, [chunk(0)], T0 + 1_000);
-    expect(result.transcripts).toHaveLength(1);
-    expect(result.transcripts[0].text).toBe('new');
-    expect(new Date(result.transcripts[0].started).getTime()).toBe(T0 + 1_500);
-    expect(result.lastTimestamp).toBe(T0 + 2_000);
-  });
-
-  test('drops segments that end before lastTimestamp', ({ expect }) => {
-    const segments: WhisperSegment[] = [
-      segment({
-        text: 'stale',
-        start: 0,
-        end: 0.5,
-        words: [{ word: 'stale', start: 0, end: 0.5 }],
-      }),
-      segment({
-        text: 'fresh',
-        start: 1,
-        end: 2,
-        words: [{ word: 'fresh', start: 1.5, end: 2 }],
-      }),
-    ];
-
-    const result = alignWhisperSegments(segments, [chunk(0)], T0 + 1_000);
-    expect(result.transcripts).toHaveLength(1);
-    expect(result.transcripts[0].text).toBe('fresh');
-  });
-});

package/src/transcriber/transcriber.ts

@@ -1,303 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { WaveFile } from 'wavefile';
-
-import { DeferredTask, Trigger } from '@dxos/async';
-import { type Context, LifecycleState, Resource } from '@dxos/context';
-import { log } from '@dxos/log';
-import { trace } from '@dxos/tracing';
-import { type ContentBlock } from '@dxos/types';
-
-import { mergeFloat64Arrays } from '../util';
-import { type AudioChunk, type AudioRecorder } from './audio-recorder';
-
-/**
- * Endpoint to the calls service.
- */
-const DEFAULT_TRANSCRIPTION_URL = 'https://calls-service.dxos.workers.dev';
-
-export type WhisperWord = {
-  word: string;
-
-  /**
-   * Time in seconds from the start of the audio.
-   */
-  start: number;
-
-  /**
-   * Time in seconds from the start of the audio.
-   */
-  end: number;
-};
-
-export type WhisperSegment = {
-  text: string;
-
-  /**
-   * Time in seconds from the start of the audio.
-   */
-  start: number;
-
-  /**
-   * Time in seconds from the start of the audio.
-   */
-  end: number;
-
-  /**
-   * Probability of no speech in the segment.
-   */
-  no_speech_prob: number;
-
-  words: WhisperWord[];
-};
-
-export type TranscribeConfig = {
-  /**
-   * Number of chunks to transcribe automatically after.
-   */
-  transcribeAfterChunksAmount: number;
-
-  /**
-   * Number of chunks to save before the user starts speaking.
-   * This is needed to catch the beginning of the user's speech.
-   */
-  prefixBufferChunksAmount: number;
-
-  /**
-   * Override the transcription endpoint base URL.
-   * Defaults to the DXOS calls service.
-   */
-  endpoint?: string;
-};
-
-/**
- * Function that converts a base64-encoded WAV payload into Whisper segments.
- * Allows callers to swap in alternative providers or mock the transport.
- */
-export type TranscribeFn = (audio: string) => Promise<WhisperSegment[]>;
-
-export type TranscriberProps = {
-  config: TranscribeConfig;
-  recorder: AudioRecorder;
-  /**
-   * Callback to handle the transcribed segments, after all segment transformers are applied.
-   * @param segments - The transcribed segments.
-   */
-  onSegments: (segments: ContentBlock.Transcript[]) => Promise<void>;
-  /**
-   * Optional override of the transcription transport. When provided, supersedes `config.endpoint`.
-   */
-  transcribe?: TranscribeFn;
-};
-
-/**
- * Handles transcription of audio chunks.
- * If user is not speaking, the last `minChunksAmount` chunks are saved and transcribed.
- * If user is speaking, the chunks are added to the buffer until the user is done talking.
- */
-export class Transcriber extends Resource {
-  private _audioChunks: AudioChunk[] = [];
-  private _lastTimestamp = 0;
-
-  private readonly _openTrigger = new Trigger({ autoReset: true });
-
-  private readonly _config: TranscribeConfig;
-  private readonly _recorder: AudioRecorder;
-  private readonly _onSegments: TranscriberProps['onSegments'];
-  private readonly _transcribeFn?: TranscribeFn;
-
-  private _recording = false;
-  private _transcribeTask?: DeferredTask = undefined;
-
-  constructor({ config, recorder, onSegments, transcribe }: TranscriberProps) {
-    super();
-    this._config = config;
-    this._recorder = recorder;
-    this._onSegments = onSegments;
-    this._transcribeFn = transcribe;
-  }
-
-  protected override async _open(ctx: Context): Promise<void> {
-    log.info('opening');
-    this._recorder.setOnChunk((chunk) => this._saveAudioChunk(chunk));
-    await this._recorder.start();
-    this._transcribeTask = new DeferredTask(ctx, async () => this._transcribe());
-    this._openTrigger.wake();
-  }
-
-  protected override async _close(): Promise<void> {
-    log.info('closing');
-    this._recording = false;
-    this._transcribeTask = undefined;
-    await this._recorder.stop();
-  }
-
-  startChunksRecording(): void {
-    log.info('starting');
-    if (this._lifecycleState !== LifecycleState.OPEN) {
-      return;
-    }
-
-    this._recording = true;
-  }
-
-  stopChunksRecording(): void {
-    if (this._lifecycleState !== LifecycleState.OPEN || !this._recording) {
-      return;
-    }
-
-    this._recording = false;
-    this._transcribeTask?.schedule();
-    log.info('stopped');
-  }
-
-  private _saveAudioChunk(chunk: AudioChunk): void {
-    log('saving audio chunk', { chunk });
-    this._audioChunks.push(chunk);
-
-    // Clean the buffer if the user is not speaking and the transcription task is not scheduled.
-    if (
-      !this._recording &&
-      (!this._transcribeTask || !this._transcribeTask.scheduled) &&
-      this._audioChunks.length >= this._config.prefixBufferChunksAmount
-    ) {
-      this._audioChunks = this._dropOldChunks();
-    } else if (this._audioChunks.length >= this._config.transcribeAfterChunksAmount && this._recording) {
-      this._transcribeTask?.schedule();
-    }
-  }
-
-  private async _transcribe(): Promise<void> {
-    log('transcribing', { chunks: this._audioChunks.length });
-    const chunks = this._audioChunks;
-    this._audioChunks = this._dropOldChunks();
-    if (chunks.length === 0) {
-      return;
-    }
-
-    const audio = await this._mergeAudioChunks(chunks);
-    const segments = await this._fetchTranscription(audio);
-    if (!Array.isArray(segments) || segments.length === 0) {
-      return;
-    }
-
-    const alignedSegments = this._alignSegments(segments, chunks);
-    if (alignedSegments.length > 0) {
-      await this._onSegments(alignedSegments);
-    }
-  }
-
-  private _dropOldChunks(): AudioChunk[] {
-    return this._config.prefixBufferChunksAmount > 0
-      ? this._audioChunks.slice(-this._config.prefixBufferChunksAmount)
-      : [];
-  }
-
-  @trace.span({ showInBrowserTimeline: true })
-  private async _mergeAudioChunks(chunks: AudioChunk[]): Promise<string> {
-    const file = new WaveFile();
-    const wavConfig = this._recorder.wavConfig;
-
-    file.fromScratch(
-      wavConfig.channels,
-      wavConfig.sampleRate,
-      wavConfig.bitDepthCode,
-      mergeFloat64Arrays(chunks.map(({ data }) => data)),
-    );
-
-    return file.toBase64();
-  }
-
-  @trace.span({ showInBrowserTimeline: true })
-  private async _fetchTranscription(audio: string): Promise<WhisperSegment[]> {
-    if (audio.length === 0) {
-      throw new Error('No audio to send for transcribing');
-    }
-
-    let segments: unknown;
-    if (this._transcribeFn) {
-      segments = await this._transcribeFn(audio);
-    } else {
-      // TODO(burdon): Create separate endpoint?
-      const endpoint = this._config.endpoint ?? DEFAULT_TRANSCRIPTION_URL;
-      const response = await fetch(`${endpoint}/transcribe`, {
-        method: 'POST',
-        body: JSON.stringify({ audio }),
-        headers: {
-          'Content-Type': 'application/json',
-        },
-      });
-
-      if (!response.ok) {
-        throw new Error(`Transcription failed: ${response.statusText}`);
-      }
-
-      ({ segments } = (await response.json()) as { segments?: unknown });
-    }
-
-    if (!Array.isArray(segments)) {
-      throw new Error('Transcription response payload is invalid');
-    }
-
-    log.info('transcription response', { segments: segments.length });
-    return segments as WhisperSegment[];
-  }
-
-  private _alignSegments(segments: WhisperSegment[], originalChunks: AudioChunk[]): ContentBlock.Transcript[] {
-    const result = alignWhisperSegments(segments, originalChunks, this._lastTimestamp);
-    if (result.lastTimestamp !== undefined) {
-      this._lastTimestamp = result.lastTimestamp;
-    }
-
-    return result.transcripts;
-  }
-}
-
-/**
- * Pure helper that filters and absolute-timestamps Whisper segments against a chunk timeline.
- * Exported for testing.
- *
- * @param segments - Raw segments returned by the Whisper service for `originalChunks`.
- * @param originalChunks - Chunks fed into the transcription request (used for the zero timestamp).
- * @param lastTimestamp - Most recent absolute timestamp already emitted; segments whose end
- *   falls strictly before this point are deduped out, as are individual words whose start does.
- */
-export const alignWhisperSegments = (
-  segments: WhisperSegment[],
-  originalChunks: AudioChunk[],
-  lastTimestamp: number,
-): { transcripts: ContentBlock.Transcript[]; lastTimestamp?: number } => {
-  if (originalChunks.length === 0) {
-    return { transcripts: [] };
-  }
-
-  const zeroTimestamp = originalChunks[0].timestamp;
-
-  const filteredSegments = segments
-    .filter((segment) => zeroTimestamp + segment.end * 1_000 >= lastTimestamp)
-    .map((segment) => {
-      const words = segment.words.filter((word) => zeroTimestamp + word.start * 1_000 >= lastTimestamp);
-      return {
-        ...segment,
-        words,
-        text: words.map((word) => word.word).join(''),
-        start: words.at(0)?.start ?? segment.end,
-      };
-    })
-    .filter((segment) => segment.words.length > 0);
-
-  if (filteredSegments.length === 0) {
-    return { transcripts: [] };
-  }
-
-  return {
-    lastTimestamp: zeroTimestamp + filteredSegments.at(-1)!.end * 1_000,
-    transcripts: filteredSegments.map((segment) => ({
-      _tag: 'transcript',
-      started: new Date(zeroTimestamp + segment.start * 1_000).toISOString(),
-      text: segment.text.trim(),
-    })),
-  };
-};

package/src/util/array.ts

@@ -1,14 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-export const mergeFloat64Arrays = (arrays: Float64Array[]) => {
-  const newArray = new Float64Array(arrays.reduce((acc, array) => array.length + acc, 0));
-  let offset = 0;
-  for (const array of arrays) {
-    newArray.set(array, offset);
-    offset += array.length;
-  }
-
-  return newArray;
-};

package/src/util/render-byline.ts

@@ -1,33 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { type Identity } from '@dxos/react-client/halo';
-import { type Message } from '@dxos/types';
-
-// TODO(thure): Move this?
-export const renderByline =
-  (identities: Identity[]) =>
-  (message: Message.Message, index: number, debug = false): string[] => {
-    if (message.sender.role === 'assistant') {
-      // Start/stop block.
-      return [message.blocks.find((block) => block._tag === 'transcript')?.text ?? '', ''];
-    }
-
-    // TODO(burdon): Use link/reference markup for users (with popover).
-    // TODO(burdon): Color and avatar.
-    const identity = identities.find((identity) => identity.did === message.sender.identityDid);
-    const name =
-      identity?.profile?.displayName ??
-      message.sender.contact?.target?.fullName ??
-      message.sender.name ??
-      message.sender.email ??
-      message.sender.identityDid;
-    const blocks = message.blocks.filter((block) => block._tag === 'transcript');
-    return [
-      // TODO(thure): Use an XML tag with the bits needed here.
-      `###### ${name}` + (debug ? ` [${index}]:${message.id}` : ''),
-      blocks.map((block) => block.text.trim()).join(' '),
-      '',
-    ];
-  };