@dxos/plugin-transcription 0.8.1 → 0.8.2-main.10c050d

package/src/segments-normalization/normalization.ts

@@ -0,0 +1,103 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Schema } from 'effect';
+
+import { DEFAULT_EDGE_MODEL } from '@dxos/ai';
+import { AISession } from '@dxos/assistant';
+import { AiService, defineFunction } from '@dxos/functions';
+import { ObjectId } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { DataType } from '@dxos/schema';
+
+const MessageWithRangeId = Schema.extend(
+  DataType.Message,
+  Schema.Struct({
+    // TODO(mykola): Move to meta in DataType.Message. Use `DataType.Message` instead.
+    rangeId: Schema.optional(Schema.Array(Schema.String)).annotations({
+      description: 'The IDs of the messages that contain the sentences.',
+    }),
+  }),
+);
+
+export interface MessageWithRangeId extends Schema.Schema.Type<typeof MessageWithRangeId> {}
+
+export const NormalizationInput = Schema.Struct({
+  // TODO(mykola): Move to meta in DataType.Message. Use `DataType.Message` instead.
+  messages: Schema.Array(MessageWithRangeId).annotations({
+    description: 'Messages to normalize into sentences.',
+  }),
+});
+
+export const NormalizationOutput = Schema.Struct({
+  sentences: Schema.Array(MessageWithRangeId.pipe(Schema.mutable)).pipe(Schema.mutable).annotations({
+    description: 'The sentences of the transcript.',
+  }),
+});
+
+const prompt = `
+You are observing a real-time transcript of a single person speaking.
+The transcription is delivered in chunks of 10 seconds or less. As a result, individual sentences may be split across multiple messages, or multiple sentences may appear within a single message. Additionally, because this is real-time transcription, punctuation and capitalization may be incorrect or missing.
+
+# Task Description:
+  - Your task is to detect and reconstruct broken or incomplete sentences by merging segments into coherent, grammatically correct sentences where appropriate.
+
+# Input Format:
+  - The input is an array of messages that contain transcription blocks.
+  - Each message is a DataType.Message, as described in the output format.
+
+# Output Format:
+  - You have been provided with the tool that defines the output format; make sure to query it.
+  - Do not output anything other than the expected format.
+
+# Segment Handling Rules:
+  - Sort messages by timestamp.
+  - Leave ID of the first message in a sentence.
+  - Each divided sentence should be added to the output as a separate message or block within existing messages, preserving the original order.
+  - If one message contains multiple sentences, you should not split them.
+  - The last sentence may be incomplete; it should still be output as a separate block or message.
+  - Keep track of the original timestamps of the messages and blocks and, use the timestamp of the first message/block of a sentence as the 'started' field of the merged message.
+  - The 'rangeId' field of the merged message should include 'messageId'-s and 'rangeId'-s of all messages that have been used to construct this message.
+
+# Punctuation and Capitalization:
+  - Do not rely on the original punctuation and capitalization—they may be incorrect.
+  - Use logical reasoning to apply appropriate punctuation (e.g., period, comma, question mark, exclamation mark) and capitalization.
+
+# Restrictions:
+  - Do not alter the order of the original messages; maintain the natural flow of speech.
+  - Do not interpret or infer meaning beyond reconstructing sentences.
+  - Do not add or remove any words or phrases.
+`;
+
+export const sentenceNormalization = defineFunction({
+  description: 'Post process of transcription for sentence normalization',
+  inputSchema: NormalizationInput,
+  outputSchema: NormalizationOutput,
+  handler: async ({ data: { messages }, context }) => {
+    log.info('input', { messages });
+    const ai = context.getService(AiService);
+    const session = new AISession({ operationModel: 'configured' });
+
+    const response = await session.runStructured(NormalizationOutput, {
+      generationOptions: { model: DEFAULT_EDGE_MODEL },
+      client: ai.client,
+      tools: [],
+      artifacts: [],
+      history: [
+        {
+          id: ObjectId.random(),
+          role: 'user',
+          content: messages.map((message) => ({ type: 'text', text: JSON.stringify(message) })),
+        },
+      ],
+      prompt,
+    });
+
+    response.sentences.forEach((sentence) => {
+      sentence.id = ObjectId.random();
+    });
+    log.info('response', { response });
+    return response;
+  },
+});

package/src/segments-normalization/utils.ts

@@ -0,0 +1,9 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type DataType } from '@dxos/schema';
+
+export const getActorId = (actor: DataType.Actor) => {
+  return actor.identityDid || actor.identityKey || actor.email || actor.name || '';
+};

package/src/testing/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './testing';

package/src/testing/testing.ts

@@ -0,0 +1,216 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Schema } from 'effect';
+import { useEffect, useMemo, useState } from 'react';
+
+import { AIServiceEdgeClient } from '@dxos/ai';
+import { AI_SERVICE_ENDPOINT } from '@dxos/ai/testing';
+import { extractionAnthropicFn, processTranscriptMessage } from '@dxos/assistant';
+import { scheduleTaskInterval } from '@dxos/async';
+import { Filter, type Queue } from '@dxos/client/echo';
+import { Context } from '@dxos/context';
+import { Type } from '@dxos/echo';
+import { create, createQueueDxn, ObjectId } from '@dxos/echo-schema';
+import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
+import { IdentityDid } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { faker } from '@dxos/random';
+import { live, makeRef, useQueue, type Space } from '@dxos/react-client/echo';
+import { DataType } from '@dxos/schema';
+import { Testing, seedTestData } from '@dxos/schema/testing';
+
+// TODO(burdon): Reconcile with plugin-markdown. Move to @dxos/schema/testing.
+export const TestItem = Schema.Struct({
+  title: Schema.String.annotations({
+    title: 'Title',
+    description: 'Product title',
+  }),
+  description: Schema.String.annotations({
+    title: 'Description',
+    description: 'Product description',
+  }),
+}).pipe(
+  Type.Obj({
+    typename: 'dxos.org/type/Test',
+    version: '0.1.0',
+  }),
+);
+
+// TODO(wittjosiah): Make builder generic and reuse for all message types.
+abstract class AbstractMessageBuilder {
+  abstract createMessage(numSegments?: number): Promise<DataType.Message>;
+}
+
+/**
+ * Generator of transcript messages.
+ */
+export class MessageBuilder extends AbstractMessageBuilder {
+  static readonly singleton = new MessageBuilder();
+
+  users = Array.from({ length: 5 }, () => ({
+    identityDid: IdentityDid.random().toString(),
+    name: faker.person.fullName(),
+    email: faker.internet.email(),
+  }));
+
+  start = new Date(Date.now() - 24 * 60 * 60 * 10_000);
+
+  constructor(private readonly _space?: Space) {
+    super();
+  }
+
+  override async createMessage(numSegments = 1): Promise<DataType.Message> {
+    return {
+      id: ObjectId.random().toString(),
+      created: this.next().toISOString(),
+      sender: faker.helpers.arrayElement(this.users),
+      blocks: Array.from({ length: numSegments }).map(() => this.createBlock()),
+    };
+  }
+
+  createBlock(): DataType.MessageBlock.Transcription {
+    let text = faker.lorem.paragraph();
+    if (this._space) {
+      const label = faker.commerce.productName();
+      const obj = this._space.db.add(live(TestItem, { title: label, description: faker.lorem.paragraph() }));
+      const dxn = makeRef(obj).dxn.toString();
+      const words = text.split(' ');
+      words.splice(Math.floor(Math.random() * words.length), 0, `[${label}][${dxn}]`);
+      text = words.join(' ');
+    }
+
+    return {
+      type: 'transcription',
+      started: this.next().toISOString(),
+      text,
+    };
+  }
+
+  next(): Date {
+    this.start = new Date(this.start.getTime() + Math.random() * 10_000);
+    return this.start;
+  }
+}
+
+// TODO(burdon): Reconcile with BlockBuilder.
+class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
+  aiService = new AIServiceEdgeClient({
+    endpoint: AI_SERVICE_ENDPOINT.REMOTE,
+  });
+
+  executor = new FunctionExecutor(new ServiceContainer().setServices({ ai: { client: this.aiService } }));
+
+  space: Space | undefined;
+  currentMessage: number = 0;
+  transcriptMessages: DataType.Message[] = [];
+
+  async connect(space: Space): Promise<void> {
+    this.space = space;
+    const { transcriptMessages } = await seedTestData(space);
+    this.transcriptMessages = transcriptMessages;
+  }
+
+  override async createMessage(): Promise<DataType.Message> {
+    if (!this.space) {
+      throw new Error('Space not connected');
+    }
+
+    const { objects } = await this.space.db
+      .query(
+        Filter.or(Filter.type(DataType.Person), Filter.type(DataType.Organization), Filter.type(Testing.DocumentType)),
+      )
+      .run();
+
+    log.info('context', { objects });
+    const message = this.transcriptMessages[this.currentMessage];
+    this.currentMessage++;
+    this.currentMessage = this.currentMessage % this.transcriptMessages.length;
+
+    const { message: enhancedMessage } = await processTranscriptMessage({
+      input: { message },
+      executor: this.executor,
+      function: extractionAnthropicFn,
+    });
+
+    return enhancedMessage;
+  }
+}
+
+type UseTestTranscriptionQueue = (
+  space: Space | undefined,
+  queueId?: ObjectId,
+  running?: boolean,
+  interval?: number,
+) => Queue<DataType.Message> | undefined;
+
+/**
+ * Test transcriptionqueue.
+ */
+export const useTestTranscriptionQueue: UseTestTranscriptionQueue = (
+  space: Space | undefined,
+  queueId?: ObjectId,
+  running = true,
+  interval = 1_000,
+) => {
+  const queueDxn = useMemo(() => (space ? createQueueDxn(space.id, queueId) : undefined), [space, queueId]);
+  const queue = useQueue<DataType.Message>(queueDxn);
+  const builder = useMemo(() => new MessageBuilder(space), [space]);
+
+  useEffect(() => {
+    if (!queue || !running) {
+      return;
+    }
+
+    const i = setInterval(() => {
+      void builder.createMessage(Math.ceil(Math.random() * 3)).then((message) => {
+        queue.append([create(DataType.Message, message)]);
+      });
+    }, interval);
+    return () => clearInterval(i);
+  }, [queue, running, interval]);
+
+  return queue;
+};
+
+/**
+ * Test transcription queue.
+ */
+// TODO(burdon): Reconcile with useTestTranscriptionQueue.
+export const useTestTranscriptionQueueWithEntityExtraction: UseTestTranscriptionQueue = (
+  space: Space | undefined,
+  queueId?: ObjectId,
+  running = true,
+  interval = 1_000,
+) => {
+  const queueDxn = useMemo(() => (space ? createQueueDxn(space.id, queueId) : undefined), [space, queueId]);
+  const queue = useQueue<DataType.Message>(queueDxn);
+  const [builder] = useState(() => new EntityExtractionMessageBuilder());
+
+  useEffect(() => {
+    if (!queue || !running) {
+      return;
+    }
+
+    if (space) {
+      void builder.connect(space);
+    }
+
+    const ctx = new Context();
+    scheduleTaskInterval(
+      ctx,
+      async () => {
+        const message = await builder.createMessage();
+        queue.append([create(DataType.Message, message)]);
+      },
+      interval,
+    );
+
+    return () => {
+      void ctx.dispose();
+    };
+  }, [space, queue, running, interval]);
+
+  return queue;
+};

package/src/transcriber/media-stream-recorder.ts

@@ -19,6 +19,13 @@ const initializeExtendableMediaRecorder = async () => {
   await register(await connect());
 };
 
+export type MediaStreamRecorderParams = {
+  mediaStreamTrack: MediaStreamTrack;
+  config: {
+    interval: number;
+  };
+};
+
 /**
  * Recorder that uses the MediaContext API and AudioNode API to record audio.
  *
@@ -30,14 +37,14 @@ export class MediaStreamRecorder implements AudioRecorder {
   /**
    * Default MediaRecorder implementation do not support wav encoding.
    */
-  private readonly _interval: number;
+  private readonly _config: MediaStreamRecorderParams['config'];
   private _mediaRecorder?: IMediaRecorder = undefined;
   private _header?: Uint8Array = undefined;
   private _onChunk?: (chunk: AudioChunk) => void;
 
-  constructor({ mediaStreamTrack, interval }: { mediaStreamTrack: MediaStreamTrack; interval: number }) {
+  constructor({ mediaStreamTrack, config }: MediaStreamRecorderParams) {
     this._mediaStreamTrack = mediaStreamTrack;
-    this._interval = interval;
+    this._config = config;
   }
 
   get wavConfig(): WavConfig {
@@ -57,7 +64,7 @@ export class MediaStreamRecorder implements AudioRecorder {
     this._onChunk = onChunk;
   }
 
-  async start() {
+  async start(): Promise<void> {
     await (initializingPromise ??= initializeExtendableMediaRecorder()).catch((err) =>
       log.info('initializeExtendableMediaRecorder', { err }),
     );
@@ -72,10 +79,10 @@ export class MediaStreamRecorder implements AudioRecorder {
     }
     invariant(this._onChunk, 'MediaStreamRecorder: onChunk is not set');
     this._mediaRecorder.ondataavailable = (event) => this._ondataavailable(event);
-    this._mediaRecorder.start(this._interval);
+    this._mediaRecorder.start(this._config.interval);
   }
 
-  async stop() {
+  async stop(): Promise<void> {
     if (this._mediaRecorder?.state !== 'recording') {
       return;
     }
@@ -83,7 +90,7 @@ export class MediaStreamRecorder implements AudioRecorder {
   }
 
   @synchronized
-  private async _ondataavailable(event: IBlobEvent) {
+  private async _ondataavailable(event: IBlobEvent): Promise<void> {
     const blob = event.data;
     const uint8Array = new Uint8Array(await blob.arrayBuffer());
 

package/src/transcriber/{transcriber.test.ts → transcriber.browser.test.ts}

@@ -9,11 +9,11 @@ import { WaveFile } from 'wavefile';
 
 import { Trigger } from '@dxos/async';
 import { log } from '@dxos/log';
+import { type DataType } from '@dxos/schema';
 import { openAndClose } from '@dxos/test-utils';
 import { trace, TRACE_PROCESSOR } from '@dxos/tracing';
 
 import { type AudioChunk, type AudioRecorder, Transcriber } from '../transcriber';
-import { type TranscriptSegment } from '../types';
 import { mergeFloat64Arrays } from '../util';
 
 // TODO(burdon): ReferenceError: Worker is not defined (Can only run with web workers).
@@ -93,7 +93,7 @@ describe.skip('Transcriber', () => {
   });
 
   test.skip('transcription of audio recording', { timeout: 10_000 }, async () => {
-    const trigger = new Trigger<TranscriptSegment[]>({ autoReset: true });
+    const trigger = new Trigger<DataType.MessageBlock.Transcription[]>({ autoReset: true });
     const recorder = new MockAudioRecorder({
       buffer: await readFile('test.wav'),
       chunkDuration: 3_000,
@@ -127,7 +127,7 @@ describe.skip('Transcriber', () => {
   });
 
   test.skip('transcription of audio recording with overlapping chunks', { timeout: 20_000 }, async () => {
-    const trigger = new Trigger<TranscriptSegment[]>({ autoReset: true });
+    const trigger = new Trigger<DataType.MessageBlock.Transcription[]>({ autoReset: true });
     const recorder = new MockAudioRecorder({
       buffer: await readFile('test.wav'),
       chunkDuration: 3_000,
@@ -194,14 +194,14 @@ class MockAudioRecorder implements AudioRecorder {
     };
   }
 
-  setOnChunk(onChunk: (chunk: AudioChunk) => void) {
+  setOnChunk(onChunk: (chunk: AudioChunk) => void): void {
     this._onChunk = onChunk;
   }
 
   /**
    * Emit a chunk in controlled manner for testing purposes.
    */
-  emitChunk() {
+  emitChunk(): void {
     const chunk = this.chunks.shift();
     if (!chunk) {
       return;
@@ -211,7 +211,7 @@ class MockAudioRecorder implements AudioRecorder {
   }
 
   @trace.span()
-  async start() {
+  async start(): Promise<void> {
     log.info('start mock audio recorder', { wavParams: this.wav.fmt });
     const now = Date.now();
     const samples = this.wav.getSamples();
@@ -226,5 +226,5 @@ class MockAudioRecorder implements AudioRecorder {
     }
   }
 
-  async stop() {}
+  async stop(): Promise<void> {}
 }

package/src/transcriber/transcriber.ts

@@ -7,10 +7,11 @@ import { WaveFile } from 'wavefile';
 import { DeferredTask, Trigger } from '@dxos/async';
 import { type Context, LifecycleState, Resource } from '@dxos/context';
 import { log } from '@dxos/log';
+import { type DataType } from '@dxos/schema';
 import { trace } from '@dxos/tracing';
 
 import { type AudioRecorder, type AudioChunk } from './audio-recorder';
-import { TRANSCRIPTION_URL, type TranscriptSegment } from '../types';
+import { TRANSCRIPTION_URL } from '../types';
 import { mergeFloat64Arrays } from '../util';
 
 type WhisperWord = {
@@ -64,7 +65,11 @@ export type TranscribeConfig = {
 export type TranscriberParams = {
   config: TranscribeConfig;
   recorder: AudioRecorder;
-  onSegments: (segments: TranscriptSegment[]) => Promise<void>;
+  /**
+   * Callback to handle the transcribed segments, after all segment transformers are applied.
+   * @param segments - The transcribed segments.
+   */
+  onSegments: (segments: DataType.MessageBlock.Transcription[]) => Promise<void>;
 };
 
 /**
@@ -92,35 +97,41 @@ export class Transcriber extends Resource {
     this._onSegments = onSegments;
   }
 
-  protected override async _open(ctx: Context) {
+  protected override async _open(ctx: Context): Promise<void> {
+    log.info('opening');
     this._recorder.setOnChunk((chunk) => this._saveAudioChunk(chunk));
     await this._recorder.start();
     this._transcribeTask = new DeferredTask(ctx, async () => this._transcribe());
     this._openTrigger.wake();
   }
 
-  protected override async _close() {
+  protected override async _close(): Promise<void> {
+    log.info('closing');
     this._recording = false;
     this._transcribeTask = undefined;
     await this._recorder.stop();
   }
 
-  startChunksRecording() {
+  startChunksRecording(): void {
+    log.info('starting');
     if (this._lifecycleState !== LifecycleState.OPEN) {
       return;
     }
+
     this._recording = true;
   }
 
-  stopChunksRecording() {
+  stopChunksRecording(): void {
     if (this._lifecycleState !== LifecycleState.OPEN || !this._recording) {
       return;
     }
+
     this._recording = false;
     this._transcribeTask?.schedule();
+    log.info('stopped');
   }
 
-  private _saveAudioChunk(chunk: AudioChunk) {
+  private _saveAudioChunk(chunk: AudioChunk): void {
     log('saving audio chunk', { chunk });
     this._audioChunks.push(chunk);
 
@@ -136,7 +147,7 @@ export class Transcriber extends Resource {
     }
   }
 
-  private async _transcribe() {
+  private async _transcribe(): Promise<void> {
     log('transcribing', { chunks: this._audioChunks.length });
     const chunks = this._audioChunks;
     this._audioChunks = this._dropOldChunks();
@@ -156,14 +167,14 @@ export class Transcriber extends Resource {
     }
   }
 
-  private _dropOldChunks() {
+  private _dropOldChunks(): AudioChunk[] {
     return this._config.prefixBufferChunksAmount > 0
       ? this._audioChunks.slice(-this._config.prefixBufferChunksAmount)
       : [];
   }
 
   @trace.span({ showInBrowserTimeline: true })
-  private async _mergeAudioChunks(chunks: AudioChunk[]) {
+  private async _mergeAudioChunks(chunks: AudioChunk[]): Promise<string> {
     const file = new WaveFile();
     const wavConfig = this._recorder.wavConfig;
 
@@ -178,7 +189,7 @@ export class Transcriber extends Resource {
   }
 
   @trace.span({ showInBrowserTimeline: true })
-  private async _fetchTranscription(audio: string) {
+  private async _fetchTranscription(audio: string): Promise<WhisperSegment[]> {
     if (audio.length === 0) {
       this._audioChunks = [];
       throw new Error('No audio to send for transcribing');
@@ -194,24 +205,25 @@ export class Transcriber extends Resource {
 
     if (!response.ok) {
       this._audioChunks = [];
-      throw new Error('Failed to transcribe');
+      throw new Error(`Transcription failed: ${response.statusText}`);
     }
 
-    const {
-      segments,
-    }: {
+    const { segments } = (await response.json()) as {
       segments: WhisperSegment[];
-    } = await response.json();
+    };
 
     log.info('transcription response', {
-      segments,
+      segments: segments.length,
       string: segments.map((segments) => segments.text).join(' '),
     });
 
     return segments;
   }
 
-  private _alignSegments(segments: WhisperSegment[], originalChunks: AudioChunk[]): TranscriptSegment[] {
+  private _alignSegments(
+    segments: WhisperSegment[],
+    originalChunks: AudioChunk[],
+  ): DataType.MessageBlock.Transcription[] {
     // Absolute zero for all relative timestamps in the segments.
     const zeroTimestamp = originalChunks.at(0)!.timestamp;
 
@@ -240,8 +252,9 @@ export class Transcriber extends Resource {
 
     // Add absolute timestamp to each segment.
     return filteredSegments.map((segment) => ({
-      started: new Date(zeroTimestamp + segment.start * 1_000),
-      text: segment.text,
+      type: 'transcription',
+      started: new Date(zeroTimestamp + segment.start * 1_000).toISOString(),
+      text: segment.text.trim(),
     }));
   }
 }