@dxos/plugin-transcription 0.8.1 → 0.8.2-main.2f9c567

package/src/testing/testing.ts

@@ -0,0 +1,221 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Schema, SchemaAST } from 'effect';
+import { useEffect, useMemo, useState } from 'react';
+
+import { AIServiceEdgeClient } from '@dxos/assistant';
+import { AI_SERVICE_ENDPOINT } from '@dxos/assistant/testing';
+import { scheduleTaskInterval } from '@dxos/async';
+import { createQueueDxn, Filter, type Queue } from '@dxos/client/echo';
+import { Context } from '@dxos/context';
+import { Type } from '@dxos/echo';
+import { create, ObjectId } from '@dxos/echo-schema';
+import { IdentityDid } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { faker } from '@dxos/random';
+import { live, makeRef, useQueue, type Space } from '@dxos/react-client/echo';
+import { DataType } from '@dxos/schema';
+import { Testing, seedTestData } from '@dxos/schema/testing';
+
+import { processTranscriptMessage } from '../entity-extraction';
+
+// TODO(burdon): Reconcile with plugin-markdown. Move to @dxos/schema/testing.
+
+export const TestItem = Schema.Struct({
+  title: Schema.String.annotations({
+    [SchemaAST.TitleAnnotationId]: 'Title',
+    [SchemaAST.DescriptionAnnotationId]: 'Product title',
+  }),
+  description: Schema.String.annotations({
+    [SchemaAST.TitleAnnotationId]: 'Description',
+    [SchemaAST.DescriptionAnnotationId]: 'Product description',
+  }),
+}).pipe(
+  Type.def({
+    typename: 'dxos.org/type/Test',
+    version: '0.1.0',
+  }),
+);
+
+// TODO(wittjosiah): Make builder generic and reuse for all message types.
+abstract class AbstractMessageBuilder {
+  abstract createMessage(numSegments?: number): Promise<DataType.Message>;
+}
+
+/**
+ * Generator of transcript messages.
+ */
+export class MessageBuilder extends AbstractMessageBuilder {
+  static readonly singleton = new MessageBuilder();
+
+  users = Array.from({ length: 5 }, () => ({
+    identityDid: IdentityDid.random().toString(),
+    name: faker.person.fullName(),
+    email: faker.internet.email(),
+  }));
+
+  start = new Date(Date.now() - 24 * 60 * 60 * 10_000);
+
+  constructor(private readonly _space?: Space) {
+    super();
+  }
+
+  override async createMessage(numSegments = 1): Promise<DataType.Message> {
+    return {
+      id: ObjectId.random().toString(),
+      created: this.next().toISOString(),
+      sender: faker.helpers.arrayElement(this.users),
+      blocks: Array.from({ length: numSegments }).map(() => this.createBlock()),
+    };
+  }
+
+  createBlock(): DataType.MessageBlock.Transcription {
+    let text = faker.lorem.paragraph();
+    if (this._space) {
+      const label = faker.commerce.productName();
+      const obj = this._space.db.add(live(TestItem, { title: label, description: faker.lorem.paragraph() }));
+      const dxn = makeRef(obj).dxn.toString();
+      const words = text.split(' ');
+      words.splice(Math.floor(Math.random() * words.length), 0, `[${label}][${dxn}]`);
+      text = words.join(' ');
+    }
+
+    return {
+      type: 'transcription',
+      started: this.next().toISOString(),
+      text,
+    };
+  }
+
+  next() {
+    this.start = new Date(this.start.getTime() + Math.random() * 10_000);
+    return this.start;
+  }
+}
+
+// TODO(burdon): Reconcile with BlockBuilder.
+class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
+  aiService = new AIServiceEdgeClient({
+    endpoint: AI_SERVICE_ENDPOINT.REMOTE,
+  });
+
+  space: Space | undefined;
+  currentMessage: number = 0;
+  transcriptMessages: DataType.Message[] = [];
+
+  async connect(space: Space) {
+    this.space = space;
+    const { transcriptMessages } = await seedTestData(space);
+    this.transcriptMessages = transcriptMessages;
+  }
+
+  override async createMessage(): Promise<DataType.Message> {
+    if (!this.space) {
+      throw new Error('Space not connected');
+    }
+
+    const { objects } = await this.space.db
+      .query(
+        Filter.or(
+          Filter.schema(DataType.Person),
+          Filter.schema(DataType.Organization),
+          Filter.schema(Testing.DocumentType),
+        ),
+      )
+      .run();
+
+    log.info('context', { objects });
+    const message = this.transcriptMessages[this.currentMessage];
+    this.currentMessage++;
+    this.currentMessage = this.currentMessage % this.transcriptMessages.length;
+
+    const { message: enhancedMessage } = await processTranscriptMessage({
+      message,
+      aiService: this.aiService,
+      context: {
+        objects,
+      },
+    });
+
+    return enhancedMessage;
+  }
+}
+
+type UseTestTranscriptionQueue = (
+  space: Space | undefined,
+  queueId?: ObjectId,
+  running?: boolean,
+  interval?: number,
+) => Queue<DataType.Message> | undefined;
+
+/**
+ * Test transcriptionqueue.
+ */
+export const useTestTranscriptionQueue: UseTestTranscriptionQueue = (
+  space: Space | undefined,
+  queueId?: ObjectId,
+  running = true,
+  interval = 1_000,
+) => {
+  const queueDxn = useMemo(() => (space ? createQueueDxn(space.id, queueId) : undefined), [space, queueId]);
+  const queue = useQueue<DataType.Message>(queueDxn);
+  const builder = useMemo(() => new MessageBuilder(space), [space]);
+
+  useEffect(() => {
+    if (!queue || !running) {
+      return;
+    }
+
+    const i = setInterval(() => {
+      void builder.createMessage(Math.ceil(Math.random() * 3)).then((message) => {
+        queue.append([create(DataType.Message, message)]);
+      });
+    }, interval);
+    return () => clearInterval(i);
+  }, [queue, running, interval]);
+
+  return queue;
+};
+
+/**
+ * Test transcription queue.
+ */
+// TODO(burdon): Reconcile with useTestTranscriptionQueue.
+export const useTestTranscriptionQueueWithEntityExtraction: UseTestTranscriptionQueue = (
+  space: Space | undefined,
+  queueId?: ObjectId,
+  running = true,
+  interval = 1_000,
+) => {
+  const queueDxn = useMemo(() => (space ? createQueueDxn(space.id, queueId) : undefined), [space, queueId]);
+  const queue = useQueue<DataType.Message>(queueDxn);
+  const [builder] = useState(() => new EntityExtractionMessageBuilder());
+
+  useEffect(() => {
+    if (!queue || !running) {
+      return;
+    }
+
+    if (space) {
+      void builder.connect(space);
+    }
+
+    const ctx = new Context();
+    scheduleTaskInterval(
+      ctx,
+      async () => {
+        const message = await builder.createMessage();
+        queue.append([create(DataType.Message, message)]);
+      },
+      interval,
+    );
+
+    return () => {
+      void ctx.dispose();
+    };
+  }, [space, queue, running, interval]);
+
+  return queue;
+};

package/src/transcriber/{transcriber.test.ts → transcriber.browser.test.ts}

@@ -9,11 +9,11 @@ import { WaveFile } from 'wavefile';
 
 import { Trigger } from '@dxos/async';
 import { log } from '@dxos/log';
+import { type DataType } from '@dxos/schema';
 import { openAndClose } from '@dxos/test-utils';
 import { trace, TRACE_PROCESSOR } from '@dxos/tracing';
 
 import { type AudioChunk, type AudioRecorder, Transcriber } from '../transcriber';
-import { type TranscriptSegment } from '../types';
 import { mergeFloat64Arrays } from '../util';
 
 // TODO(burdon): ReferenceError: Worker is not defined (Can only run with web workers).
@@ -93,7 +93,7 @@ describe.skip('Transcriber', () => {
   });
 
   test.skip('transcription of audio recording', { timeout: 10_000 }, async () => {
-    const trigger = new Trigger<TranscriptSegment[]>({ autoReset: true });
+    const trigger = new Trigger<DataType.MessageBlock.Transcription[]>({ autoReset: true });
     const recorder = new MockAudioRecorder({
       buffer: await readFile('test.wav'),
       chunkDuration: 3_000,
@@ -127,7 +127,7 @@ describe.skip('Transcriber', () => {
   });
 
   test.skip('transcription of audio recording with overlapping chunks', { timeout: 20_000 }, async () => {
-    const trigger = new Trigger<TranscriptSegment[]>({ autoReset: true });
+    const trigger = new Trigger<DataType.MessageBlock.Transcription[]>({ autoReset: true });
     const recorder = new MockAudioRecorder({
       buffer: await readFile('test.wav'),
       chunkDuration: 3_000,

package/src/transcriber/transcriber.ts

@@ -7,10 +7,11 @@ import { WaveFile } from 'wavefile';
 import { DeferredTask, Trigger } from '@dxos/async';
 import { type Context, LifecycleState, Resource } from '@dxos/context';
 import { log } from '@dxos/log';
+import { type DataType } from '@dxos/schema';
 import { trace } from '@dxos/tracing';
 
 import { type AudioRecorder, type AudioChunk } from './audio-recorder';
-import { TRANSCRIPTION_URL, type TranscriptSegment } from '../types';
+import { TRANSCRIPTION_URL } from '../types';
 import { mergeFloat64Arrays } from '../util';
 
 type WhisperWord = {
@@ -64,7 +65,7 @@ export type TranscribeConfig = {
 export type TranscriberParams = {
   config: TranscribeConfig;
   recorder: AudioRecorder;
-  onSegments: (segments: TranscriptSegment[]) => Promise<void>;
+  onSegments: (segments: DataType.MessageBlock.Transcription[]) => Promise<void>;
 };
 
 /**
@@ -93,6 +94,7 @@ export class Transcriber extends Resource {
   }
 
   protected override async _open(ctx: Context) {
+    log.info('opening');
     this._recorder.setOnChunk((chunk) => this._saveAudioChunk(chunk));
     await this._recorder.start();
     this._transcribeTask = new DeferredTask(ctx, async () => this._transcribe());
@@ -100,15 +102,18 @@ export class Transcriber extends Resource {
   }
 
   protected override async _close() {
+    log.info('closing');
     this._recording = false;
     this._transcribeTask = undefined;
     await this._recorder.stop();
   }
 
   startChunksRecording() {
+    log.info('starting');
     if (this._lifecycleState !== LifecycleState.OPEN) {
       return;
     }
+
     this._recording = true;
   }
 
@@ -116,8 +121,10 @@ export class Transcriber extends Resource {
     if (this._lifecycleState !== LifecycleState.OPEN || !this._recording) {
       return;
     }
+
     this._recording = false;
     this._transcribeTask?.schedule();
+    log.info('stopped');
   }
 
   private _saveAudioChunk(chunk: AudioChunk) {
@@ -194,24 +201,25 @@ export class Transcriber extends Resource {
 
     if (!response.ok) {
       this._audioChunks = [];
-      throw new Error('Failed to transcribe');
+      throw new Error(`Transcription failed: ${response.statusText}`);
     }
 
-    const {
-      segments,
-    }: {
+    const { segments } = (await response.json()) as {
       segments: WhisperSegment[];
-    } = await response.json();
+    };
 
     log.info('transcription response', {
-      segments,
+      segments: segments.length,
       string: segments.map((segments) => segments.text).join(' '),
     });
 
     return segments;
   }
 
-  private _alignSegments(segments: WhisperSegment[], originalChunks: AudioChunk[]): TranscriptSegment[] {
+  private _alignSegments(
+    segments: WhisperSegment[],
+    originalChunks: AudioChunk[],
+  ): DataType.MessageBlock.Transcription[] {
     // Absolute zero for all relative timestamps in the segments.
     const zeroTimestamp = originalChunks.at(0)!.timestamp;
 
@@ -240,8 +248,9 @@ export class Transcriber extends Resource {
 
     // Add absolute timestamp to each segment.
     return filteredSegments.map((segment) => ({
-      started: new Date(zeroTimestamp + segment.start * 1_000),
-      text: segment.text,
+      type: 'transcription',
+      started: new Date(zeroTimestamp + segment.start * 1_000).toISOString(),
+      text: segment.text.trim(),
     }));
   }
 }

package/src/transcriber/transcription-manager.ts

@@ -5,15 +5,14 @@
 import { synchronized } from '@dxos/async';
 import { Resource } from '@dxos/context';
 import { QueueImpl, type Queue } from '@dxos/echo-db';
-import { createStatic } from '@dxos/echo-schema';
-import { invariant } from '@dxos/invariant';
-import { DXN } from '@dxos/keys';
+import { create } from '@dxos/echo-schema';
+import { type DXN } from '@dxos/keys';
+import { log } from '@dxos/log';
 import { type EdgeHttpClient } from '@dxos/react-edge-client';
-import { type HuePalette } from '@dxos/react-ui-theme';
+import { DataType } from '@dxos/schema';
 
 import { MediaStreamRecorder } from './media-stream-recorder';
 import { Transcriber } from './transcriber';
-import { TranscriptBlock, type TranscriptSegment } from '../types';
 
 /**
  * Length of the chunk in ms.
@@ -31,23 +30,34 @@ const PREFIXED_CHUNKS_AMOUNT = 10;
  */
 const TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
 
+export type TranscriptMessageEnricher = (message: DataType.Message) => Promise<DataType.Message>;
+
+export type TranscriptionManagerOptions = {
+  edgeClient: EdgeHttpClient;
+
+  /**
+   * Enrich the message before it is written to the transcription queue.
+   */
+  messageEnricher?: TranscriptMessageEnricher;
+};
+
 /**
  * Manages transcription state for a meeting.
  */
-// TODO(mykola): Reconcile with transcriber capability.
 export class TranscriptionManager extends Resource {
   private readonly _edgeClient: EdgeHttpClient;
+  private readonly _messageEnricher?: TranscriptMessageEnricher;
   private _audioStreamTrack?: MediaStreamTrack = undefined;
-  private _name?: string = undefined;
-  private _hue?: HuePalette = undefined;
+  private _identityDid?: string = undefined;
   private _mediaRecorder?: MediaStreamRecorder = undefined;
   private _transcriber?: Transcriber = undefined;
+  private _queue?: Queue<DataType.Message> = undefined;
   private _enabled = false;
-  private _queue?: Queue<TranscriptBlock> = undefined;
 
-  constructor(edgeClient: EdgeHttpClient) {
+  constructor(options: TranscriptionManagerOptions) {
     super();
-    this._edgeClient = edgeClient;
+    this._edgeClient = options.edgeClient;
+    this._messageEnricher = options.messageEnricher;
   }
 
   protected override async _open() {
@@ -58,10 +68,24 @@ export class TranscriptionManager extends Resource {
     void this._transcriber?.close();
   }
 
-  @synchronized
-  setRecording(recording?: boolean) {
+  setQueue(queueDxn: DXN): TranscriptionManager {
+    if (this._queue?.dxn.toString() !== queueDxn.toString()) {
+      log.info('setQueue', { queueDxn: queueDxn.toString() });
+      this._queue = new QueueImpl<DataType.Message>(this._edgeClient, queueDxn);
+    }
+    return this;
+  }
+
+  setIdentityDid(did: string): TranscriptionManager {
+    if (this._identityDid !== did) {
+      this._identityDid = did;
+    }
+    return this;
+  }
+
+  setRecording(recording?: boolean): TranscriptionManager {
     if (!this.isOpen || !this._enabled) {
-      return;
+      return this;
     }
 
     if (recording) {
@@ -69,6 +93,7 @@ export class TranscriptionManager extends Resource {
     } else {
       this._transcriber?.stopChunksRecording();
     }
+    return this;
   }
 
   /**
@@ -80,7 +105,9 @@ export class TranscriptionManager extends Resource {
     if (this._enabled === enabled) {
       return;
     }
+
     this._enabled = enabled ?? false;
+    // TODO(burdon): Why is toggle called here?
     this.isOpen && (await this._toggleTranscriber());
   }
 
@@ -89,60 +116,33 @@ export class TranscriptionManager extends Resource {
     if (this._audioStreamTrack === track) {
       return;
     }
+
     this._audioStreamTrack = track;
     this.isOpen && (await this._toggleTranscriber());
   }
 
-  /**
-   * Set the queue to save the transcription to.
-   * @param queue - The queue to save the transcription to or the DXN of the queue.
-   */
-  @synchronized
-  setQueue(queue?: Queue<TranscriptBlock> | string) {
-    switch (typeof queue) {
-      case 'string':
-        if (this._queue?.dxn.toString() === queue) {
-          return;
-        }
-        this._queue = new QueueImpl<TranscriptBlock>(this._edgeClient, DXN.parse(queue));
-        break;
-      case 'object':
-        if (this._queue === queue) {
-          return;
-        }
-        invariant(queue instanceof QueueImpl);
-        this._queue = queue;
-        break;
-      case 'undefined':
-        this._queue = undefined;
-        break;
-    }
-  }
-
-  @synchronized
-  setName(name: string) {
-    if (this._name === name) {
-      return;
-    }
-    this._name = name;
-  }
-
-  @synchronized
-  setHue(hue: HuePalette) {
-    if (this._hue === hue) {
-      return;
-    }
-    this._hue = hue;
-  }
-
+  // TODO(burdon): Change this to setEnables (explicit), not toggle.
   private async _toggleTranscriber() {
     await this._maybeReinitTranscriber();
 
     // Open or close transcriber if transcription is enabled or disabled.
     if (this._enabled) {
       await this._transcriber?.open();
+      // TODO(burdon): Started and stopped blocks appear twice.
+      const block = create(DataType.Message, {
+        created: new Date().toISOString(),
+        blocks: [{ type: 'transcription', text: 'Started', started: new Date().toISOString() }],
+        sender: { role: 'assistant' },
+      });
+      this._queue?.append([block]);
     } else {
       await this._transcriber?.close();
+      const block = create(DataType.Message, {
+        created: new Date().toISOString(),
+        blocks: [{ type: 'transcription', text: 'Stopped', started: new Date().toISOString() }],
+        sender: { role: 'assistant' },
+      });
+      this._queue?.append([block]);
     }
   }
 
@@ -173,12 +173,21 @@ export class TranscriptionManager extends Resource {
     }
   }
 
-  private async _onSegments(segments: TranscriptSegment[]) {
+  private async _onSegments(segments: DataType.MessageBlock.Transcription[]) {
     if (!this.isOpen || !this._queue) {
       return;
     }
 
-    const block = createStatic(TranscriptBlock, { authorName: this._name, authorHue: this._hue, segments });
+    let block = create(DataType.Message, {
+      created: new Date().toISOString(),
+      blocks: segments,
+      sender: { identityDid: this._identityDid },
+    });
+
+    if (this._messageEnricher) {
+      block = await this._messageEnricher(block);
+    }
+
     this._queue.append([block]);
   }
 }

package/src/translations.ts

@@ -22,6 +22,8 @@ export default [
         'delete button': 'Delete',
         'bookmark button': 'Bookmark',
         'scroll to end label': 'Scroll to latest',
+
+        'settings entity extraction label': 'Entity extraction',
       },
     },
   },

package/src/types/schema.ts

@@ -2,17 +2,19 @@
 // Copyright 2025 DXOS.org
 //
 
-import { EchoObject, Expando, Ref, S, TypedObject } from '@dxos/echo-schema';
+import { Schema } from 'effect';
+
+import { Expando, Ref, TypedObject } from '@dxos/echo-schema';
 
 /**
  * Root transcript object created when the user starts a transcription.
  */
-export const TranscriptSchema = S.Struct({
-  name: S.String,
+export const TranscriptSchema = Schema.Struct({
+  name: Schema.String,
 
-  // TODO(burdon): Use string for dates?
-  started: S.optional(S.Date),
-  ended: S.optional(S.Date),
+  // TODO(burdon): Use Date or string?
+  started: Schema.optional(Schema.String),
+  ended: Schema.optional(Schema.String),
 
   /**
    * Queue containing TranscriptBlock objects.
@@ -32,31 +34,8 @@ export class TranscriptType extends TypedObject({
  * First message in queue.
  * Contains metadata for the recording and transcript.
  */
-const TranscriptHeader = S.Struct({
-  started: S.optional(S.Date),
+const TranscriptHeader = Schema.Struct({
+  started: Schema.optional(Schema.String),
 });
 
-export type TranscriptHeader = S.Schema.Type<typeof TranscriptHeader>;
-
-/**
- * Transcription fragment.
- */
-const TranscriptSegment = S.Struct({
-  // TODO(burdon): TS from service is not Unix TS (x1000).
-  started: S.Date,
-  text: S.String,
-});
-
-export type TranscriptSegment = S.Schema.Type<typeof TranscriptSegment>;
-
-/**
- * Transcription block (from single speaker).
- */
-export const TranscriptBlock = S.Struct({
-  id: S.String,
-  authorName: S.optional(S.String), // TODO(burdon): IdentityDid.
-  authorHue: S.optional(S.String),
-  segments: S.Array(TranscriptSegment),
-}).pipe(EchoObject({ typename: 'dxos.org/type/TranscriptBlock', version: '0.1.0' }));
-
-export type TranscriptBlock = S.Schema.Type<typeof TranscriptBlock>;
+export type TranscriptHeader = Schema.Schema.Type<typeof TranscriptHeader>;

package/src/types/types.ts

@@ -2,8 +2,10 @@
 // Copyright 2025 DXOS.org
 //
 
-import { S, isInstanceOf } from '@dxos/echo-schema';
-import { isReactiveObject } from '@dxos/react-client/echo';
+import { Schema } from 'effect';
+
+import { SpaceIdSchema, isInstanceOf } from '@dxos/echo-schema';
+import { isLiveObject } from '@dxos/react-client/echo';
 
 import { TranscriptType } from './schema';
 import { TRANSCRIPTION_PLUGIN } from '../meta';
@@ -19,18 +21,26 @@ export const TRANSCRIPTION_URL = 'https://calls-service.dxos.workers.dev';
 export namespace TranscriptionAction {
   const TRANSCRIPTION_ACTION = `${TRANSCRIPTION_PLUGIN}/action`;
 
-  export class Create extends S.TaggedClass<Create>()(`${TRANSCRIPTION_ACTION}/create`, {
-    input: S.Struct({
-      name: S.optional(S.String),
-      // TODO(wittjosiah): SpaceId.
-      spaceId: S.String,
+  export class Create extends Schema.TaggedClass<Create>()(`${TRANSCRIPTION_ACTION}/create`, {
+    input: Schema.Struct({
+      name: Schema.optional(Schema.String),
+      spaceId: SpaceIdSchema,
     }),
-    output: S.Struct({
+    output: Schema.Struct({
       object: TranscriptType,
     }),
   }) {}
 }
 
 export const isTranscript = (object: unknown): object is typeof TranscriptType => {
-  return isReactiveObject(object) && isInstanceOf(TranscriptType, object);
+  return isLiveObject(object) && isInstanceOf(TranscriptType, object);
 };
+
+// TODO(burdon): Create with decode consistently: Schema.decodeSync(TranscriptionSettingsSchema)({}))
+export const TranscriptionSettingsSchema = Schema.mutable(
+  Schema.Struct({
+    entityExtraction: Schema.optional(Schema.Boolean).pipe(Schema.withConstructorDefault(() => true)),
+  }),
+);
+
+export type TranscriptionSettingsProps = Schema.Schema.Type<typeof TranscriptionSettingsSchema>;

package/src/util/index.ts

@@ -5,4 +5,3 @@
 export * from './array';
 export * from './get-time-str';
 export * from './monitor-audio-level';
-export * from './random-queue-dxn';