@dxos/plugin-transcription 0.8.2-main.f11618f → 0.8.2-staging.7ac8446

package/src/hooks/index.ts

@@ -4,6 +4,5 @@
 
 export * from './useAudioFile';
 export * from './useAudioTrack';
-export * from './useQueueModelAdapter';
 export * from './useTranscriber';
 export * from './useVoiceInput';

package/src/hooks/useTranscriber.ts

@@ -15,7 +15,6 @@ import { type Transcriber } from '../transcriber';
 export const useTranscriber = ({
   audioStreamTrack,
   onSegments,
-  config,
 }: Partial<TranscriptionCapabilities.GetTranscriberProps>) => {
   const [getTranscriber] = useCapabilities(TranscriptionCapabilities.Transcriber);
 
@@ -25,8 +24,8 @@ export const useTranscriber = ({
       return undefined;
     }
 
-    return getTranscriber({ audioStreamTrack, onSegments, config });
-  }, [audioStreamTrack, onSegments, getTranscriber, config]);
+    return getTranscriber({ audioStreamTrack, onSegments });
+  }, [audioStreamTrack, onSegments, getTranscriber]);
 
   useEffect(() => {
     return () => {

package/src/transcriber/{transcriber.browser.test.ts → transcriber.test.ts}

@@ -9,11 +9,11 @@ import { WaveFile } from 'wavefile';
 
 import { Trigger } from '@dxos/async';
 import { log } from '@dxos/log';
-import { type TranscriptionContentBlock } from '@dxos/schema';
 import { openAndClose } from '@dxos/test-utils';
 import { trace, TRACE_PROCESSOR } from '@dxos/tracing';
 
 import { type AudioChunk, type AudioRecorder, Transcriber } from '../transcriber';
+import { type TranscriptSegment } from '../types';
 import { mergeFloat64Arrays } from '../util';
 
 // TODO(burdon): ReferenceError: Worker is not defined (Can only run with web workers).
@@ -93,7 +93,7 @@ describe.skip('Transcriber', () => {
   });
 
   test.skip('transcription of audio recording', { timeout: 10_000 }, async () => {
-    const trigger = new Trigger<TranscriptionContentBlock[]>({ autoReset: true });
+    const trigger = new Trigger<TranscriptSegment[]>({ autoReset: true });
     const recorder = new MockAudioRecorder({
       buffer: await readFile('test.wav'),
       chunkDuration: 3_000,
@@ -127,7 +127,7 @@ describe.skip('Transcriber', () => {
   });
 
   test.skip('transcription of audio recording with overlapping chunks', { timeout: 20_000 }, async () => {
-    const trigger = new Trigger<TranscriptionContentBlock[]>({ autoReset: true });
+    const trigger = new Trigger<TranscriptSegment[]>({ autoReset: true });
     const recorder = new MockAudioRecorder({
       buffer: await readFile('test.wav'),
       chunkDuration: 3_000,

package/src/transcriber/transcriber.ts

@@ -7,11 +7,10 @@ import { WaveFile } from 'wavefile';
 import { DeferredTask, Trigger } from '@dxos/async';
 import { type Context, LifecycleState, Resource } from '@dxos/context';
 import { log } from '@dxos/log';
-import { type TranscriptionContentBlock } from '@dxos/schema';
 import { trace } from '@dxos/tracing';
 
 import { type AudioRecorder, type AudioChunk } from './audio-recorder';
-import { TRANSCRIPTION_URL } from '../types';
+import { TRANSCRIPTION_URL, type TranscriptSegment } from '../types';
 import { mergeFloat64Arrays } from '../util';
 
 type WhisperWord = {
@@ -65,7 +64,7 @@ export type TranscribeConfig = {
 export type TranscriberParams = {
   config: TranscribeConfig;
   recorder: AudioRecorder;
-  onSegments: (segments: TranscriptionContentBlock[]) => Promise<void>;
+  onSegments: (segments: TranscriptSegment[]) => Promise<void>;
 };
 
 /**
@@ -94,7 +93,6 @@ export class Transcriber extends Resource {
   }
 
   protected override async _open(ctx: Context) {
-    log.info('opening');
     this._recorder.setOnChunk((chunk) => this._saveAudioChunk(chunk));
     await this._recorder.start();
     this._transcribeTask = new DeferredTask(ctx, async () => this._transcribe());
@@ -102,18 +100,15 @@ export class Transcriber extends Resource {
   }
 
   protected override async _close() {
-    log.info('closing');
     this._recording = false;
     this._transcribeTask = undefined;
     await this._recorder.stop();
   }
 
   startChunksRecording() {
-    log.info('starting');
     if (this._lifecycleState !== LifecycleState.OPEN) {
       return;
     }
-
     this._recording = true;
   }
 
@@ -121,10 +116,8 @@ export class Transcriber extends Resource {
     if (this._lifecycleState !== LifecycleState.OPEN || !this._recording) {
       return;
     }
-
     this._recording = false;
     this._transcribeTask?.schedule();
-    log.info('stopped');
   }
 
   private _saveAudioChunk(chunk: AudioChunk) {
@@ -201,22 +194,24 @@ export class Transcriber extends Resource {
 
     if (!response.ok) {
       this._audioChunks = [];
-      throw new Error(`Transcription failed: ${response.statusText}`);
+      throw new Error('Failed to transcribe');
     }
 
-    const { segments } = (await response.json()) as {
+    const {
+      segments,
+    }: {
       segments: WhisperSegment[];
-    };
+    } = await response.json();
 
     log.info('transcription response', {
-      segments: segments.length,
+      segments,
       string: segments.map((segments) => segments.text).join(' '),
     });
 
     return segments;
   }
 
-  private _alignSegments(segments: WhisperSegment[], originalChunks: AudioChunk[]): TranscriptionContentBlock[] {
+  private _alignSegments(segments: WhisperSegment[], originalChunks: AudioChunk[]): TranscriptSegment[] {
     // Absolute zero for all relative timestamps in the segments.
     const zeroTimestamp = originalChunks.at(0)!.timestamp;
 
@@ -245,9 +240,8 @@ export class Transcriber extends Resource {
 
     // Add absolute timestamp to each segment.
     return filteredSegments.map((segment) => ({
-      type: 'transcription',
-      started: new Date(zeroTimestamp + segment.start * 1_000).toISOString(),
-      text: segment.text.trim(),
+      started: new Date(zeroTimestamp + segment.start * 1_000),
+      text: segment.text,
     }));
   }
 }

package/src/transcriber/transcription-manager.ts

@@ -5,14 +5,15 @@
 import { synchronized } from '@dxos/async';
 import { Resource } from '@dxos/context';
 import { QueueImpl, type Queue } from '@dxos/echo-db';
-import { create } from '@dxos/echo-schema';
-import { type DXN } from '@dxos/keys';
-import { log } from '@dxos/log';
+import { createStatic } from '@dxos/echo-schema';
+import { invariant } from '@dxos/invariant';
+import { DXN } from '@dxos/keys';
 import { type EdgeHttpClient } from '@dxos/react-edge-client';
-import { MessageType, type TranscriptionContentBlock } from '@dxos/schema';
+import { type HuePalette } from '@dxos/react-ui-theme';
 
 import { MediaStreamRecorder } from './media-stream-recorder';
 import { Transcriber } from './transcriber';
+import { TranscriptBlock, type TranscriptSegment } from '../types';
 
 /**
  * Length of the chunk in ms.
@@ -30,34 +31,23 @@ const PREFIXED_CHUNKS_AMOUNT = 10;
  */
 const TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
 
-export type TranscriptMessageEnricher = (message: MessageType) => Promise<MessageType>;
-
-export type TranscriptionManagerOptions = {
-  edgeClient: EdgeHttpClient;
-
-  /**
-   * Enrich the message before it is written to the transcription queue.
-   */
-  messageEnricher?: TranscriptMessageEnricher;
-};
-
 /**
  * Manages transcription state for a meeting.
  */
+// TODO(mykola): Reconcile with transcriber capability.
 export class TranscriptionManager extends Resource {
   private readonly _edgeClient: EdgeHttpClient;
-  private readonly _messageEnricher?: TranscriptMessageEnricher;
   private _audioStreamTrack?: MediaStreamTrack = undefined;
-  private _identityDid?: string = undefined;
+  private _name?: string = undefined;
+  private _hue?: HuePalette = undefined;
   private _mediaRecorder?: MediaStreamRecorder = undefined;
   private _transcriber?: Transcriber = undefined;
-  private _queue?: Queue<MessageType> = undefined;
   private _enabled = false;
+  private _queue?: Queue<TranscriptBlock> = undefined;
 
-  constructor(options: TranscriptionManagerOptions) {
+  constructor(edgeClient: EdgeHttpClient) {
     super();
-    this._edgeClient = options.edgeClient;
-    this._messageEnricher = options.messageEnricher;
+    this._edgeClient = edgeClient;
   }
 
   protected override async _open() {
@@ -68,24 +58,10 @@ export class TranscriptionManager extends Resource {
     void this._transcriber?.close();
   }
 
-  setQueue(queueDxn: DXN): TranscriptionManager {
-    if (this._queue?.dxn.toString() !== queueDxn.toString()) {
-      log.info('setQueue', { queueDxn: queueDxn.toString() });
-      this._queue = new QueueImpl<MessageType>(this._edgeClient, queueDxn);
-    }
-    return this;
-  }
-
-  setIdentityDid(did: string): TranscriptionManager {
-    if (this._identityDid !== did) {
-      this._identityDid = did;
-    }
-    return this;
-  }
-
-  setRecording(recording?: boolean): TranscriptionManager {
+  @synchronized
+  setRecording(recording?: boolean) {
     if (!this.isOpen || !this._enabled) {
-      return this;
+      return;
     }
 
     if (recording) {
@@ -93,7 +69,6 @@ export class TranscriptionManager extends Resource {
     } else {
       this._transcriber?.stopChunksRecording();
     }
-    return this;
   }
 
   /**
@@ -105,9 +80,7 @@ export class TranscriptionManager extends Resource {
     if (this._enabled === enabled) {
       return;
     }
-
     this._enabled = enabled ?? false;
-    // TODO(burdon): Why is toggle called here?
     this.isOpen && (await this._toggleTranscriber());
   }
 
@@ -116,33 +89,60 @@ export class TranscriptionManager extends Resource {
     if (this._audioStreamTrack === track) {
       return;
     }
-
     this._audioStreamTrack = track;
     this.isOpen && (await this._toggleTranscriber());
   }
 
-  // TODO(burdon): Change this to setEnables (explicit), not toggle.
+  /**
+   * Set the queue to save the transcription to.
+   * @param queue - The queue to save the transcription to or the DXN of the queue.
+   */
+  @synchronized
+  setQueue(queue?: Queue<TranscriptBlock> | string) {
+    switch (typeof queue) {
+      case 'string':
+        if (this._queue?.dxn.toString() === queue) {
+          return;
+        }
+        this._queue = new QueueImpl<TranscriptBlock>(this._edgeClient, DXN.parse(queue));
+        break;
+      case 'object':
+        if (this._queue === queue) {
+          return;
+        }
+        invariant(queue instanceof QueueImpl);
+        this._queue = queue;
+        break;
+      case 'undefined':
+        this._queue = undefined;
+        break;
+    }
+  }
+
+  @synchronized
+  setName(name: string) {
+    if (this._name === name) {
+      return;
+    }
+    this._name = name;
+  }
+
+  @synchronized
+  setHue(hue: HuePalette) {
+    if (this._hue === hue) {
+      return;
+    }
+    this._hue = hue;
+  }
+
   private async _toggleTranscriber() {
     await this._maybeReinitTranscriber();
 
     // Open or close transcriber if transcription is enabled or disabled.
     if (this._enabled) {
       await this._transcriber?.open();
-      // TODO(burdon): Started and stopped blocks appear twice.
-      const block = create(MessageType, {
-        created: new Date().toISOString(),
-        blocks: [{ type: 'transcription', text: 'Started', started: new Date().toISOString() }],
-        sender: { role: 'assistant' },
-      });
-      this._queue?.append([block]);
     } else {
       await this._transcriber?.close();
-      const block = create(MessageType, {
-        created: new Date().toISOString(),
-        blocks: [{ type: 'transcription', text: 'Stopped', started: new Date().toISOString() }],
-        sender: { role: 'assistant' },
-      });
-      this._queue?.append([block]);
     }
   }
 
@@ -173,21 +173,12 @@ export class TranscriptionManager extends Resource {
     }
   }
 
-  private async _onSegments(segments: TranscriptionContentBlock[]) {
+  private async _onSegments(segments: TranscriptSegment[]) {
     if (!this.isOpen || !this._queue) {
       return;
     }
 
-    let block = create(MessageType, {
-      created: new Date().toISOString(),
-      blocks: segments,
-      sender: { identityDid: this._identityDid },
-    });
-
-    if (this._messageEnricher) {
-      block = await this._messageEnricher(block);
-    }
-
+    const block = createStatic(TranscriptBlock, { authorName: this._name, authorHue: this._hue, segments });
     this._queue.append([block]);
   }
 }

package/src/translations.ts

@@ -22,8 +22,6 @@ export default [
         'delete button': 'Delete',
         'bookmark button': 'Bookmark',
         'scroll to end label': 'Scroll to latest',
-
-        'settings entity extraction label': 'Entity extraction',
       },
     },
   },

package/src/types/schema.ts

@@ -2,7 +2,7 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Expando, Ref, S, TypedObject } from '@dxos/echo-schema';
+import { EchoObject, Expando, Ref, S, TypedObject } from '@dxos/echo-schema';
 
 /**
  * Root transcript object created when the user starts a transcription.
@@ -10,9 +10,9 @@ import { Expando, Ref, S, TypedObject } from '@dxos/echo-schema';
 export const TranscriptSchema = S.Struct({
   name: S.String,
 
-  // TODO(burdon): Use Date or string?
-  started: S.optional(S.String),
-  ended: S.optional(S.String),
+  // TODO(burdon): Use string for dates?
+  started: S.optional(S.Date),
+  ended: S.optional(S.Date),
 
   /**
    * Queue containing TranscriptBlock objects.
@@ -33,7 +33,30 @@ export class TranscriptType extends TypedObject({
  * Contains metadata for the recording and transcript.
  */
 const TranscriptHeader = S.Struct({
-  started: S.optional(S.String),
+  started: S.optional(S.Date),
 });
 
 export type TranscriptHeader = S.Schema.Type<typeof TranscriptHeader>;
+
+/**
+ * Transcription fragment.
+ */
+const TranscriptSegment = S.Struct({
+  // TODO(burdon): TS from service is not Unix TS (x1000).
+  started: S.Date,
+  text: S.String,
+});
+
+export type TranscriptSegment = S.Schema.Type<typeof TranscriptSegment>;
+
+/**
+ * Transcription block (from single speaker).
+ */
+export const TranscriptBlock = S.Struct({
+  id: S.String,
+  authorName: S.optional(S.String), // TODO(burdon): IdentityDid.
+  authorHue: S.optional(S.String),
+  segments: S.Array(TranscriptSegment),
+}).pipe(EchoObject({ typename: 'dxos.org/type/TranscriptBlock', version: '0.1.0' }));
+
+export type TranscriptBlock = S.Schema.Type<typeof TranscriptBlock>;

package/src/types/types.ts

@@ -2,8 +2,8 @@
 // Copyright 2025 DXOS.org
 //
 
-import { SpaceIdSchema, S, isInstanceOf } from '@dxos/echo-schema';
-import { isLiveObject } from '@dxos/react-client/echo';
+import { S, isInstanceOf } from '@dxos/echo-schema';
+import { isReactiveObject } from '@dxos/react-client/echo';
 
 import { TranscriptType } from './schema';
 import { TRANSCRIPTION_PLUGIN } from '../meta';
@@ -22,7 +22,8 @@ export namespace TranscriptionAction {
   export class Create extends S.TaggedClass<Create>()(`${TRANSCRIPTION_ACTION}/create`, {
     input: S.Struct({
       name: S.optional(S.String),
-      spaceId: SpaceIdSchema,
+      // TODO(wittjosiah): SpaceId.
+      spaceId: S.String,
     }),
     output: S.Struct({
       object: TranscriptType,
@@ -31,14 +32,5 @@ export namespace TranscriptionAction {
 }
 
 export const isTranscript = (object: unknown): object is typeof TranscriptType => {
-  return isLiveObject(object) && isInstanceOf(TranscriptType, object);
+  return isReactiveObject(object) && isInstanceOf(TranscriptType, object);
 };
-
-// TODO(burdon): Create with decode consistently: S.decodeSync(TranscriptionSettingsSchema)({}))
-export const TranscriptionSettingsSchema = S.mutable(
-  S.Struct({
-    entityExtraction: S.optional(S.Boolean).pipe(S.withConstructorDefault(() => true)),
-  }),
-);
-
-export type TranscriptionSettingsProps = S.Schema.Type<typeof TranscriptionSettingsSchema>;

package/src/util/index.ts

@@ -5,3 +5,4 @@
 export * from './array';
 export * from './get-time-str';
 export * from './monitor-audio-level';
+export * from './random-queue-dxn';

package/src/util/random-queue-dxn.ts

@@ -0,0 +1,9 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { ObjectId } from '@dxos/echo-schema';
+import { DXN, QueueSubspaceTags, SpaceId } from '@dxos/keys';
+
+export const randomQueueDxn = (spaceId?: SpaceId) =>
+  new DXN(DXN.kind.QUEUE, [QueueSubspaceTags.DATA, spaceId ?? SpaceId.random(), ObjectId.random()]);