@dxos/plugin-transcription 0.8.4-main.abd8ff62ef → 0.8.4-main.bc2380dfbc

package/src/enrichment/stream-simulator.ts

@@ -0,0 +1,125 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+// Turns a parsed conversation into a chronologically-ordered sequence of Whisper-shaped
+// transcript batches, one per (speaker, utterance) pair. Each batch carries a virtual
+// timestamp (ms since the conversation start) and the speaker label so consumers can fan
+// out per-stream — analogous to two microphones being transcribed in parallel.
+//
+// The simulator does NOT do any wall-clock waiting. Tests drive virtual time via
+// `vi.useFakeTimers()` or step through the batches directly; production code wouldn't use
+// this module at all.
+
+import { type ContentBlock } from '@dxos/types';
+
+import type { ConversationUtterance, ParsedConversation } from './parser';
+
+export type StreamSimulatorOptions = {
+  /**
+   * Average speaking rate, words per second. Used to assign a virtual duration to each
+   * utterance so consecutive utterances land at realistic offsets.
+   * @default 2.5 (≈150 wpm — typical conversational pace)
+   */
+  wordsPerSecond?: number;
+
+  /**
+   * Gap between consecutive utterances in seconds. Models a brief pause for turn-taking
+   * even when speakers don't actually overlap.
+   * @default 0.5
+   */
+  gapSeconds?: number;
+
+  /**
+   * Whisper batches typically arrive in chunks of N seconds rather than per-utterance.
+   * When set, utterances shorter than `batchSeconds` are merged into the same batch as
+   * the following utterance from the same speaker. `0` disables merging — every utterance
+   * is its own batch.
+   * @default 0
+   */
+  batchSeconds?: number;
+};
+
+export type SimulatedBatch = {
+  /** Speaker label from the fixture (`A`, `B`, …). Maps to a virtual `identityDid`. */
+  speaker: string;
+  /** ms offset from the conversation start when this batch is delivered. */
+  offsetMs: number;
+  /** Whisper-shaped transcript blocks (raw text, no enrichment). */
+  blocks: ContentBlock.Transcript[];
+};
+
+/**
+ * Build the chronological batch schedule for a parsed conversation. The result is a
+ * pure data structure — consumers iterate and drive virtual time themselves.
+ */
+export const simulateStream = (
+  conversation: ParsedConversation,
+  options: StreamSimulatorOptions = {},
+): SimulatedBatch[] => {
+  const wordsPerSecond = options.wordsPerSecond ?? 2.5;
+  const gapSeconds = options.gapSeconds ?? 0.5;
+  const batchSeconds = options.batchSeconds ?? 0;
+
+  // Guard against divide-by-zero / NaN / negative inputs — `wordsPerSecond` divides into a
+  // duration below, and the other knobs feed into offsets that must be finite.
+  if (!Number.isFinite(wordsPerSecond) || wordsPerSecond <= 0) {
+    throw new Error(`simulateStream: wordsPerSecond must be a finite number > 0 (got ${wordsPerSecond}).`);
+  }
+  if (!Number.isFinite(gapSeconds) || gapSeconds < 0) {
+    throw new Error(`simulateStream: gapSeconds must be a finite number >= 0 (got ${gapSeconds}).`);
+  }
+  if (!Number.isFinite(batchSeconds) || batchSeconds < 0) {
+    throw new Error(`simulateStream: batchSeconds must be a finite number >= 0 (got ${batchSeconds}).`);
+  }
+
+  const conversationStart = Date.UTC(2024, 0, 1, 12, 0, 0); // deterministic, ignored by callers
+  let cursorMs = 0;
+  const batches: SimulatedBatch[] = [];
+
+  // Optionally merge same-speaker consecutive utterances until they reach `batchSeconds`.
+  const grouped: { speaker: string; utterances: ConversationUtterance[] }[] = [];
+  if (batchSeconds === 0) {
+    for (const u of conversation.utterances) {
+      grouped.push({ speaker: u.speaker, utterances: [u] });
+    }
+  } else {
+    let pending: { speaker: string; utterances: ConversationUtterance[]; words: number } | undefined;
+    for (const u of conversation.utterances) {
+      const words = utteranceWordCount(u.text);
+      if (pending && pending.speaker === u.speaker && pending.words / wordsPerSecond < batchSeconds) {
+        pending.utterances.push(u);
+        pending.words += words;
+      } else {
+        if (pending) {
+          grouped.push({ speaker: pending.speaker, utterances: pending.utterances });
+        }
+        pending = { speaker: u.speaker, utterances: [u], words };
+      }
+    }
+    if (pending) {
+      grouped.push({ speaker: pending.speaker, utterances: pending.utterances });
+    }
+  }
+
+  for (const group of grouped) {
+    const startMs = cursorMs;
+    const blocks: ContentBlock.Transcript[] = [];
+    let segmentCursorMs = cursorMs;
+    for (const u of group.utterances) {
+      const durationMs = Math.max(500, (utteranceWordCount(u.text) / wordsPerSecond) * 1_000);
+      blocks.push({
+        _tag: 'transcript',
+        started: new Date(conversationStart + segmentCursorMs).toISOString(),
+        text: u.text,
+      });
+      segmentCursorMs += durationMs;
+    }
+    batches.push({ speaker: group.speaker, offsetMs: startMs, blocks });
+    cursorMs = segmentCursorMs + gapSeconds * 1_000;
+  }
+
+  return batches;
+};
+
+const utteranceWordCount = (text: string): number => text.trim().split(/\s+/).filter(Boolean).length;

package/src/hooks/index.ts

@@ -4,6 +4,7 @@
 
 export * from './useAudioFile';
 export * from './useAudioTrack';
-export * from './useQueueModelAdapter';
+export * from './useFeedModelAdapter';
 export * from './useTranscriber';
+export * from './useTranscriptionRecording';
 export * from './useVoiceInput';

package/src/hooks/useFeedModelAdapter.ts

@@ -0,0 +1,46 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { useEffect, useMemo } from 'react';
+
+import type { Obj } from '@dxos/echo';
+
+import { type ChunkRenderer, SerializationModel } from '../model';
+
+/**
+ * Feeds an array of objects (typically from `useQuery(db, Query.from(feed))`) into a
+ * `SerializationModel`, appending each new tail item as the array grows.
+ *
+ * @param renderer - chunk renderer.
+ * @param objects - current snapshot of feed items.
+ * @param initialChunks - chunks to seed the model with on first render.
+ */
+export const useFeedModelAdapter = <T extends Obj.Unknown>(
+  renderer: ChunkRenderer<T>,
+  objects: readonly T[],
+  initialChunks: T[] = [],
+): SerializationModel<T> => {
+  const model = useMemo(() => new SerializationModel<T>(renderer, initialChunks), [renderer]);
+
+  // Seed the model with the current snapshot on first render and whenever the
+  // identity of `objects` changes (e.g. feed swapped).
+  useEffect(() => {
+    for (const chunk of objects) {
+      model.appendChunk(chunk);
+    }
+  }, [model, objects]);
+
+  // Append new tail items as the array grows.
+  useEffect(() => {
+    if (model.chunks.length === objects.length) {
+      return;
+    }
+    const chunk = objects.at(-1);
+    if (chunk) {
+      model.appendChunk(chunk);
+    }
+  }, [model, objects.length]);
+
+  return model;
+};

package/src/hooks/useTranscriptionRecording.ts

@@ -0,0 +1,87 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+import { useCallback, useEffect, useState } from 'react';
+
+import { createFeedServiceLayer } from '@dxos/client/echo';
+import { Feed, Obj } from '@dxos/echo';
+import { runAndForwardErrors } from '@dxos/effect';
+import { log } from '@dxos/log';
+import { getSpace } from '@dxos/react-client/echo';
+import { useIdentity } from '@dxos/react-client/halo';
+import { Message, type Transcript } from '@dxos/types';
+
+import { useAudioTrack } from './useAudioTrack';
+import { useTranscriber } from './useTranscriber';
+
+export type TranscriptionRecording = {
+  recording: boolean;
+  toggleRecording: () => void;
+};
+
+/**
+ * Drives the recording lifecycle for a transcript: opens an audio track, feeds it through the
+ * transcriber, and appends transcribed message batches to the transcript's feed. Returns the
+ * recording flag and a toggle callback for the caller to wire to UI.
+ */
+export const useTranscriptionRecording = (transcript: Transcript.Transcript): TranscriptionRecording => {
+  const space = getSpace(transcript);
+  const identity = useIdentity();
+  const feed = transcript.feed.target;
+
+  const [recording, setRecording] = useState(false);
+  const track = useAudioTrack(recording);
+
+  // Append a transcribed message batch to the transcript's feed.
+  const handleSegments = useCallback(
+    async (blocks: Message.Message['blocks']) => {
+      if (!space || !feed || blocks.length === 0) {
+        return;
+      }
+      const sender = identity ? { identityDid: identity.did, name: identity.profile?.displayName } : {};
+      const message = Obj.make(Message.Message, {
+        sender,
+        created: new Date().toISOString(),
+        blocks,
+      });
+      const feedServiceLayer = createFeedServiceLayer(space.queues);
+      await Feed.append(feed, [message]).pipe(Effect.provide(feedServiceLayer), runAndForwardErrors);
+    },
+    [space, feed, identity],
+  );
+
+  // Drive the transcriber lifecycle off the recording flag + audio track.
+  const transcriber = useTranscriber({ audioStreamTrack: track, onSegments: handleSegments });
+  useEffect(() => {
+    if (!transcriber) {
+      return;
+    }
+    let cancelled = false;
+    // If `open()` rejects (e.g. mic permission denied, device init failure) we must
+    // clear `recording` — otherwise the toolbar button reports "stop" forever while
+    // nothing is actually being captured.
+    void transcriber.open().then(
+      () => {
+        if (!cancelled && recording) {
+          transcriber.startChunksRecording();
+        }
+      },
+      (err) => {
+        if (!cancelled) {
+          log.catch(err);
+          setRecording(false);
+        }
+      },
+    );
+    return () => {
+      cancelled = true;
+      transcriber.stopChunksRecording();
+    };
+  }, [transcriber, recording]);
+
+  const toggleRecording = useCallback(() => setRecording((value) => !value), []);
+
+  return { recording, toggleRecording };
+};

package/src/index.ts

@@ -2,16 +2,10 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Plugin } from '@dxos/app-framework';
-
-import { meta } from './meta';
-
-export const TranscriptionPlugin = Plugin.lazy(meta, () => import('#plugin'));
-
-export { TranscriptionCapabilities } from './types';
-
 // TODO(wittjosiah): Hooks should not be exported from the plugin package at all.
 //   Either refactor callers to not need them or factor them out to a shared package.
-export * from './meta';
+export * from './blueprints';
 export * from './hooks';
+export * from './meta';
 export * from './transcriber';
+export * from './types';

package/src/normalization/message-normalizer.ts

@@ -2,12 +2,16 @@
 // Copyright 2025 DXOS.org
 //
 
+import * as Effect from 'effect/Effect';
+import type * as Runtime from 'effect/Runtime';
+
 import { DeferredTask, asyncTimeout } from '@dxos/async';
 import { LifecycleState, Resource } from '@dxos/context';
-import { type Queue } from '@dxos/echo-db';
+import { Feed, Filter } from '@dxos/echo';
+import { runAndForwardErrors } from '@dxos/effect';
 import { type FunctionExecutor } from '@dxos/functions-runtime';
 import { log } from '@dxos/log';
-import { type Message } from '@dxos/types';
+import { Message } from '@dxos/types';
 
 import { type MessageWithRangeId, type NormalizationOutput, sentenceNormalization } from './normalization';
 import { getActorId } from './utils';
@@ -17,7 +21,8 @@ const MAX_RANGE_ID_COUNT = 10;
 
 export type SegmentsNormalizerProps = {
   functionExecutor: FunctionExecutor;
-  queue: Queue<Message.Message>;
+  feed: Feed.Feed;
+  feedRuntime: Runtime.Runtime<Feed.FeedService>;
   startingCursor: QueueCursor;
 };
 
@@ -30,28 +35,35 @@ export type QueueCursor = {
 export class MessageNormalizer extends Resource {
   private readonly _functionExecutor: FunctionExecutor;
 
-  private _queue: Queue<Message.Message>;
+  private _feed: Feed.Feed;
+  private _feedRuntime: Runtime.Runtime<Feed.FeedService>;
   private _cursor: QueueCursor;
   private _messagesToProcess: MessageWithRangeId[] = [];
   private _normalizationTask?: DeferredTask;
   private _lastProcessedMessageIds?: string[];
 
-  constructor({ functionExecutor, queue, startingCursor }: SegmentsNormalizerProps) {
+  constructor({ functionExecutor, feed, feedRuntime, startingCursor }: SegmentsNormalizerProps) {
     super();
     this._functionExecutor = functionExecutor;
-    this._queue = queue;
+    this._feed = feed;
+    this._feedRuntime = feedRuntime;
     this._cursor = startingCursor;
   }
 
   protected override async _open(): Promise<void> {
     this._normalizationTask = new DeferredTask(this._ctx, () => this._processMessages());
 
+    const queryResult = await Feed.query(this._feed, Filter.type(Message.Message)).pipe(
+      Effect.provide(this._feedRuntime),
+      runAndForwardErrors,
+    );
+
     const updateMessages = () => {
       if (this._lifecycleState !== LifecycleState.OPEN) {
         return;
       }
 
-      this._messagesToProcess = this._queue.objects.filter((message) => {
+      this._messagesToProcess = (queryResult.results as Message.Message[]).filter((message) => {
         const actorId = getActorId(message.sender);
         return actorId === this._cursor.actorId && message.created >= this._cursor.timestamp;
       });
@@ -62,8 +74,8 @@ export class MessageNormalizer extends Resource {
     // Initial update.
     updateMessages();
 
-    // Subscribe to queue changes.
-    const unsubscribe = this._queue.subscribe(updateMessages);
+    // Subscribe to feed changes.
+    const unsubscribe = queryResult.subscribe(updateMessages);
     this._ctx.onDispose(unsubscribe);
   }
 
@@ -87,7 +99,7 @@ export class MessageNormalizer extends Resource {
         PROCESSING_TIMEOUT,
       );
 
-      this._writeMessages(response.sentences);
+      await this._writeMessages(response.sentences);
       this._lastProcessedMessageIds.push(...response.sentences.map((sentence) => sentence.id));
       if (response.sentences.length === 1 && response.sentences.at(-1)!.rangeId!.length > MAX_RANGE_ID_COUNT) {
         log.warn('Sentence is too long', { messages });
@@ -96,14 +108,14 @@ export class MessageNormalizer extends Resource {
     } catch {
       log.error('Failed to normalize segments', { messages });
       // If processing failed, emit the segments as is.
-      this._writeMessages(messages);
+      await this._writeMessages(messages);
     }
   }
 
-  private _writeMessages(messages: MessageWithRangeId[]): void {
+  private async _writeMessages(messages: MessageWithRangeId[]): Promise<void> {
     log.info('writing messages', { messages });
     const lastMessage = messages[messages.length - 1];
     this._cursor.timestamp = lastMessage.created;
-    void this._queue.append(messages);
+    await Feed.append(this._feed, messages).pipe(Effect.provide(this._feedRuntime), runAndForwardErrors);
   }
 }

package/src/normalization/normalization.test.ts

@@ -4,18 +4,12 @@
 
 import { describe, test } from 'vitest';
 
-import { scheduleTaskInterval } from '@dxos/async';
-import { Context } from '@dxos/context';
 import { Obj } from '@dxos/echo';
-import { MemoryQueue } from '@dxos/echo-db';
-import { createQueueDXN } from '@dxos/echo/internal';
 import { FunctionExecutor, ServiceContainer } from '@dxos/functions-runtime';
 import { log } from '@dxos/log';
 import { type Actor, Message } from '@dxos/types';
 
-import { MessageNormalizer } from './message-normalizer';
 import { type MessageWithRangeId, sentenceNormalization } from './normalization';
-import { getActorId } from './utils';
 
 const sender: Actor.Actor = {
   identityDid: 'did:key:123',
@@ -104,35 +98,7 @@ describe.skip('SentenceNormalization', () => {
     throw new Error('test');
   });
 
-  test.only('queue', { timeout: 120_000 }, async () => {
-    // Create queue.
-    // TODO(dmaretskyi): Use space.queues.create() instead.
-    const queue = new MemoryQueue<Message.Message>(createQueueDXN());
-    const ctx = new Context();
-    let idx = 0;
-    scheduleTaskInterval(
-      ctx,
-      async () => {
-        if (idx >= messages.length) {
-          void ctx.dispose();
-          return;
-        }
-        await queue.append([messages[idx]]);
-        idx++;
-      },
-      2_000,
-    );
-
-    // Create normalizer.
-    const normalizer = new MessageNormalizer({
-      functionExecutor: getExecutor(),
-      queue,
-      startingCursor: { actorId: getActorId(sender), timestamp: '0' },
-    });
-
-    // Start normalizer.
-    await normalizer.open();
-
-    await new Promise(() => {});
-  });
+  // TODO(burdon): Restore a Feed-based version once `MessageNormalizer` has a
+  //   space-backed test fixture available; the previous `MemoryQueue`-based
+  //   test was incompatible with the `Feed.Feed` + `FeedService` refactor.
 });

package/src/normalization/normalization.ts

@@ -9,12 +9,11 @@ import { Operation } from '@dxos/compute';
 import { log } from '@dxos/log';
 import { trim } from '@dxos/util';
 
-import {
-  SentenceNormalization,
-  SentenceNormalizationOutput,
-  type MessageWithRangeIdType,
-  type SentenceNormalizationInputType,
-} from '../operations/definitions';
+import { TranscriptOperation } from '../types';
+
+type MessageWithRangeIdType = TranscriptOperation.MessageWithRangeIdType;
+type SentenceNormalizationInputType = TranscriptOperation.SentenceNormalizationInputType;
+const { SentenceNormalization, SentenceNormalizationOutput } = TranscriptOperation;
 
 export type MessageWithRangeId = MessageWithRangeIdType;
 
@@ -35,7 +34,7 @@ export const sentenceNormalization = SentenceNormalization.pipe(
       // TODO(dmaretskyi): runStructured was removed from AiRequest. Reimplement using new API.
       return yield* Effect.die(
         new Error(
-          'Sentence normalization needs to be reimplemented - runStructured was removed from AiRequest. Use AiRequest.run or LanguageModel.generateObject.',
+          'Sentence normalization needs to be reimplemented - runStructured was removed from AiRequest.Request. Use AiRequest.run or LanguageModel.generateObject.',
         ),
       );
     }),

package/src/operations/create.ts

@@ -8,9 +8,9 @@ import { Operation } from '@dxos/compute';
 import { Feed, Ref } from '@dxos/echo';
 import { Transcript } from '@dxos/types';
 
-import { Create } from './definitions';
+import { TranscriptOperation } from '../types';
 
-const handler: Operation.WithHandler<typeof Create> = Create.pipe(
+const handler: Operation.WithHandler<typeof TranscriptOperation.Create> = TranscriptOperation.Create.pipe(
   Operation.withHandler(
     Effect.fn(function* ({ space }) {
       const feed = space.db.add(Feed.make());

package/src/operations/index.ts

@@ -4,8 +4,6 @@
 
 import { OperationHandlerSet } from '@dxos/compute';
 
-export * as TranscriptOperation from './definitions';
-
 export const TranscriptionOperationHandlerSet = OperationHandlerSet.lazy(
   () => import('./create'),
   () => import('./open'),

package/src/operations/open.ts

@@ -8,10 +8,10 @@ import { Operation } from '@dxos/compute';
 import { Database, Feed, Filter } from '@dxos/echo';
 import { Message } from '@dxos/types';
 
+import { TranscriptOperation } from '../types';
 import { renderByline } from '../util';
-import { Open } from './definitions';
 
-const handler: Operation.WithHandler<typeof Open> = Open.pipe(
+const handler: Operation.WithHandler<typeof TranscriptOperation.Open> = TranscriptOperation.Open.pipe(
   Operation.withHandler(
     Effect.fn(function* ({ transcript }) {
       const transcriptObj = yield* Database.load(transcript);

package/src/operations/summarize.ts

@@ -14,16 +14,16 @@ import { Trace, Operation, OperationRegistry } from '@dxos/compute';
 import { Database } from '@dxos/echo';
 import { trim } from '@dxos/util';
 
-import { Summarize } from './definitions';
+import { TranscriptOperation } from '../types';
 
 /**
  * Summarize a transcript of a meeting.
  */
-const handler: Operation.WithHandler<typeof Summarize> = Summarize.pipe(
+const handler: Operation.WithHandler<typeof TranscriptOperation.Summarize> = TranscriptOperation.Summarize.pipe(
   Operation.withHandler(
     Effect.fnUntraced(
       function* ({ transcript, notes }) {
-        const result = yield* new AiRequest({
+        const result = yield* new AiRequest.Request({
           observer: GenerationObserver.fromPrinter(new ConsolePrinter({ tag: 'summarize' })),
         }).run({
           prompt: `Transcript: ${transcript}\n\nNotes: ${notes}`,

package/src/plugin.ts

@@ -0,0 +1,11 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Plugin } from '@dxos/app-framework';
+
+import { meta } from './meta';
+
+export const TranscriptionPlugin = Plugin.lazy(meta, () => import('#plugin'));
+
+export { TranscriptionOperationHandlerSet } from './operations';