@dxos/plugin-transcription 0.8.4-main.fd6878d → 0.9.0

package/src/hooks/useTranscriber.ts

@@ -4,9 +4,10 @@
 
 import { useEffect, useMemo } from 'react';
 
-import { useCapabilities } from '@dxos/app-framework';
+import { useCapabilities } from '@dxos/app-framework/ui';
+
+import { TranscriptionCapabilities } from '#types';
 
-import { TranscriptionCapabilities } from '../capabilities';
 import { type Transcriber } from '../transcriber';
 
 /**
@@ -17,17 +18,24 @@ export const useTranscriber = ({
   recorderConfig,
   transcriberConfig,
   onSegments,
-}: Partial<TranscriptionCapabilities.GetTranscriberProps>) => {
-  const [getTranscriber] = useCapabilities(TranscriptionCapabilities.Transcriber);
+  transcribe,
+}: Partial<TranscriptionCapabilities.TranscriberProviderProps>) => {
+  const [transcriberProvider] = useCapabilities(TranscriptionCapabilities.TranscriberProvider);
 
   // Initialize audio transcription.
   const transcriber = useMemo<Transcriber | undefined>(() => {
-    if (!getTranscriber || !audioStreamTrack || !onSegments) {
+    if (!transcriberProvider || !audioStreamTrack || !onSegments) {
       return undefined;
     }
 
-    return getTranscriber({ audioStreamTrack, recorderConfig, transcriberConfig, onSegments });
-  }, [getTranscriber, audioStreamTrack, recorderConfig, transcriberConfig, onSegments]);
+    return transcriberProvider({
+      audioStreamTrack,
+      recorderConfig,
+      transcriberConfig,
+      onSegments,
+      transcribe,
+    });
+  }, [transcriberProvider, audioStreamTrack, recorderConfig, transcriberConfig, onSegments, transcribe]);
 
   useEffect(() => {
     return () => {

package/src/hooks/useTranscriptionRecording.ts

@@ -0,0 +1,87 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+import { useCallback, useEffect, useState } from 'react';
+
+import { createFeedServiceLayer } from '@dxos/client/echo';
+import { Feed, Obj } from '@dxos/echo';
+import { EffectEx } from '@dxos/effect';
+import { log } from '@dxos/log';
+import { getSpace } from '@dxos/react-client/echo';
+import { useIdentity } from '@dxos/react-client/halo';
+import { Message, type Transcript } from '@dxos/types';
+
+import { useAudioTrack } from './useAudioTrack';
+import { useTranscriber } from './useTranscriber';
+
+export type TranscriptionRecording = {
+  recording: boolean;
+  toggleRecording: () => void;
+};
+
+/**
+ * Drives the recording lifecycle for a transcript: opens an audio track, feeds it through the
+ * transcriber, and appends transcribed message batches to the transcript's feed. Returns the
+ * recording flag and a toggle callback for the caller to wire to UI.
+ */
+export const useTranscriptionRecording = (transcript: Transcript.Transcript): TranscriptionRecording => {
+  const space = getSpace(transcript);
+  const identity = useIdentity();
+  const feed = transcript.feed.target;
+
+  const [recording, setRecording] = useState(false);
+  const track = useAudioTrack(recording);
+
+  // Append a transcribed message batch to the transcript's feed.
+  const handleSegments = useCallback(
+    async (blocks: Message.Message['blocks']) => {
+      if (!space || !feed || blocks.length === 0) {
+        return;
+      }
+      const sender = identity ? { identityDid: identity.did, name: identity.profile?.displayName } : {};
+      const message = Obj.make(Message.Message, {
+        sender,
+        created: new Date().toISOString(),
+        blocks,
+      });
+      const feedServiceLayer = createFeedServiceLayer(space.queues);
+      await Feed.append(feed, [message]).pipe(Effect.provide(feedServiceLayer), EffectEx.runAndForwardErrors);
+    },
+    [space, feed, identity],
+  );
+
+  // Drive the transcriber lifecycle off the recording flag + audio track.
+  const transcriber = useTranscriber({ audioStreamTrack: track, onSegments: handleSegments });
+  useEffect(() => {
+    if (!transcriber) {
+      return;
+    }
+    let cancelled = false;
+    // If `open()` rejects (e.g. mic permission denied, device init failure) we must
+    // clear `recording` — otherwise the toolbar button reports "stop" forever while
+    // nothing is actually being captured.
+    void transcriber.open().then(
+      () => {
+        if (!cancelled && recording) {
+          transcriber.startChunksRecording();
+        }
+      },
+      (err) => {
+        if (!cancelled) {
+          log.catch(err);
+          setRecording(false);
+        }
+      },
+    );
+    return () => {
+      cancelled = true;
+      transcriber.stopChunksRecording();
+    };
+  }, [transcriber, recording]);
+
+  const toggleRecording = useCallback(() => setRecording((value) => !value), []);
+
+  return { recording, toggleRecording };
+};

package/src/hooks/useVoiceInput.ts

@@ -7,10 +7,9 @@ import { useCallback, useEffect, useState } from 'react';
 import { scheduleMicroTask } from '@dxos/async';
 import { Context } from '@dxos/context';
 import { log } from '@dxos/log';
-import { useSoundEffect } from '@dxos/react-ui-sfx';
-
-import { type TranscriberParams } from '../transcriber';
+import { useSoundEffect } from '@dxos/react-ui-audio';
 
+import { type TranscriberProps } from '../transcriber';
 import { useAudioTrack } from './useAudioTrack';
 import { useTranscriber } from './useTranscriber';
 
@@ -26,7 +25,7 @@ export const useVoiceInput = ({ active, onUpdate }: UseVoiceInputProps) => {
   const soundStart = useSoundEffect('StartRecording');
   const soundStop = useSoundEffect('StopRecording');
 
-  const handleSegments = useCallback<TranscriberParams['onSegments']>(async (segments) => {
+  const handleSegments = useCallback<TranscriberProps['onSegments']>(async (segments) => {
     const text = segments.map((str) => str.text.trim().replace(/[^\w\s]/g, '')).join(' ');
     onUpdate(text);
   }, []);

package/src/index.ts

@@ -2,11 +2,10 @@
 // Copyright 2025 DXOS.org
 //
 
-export { TranscriptionCapabilities } from './capabilities';
-
-export * from './components';
+// TODO(wittjosiah): Hooks should not be exported from the plugin package at all.
+//   Either refactor callers to not need them or factor them out to a shared package.
+export * from './blueprints';
 export * from './hooks';
 export * from './meta';
 export * from './transcriber';
-
-export * from './TranscriptionPlugin';
+export * from './types';

package/src/meta.ts

@@ -2,16 +2,41 @@
 // Copyright 2023 DXOS.org
 //
 
-import { type PluginMeta } from '@dxos/app-framework';
+import { Plugin } from '@dxos/app-framework';
+import { DXN } from '@dxos/keys';
+import { trim } from '@dxos/util';
 
-export const TRANSCRIPTION_PLUGIN = 'dxos.org/plugin/transcription';
-
-export const meta: PluginMeta = {
-  id: TRANSCRIPTION_PLUGIN,
+export const meta = Plugin.makeMeta({
+  key: DXN.make('org.dxos.plugin.transcription'),
   name: 'Transcription',
-  description: 'Voice transcription.',
+  author: 'DXOS',
+  description: trim`
+    Real-time voice-to-text transcription for DXOS Composer.
+    Captures microphone audio in short PCM chunks, batches them into WAV payloads, and streams
+    the results to a Whisper-compatible speech-recognition endpoint.
+    Transcribed segments are timestamped, deduplicated across chunk boundaries, and persisted
+    as Message objects in an ECHO Feed so that every participant in a shared space sees the
+    live transcript without any manual sync step.
+
+    Each recording session is represented as a Transcript object linked to its Feed.
+    The read-only editor renders messages with speaker bylines and wall-clock timestamps,
+    supporting markdown decoration and document preview.
+    An optional message enricher pipeline lets plugins annotate messages before they are
+    committed — for example, to attach speaker identity from a calls service.
+
+    The plugin ships two blueprint-compatible operations: Open (loads and formats a transcript
+    for reading by an AI assistant) and Summarize (sends the transcript text to an LLM and
+    returns a structured Markdown summary with key points, verbatim quotes, and actionable
+    tasks extracted from the conversation).
+
+    A sentence-normalization pass merges raw word-level tokens from the recognizer into
+    grammatically complete sentences, improving readability of the stored transcript without
+    altering the underlying audio timeline.
+  `,
   icon: 'ph--microphone--regular',
+  iconHue: 'sky',
   source: 'https://github.com/dxos/dxos/tree/main/packages/plugins/plugin-transcription',
+  spec: 'PLUGIN.mdl',
   tags: ['labs'],
   screenshots: [],
-};
+});

package/src/model/model.test.ts

@@ -6,13 +6,13 @@ import { EditorView } from '@codemirror/view';
 import { describe, test } from 'vitest';
 
 import { Obj } from '@dxos/echo';
-import type { ObjectId } from '@dxos/keys';
-import { DataType } from '@dxos/schema';
+import type { EntityId } from '@dxos/keys';
+import { Message } from '@dxos/types';
 
 import { type ChunkRenderer, DocumentAdapter, SerializationModel } from './model';
 
-const blockToMarkdown: ChunkRenderer<DataType.Message> = (
-  message: DataType.Message,
+const blockToMarkdown: ChunkRenderer<Message.Message> = (
+  message: Message.Message,
   index: number,
   debug = true,
 ): string[] => {
@@ -27,12 +27,12 @@ const createDate = () => new Date().toISOString();
 
 describe('SerializationModel', () => {
   test('basic', ({ expect }) => {
-    const model = new SerializationModel<DataType.Message>(blockToMarkdown);
+    const model = new SerializationModel<Message.Message>(blockToMarkdown);
     expect(model.chunks.length).to.eq(0);
     expect(model.doc.toString()).to.eq('');
 
     // Create message.
-    const message = Obj.make(DataType.Message, {
+    const message = Obj.make(Message.Message, {
       created: createDate(),
       sender: { name: 'Alice' },
       blocks: [
@@ -51,7 +51,9 @@ describe('SerializationModel', () => {
     }
 
     // Update message.
-    message.blocks.push({ _tag: 'transcript', started: createDate(), text: 'Hello again!' });
+    Obj.update(message, (message) => {
+      message.blocks.push({ _tag: 'transcript', started: createDate(), text: 'Hello again!' });
+    });
     model.updateChunk(message);
     {
       const text = model.doc.toString();
@@ -62,14 +64,14 @@ describe('SerializationModel', () => {
 
   test('sync - append', ({ expect }) => {
     const view = new EditorView({ extensions: [], doc: '' });
-    const model = new SerializationModel<DataType.Message>(blockToMarkdown);
+    const model = new SerializationModel<Message.Message>(blockToMarkdown);
     const adapter = new DocumentAdapter(view);
     expect(adapter.lineCount()).to.eq(0);
     expect(view.state.doc.toString()).to.eq('');
 
     // Append message.
     {
-      const message = Obj.make(DataType.Message, {
+      const message = Obj.make(Message.Message, {
         created: createDate(),
         sender: { name: 'Alice' },
         blocks: [
@@ -87,7 +89,7 @@ describe('SerializationModel', () => {
 
     // Append message.
     {
-      const message = Obj.make(DataType.Message, {
+      const message = Obj.make(Message.Message, {
         created: createDate(),
         sender: { name: 'Bob' },
         blocks: [
@@ -106,15 +108,15 @@ describe('SerializationModel', () => {
 
   test('sync - append, update, delete', ({ expect }) => {
     const view = new EditorView({ extensions: [], doc: '' });
-    const model = new SerializationModel<DataType.Message>(blockToMarkdown);
+    const model = new SerializationModel<Message.Message>(blockToMarkdown);
     const adapter = new DocumentAdapter(view);
     expect(adapter.lineCount()).to.eq(0);
 
-    let msgId: ObjectId;
+    let msgId: EntityId;
 
     // Append message.
     {
-      const message = Obj.make(DataType.Message, {
+      const message = Obj.make(Message.Message, {
         created: createDate(),
         sender: { name: 'Alice' },
         blocks: [
@@ -131,13 +133,17 @@ describe('SerializationModel', () => {
       expect(view.state.doc.toString()).to.eq('###### Alice\nHello world!\n\n');
 
       // Update message (add block).
-      message.blocks.push({ _tag: 'transcript', started: createDate(), text: 'Hello again!' });
+      Obj.update(message, (message) => {
+        message.blocks.push({ _tag: 'transcript', started: createDate(), text: 'Hello again!' });
+      });
       model.updateChunk(message);
       model.sync(adapter);
       expect(view.state.doc.toString()).to.eq('###### Alice\nHello world!\nHello again!\n\n');
 
       // Update message (remove block).
-      message.blocks.splice(0, 1);
+      Obj.update(message, (message) => {
+        message.blocks.splice(0, 1);
+      });
       model.updateChunk(message);
       model.sync(adapter);
       expect(view.state.doc.toString()).to.eq('###### Alice\nHello again!\n\n');
@@ -147,7 +153,7 @@ describe('SerializationModel', () => {
 
     // Append message.
     {
-      const message = Obj.make(DataType.Message, {
+      const message = Obj.make(Message.Message, {
         created: createDate(),
         sender: { name: 'Bob' },
         blocks: [{ _tag: 'transcript', started: createDate(), text: 'Hello again!' }],

package/src/model/model.ts

@@ -80,6 +80,9 @@ type ChunkChange<T extends Chunk> =
  * Instead this model tracks changes and syncs them with the ChunkDocument.
  */
 export class SerializationModel<T extends Chunk> {
+  /** Emits when the document is updated. */
+  public readonly update = new Event<void>();
+
   /** Ordered set of chunks. */
   private readonly _chunks: T[] = [];
 
@@ -92,9 +95,6 @@ export class SerializationModel<T extends Chunk> {
   /** Track chunk changes since last sync. */
   private readonly _changes: Array<ChunkChange<T>> = [];
 
-  /** Emits when the document is updated. */
-  public readonly update = new Event<void>();
-
   constructor(
     private readonly _renderer: ChunkRenderer<T>,
     initialChunks: T[] = [],

package/src/{segments-normalization → normalization}/message-normalizer.ts

@@ -2,28 +2,27 @@
 // Copyright 2025 DXOS.org
 //
 
-//
-// Copyright 2025 DXOS.org
-//
-
-import { effect } from '@preact/signals-core';
+import * as Effect from 'effect/Effect';
+import type * as Runtime from 'effect/Runtime';
 
 import { DeferredTask, asyncTimeout } from '@dxos/async';
 import { LifecycleState, Resource } from '@dxos/context';
-import { type Queue } from '@dxos/echo-db';
-import { type FunctionExecutor } from '@dxos/functions';
+import { Feed, Filter } from '@dxos/echo';
+import { EffectEx } from '@dxos/effect';
+import { type FunctionExecutor } from '@dxos/functions-runtime';
 import { log } from '@dxos/log';
-import { type DataType } from '@dxos/schema';
+import { Message } from '@dxos/types';
 
-import { type MessageWithRangeId, sentenceNormalization } from './normalization';
+import { type MessageWithRangeId, type NormalizationOutput, sentenceNormalization } from './normalization';
 import { getActorId } from './utils';
 
 const PROCESSING_TIMEOUT = 20_000; // ms
 const MAX_RANGE_ID_COUNT = 10;
 
-export type SegmentsNormalizerParams = {
+export type SegmentsNormalizerProps = {
   functionExecutor: FunctionExecutor;
-  queue: Queue<DataType.Message>;
+  feed: Feed.Feed;
+  feedRuntime: Runtime.Runtime<Feed.FeedService>;
   startingCursor: QueueCursor;
 };
 
@@ -33,37 +32,51 @@ export type QueueCursor = {
   timestamp: string;
 };
 
-// TODO(mykola): .
 export class MessageNormalizer extends Resource {
   private readonly _functionExecutor: FunctionExecutor;
-  private _queue: Queue<DataType.Message>;
+
+  private _feed: Feed.Feed;
+  private _feedRuntime: Runtime.Runtime<Feed.FeedService>;
   private _cursor: QueueCursor;
   private _messagesToProcess: MessageWithRangeId[] = [];
   private _normalizationTask?: DeferredTask;
   private _lastProcessedMessageIds?: string[];
 
-  constructor({ functionExecutor, queue, startingCursor }: SegmentsNormalizerParams) {
+  constructor({ functionExecutor, feed, feedRuntime, startingCursor }: SegmentsNormalizerProps) {
     super();
     this._functionExecutor = functionExecutor;
-    this._queue = queue;
+    this._feed = feed;
+    this._feedRuntime = feedRuntime;
     this._cursor = startingCursor;
   }
 
   protected override async _open(): Promise<void> {
     this._normalizationTask = new DeferredTask(this._ctx, () => this._processMessages());
-    const unsubscribe = effect(() => {
+
+    const queryResult = await Feed.query(this._feed, Filter.type(Message.Message)).pipe(
+      Effect.provide(this._feedRuntime),
+      EffectEx.runAndForwardErrors,
+    );
+
+    const updateMessages = () => {
       if (this._lifecycleState !== LifecycleState.OPEN) {
         return;
       }
 
-      this._messagesToProcess = this._queue.objects.filter((message) => {
+      this._messagesToProcess = (queryResult.results as Message.Message[]).filter((message) => {
         const actorId = getActorId(message.sender);
         return actorId === this._cursor.actorId && message.created >= this._cursor.timestamp;
       });
 
       this._normalizationTask!.schedule();
-    });
-    this._ctx.onDispose(() => unsubscribe());
+    };
+
+    // Initial update.
+    updateMessages();
+
+    // Subscribe to feed changes.
+    const unsubscribe = queryResult.subscribe(updateMessages);
+    this._ctx.onDispose(unsubscribe);
   }
 
   // Need to unpack strings from blocks from messages run them through the function and then pack them back into blocks into messages.
@@ -81,12 +94,12 @@ export class MessageNormalizer extends Resource {
 
     try {
       // TODO(mykola): Executor should support timeout.
-      const response = await asyncTimeout(
+      const response: NormalizationOutput = await asyncTimeout(
         this._functionExecutor.invoke(sentenceNormalization, { messages }),
         PROCESSING_TIMEOUT,
       );
 
-      this._writeMessages(response.sentences);
+      await this._writeMessages(response.sentences);
       this._lastProcessedMessageIds.push(...response.sentences.map((sentence) => sentence.id));
       if (response.sentences.length === 1 && response.sentences.at(-1)!.rangeId!.length > MAX_RANGE_ID_COUNT) {
         log.warn('Sentence is too long', { messages });
@@ -95,14 +108,14 @@ export class MessageNormalizer extends Resource {
     } catch {
       log.error('Failed to normalize segments', { messages });
       // If processing failed, emit the segments as is.
-      this._writeMessages(messages);
+      await this._writeMessages(messages);
     }
   }
 
-  private _writeMessages(messages: MessageWithRangeId[]): void {
+  private async _writeMessages(messages: MessageWithRangeId[]): Promise<void> {
     log.info('writing messages', { messages });
     const lastMessage = messages[messages.length - 1];
     this._cursor.timestamp = lastMessage.created;
-    void this._queue.append(messages);
+    await Feed.append(this._feed, messages).pipe(Effect.provide(this._feedRuntime), EffectEx.runAndForwardErrors);
   }
 }

package/src/{segments-normalization → normalization}/normalization.test.ts

@@ -2,23 +2,16 @@
 // Copyright 2025 DXOS.org
 //
 
-import { effect } from '@preact/signals-core';
 import { describe, test } from 'vitest';
 
-import { scheduleTaskInterval } from '@dxos/async';
-import { Context } from '@dxos/context';
 import { Obj } from '@dxos/echo';
-import { MemoryQueue } from '@dxos/echo-db';
-import { createQueueDXN } from '@dxos/echo-schema';
-import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
+import { FunctionExecutor, ServiceContainer } from '@dxos/functions-runtime';
 import { log } from '@dxos/log';
-import { DataType } from '@dxos/schema';
+import { type Actor, Message } from '@dxos/types';
 
-import { MessageNormalizer } from './message-normalizer';
 import { type MessageWithRangeId, sentenceNormalization } from './normalization';
-import { getActorId } from './utils';
 
-const sender: DataType.Actor = {
+const sender: Actor.Actor = {
   identityDid: 'did:key:123',
 };
 
@@ -48,7 +41,7 @@ const messages: MessageWithRangeId[] = [
   // No punctuation.
   'in classical physics objects have well-defined properties such as position speed and momentum',
 ].map((string, index) =>
-  Obj.make(DataType.Message, {
+  Obj.make(Message.Message, {
     created: new Date(Date.now() + 1_000 * index).toISOString(),
     sender,
     blocks: [{ _tag: 'transcript', started: new Date(Date.now() + 1_000 * index).toISOString(), text: string }],
@@ -66,7 +59,7 @@ describe.skip('SentenceNormalization', () => {
         //     ? new Edge AiServiceClient({
         //         endpoint: AI_SERVICE_ENDPOINT.REMOTE,
         //         defaultGenerationOptions: {
-        //           model: '@anthropic/claude-3-5-sonnet-20241022',
+        //           model: 'ai.claude.model.claude-3-5-sonnet-20241022',
         //         },
         //       })
         //     : new Ollama AiServiceClient({
@@ -99,45 +92,13 @@ describe.skip('SentenceNormalization', () => {
     }
 
     log.info('sentences', {
-      originalMessages: JSON.stringify(messages, null, 2),
+      messages: JSON.stringify(messages, null, 2),
       sentences: JSON.stringify(sentences, null, 2),
     });
     throw new Error('test');
   });
 
-  test.only('queue', { timeout: 120_000 }, async () => {
-    // Create queue.
-    // TODO(dmaretskyi): Use space.queues.create() instead.
-    const queue = new MemoryQueue<DataType.Message>(createQueueDXN());
-    const ctx = new Context();
-    let idx = 0;
-    scheduleTaskInterval(
-      ctx,
-      async () => {
-        if (idx >= messages.length) {
-          void ctx.dispose();
-          return;
-        }
-        await queue.append([messages[idx]]);
-        idx++;
-      },
-      2_000,
-    );
-
-    // Create normalizer.
-    const normalizer = new MessageNormalizer({
-      functionExecutor: getExecutor(),
-      queue,
-      startingCursor: { actorId: getActorId(sender), timestamp: '0' },
-    });
-
-    // Start normalizer.
-    await normalizer.open();
-    effect(() => {
-      log.info('normalizer');
-      log.info(JSON.stringify(queue.objects, null, 2));
-    });
-
-    await new Promise(() => {});
-  });
+  // TODO(burdon): Restore a Feed-based version once `MessageNormalizer` has a
+  //   space-backed test fixture available; the previous `MemoryQueue`-based
+  //   test was incompatible with the `Feed.Feed` + `FeedService` refactor.
 });

package/src/normalization/normalization.ts

@@ -0,0 +1,76 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+import * as Schema from 'effect/Schema';
+
+import { Operation } from '@dxos/compute';
+import { log } from '@dxos/log';
+import { trim } from '@dxos/util';
+
+import { TranscriptOperation } from '../types';
+
+type MessageWithRangeIdType = TranscriptOperation.MessageWithRangeIdType;
+type SentenceNormalizationInputType = TranscriptOperation.SentenceNormalizationInputType;
+const { SentenceNormalization, SentenceNormalizationOutput } = TranscriptOperation;
+
+export type MessageWithRangeId = MessageWithRangeIdType;
+
+/** @deprecated Use SentenceNormalizationInputType */
+export type NormalizationInput = SentenceNormalizationInputType;
+
+/** @deprecated Use SentenceNormalizationOutput */
+export type NormalizationOutput = Schema.Schema.Type<typeof SentenceNormalizationOutput>;
+
+/**
+ * Sentence normalization for transcription.
+ * TODO(dmaretskyi): Reimplement using AiRequest.run or LanguageModel.generateObject - runStructured was removed.
+ */
+export const sentenceNormalization = SentenceNormalization.pipe(
+  Operation.withHandler(
+    Effect.fn(function* ({ messages }) {
+      log.info('input', { messages });
+      // TODO(dmaretskyi): runStructured was removed from AiRequest. Reimplement using new API.
+      return yield* Effect.die(
+        new Error(
+          'Sentence normalization needs to be reimplemented - runStructured was removed from AiRequest.Request. Use AiRequest.run or LanguageModel.generateObject.',
+        ),
+      );
+    }),
+  ),
+);
+
+const _unused = trim`
+  You are observing a real-time transcript of a single person speaking.
+  The transcription is delivered in chunks of 10 seconds or less. As a result, individual sentences may be split across multiple messages, or multiple sentences may appear within a single message. Additionally, because this is real-time transcription, punctuation and capitalization may be incorrect or missing.
+
+  # Task Description:
+    - Your task is to detect and reconstruct broken or incomplete sentences by merging segments into coherent, grammatically correct sentences where appropriate.
+
+  # Input Format:
+    - The input is an array of messages that contain transcription blocks.
+    - Each message is a Message.Message, as described in the output format.
+
+  # Output Format:
+    - You have been provided with the tool that defines the output format; make sure to query it.
+    - Do not output anything other than the expected format.
+
+  # Segment Handling Rules:
+    - Sort messages by timestamp.
+    - Leave ID of the first message in a sentence.
+    - Each divided sentence should be added to the output as a separate message or block within existing messages, preserving the original order.
+    - If one message contains multiple sentences, you should not split them.
+    - The last sentence may be incomplete; it should still be output as a separate block or message.
+    - Keep track of the original timestamps of the messages and blocks and, use the timestamp of the first message/block of a sentence as the 'started' field of the merged message.
+    - The 'rangeId' field of the merged message should include 'messageId'-s and 'rangeId'-s of all messages that have been used to construct this message.
+
+  # Punctuation and Capitalization:
+    - Do not rely on the original punctuation and capitalization—they may be incorrect.
+    - Use logical reasoning to apply appropriate punctuation (e.g., period, comma, question mark, exclamation mark) and capitalization.
+
+  # Restrictions:
+    - Do not alter the order of the original messages; maintain the natural flow of speech.
+    - Do not interpret or infer meaning beyond reconstructing sentences.
+    - Do not add or remove any words or phrases.
+`;

package/src/{segments-normalization → normalization}/utils.ts

@@ -2,8 +2,8 @@
 // Copyright 2025 DXOS.org
 //
 
-import { type DataType } from '@dxos/schema';
+import { type Actor } from '@dxos/types';
 
-export const getActorId = (actor: DataType.Actor) => {
+export const getActorId = (actor: Actor.Actor) => {
   return actor.identityDid || actor.identityKey || actor.email || actor.name || '';
 };