@dxos/plugin-transcription 0.8.4-main.ae835ea → 0.8.4-main.bc2380dfbc

package/src/{segments-normalization → normalization}/message-normalizer.ts

@@ -2,28 +2,27 @@
 // Copyright 2025 DXOS.org
 //
 
-//
-// Copyright 2025 DXOS.org
-//
-
-import { effect } from '@preact/signals-core';
+import * as Effect from 'effect/Effect';
+import type * as Runtime from 'effect/Runtime';
 
 import { DeferredTask, asyncTimeout } from '@dxos/async';
 import { LifecycleState, Resource } from '@dxos/context';
-import { type Queue } from '@dxos/echo-db';
-import { type FunctionExecutor } from '@dxos/functions';
+import { Feed, Filter } from '@dxos/echo';
+import { runAndForwardErrors } from '@dxos/effect';
+import { type FunctionExecutor } from '@dxos/functions-runtime';
 import { log } from '@dxos/log';
-import { type DataType } from '@dxos/schema';
+import { Message } from '@dxos/types';
 
-import { type MessageWithRangeId, sentenceNormalization } from './normalization';
+import { type MessageWithRangeId, type NormalizationOutput, sentenceNormalization } from './normalization';
 import { getActorId } from './utils';
 
 const PROCESSING_TIMEOUT = 20_000; // ms
 const MAX_RANGE_ID_COUNT = 10;
 
-export type SegmentsNormalizerParams = {
+export type SegmentsNormalizerProps = {
   functionExecutor: FunctionExecutor;
-  queue: Queue<DataType.Message>;
+  feed: Feed.Feed;
+  feedRuntime: Runtime.Runtime<Feed.FeedService>;
   startingCursor: QueueCursor;
 };
 
@@ -33,37 +32,51 @@ export type QueueCursor = {
   timestamp: string;
 };
 
-// TODO(mykola): .
 export class MessageNormalizer extends Resource {
   private readonly _functionExecutor: FunctionExecutor;
-  private _queue: Queue<DataType.Message>;
+
+  private _feed: Feed.Feed;
+  private _feedRuntime: Runtime.Runtime<Feed.FeedService>;
   private _cursor: QueueCursor;
   private _messagesToProcess: MessageWithRangeId[] = [];
   private _normalizationTask?: DeferredTask;
   private _lastProcessedMessageIds?: string[];
 
-  constructor({ functionExecutor, queue, startingCursor }: SegmentsNormalizerParams) {
+  constructor({ functionExecutor, feed, feedRuntime, startingCursor }: SegmentsNormalizerProps) {
     super();
     this._functionExecutor = functionExecutor;
-    this._queue = queue;
+    this._feed = feed;
+    this._feedRuntime = feedRuntime;
     this._cursor = startingCursor;
   }
 
   protected override async _open(): Promise<void> {
     this._normalizationTask = new DeferredTask(this._ctx, () => this._processMessages());
-    const unsubscribe = effect(() => {
+
+    const queryResult = await Feed.query(this._feed, Filter.type(Message.Message)).pipe(
+      Effect.provide(this._feedRuntime),
+      runAndForwardErrors,
+    );
+
+    const updateMessages = () => {
       if (this._lifecycleState !== LifecycleState.OPEN) {
         return;
       }
 
-      this._messagesToProcess = this._queue.objects.filter((message) => {
+      this._messagesToProcess = (queryResult.results as Message.Message[]).filter((message) => {
         const actorId = getActorId(message.sender);
         return actorId === this._cursor.actorId && message.created >= this._cursor.timestamp;
       });
 
       this._normalizationTask!.schedule();
-    });
-    this._ctx.onDispose(() => unsubscribe());
+    };
+
+    // Initial update.
+    updateMessages();
+
+    // Subscribe to feed changes.
+    const unsubscribe = queryResult.subscribe(updateMessages);
+    this._ctx.onDispose(unsubscribe);
   }
 
   // Need to unpack strings from blocks from messages run them through the function and then pack them back into blocks into messages.
@@ -81,12 +94,12 @@ export class MessageNormalizer extends Resource {
 
     try {
       // TODO(mykola): Executor should support timeout.
-      const response = await asyncTimeout(
+      const response: NormalizationOutput = await asyncTimeout(
         this._functionExecutor.invoke(sentenceNormalization, { messages }),
         PROCESSING_TIMEOUT,
       );
 
-      this._writeMessages(response.sentences);
+      await this._writeMessages(response.sentences);
       this._lastProcessedMessageIds.push(...response.sentences.map((sentence) => sentence.id));
       if (response.sentences.length === 1 && response.sentences.at(-1)!.rangeId!.length > MAX_RANGE_ID_COUNT) {
         log.warn('Sentence is too long', { messages });
@@ -95,14 +108,14 @@ export class MessageNormalizer extends Resource {
     } catch {
       log.error('Failed to normalize segments', { messages });
       // If processing failed, emit the segments as is.
-      this._writeMessages(messages);
+      await this._writeMessages(messages);
     }
   }
 
-  private _writeMessages(messages: MessageWithRangeId[]): void {
+  private async _writeMessages(messages: MessageWithRangeId[]): Promise<void> {
     log.info('writing messages', { messages });
     const lastMessage = messages[messages.length - 1];
     this._cursor.timestamp = lastMessage.created;
-    void this._queue.append(messages);
+    await Feed.append(this._feed, messages).pipe(Effect.provide(this._feedRuntime), runAndForwardErrors);
   }
 }

package/src/{segments-normalization → normalization}/normalization.test.ts

@@ -2,23 +2,16 @@
 // Copyright 2025 DXOS.org
 //
 
-import { effect } from '@preact/signals-core';
 import { describe, test } from 'vitest';
 
-import { scheduleTaskInterval } from '@dxos/async';
-import { Context } from '@dxos/context';
 import { Obj } from '@dxos/echo';
-import { createQueueDXN } from '@dxos/echo/internal';
-import { MemoryQueue } from '@dxos/echo-db';
-import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
+import { FunctionExecutor, ServiceContainer } from '@dxos/functions-runtime';
 import { log } from '@dxos/log';
-import { DataType } from '@dxos/schema';
+import { type Actor, Message } from '@dxos/types';
 
-import { MessageNormalizer } from './message-normalizer';
 import { type MessageWithRangeId, sentenceNormalization } from './normalization';
-import { getActorId } from './utils';
 
-const sender: DataType.Actor = {
+const sender: Actor.Actor = {
   identityDid: 'did:key:123',
 };
 
@@ -48,7 +41,7 @@ const messages: MessageWithRangeId[] = [
   // No punctuation.
   'in classical physics objects have well-defined properties such as position speed and momentum',
 ].map((string, index) =>
-  Obj.make(DataType.Message, {
+  Obj.make(Message.Message, {
     created: new Date(Date.now() + 1_000 * index).toISOString(),
     sender,
     blocks: [{ _tag: 'transcript', started: new Date(Date.now() + 1_000 * index).toISOString(), text: string }],
@@ -99,45 +92,13 @@ describe.skip('SentenceNormalization', () => {
     }
 
     log.info('sentences', {
-      originalMessages: JSON.stringify(messages, null, 2),
+      messages: JSON.stringify(messages, null, 2),
       sentences: JSON.stringify(sentences, null, 2),
     });
     throw new Error('test');
   });
 
-  test.only('queue', { timeout: 120_000 }, async () => {
-    // Create queue.
-    // TODO(dmaretskyi): Use space.queues.create() instead.
-    const queue = new MemoryQueue<DataType.Message>(createQueueDXN());
-    const ctx = new Context();
-    let idx = 0;
-    scheduleTaskInterval(
-      ctx,
-      async () => {
-        if (idx >= messages.length) {
-          void ctx.dispose();
-          return;
-        }
-        await queue.append([messages[idx]]);
-        idx++;
-      },
-      2_000,
-    );
-
-    // Create normalizer.
-    const normalizer = new MessageNormalizer({
-      functionExecutor: getExecutor(),
-      queue,
-      startingCursor: { actorId: getActorId(sender), timestamp: '0' },
-    });
-
-    // Start normalizer.
-    await normalizer.open();
-    effect(() => {
-      log.info('normalizer');
-      log.info(JSON.stringify(queue.objects, null, 2));
-    });
-
-    await new Promise(() => {});
-  });
+  // TODO(burdon): Restore a Feed-based version once `MessageNormalizer` has a
+  //   space-backed test fixture available; the previous `MemoryQueue`-based
+  //   test was incompatible with the `Feed.Feed` + `FeedService` refactor.
 });

package/src/normalization/normalization.ts

@@ -0,0 +1,76 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+import * as Schema from 'effect/Schema';
+
+import { Operation } from '@dxos/compute';
+import { log } from '@dxos/log';
+import { trim } from '@dxos/util';
+
+import { TranscriptOperation } from '../types';
+
+type MessageWithRangeIdType = TranscriptOperation.MessageWithRangeIdType;
+type SentenceNormalizationInputType = TranscriptOperation.SentenceNormalizationInputType;
+const { SentenceNormalization, SentenceNormalizationOutput } = TranscriptOperation;
+
+export type MessageWithRangeId = MessageWithRangeIdType;
+
+/** @deprecated Use SentenceNormalizationInputType */
+export type NormalizationInput = SentenceNormalizationInputType;
+
+/** @deprecated Use SentenceNormalizationOutput */
+export type NormalizationOutput = Schema.Schema.Type<typeof SentenceNormalizationOutput>;
+
+/**
+ * Sentence normalization for transcription.
+ * TODO(dmaretskyi): Reimplement using AiRequest.run or LanguageModel.generateObject - runStructured was removed.
+ */
+export const sentenceNormalization = SentenceNormalization.pipe(
+  Operation.withHandler(
+    Effect.fn(function* ({ messages }) {
+      log.info('input', { messages });
+      // TODO(dmaretskyi): runStructured was removed from AiRequest. Reimplement using new API.
+      return yield* Effect.die(
+        new Error(
+          'Sentence normalization needs to be reimplemented - runStructured was removed from AiRequest.Request. Use AiRequest.run or LanguageModel.generateObject.',
+        ),
+      );
+    }),
+  ),
+);
+
+const _unused = trim`
+  You are observing a real-time transcript of a single person speaking.
+  The transcription is delivered in chunks of 10 seconds or less. As a result, individual sentences may be split across multiple messages, or multiple sentences may appear within a single message. Additionally, because this is real-time transcription, punctuation and capitalization may be incorrect or missing.
+
+  # Task Description:
+    - Your task is to detect and reconstruct broken or incomplete sentences by merging segments into coherent, grammatically correct sentences where appropriate.
+
+  # Input Format:
+    - The input is an array of messages that contain transcription blocks.
+    - Each message is a Message.Message, as described in the output format.
+
+  # Output Format:
+    - You have been provided with the tool that defines the output format; make sure to query it.
+    - Do not output anything other than the expected format.
+
+  # Segment Handling Rules:
+    - Sort messages by timestamp.
+    - Leave ID of the first message in a sentence.
+    - Each divided sentence should be added to the output as a separate message or block within existing messages, preserving the original order.
+    - If one message contains multiple sentences, you should not split them.
+    - The last sentence may be incomplete; it should still be output as a separate block or message.
+    - Keep track of the original timestamps of the messages and blocks and, use the timestamp of the first message/block of a sentence as the 'started' field of the merged message.
+    - The 'rangeId' field of the merged message should include 'messageId'-s and 'rangeId'-s of all messages that have been used to construct this message.
+
+  # Punctuation and Capitalization:
+    - Do not rely on the original punctuation and capitalization—they may be incorrect.
+    - Use logical reasoning to apply appropriate punctuation (e.g., period, comma, question mark, exclamation mark) and capitalization.
+
+  # Restrictions:
+    - Do not alter the order of the original messages; maintain the natural flow of speech.
+    - Do not interpret or infer meaning beyond reconstructing sentences.
+    - Do not add or remove any words or phrases.
+`;

package/src/{segments-normalization → normalization}/utils.ts

@@ -2,8 +2,8 @@
 // Copyright 2025 DXOS.org
 //
 
-import { type DataType } from '@dxos/schema';
+import { type Actor } from '@dxos/types';
 
-export const getActorId = (actor: DataType.Actor) => {
+export const getActorId = (actor: Actor.Actor) => {
   return actor.identityDid || actor.identityKey || actor.email || actor.name || '';
 };

package/src/operations/create.ts

@@ -0,0 +1,24 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+
+import { Operation } from '@dxos/compute';
+import { Feed, Ref } from '@dxos/echo';
+import { Transcript } from '@dxos/types';
+
+import { TranscriptOperation } from '../types';
+
+const handler: Operation.WithHandler<typeof TranscriptOperation.Create> = TranscriptOperation.Create.pipe(
+  Operation.withHandler(
+    Effect.fn(function* ({ space }) {
+      const feed = space.db.add(Feed.make());
+      return {
+        object: Transcript.make(Ref.make(feed)),
+      };
+    }),
+  ),
+);
+
+export default handler;

package/src/operations/index.ts

@@ -0,0 +1,11 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { OperationHandlerSet } from '@dxos/compute';
+
+export const TranscriptionOperationHandlerSet = OperationHandlerSet.lazy(
+  () => import('./create'),
+  () => import('./open'),
+  () => import('./summarize'),
+);

package/src/operations/open.ts

@@ -0,0 +1,28 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+
+import { Operation } from '@dxos/compute';
+import { Database, Feed, Filter } from '@dxos/echo';
+import { Message } from '@dxos/types';
+
+import { TranscriptOperation } from '../types';
+import { renderByline } from '../util';
+
+const handler: Operation.WithHandler<typeof TranscriptOperation.Open> = TranscriptOperation.Open.pipe(
+  Operation.withHandler(
+    Effect.fn(function* ({ transcript }) {
+      const transcriptObj = yield* Database.load(transcript);
+      const feed = yield* Database.load(transcriptObj.feed);
+      const messages = yield* Feed.runQuery(feed, Filter.type(Message.Message));
+      const content = messages
+        .flatMap((message: Message.Message, index: number) => renderByline([])(message, index))
+        .join('\n\n');
+      return { content };
+    }),
+  ),
+);
+
+export default handler;

package/src/operations/summarize.ts

@@ -0,0 +1,101 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Array from 'effect/Array';
+import * as Effect from 'effect/Effect';
+import * as Function from 'effect/Function';
+import * as Layer from 'effect/Layer';
+import * as Option from 'effect/Option';
+
+import { AiService, ConsolePrinter, ToolExecutionService, ToolResolverService } from '@dxos/ai';
+import { AiRequest, GenerationObserver } from '@dxos/assistant';
+import { Trace, Operation, OperationRegistry } from '@dxos/compute';
+import { Database } from '@dxos/echo';
+import { trim } from '@dxos/util';
+
+import { TranscriptOperation } from '../types';
+
+/**
+ * Summarize a transcript of a meeting.
+ */
+const handler: Operation.WithHandler<typeof TranscriptOperation.Summarize> = TranscriptOperation.Summarize.pipe(
+  Operation.withHandler(
+    Effect.fnUntraced(
+      function* ({ transcript, notes }) {
+        const result = yield* new AiRequest.Request({
+          observer: GenerationObserver.fromPrinter(new ConsolePrinter({ tag: 'summarize' })),
+        }).run({
+          prompt: `Transcript: ${transcript}\n\nNotes: ${notes}`,
+          history: [],
+          system: systemPrompt,
+        });
+
+        const summary = Function.pipe(
+          result,
+          Array.findLast((msg) => msg.sender.role === 'assistant' && msg.blocks.some((block) => block._tag === 'text')),
+          Option.flatMap((msg) =>
+            Function.pipe(
+              msg.blocks,
+              Array.findLast((block) => block._tag === 'text'),
+              Option.map((block) => block.text),
+            ),
+          ),
+          Option.getOrThrowWith(() => new Error('No summary found')),
+        );
+
+        return { summary };
+      },
+      Effect.provide(
+        Layer.mergeAll(
+          AiService.model('@anthropic/claude-sonnet-4-0'),
+          ToolResolverService.layerEmpty,
+          ToolExecutionService.layerEmpty,
+          Trace.writerLayerNoop,
+          Database.notAvailable,
+          Layer.succeed(Operation.Service, {
+            invoke: () => Effect.die('Not available.'),
+            schedule: () => Effect.die('Not available.'),
+            invokePromise: async () => ({ error: new Error('Not available.') }),
+          } as any),
+          Layer.succeed(OperationRegistry.Service, { resolve: () => Effect.succeed(undefined) } as any),
+        ),
+      ),
+    ),
+  ),
+);
+
+export default handler;
+
+const systemPrompt = trim`
+  You are a helpful assistant that summarizes transcripts of meetings.
+
+  # Goal
+  Create a markdown summary of the meeting transcript with text notes provided.
+  Notes are very important so make sure to include them in the summary if they contain meaningful information.
+
+  # Formatting
+  - Format the summary as a markdown document without extra comments like "Here is the summary of the transcript:".
+  - Use markdown formatting for headings and bullet points.
+  - Format the summary as a list of key points and takeaways.
+  - All names of people should be in bold.
+
+  # Note Taking
+  - Correlate items in the summary with the person of origin to build a coherent narrative.
+  - Include short quotes verbatim where appropriate. Especially when concerned with design decisions and problem descriptions.
+
+  # Tasks
+  At the end of the summary include tasks.
+  Extract only the tasks that are:
+  - Directly actionable.
+  - Clearly assigned to a person or team (or can easily be inferred).
+  - Strongly implied by the conversation and/or user note (no speculative tasks).
+  - Specific enough that someone reading them would know exactly what to do next.
+
+  Format all tasks as markdown checkboxes using the syntax:
+  - [ ] Task description.
+
+  Additional information can be included (indented).
+
+  If no actionable tasks are found, omit this tasks section.
+`;

package/src/plugin.ts

@@ -0,0 +1,11 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Plugin } from '@dxos/app-framework';
+
+import { meta } from './meta';
+
+export const TranscriptionPlugin = Plugin.lazy(meta, () => import('#plugin'));
+
+export { TranscriptionOperationHandlerSet } from './operations';

package/src/stories/FileTranscription.stories.tsx

@@ -0,0 +1,143 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type Meta, type StoryObj } from '@storybook/react-vite';
+import React, { useCallback, useEffect, useRef, useState } from 'react';
+
+import { useAudioFile } from '#hooks';
+
+import { type MediaStreamRecorderProps, type TranscriberProps } from '../transcriber';
+import {
+  useIsSpeaking,
+  createStoryDecorators,
+  useStoryAppendSegments,
+  useStoryMessageModel,
+  useStoryTranscriber,
+} from './common';
+import { TranscriptionStory } from './TranscriptionStory';
+
+const DEFAULT_TRANSCRIBER_CONFIG = {
+  transcribeAfterChunksAmount: 100,
+  prefixBufferChunksAmount: 50,
+  normalizeSentences: true,
+};
+
+const DEFAULT_RECORDER_CONFIG = {
+  interval: 200,
+};
+
+type DefaultStoryProps = {
+  detectSpeaking?: boolean;
+  normalizeSentences?: boolean;
+  audioUrl: string;
+  transcriberConfig?: TranscriberProps['config'];
+  recorderConfig?: MediaStreamRecorderProps['config'];
+  audioConstraints?: MediaTrackConstraints;
+};
+
+const DefaultStory = ({
+  detectSpeaking,
+  audioUrl,
+  normalizeSentences,
+  transcriberConfig = DEFAULT_TRANSCRIBER_CONFIG,
+  recorderConfig = DEFAULT_RECORDER_CONFIG,
+  audioConstraints,
+}: DefaultStoryProps) => {
+  const [running, setRunning] = useState(false);
+
+  // Optional uploaded file overrides the default URL; revoke the previous URL on change/unmount.
+  const [uploadedUrl, setUploadedUrl] = useState<string>();
+  useEffect(() => {
+    return () => {
+      if (uploadedUrl) {
+        URL.revokeObjectURL(uploadedUrl);
+      }
+    };
+  }, [uploadedUrl]);
+
+  const handleUpload = useCallback((file: File) => {
+    setRunning(false);
+    setUploadedUrl(URL.createObjectURL(file));
+  }, []);
+
+  const sourceUrl = uploadedUrl ?? audioUrl;
+  const { audio, track, stream } = useAudioFile(sourceUrl, audioConstraints);
+  const isSpeaking = detectSpeaking ? useIsSpeaking(track) : true;
+
+  // Pipe the decoded stream into the <audio> element so the user hears playback.
+  const ref = useRef<HTMLAudioElement>(null);
+  useEffect(() => {
+    if (ref.current && stream) {
+      ref.current.srcObject = stream;
+    }
+  }, [stream]);
+
+  // Mirror the toolbar's running flag to the underlying HTMLAudioElement.
+  useEffect(() => {
+    if (!audio) {
+      return;
+    }
+
+    if (running) {
+      void audio.play();
+    } else {
+      void audio.pause();
+    }
+  }, [audio, running]);
+
+  const { model, appendMessage } = useStoryMessageModel();
+  const handleSegments = useStoryAppendSegments(appendMessage);
+
+  useStoryTranscriber({
+    audioStreamTrack: track,
+    running,
+    isSpeaking,
+    transcriberConfig,
+    recorderConfig,
+    onSegments: handleSegments,
+  });
+
+  // TODO(burdon): Sentence normalization moved to require a real space-backed Feed + FeedService runtime.
+  //  Re-enable here once the story has access to one.
+  void normalizeSentences;
+
+  return (
+    <TranscriptionStory
+      audioRef={ref}
+      disabled={!stream}
+      model={model}
+      running={running}
+      onRunningChange={setRunning}
+      onUpload={handleUpload}
+    />
+  );
+};
+
+const meta = {
+  title: 'plugins/plugin-transcription/stories/FileTranscription',
+  render: DefaultStory,
+  decorators: createStoryDecorators(),
+} satisfies Meta<DefaultStoryProps>;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+export const Default: Story = {
+  args: {
+    detectSpeaking: true,
+    // https://learnenglish.britishcouncil.org/general-english/audio-zone/living-london
+    audioUrl: 'https://dxos.network/audio-london.m4a',
+    // textUrl: 'https://dxos.network/audio-london.txt',
+  },
+};
+
+// TODO(mykola): Fix sentence normalization.
+// export const WithSentenceNormalization: StoryObj<typeof AudioFile> = {
+//   args: {
+//     detectSpeaking: true,
+//     normalizeSentences: true,
+//     audioUrl: 'https://dxos.network/audio-london.m4a',
+//   },
+// };