@dxos/plugin-transcription 0.8.3 → 0.8.4-main.1c7ec43d41

package/src/normalization/normalization.ts

@@ -0,0 +1,77 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+import * as Schema from 'effect/Schema';
+
+import { Operation } from '@dxos/compute';
+import { log } from '@dxos/log';
+import { trim } from '@dxos/util';
+
+import {
+  SentenceNormalization,
+  SentenceNormalizationOutput,
+  type MessageWithRangeIdType,
+  type SentenceNormalizationInputType,
+} from '../operations/definitions';
+
+export type MessageWithRangeId = MessageWithRangeIdType;
+
+/** @deprecated Use SentenceNormalizationInputType */
+export type NormalizationInput = SentenceNormalizationInputType;
+
+/** @deprecated Use SentenceNormalizationOutput */
+export type NormalizationOutput = Schema.Schema.Type<typeof SentenceNormalizationOutput>;
+
+/**
+ * Sentence normalization for transcription.
+ * TODO(dmaretskyi): Reimplement using AiRequest.run or LanguageModel.generateObject - runStructured was removed.
+ */
+export const sentenceNormalization = SentenceNormalization.pipe(
+  Operation.withHandler(
+    Effect.fn(function* ({ messages }) {
+      log.info('input', { messages });
+      // TODO(dmaretskyi): runStructured was removed from AiRequest. Reimplement using new API.
+      return yield* Effect.die(
+        new Error(
+          'Sentence normalization needs to be reimplemented - runStructured was removed from AiRequest. Use AiRequest.run or LanguageModel.generateObject.',
+        ),
+      );
+    }),
+  ),
+);
+
+const _unused = trim`
+  You are observing a real-time transcript of a single person speaking.
+  The transcription is delivered in chunks of 10 seconds or less. As a result, individual sentences may be split across multiple messages, or multiple sentences may appear within a single message. Additionally, because this is real-time transcription, punctuation and capitalization may be incorrect or missing.
+
+  # Task Description:
+    - Your task is to detect and reconstruct broken or incomplete sentences by merging segments into coherent, grammatically correct sentences where appropriate.
+
+  # Input Format:
+    - The input is an array of messages that contain transcription blocks.
+    - Each message is a Message.Message, as described in the output format.
+
+  # Output Format:
+    - You have been provided with the tool that defines the output format; make sure to query it.
+    - Do not output anything other than the expected format.
+
+  # Segment Handling Rules:
+    - Sort messages by timestamp.
+    - Leave ID of the first message in a sentence.
+    - Each divided sentence should be added to the output as a separate message or block within existing messages, preserving the original order.
+    - If one message contains multiple sentences, you should not split them.
+    - The last sentence may be incomplete; it should still be output as a separate block or message.
+    - Keep track of the original timestamps of the messages and blocks and, use the timestamp of the first message/block of a sentence as the 'started' field of the merged message.
+    - The 'rangeId' field of the merged message should include 'messageId'-s and 'rangeId'-s of all messages that have been used to construct this message.
+
+  # Punctuation and Capitalization:
+    - Do not rely on the original punctuation and capitalization—they may be incorrect.
+    - Use logical reasoning to apply appropriate punctuation (e.g., period, comma, question mark, exclamation mark) and capitalization.
+
+  # Restrictions:
+    - Do not alter the order of the original messages; maintain the natural flow of speech.
+    - Do not interpret or infer meaning beyond reconstructing sentences.
+    - Do not add or remove any words or phrases.
+`;

package/src/{segments-normalization → normalization}/utils.ts

@@ -2,8 +2,8 @@
 // Copyright 2025 DXOS.org
 //
 
-import { type DataType } from '@dxos/schema';
+import { type Actor } from '@dxos/types';
 
-export const getActorId = (actor: DataType.Actor) => {
+export const getActorId = (actor: Actor.Actor) => {
   return actor.identityDid || actor.identityKey || actor.email || actor.name || '';
 };

package/src/operations/create.ts

@@ -0,0 +1,24 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+
+import { Operation } from '@dxos/compute';
+import { Feed, Ref } from '@dxos/echo';
+import { Transcript } from '@dxos/types';
+
+import { Create } from './definitions';
+
+const handler: Operation.WithHandler<typeof Create> = Create.pipe(
+  Operation.withHandler(
+    Effect.fn(function* ({ space }) {
+      const feed = space.db.add(Feed.make());
+      return {
+        object: Transcript.make(Ref.make(feed)),
+      };
+    }),
+  ),
+);
+
+export default handler;

package/src/operations/definitions.ts

@@ -0,0 +1,101 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+import { AiService } from '@dxos/ai';
+import { SpaceSchema } from '@dxos/client/echo';
+import { Operation } from '@dxos/compute';
+import { Database, Feed, Ref } from '@dxos/echo';
+import { Message, Transcript } from '@dxos/types';
+
+import { meta } from '#meta';
+
+const TRANSCRIPT_OPERATION = `${meta.id}.operation`;
+
+export const Create = Operation.make({
+  meta: { key: `${TRANSCRIPT_OPERATION}.create`, name: 'Create Transcript' },
+  input: Schema.Struct({
+    name: Schema.optional(Schema.String),
+    space: SpaceSchema,
+  }),
+  output: Schema.Struct({
+    object: Transcript.Transcript,
+  }),
+});
+
+export const MessageWithRangeId = Schema.extend(
+  Message.Message,
+  Schema.Struct({
+    rangeId: Schema.optional(Schema.Array(Schema.String)).annotations({
+      description: 'The IDs of the messages that contain the sentences.',
+    }),
+  }),
+);
+
+export type MessageWithRangeIdType = Schema.Schema.Type<typeof MessageWithRangeId>;
+
+export const Open = Operation.make({
+  meta: {
+    key: 'org.dxos.function.transcription.open',
+    name: 'Open',
+    description: 'Opens and reads the contents of a transcription object.',
+  },
+  input: Schema.Struct({
+    transcript: Ref.Ref(Transcript.Transcript).annotations({
+      description: 'The ID of the transcription object.',
+    }),
+  }),
+  output: Schema.Struct({
+    content: Schema.String,
+  }),
+  services: [Database.Service, Feed.FeedService],
+});
+
+export const Summarize = Operation.make({
+  meta: {
+    key: 'org.dxos.function.transcription.summarize',
+    name: 'Summarize',
+    description: 'Summarize a transcript of a meeting.',
+  },
+  input: Schema.Struct({
+    transcript: Schema.String.annotations({
+      description: 'The transcript of the meeting.',
+    }),
+    notes: Schema.optional(Schema.String).annotations({
+      description: 'Additional notes from the participants.',
+    }),
+  }),
+  output: Schema.Struct({
+    summary: Schema.String.annotations({
+      description: 'The summary of the transcript.',
+    }),
+  }),
+  services: [AiService.AiService],
+});
+
+export const SentenceNormalizationInput = Schema.Struct({
+  messages: Schema.Array(MessageWithRangeId).annotations({
+    description: 'Messages to normalize into sentences.',
+  }),
+});
+
+export type SentenceNormalizationInputType = Schema.Schema.Type<typeof SentenceNormalizationInput>;
+
+export const SentenceNormalizationOutput = Schema.Struct({
+  sentences: Schema.Array(MessageWithRangeId.pipe(Schema.mutable)).pipe(Schema.mutable).annotations({
+    description: 'The sentences of the transcript.',
+  }),
+});
+
+export const SentenceNormalization = Operation.make({
+  meta: {
+    key: 'org.dxos.function.transcription.sentence-normalization',
+    name: 'Sentence Normalization',
+    description: 'Post process of transcription for sentence normalization',
+  },
+  input: SentenceNormalizationInput,
+  output: SentenceNormalizationOutput,
+  services: [],
+});

package/src/operations/index.ts

@@ -0,0 +1,13 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { OperationHandlerSet } from '@dxos/compute';
+
+export * as TranscriptOperation from './definitions';
+
+export const TranscriptionOperationHandlerSet = OperationHandlerSet.lazy(
+  () => import('./create'),
+  () => import('./open'),
+  () => import('./summarize'),
+);

package/src/operations/open.ts

@@ -0,0 +1,28 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+
+import { Operation } from '@dxos/compute';
+import { Database, Feed, Filter } from '@dxos/echo';
+import { Message } from '@dxos/types';
+
+import { renderByline } from '../util';
+import { Open } from './definitions';
+
+const handler: Operation.WithHandler<typeof Open> = Open.pipe(
+  Operation.withHandler(
+    Effect.fn(function* ({ transcript }) {
+      const transcriptObj = yield* Database.load(transcript);
+      const feed = yield* Database.load(transcriptObj.feed);
+      const messages = yield* Feed.runQuery(feed, Filter.type(Message.Message));
+      const content = messages
+        .flatMap((message: Message.Message, index: number) => renderByline([])(message, index))
+        .join('\n\n');
+      return { content };
+    }),
+  ),
+);
+
+export default handler;

package/src/operations/summarize.ts

@@ -0,0 +1,101 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Array from 'effect/Array';
+import * as Effect from 'effect/Effect';
+import * as Function from 'effect/Function';
+import * as Layer from 'effect/Layer';
+import * as Option from 'effect/Option';
+
+import { AiService, ConsolePrinter, ToolExecutionService, ToolResolverService } from '@dxos/ai';
+import { AiRequest, GenerationObserver } from '@dxos/assistant';
+import { Trace, Operation, OperationRegistry } from '@dxos/compute';
+import { Database } from '@dxos/echo';
+import { trim } from '@dxos/util';
+
+import { Summarize } from './definitions';
+
+/**
+ * Summarize a transcript of a meeting.
+ */
+const handler: Operation.WithHandler<typeof Summarize> = Summarize.pipe(
+  Operation.withHandler(
+    Effect.fnUntraced(
+      function* ({ transcript, notes }) {
+        const result = yield* new AiRequest({
+          observer: GenerationObserver.fromPrinter(new ConsolePrinter({ tag: 'summarize' })),
+        }).run({
+          prompt: `Transcript: ${transcript}\n\nNotes: ${notes}`,
+          history: [],
+          system: systemPrompt,
+        });
+
+        const summary = Function.pipe(
+          result,
+          Array.findLast((msg) => msg.sender.role === 'assistant' && msg.blocks.some((block) => block._tag === 'text')),
+          Option.flatMap((msg) =>
+            Function.pipe(
+              msg.blocks,
+              Array.findLast((block) => block._tag === 'text'),
+              Option.map((block) => block.text),
+            ),
+          ),
+          Option.getOrThrowWith(() => new Error('No summary found')),
+        );
+
+        return { summary };
+      },
+      Effect.provide(
+        Layer.mergeAll(
+          AiService.model('@anthropic/claude-sonnet-4-0'),
+          ToolResolverService.layerEmpty,
+          ToolExecutionService.layerEmpty,
+          Trace.writerLayerNoop,
+          Database.notAvailable,
+          Layer.succeed(Operation.Service, {
+            invoke: () => Effect.die('Not available.'),
+            schedule: () => Effect.die('Not available.'),
+            invokePromise: async () => ({ error: new Error('Not available.') }),
+          } as any),
+          Layer.succeed(OperationRegistry.Service, { resolve: () => Effect.succeed(undefined) } as any),
+        ),
+      ),
+    ),
+  ),
+);
+
+export default handler;
+
+const systemPrompt = trim`
+  You are a helpful assistant that summarizes transcripts of meetings.
+
+  # Goal
+  Create a markdown summary of the meeting transcript with text notes provided.
+  Notes are very important so make sure to include them in the summary if they contain meaningful information.
+
+  # Formatting
+  - Format the summary as a markdown document without extra comments like "Here is the summary of the transcript:".
+  - Use markdown formatting for headings and bullet points.
+  - Format the summary as a list of key points and takeaways.
+  - All names of people should be in bold.
+
+  # Note Taking
+  - Correlate items in the summary with the person of origin to build a coherent narrative.
+  - Include short quotes verbatim where appropriate. Especially when concerned with design decisions and problem descriptions.
+
+  # Tasks
+  At the end of the summary include tasks.
+  Extract only the tasks that are:
+  - Directly actionable.
+  - Clearly assigned to a person or team (or can easily be inferred).
+  - Strongly implied by the conversation and/or user note (no speculative tasks).
+  - Specific enough that someone reading them would know exactly what to do next.
+
+  Format all tasks as markdown checkboxes using the syntax:
+  - [ ] Task description.
+
+  Additional information can be included (indented).
+
+  If no actionable tasks are found, omit this tasks section.
+`;

package/src/{components/stories → stories}/FileTranscription.stories.tsx

@@ -2,40 +2,37 @@
 // Copyright 2025 DXOS.org
 //
 
-import '@dxos-theme';
-
-import { type Meta, type StoryObj } from '@storybook/react';
+import { type Meta, type StoryObj } from '@storybook/react-vite';
+import * as Effect from 'effect/Effect';
 import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
 
-import { EdgeAiServiceClient } from '@dxos/ai';
-import { AI_SERVICE_ENDPOINT } from '@dxos/ai/testing';
-import { Events, IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
 import { withPluginManager } from '@dxos/app-framework/testing';
+import { AppActivationEvents } from '@dxos/app-toolkit/events';
 import { scheduleTask } from '@dxos/async';
 import { Context } from '@dxos/context';
 import { Obj } from '@dxos/echo';
 import { MemoryQueue } from '@dxos/echo-db';
-import { createQueueDXN } from '@dxos/echo-schema';
-import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
+import { createQueueDXN } from '@dxos/echo/internal';
+import { FunctionExecutor, ServiceContainer } from '@dxos/functions-runtime';
 import { log } from '@dxos/log';
 import { ClientPlugin } from '@dxos/plugin-client';
+import { initializeIdentity } from '@dxos/plugin-client/testing';
 import { PreviewPlugin } from '@dxos/plugin-preview';
-import { SpacePlugin } from '@dxos/plugin-space';
-import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
-import { ThemePlugin } from '@dxos/plugin-theme';
-import { defaultTx } from '@dxos/react-ui-theme';
-import { DataType } from '@dxos/schema';
-import { seedTestData, Testing } from '@dxos/schema/testing';
-import { withLayout, withTheme } from '@dxos/storybook-utils';
+import { StorybookPlugin, corePlugins } from '@dxos/plugin-testing';
+import { withLayout } from '@dxos/react-ui/testing';
+import { TestSchema } from '@dxos/schema/testing';
+import { type Actor, Message, Organization, Person } from '@dxos/types';
+import { seedTestData } from '@dxos/types/testing';
+
+import { useAudioFile, useQueueModelAdapter, useTranscriber } from '#hooks';
+import { TestItem } from '#testing';
 
+import { MessageNormalizer, getActorId } from '../normalization';
+import { type MediaStreamRecorderProps, type TranscriberProps } from '../transcriber';
+import { TranscriptionPlugin } from '../TranscriptionPlugin';
+import { renderByline } from '../util';
 import { TranscriptionStory } from './TranscriptionStory';
 import { useIsSpeaking } from './useIsSpeaking';
-import { TranscriptionPlugin } from '../../TranscriptionPlugin';
-import { useAudioFile, useQueueModelAdapter, useTranscriber } from '../../hooks';
-import { getActorId, MessageNormalizer } from '../../segments-normalization';
-import { TestItem } from '../../testing';
-import { type MediaStreamRecorderParams, type TranscriberParams } from '../../transcriber';
-import { renderMarkdown } from '../Transcript';
 
 const AudioFile = ({
   detectSpeaking,
@@ -48,15 +45,33 @@ const AudioFile = ({
   detectSpeaking?: boolean;
   normalizeSentences?: boolean;
   audioUrl: string;
-  transcriberConfig?: TranscriberParams['config'];
-  recorderConfig?: MediaStreamRecorderParams['config'];
+  transcriberConfig?: TranscriberProps['config'];
+  recorderConfig?: MediaStreamRecorderProps['config'];
   audioConstraints?: MediaTrackConstraints;
 }) => {
   const [running, setRunning] = useState(false);
-  const actor: DataType.Actor = useMemo(() => ({ name: 'You' }), []);
+  const actor: Actor.Actor = useMemo(() => ({ name: 'You' }), []);
+
+  // Optional uploaded file overrides the default URL.
+  // The useEffect below revokes the previous URL whenever it changes (and on unmount).
+  const [uploadedUrl, setUploadedUrl] = useState<string>();
+  useEffect(() => {
+    return () => {
+      if (uploadedUrl) {
+        URL.revokeObjectURL(uploadedUrl);
+      }
+    };
+  }, [uploadedUrl]);
+
+  const handleUpload = useCallback((file: File) => {
+    setRunning(false);
+    setUploadedUrl(URL.createObjectURL(file));
+  }, []);
+
+  const sourceUrl = uploadedUrl ?? audioUrl;
 
   // Audio.
-  const { audio, track, stream } = useAudioFile(audioUrl, audioConstraints);
+  const { audio, track, stream } = useAudioFile(sourceUrl, audioConstraints);
 
   // Speaking.
   const isSpeaking = detectSpeaking ? useIsSpeaking(track) : true;
@@ -69,7 +84,6 @@ const AudioFile = ({
 
   useEffect(() => {
     if (!audio) {
-      log.warn('no audio');
       return;
     }
 
@@ -83,13 +97,18 @@ const AudioFile = ({
   // Transcriber.
   // TODO(dmaretskyi): Use space.queues.create() instead.
   const queueDxn = useMemo(() => createQueueDXN(), []);
-  const queue = useMemo(() => new MemoryQueue<DataType.Message>(queueDxn), [queueDxn]);
+  const queue = useMemo(() => new MemoryQueue<Message.Message>(queueDxn), [queueDxn]);
 
-  const model = useQueueModelAdapter(renderMarkdown([]), queue);
-  const handleSegments = useCallback<TranscriberParams['onSegments']>(
+  const model = useQueueModelAdapter(renderByline([]), queue);
+  const handleSegments = useCallback<TranscriberProps['onSegments']>(
     async (blocks) => {
-      const message = Obj.make(DataType.Message, { sender: actor, created: new Date().toISOString(), blocks });
-      void queue?.append([message]);
+      void queue?.append([
+        Obj.make(Message.Message, {
+          created: new Date().toISOString(),
+          sender: actor,
+          blocks,
+        }),
+      ]);
     },
     [queue],
   );
@@ -108,14 +127,14 @@ const AudioFile = ({
     }
     const executor = new FunctionExecutor(
       new ServiceContainer().setServices({
-        ai: {
-          client: new EdgeAiServiceClient({
-            endpoint: AI_SERVICE_ENDPOINT.REMOTE,
-            defaultGenerationOptions: {
-              model: '@anthropic/claude-3-5-sonnet-20241022',
-            },
-          }),
-        },
+        // ai: {
+        //   client: new Edge AiServiceClient({
+        //     endpoint: AI_SERVICE_ENDPOINT.REMOTE,
+        //     defaultGenerationOptions: {
+        //       model: '@anthropic/claude-3-5-sonnet-20241022',
+        //     },
+        //   }),
+        // },
       }),
     );
 
@@ -143,10 +162,10 @@ const AudioFile = ({
   }, [normalizer]);
 
   const manageChunkRecording = () => {
-    if (running && isSpeaking) {
+    if (running && isSpeaking && transcriber) {
       log.info('starting transcription');
       transcriber?.startChunksRecording();
-    } else if (!isSpeaking || !running) {
+    } else if ((!isSpeaking || !running) && transcriber) {
       log.info('stopping transcription');
       transcriber?.stopChunksRecording();
     }
@@ -172,40 +191,47 @@ const AudioFile = ({
     manageChunkRecording();
   }, [isSpeaking, stream]);
 
-  return <TranscriptionStory model={model} running={running} onRunningChange={setRunning} audioRef={ref} />;
+  return (
+    <TranscriptionStory
+      disabled={!stream}
+      model={model}
+      running={running}
+      onRunningChange={setRunning}
+      audioRef={ref}
+      onUpload={handleUpload}
+    />
+  );
 };
 
-const meta: Meta<typeof AudioFile> = {
-  title: 'plugins/plugin-transcription/FileTranscription',
+const meta = {
+  title: 'plugins/plugin-transcription/components/FileTranscription',
   decorators: [
+    withLayout({ layout: 'column' }),
     withPluginManager({
       plugins: [
-        ThemePlugin({ tx: defaultTx }),
-        StorybookLayoutPlugin(),
+        ...corePlugins(),
+        StorybookPlugin({}),
         ClientPlugin({
-          types: [TestItem, DataType.Person, DataType.Organization, Testing.DocumentType],
-          onClientInitialized: async (_, client) => {
-            await client.halo.createIdentity();
-            await client.spaces.waitUntilReady();
-            await client.spaces.default.waitUntilReady();
-            await seedTestData(client.spaces.default);
-          },
+          types: [TestItem, Person.Person, Organization.Organization, TestSchema.DocumentType],
+          onClientInitialized: ({ client }) =>
+            Effect.gen(function* () {
+              const { personalSpace } = yield* initializeIdentity(client);
+              yield* Effect.promise(() => seedTestData(personalSpace));
+            }),
         }),
-        SpacePlugin(),
-        SettingsPlugin(),
+
         PreviewPlugin(),
-        IntentPlugin(),
         TranscriptionPlugin(),
       ],
-      fireEvents: [Events.SetupAppGraph],
+      fireEvents: [AppActivationEvents.SetupAppGraph],
     }),
-    withTheme,
-    withLayout({ fullscreen: true, classNames: 'justify-center' }),
   ],
-};
+} satisfies Meta;
 
 export default meta;
 
+type Story = StoryObj<typeof meta>;
+
 const TRANSCRIBER_CONFIG = {
   transcribeAfterChunksAmount: 100,
   prefixBufferChunksAmount: 50,
@@ -228,15 +254,16 @@ export const Default: StoryObj<typeof AudioFile> = {
   },
 };
 
-export const WithSentenceNormalization: StoryObj<typeof AudioFile> = {
-  render: AudioFile,
-  args: {
-    detectSpeaking: true,
-    normalizeSentences: true,
-    // https://learnenglish.britishcouncil.org/general-english/audio-zone/living-london
-    audioUrl: 'https://dxos.network/audio-london.m4a',
-    // textUrl: 'https://dxos.network/audio-london.txt',
-    transcriberConfig: TRANSCRIBER_CONFIG,
-    recorderConfig: RECORDER_CONFIG,
-  },
-};
+// TODO(mykola): Fix sentence normalization.
+// export const WithSentenceNormalization: StoryObj<typeof AudioFile> = {
+//   render: AudioFile,
+//   args: {
+//     detectSpeaking: true,
+//     normalizeSentences: true,
+//     // https://learnenglish.britishcouncil.org/general-english/audio-zone/living-london
+//     audioUrl: 'https://dxos.network/audio-london.m4a',
+//     // textUrl: 'https://dxos.network/audio-london.txt',
+//     transcriberConfig: TRANSCRIBER_CONFIG,
+//     recorderConfig: RECORDER_CONFIG,
+//   },
+// };