@dxos/plugin-transcription 0.8.4-main.bc674ce → 0.8.4-main.bcb3aa67d6

package/src/components/index.ts

@@ -2,9 +2,4 @@
 // Copyright 2025 DXOS.org
 //
 
-import { lazy } from 'react';
-
-export * from './Transcript';
-export * from './TranscriptContainer';
-
-export const TranscriptContainer = lazy(() => import('./TranscriptContainer'));
+export * from './Transcription';

package/src/components/stories/FileTranscription.stories.tsx

@@ -6,8 +6,8 @@ import { type Meta, type StoryObj } from '@storybook/react-vite';
 import * as Effect from 'effect/Effect';
 import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
 
-import { Common } from '@dxos/app-framework';
 import { withPluginManager } from '@dxos/app-framework/testing';
+import { AppActivationEvents } from '@dxos/app-toolkit';
 import { scheduleTask } from '@dxos/async';
 import { Context } from '@dxos/context';
 import { Obj } from '@dxos/echo';
@@ -16,16 +16,17 @@ import { MemoryQueue } from '@dxos/echo-db';
 import { FunctionExecutor, ServiceContainer } from '@dxos/functions-runtime';
 import { log } from '@dxos/log';
 import { ClientPlugin } from '@dxos/plugin-client';
+import { initializeIdentity } from '@dxos/plugin-client/testing';
 import { PreviewPlugin } from '@dxos/plugin-preview';
 import { StorybookPlugin, corePlugins } from '@dxos/plugin-testing';
-import { withLayout, withTheme } from '@dxos/react-ui/testing';
+import { withLayout } from '@dxos/react-ui/testing';
 import { TestSchema } from '@dxos/schema/testing';
 import { type Actor, Message, Organization, Person } from '@dxos/types';
 import { seedTestData } from '@dxos/types/testing';
 
-import { useAudioFile, useQueueModelAdapter, useTranscriber } from '../../hooks';
+import { useAudioFile, useQueueModelAdapter, useTranscriber } from '#hooks';
 import { MessageNormalizer, getActorId } from '../../segments-normalization';
-import { TestItem } from '../../testing';
+import { TestItem } from '#testing';
 import { type MediaStreamRecorderProps, type TranscriberProps } from '../../transcriber';
 import { TranscriptionPlugin } from '../../TranscriptionPlugin';
 import { renderByline } from '../../util';
@@ -184,9 +185,8 @@ const AudioFile = ({
 };
 
 const meta = {
-  title: 'plugins/plugin-transcription/FileTranscription',
+  title: 'plugins/plugin-transcription/components/FileTranscription',
   decorators: [
-    withTheme,
     withLayout({ layout: 'column' }),
     withPluginManager({
       plugins: [
@@ -196,17 +196,15 @@ const meta = {
           types: [TestItem, Person.Person, Organization.Organization, TestSchema.DocumentType],
           onClientInitialized: ({ client }) =>
             Effect.gen(function* () {
-              yield* Effect.promise(() => client.halo.createIdentity());
-              yield* Effect.promise(() => client.spaces.waitUntilReady());
-              yield* Effect.promise(() => client.spaces.default.waitUntilReady());
-              yield* Effect.promise(() => seedTestData(client.spaces.default));
+              const { personalSpace } = yield* initializeIdentity(client);
+              yield* Effect.promise(() => seedTestData(personalSpace));
             }),
         }),
 
         PreviewPlugin(),
         TranscriptionPlugin(),
       ],
-      fireEvents: [Common.ActivationEvent.SetupAppGraph],
+      fireEvents: [AppActivationEvents.SetupAppGraph],
     }),
   ],
 } satisfies Meta;

package/src/components/stories/KeyWordRecognition.stories.tsx

@@ -15,11 +15,11 @@ type Word = {
   matched: boolean;
 };
 
-type StoryProps = {
+type DefaultStoryProps = {
   keywords: string[];
 };
 
-const DefaultStory = ({ keywords }: StoryProps) => {
+const DefaultStory = ({ keywords }: DefaultStoryProps) => {
   const [running, setRunning] = useState(false);
   const [matchingWords, setMatchingWords] = useState<Word[]>(keywords.map((word) => ({ text: word, matched: false })));
 
@@ -86,7 +86,7 @@ const DefaultStory = ({ keywords }: StoryProps) => {
   }, [running, recognition]);
 
   return (
-    <div className={mx('flex flex-col gap-2 is-[30rem]')}>
+    <div className={mx('flex flex-col gap-2 w-[30rem]')}>
       <Toolbar.Root>
         <Toolbar.IconButton
           iconOnly
@@ -101,7 +101,7 @@ const DefaultStory = ({ keywords }: StoryProps) => {
           <span
             key={index}
             className={mx(
-              'pli-3 plb-1 m-2 rounded border',
+              'px-3 py-1 m-2 rounded-sm border',
               word.matched ? 'bg-red-200 text-red-800 border-red-300' : 'bg-gray-200 text-gray-600 border-gray-300',
             )}
           >
@@ -116,9 +116,9 @@ const DefaultStory = ({ keywords }: StoryProps) => {
 };
 
 const meta = {
-  title: 'plugins/plugin-transcription/KeyWordDetection',
+  title: 'plugins/plugin-transcription/components/KeyWordDetection',
   render: DefaultStory,
-  decorators: [withTheme, withLayout({ layout: 'column' })],
+  decorators: [withTheme(), withLayout({ layout: 'column' })],
 } satisfies Meta<typeof DefaultStory>;
 
 export default meta;

package/src/components/stories/MicrophoneTranscription.stories.tsx

@@ -6,8 +6,8 @@ import { type Meta, type StoryObj } from '@storybook/react-vite';
 import * as Effect from 'effect/Effect';
 import React, { useCallback, useEffect, useMemo, useState } from 'react';
 
-import { Common } from '@dxos/app-framework';
 import { withPluginManager } from '@dxos/app-framework/testing';
+import { AppActivationEvents } from '@dxos/app-toolkit';
 import {
   type ExtractionFunction,
   extractionAnthropicFunction,
@@ -18,20 +18,20 @@ import {
 import { Filter, type Obj } from '@dxos/echo';
 import { createQueueDXN } from '@dxos/echo/internal';
 import { MemoryQueue } from '@dxos/echo-db';
-import { FunctionExecutor, ServiceContainer } from '@dxos/functions-runtime';
 import { invariant } from '@dxos/invariant';
 import { log } from '@dxos/log';
 import { ClientPlugin } from '@dxos/plugin-client';
+import { initializeIdentity } from '@dxos/plugin-client/testing';
 import { PreviewPlugin } from '@dxos/plugin-preview';
 import { StorybookPlugin, corePlugins } from '@dxos/plugin-testing';
 import { IndexKind, useSpace } from '@dxos/react-client/echo';
-import { withLayout, withTheme } from '@dxos/react-ui/testing';
+import { withLayout } from '@dxos/react-ui/testing';
 import { TestSchema } from '@dxos/schema/testing';
 import { Message, Organization, Person } from '@dxos/types';
 import { seedTestData } from '@dxos/types/testing';
 
-import { useAudioTrack, useQueueModelAdapter, useTranscriber } from '../../hooks';
-import { TestItem } from '../../testing';
+import { useAudioTrack, useQueueModelAdapter, useTranscriber } from '#hooks';
+import { TestItem } from '#testing';
 import { type MediaStreamRecorderProps, type TranscriberProps } from '../../transcriber';
 import { TranscriptionPlugin } from '../../TranscriptionPlugin';
 import { renderByline } from '../../util';
@@ -48,7 +48,7 @@ const RECORDER_CONFIG = {
   interval: 200,
 };
 
-type StoryProps = {
+type DefaultStoryProps = {
   detectSpeaking?: boolean;
   transcriberConfig: TranscriberProps['config'];
   recorderConfig: MediaStreamRecorderProps['config'];
@@ -62,7 +62,7 @@ const DefaultStory = ({
   transcriberConfig,
   recorderConfig,
   audioConstraints,
-}: StoryProps) => {
+}: DefaultStoryProps) => {
   const [running, setRunning] = useState(false);
 
   // Audio.
@@ -85,13 +85,12 @@ const DefaultStory = ({
   }, [space]);
 
   // Entity extraction.
-  const { extractionFunction, executor, objects } = useMemo(() => {
+  const { extractionFunction, objects } = useMemo(() => {
     if (!space) {
       log.warn('no space');
       return {};
     }
 
-    let executor: FunctionExecutor | undefined;
     let extractionFunction: ExtractionFunction | undefined;
     let objects: Promise<Obj.Unknown[]> | undefined;
 
@@ -111,20 +110,8 @@ const DefaultStory = ({
         )
         .run();
     }
-    if (entityExtraction !== 'none') {
-      executor = new FunctionExecutor(
-        new ServiceContainer().setServices({
-          // ai: {
-          //   client: new Edge AiServiceClient({
-          //     endpoint: AI_SERVICE_ENDPOINT.REMOTE,
-          //   }),
-          // },
-          // database: { db: space!.db },
-        }),
-      );
-    }
 
-    return { extractionFunction, executor, objects };
+    return { extractionFunction, objects };
   }, [entityExtraction, space]);
 
   // Transcriber.
@@ -142,9 +129,7 @@ const DefaultStory = ({
 
       if (entityExtraction !== 'none') {
         invariant(extractionFunction, 'extractionFunction is required');
-        invariant(executor, 'executor is required');
         const result = await processTranscriptMessage({
-          executor,
           function: extractionFunction,
           input: {
             message,
@@ -190,10 +175,9 @@ const DefaultStory = ({
 };
 
 const meta = {
-  title: 'plugins/plugin-transcription/MicrophoneTranscription',
+  title: 'plugins/plugin-transcription/components/MicrophoneTranscription',
   render: DefaultStory,
   decorators: [
-    withTheme,
     withLayout({ layout: 'column' }),
     withPluginManager({
       plugins: [
@@ -203,9 +187,7 @@ const meta = {
           types: [TestItem, Person.Person, Organization.Organization, TestSchema.DocumentType],
           onClientInitialized: ({ client }) =>
             Effect.gen(function* () {
-              yield* Effect.promise(() => client.halo.createIdentity());
-              yield* Effect.promise(() => client.spaces.waitUntilReady());
-              yield* Effect.promise(() => client.spaces.default.waitUntilReady());
+              const { personalSpace } = yield* initializeIdentity(client);
               // TODO(mykola): Make API easier to use.
               // TODO(mykola): Delete after enabling vector indexing by default.
               // Enable vector indexing.
@@ -221,14 +203,14 @@ const meta = {
                 }),
               );
               yield* Effect.promise(() => client.services.services.QueryService!.reindex());
-              yield* Effect.promise(() => seedTestData(client.spaces.default));
+              yield* Effect.promise(() => seedTestData(personalSpace));
             }),
         }),
 
         PreviewPlugin(),
         TranscriptionPlugin(),
       ],
-      fireEvents: [Common.ActivationEvent.SetupAppGraph],
+      fireEvents: [AppActivationEvents.SetupAppGraph],
     }),
   ],
 } satisfies Meta<typeof DefaultStory>;

package/src/components/stories/TranscriptionStory.tsx

@@ -10,7 +10,7 @@ import { IconButton, ScrollContainer, Toolbar } from '@dxos/react-ui';
 import { type Message } from '@dxos/types';
 
 import { type SerializationModel } from '../../model';
-import { TranscriptView } from '../Transcript';
+import { Transcription } from '../Transcription';
 
 export const TranscriptionStory: FC<{
   model: SerializationModel<Message.Message>;
@@ -20,7 +20,7 @@ export const TranscriptionStory: FC<{
   audioRef?: RefObject<HTMLAudioElement | null>;
 }> = ({ model, running, onRunningChange, audioRef, disabled }) => {
   return (
-    <div className='flex flex-col is-[30rem]'>
+    <div className='flex flex-col w-[30rem]'>
       {audioRef && <audio ref={audioRef} autoPlay />}
       <Toolbar.Root>
         <IconButton
@@ -32,9 +32,11 @@ export const TranscriptionStory: FC<{
         />
       </Toolbar.Root>
       <ScrollContainer.Root pin>
-        <ScrollContainer.Viewport>
-          <TranscriptView model={model} />
-        </ScrollContainer.Viewport>
+        <ScrollContainer.Content>
+          <ScrollContainer.Viewport>
+            <Transcription model={model} />
+          </ScrollContainer.Viewport>
+        </ScrollContainer.Content>
       </ScrollContainer.Root>
     </div>
   );

package/src/containers/TranscriptionContainer/TranscriptionContainer.tsx

@@ -0,0 +1,32 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import React from 'react';
+
+import { type ObjectSurfaceProps } from '@dxos/app-toolkit/ui';
+import { Obj } from '@dxos/echo';
+import { useMembers, useQueue } from '@dxos/react-client/echo';
+import { Panel } from '@dxos/react-ui';
+import { type Message, type Transcript } from '@dxos/types';
+
+import { Transcription } from '#components';
+import { useQueueModelAdapter } from '#hooks';
+import { renderByline } from '../../util';
+
+export type TranscriptionContainerProps = ObjectSurfaceProps<Transcript.Transcript>;
+
+export const TranscriptionContainer = ({ role, subject: transcript, attendableId }: TranscriptionContainerProps) => {
+  const db = Obj.getDatabase(transcript);
+  const members = useMembers(db?.spaceId).map((member) => member.identity);
+  const queue = useQueue<Message.Message>(transcript.queue.dxn, { pollInterval: 1_000 });
+  const model = useQueueModelAdapter(renderByline(members), queue);
+
+  return (
+    <Panel.Root role={role}>
+      <Panel.Content asChild>
+        <Transcription attendableId={attendableId} model={model} transcript={transcript} />
+      </Panel.Content>
+    </Panel.Root>
+  );
+};

package/src/containers/TranscriptionContainer/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { TranscriptionContainer } from './TranscriptionContainer';
+
+export default TranscriptionContainer;

package/src/containers/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { type ComponentType, lazy } from 'react';
+
+export const TranscriptionContainer: ComponentType<any> = lazy(() => import('./TranscriptionContainer'));

package/src/hooks/useTranscriber.ts

@@ -4,10 +4,10 @@
 
 import { useEffect, useMemo } from 'react';
 
-import { useCapabilities } from '@dxos/app-framework/react';
+import { useCapabilities } from '@dxos/app-framework/ui';
 
 import { type Transcriber } from '../transcriber';
-import { TranscriptionCapabilities } from '../types';
+import { TranscriptionCapabilities } from '#types';
 
 /**
  * Records audio while user is speaking and transcribes it after user is done speaking.

package/src/meta.ts

@@ -6,7 +6,7 @@ import { type Plugin } from '@dxos/app-framework';
 import { trim } from '@dxos/util';
 
 export const meta: Plugin.Meta = {
-  id: 'dxos.org/plugin/transcription',
+  id: 'org.dxos.plugin.transcription',
   name: 'Transcription',
   description: trim`
     Real-time voice-to-text transcription service for capturing spoken content.

package/src/model/model.test.ts

@@ -51,8 +51,8 @@ describe('SerializationModel', () => {
     }
 
     // Update message.
-    Obj.change(message, (m) => {
-      m.blocks.push({ _tag: 'transcript', started: createDate(), text: 'Hello again!' });
+    Obj.change(message, (obj) => {
+      obj.blocks.push({ _tag: 'transcript', started: createDate(), text: 'Hello again!' });
     });
     model.updateChunk(message);
     {
@@ -133,16 +133,16 @@ describe('SerializationModel', () => {
       expect(view.state.doc.toString()).to.eq('###### Alice\nHello world!\n\n');
 
       // Update message (add block).
-      Obj.change(message, (m) => {
-        m.blocks.push({ _tag: 'transcript', started: createDate(), text: 'Hello again!' });
+      Obj.change(message, (obj) => {
+        obj.blocks.push({ _tag: 'transcript', started: createDate(), text: 'Hello again!' });
       });
       model.updateChunk(message);
       model.sync(adapter);
       expect(view.state.doc.toString()).to.eq('###### Alice\nHello world!\nHello again!\n\n');
 
       // Update message (remove block).
-      Obj.change(message, (m) => {
-        m.blocks.splice(0, 1);
+      Obj.change(message, (obj) => {
+        obj.blocks.splice(0, 1);
       });
       model.updateChunk(message);
       model.sync(adapter);

package/src/operations/create.ts

@@ -0,0 +1,22 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+
+import { Operation } from '@dxos/operation';
+import { Transcript } from '@dxos/types';
+
+import { Create } from './definitions';
+
+const handler: Operation.WithHandler<typeof Create> = Create.pipe(
+  Operation.withHandler(
+    Effect.fn(function* ({ space }) {
+      return {
+        object: Transcript.make(space.queues.create().dxn),
+      };
+    }),
+  ),
+);
+
+export default handler;

package/src/operations/definitions.ts

@@ -0,0 +1,102 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+import { AiService } from '@dxos/ai';
+import { SpaceSchema } from '@dxos/client/echo';
+import { Database, Ref } from '@dxos/echo';
+import { QueueService } from '@dxos/functions';
+import { Operation } from '@dxos/operation';
+import { Message, Transcript } from '@dxos/types';
+
+import { meta } from '#meta';
+
+const TRANSCRIPT_OPERATION = `${meta.id}.operation`;
+
+export const Create = Operation.make({
+  meta: { key: `${TRANSCRIPT_OPERATION}.create`, name: 'Create Transcript' },
+  input: Schema.Struct({
+    name: Schema.optional(Schema.String),
+    space: SpaceSchema,
+  }),
+  output: Schema.Struct({
+    object: Transcript.Transcript,
+  }),
+});
+
+export const MessageWithRangeId = Schema.extend(
+  Message.Message,
+  Schema.Struct({
+    rangeId: Schema.optional(Schema.Array(Schema.String)).annotations({
+      description: 'The IDs of the messages that contain the sentences.',
+    }),
+  }),
+);
+
+export type MessageWithRangeIdType = Schema.Schema.Type<typeof MessageWithRangeId>;
+
+export const Open = Operation.make({
+  meta: {
+    key: 'org.dxos.function.transcription.open',
+    name: 'Open',
+    description: 'Opens and reads the contents of a transcription object.',
+  },
+  input: Schema.Struct({
+    transcript: Ref.Ref(Transcript.Transcript).annotations({
+      description: 'The ID of the transcription object.',
+    }),
+  }),
+  output: Schema.Struct({
+    content: Schema.String,
+  }),
+  services: [Database.Service, QueueService],
+});
+
+export const Summarize = Operation.make({
+  meta: {
+    key: 'org.dxos.function.transcription.summarize',
+    name: 'Summarize',
+    description: 'Summarize a transcript of a meeting.',
+  },
+  input: Schema.Struct({
+    transcript: Schema.String.annotations({
+      description: 'The transcript of the meeting.',
+    }),
+    notes: Schema.optional(Schema.String).annotations({
+      description: 'Additional notes from the participants.',
+    }),
+  }),
+  output: Schema.Struct({
+    summary: Schema.String.annotations({
+      description: 'The summary of the transcript.',
+    }),
+  }),
+  services: [AiService.AiService],
+});
+
+export const SentenceNormalizationInput = Schema.Struct({
+  messages: Schema.Array(MessageWithRangeId).annotations({
+    description: 'Messages to normalize into sentences.',
+  }),
+});
+
+export type SentenceNormalizationInputType = Schema.Schema.Type<typeof SentenceNormalizationInput>;
+
+export const SentenceNormalizationOutput = Schema.Struct({
+  sentences: Schema.Array(MessageWithRangeId.pipe(Schema.mutable)).pipe(Schema.mutable).annotations({
+    description: 'The sentences of the transcript.',
+  }),
+});
+
+export const SentenceNormalization = Operation.make({
+  meta: {
+    key: 'org.dxos.function.transcription.sentence-normalization',
+    name: 'Sentence Normalization',
+    description: 'Post process of transcription for sentence normalization',
+  },
+  input: SentenceNormalizationInput,
+  output: SentenceNormalizationOutput,
+  services: [],
+});

package/src/operations/index.ts

@@ -0,0 +1,13 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { OperationHandlerSet } from '@dxos/operation';
+
+export * as TranscriptOperation from './definitions';
+
+export const TranscriptionOperationHandlerSet = OperationHandlerSet.lazy(
+  () => import('./create'),
+  () => import('./open'),
+  () => import('./summarize'),
+);

package/src/operations/open.ts

@@ -0,0 +1,31 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+
+import { Database, Obj } from '@dxos/echo';
+import { QueueService } from '@dxos/functions';
+import { Operation } from '@dxos/operation';
+import { Message } from '@dxos/types';
+
+import { renderByline } from '../util';
+import { Open } from './definitions';
+
+const handler: Operation.WithHandler<typeof Open> = Open.pipe(
+  Operation.withHandler(
+    Effect.fn(function* ({ transcript }) {
+      const transcriptObj = yield* Database.load(transcript);
+      const { dxn } = yield* Effect.promise(() => transcriptObj.queue.load());
+      const queue = yield* QueueService.getQueue(dxn);
+      yield* Effect.promise(() => queue?.queryObjects());
+      const content = queue?.objects
+        .filter((message: unknown) => Obj.instanceOf(Message.Message, message))
+        .flatMap((message: Message.Message, index: number) => renderByline([])(message, index))
+        .join('\n\n');
+      return { content };
+    }),
+  ),
+);
+
+export default handler;