@dxos/plugin-transcription 0.8.2-main.fbd8ed0 → 0.8.2-staging.7ac8446

package/src/capabilities/app-graph-builder.ts

@@ -2,42 +2,45 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Rx } from '@effect-rx/rx-react';
-import { Option, pipe, type Schema } from 'effect';
-
-import { Capabilities, contributes, createIntent, type PluginContext } from '@dxos/app-framework';
-import { AIServiceEdgeClient, type AIServiceClient } from '@dxos/assistant';
-import { AI_SERVICE_ENDPOINT } from '@dxos/assistant/testing';
-import { Filter, fullyQualifiedId, getSpace, makeRef, Query, type Space } from '@dxos/client/echo';
-import { getSchemaTypename, isInstanceOf, type BaseEchoObject } from '@dxos/echo-schema';
+import { Capabilities, contributes, createIntent, type PluginsContext } from '@dxos/app-framework';
+import { fullyQualifiedId, getSpace, makeRef, type Space } from '@dxos/client/echo';
+import { generateName } from '@dxos/display-name';
+import { getSchemaTypename, isInstanceOf } from '@dxos/echo-schema';
 import { invariant } from '@dxos/invariant';
-import { DXN } from '@dxos/keys';
-import { log } from '@dxos/log';
 import { ClientCapabilities } from '@dxos/plugin-client';
 import { PLANK_COMPANION_TYPE, ATTENDABLE_PATH_SEPARATOR } from '@dxos/plugin-deck/types';
-import { createExtension, rxFromSignal } from '@dxos/plugin-graph';
-import { DocumentType } from '@dxos/plugin-markdown/types';
+import { createExtension, type Node } from '@dxos/plugin-graph';
 import { MeetingCapabilities, type CallState, type MediaState } from '@dxos/plugin-meeting';
-import { MeetingType, type MeetingCallProperties } from '@dxos/plugin-meeting/types';
-import { type buf } from '@dxos/protocols/buf';
-import { type TranscriptionPayloadSchema } from '@dxos/protocols/buf/dxos/edge/calls_pb';
-import { DataType } from '@dxos/schema';
+import { MeetingType } from '@dxos/plugin-meeting/types';
+import { buf } from '@dxos/protocols/buf';
+import {
+  TranscriptionPayloadSchema,
+  type TranscriptionPayload as TranscriptionPayloadProto,
+} from '@dxos/protocols/buf/dxos/edge/calls_pb';
+import { keyToFallback } from '@dxos/util';
 
 import { TranscriptionCapabilities } from './capabilities';
-import { processTranscriptMessage } from '../entity-extraction';
 import { TRANSCRIPTION_PLUGIN } from '../meta';
 import { TranscriptionManager } from '../transcriber';
-import { TranscriptionAction, type TranscriptionSettingsProps, TranscriptType } from '../types';
+import { TranscriptionAction, TranscriptType } from '../types';
 
 // TODO(wittjosiah): Factor out.
 // TODO(wittjosiah): Can we stop using protobuf for this?
 type TranscriptionPayload = buf.MessageInitShape<typeof TranscriptionPayloadSchema>;
+const codec = {
+  encode: (message: TranscriptionPayload): Uint8Array => {
+    return buf.toBinary(TranscriptionPayloadSchema, buf.create(TranscriptionPayloadSchema, message));
+  },
+  decode: (message: Uint8Array): TranscriptionPayloadProto => {
+    return buf.fromBinary(TranscriptionPayloadSchema, message);
+  },
+};
 
 const getMeetingTranscript = async (
-  context: PluginContext,
+  context: PluginsContext,
   meeting: MeetingType,
 ): Promise<TranscriptType | undefined> => {
-  const { dispatchPromise: dispatch } = context.getCapability(Capabilities.IntentDispatcher);
+  const { dispatchPromise: dispatch } = context.requestCapability(Capabilities.IntentDispatcher);
 
   const typename = getSchemaTypename(TranscriptType)!;
   const existing = await meeting.artifacts[typename]?.load();
@@ -53,181 +56,97 @@ const getMeetingTranscript = async (
 };
 
 // TODO(wittjosiah): Introduce a sound that plays when transcription is activated/deactivated.
-export default (context: PluginContext) =>
+export default (context: PluginsContext) =>
   contributes(Capabilities.AppGraphBuilder, [
     createExtension({
       id: `${TRANSCRIPTION_PLUGIN}/meeting-transcript`,
-      actions: (node) =>
-        Rx.make((get) =>
-          pipe(
-            get(node),
-            Option.flatMap((node) => (isInstanceOf(MeetingType, node.data) ? Option.some(node.data) : Option.none())),
-            Option.map((meeting) => {
-              const state = context.getCapability(TranscriptionCapabilities.MeetingTranscriptionState);
-              const enabled = get(rxFromSignal(() => state.enabled));
-              return [
-                {
-                  id: `${fullyQualifiedId(meeting)}/action/start-stop-transcription`,
-                  data: async () => {
-                    // NOTE: We are not saving the state of the transcription manager here.
-                    // We expect the state to be updated through `onCallStateUpdated` once it is propagated through Swarm.
-                    // This is done to avoid race conditions and to not handle optimistic updates.
-                    const transcript = await getMeetingTranscript(context, meeting);
-                    invariant(transcript, 'Failed to create transcript');
-
-                    const callManager = context.getCapability(MeetingCapabilities.CallManager);
-                    callManager.setActivity(getSchemaTypename(TranscriptType)!, {
-                      queueDxn: transcript.queue.dxn.toString(),
-                      enabled: !enabled,
-                    });
-                  },
-                  properties: {
-                    label: enabled
-                      ? ['stop transcription label', { ns: TRANSCRIPTION_PLUGIN }]
-                      : ['start transcription label', { ns: TRANSCRIPTION_PLUGIN }],
-                    icon: 'ph--subtitles--regular',
-                    disposition: 'toolbar',
-                    classNames: enabled ? 'bg-callAlert' : '',
-                  },
-                },
-              ];
-            }),
-            Option.getOrElse(() => []),
-          ),
-        ),
-      connector: (node) =>
-        Rx.make((get) =>
-          pipe(
-            get(node),
-            Option.flatMap((node) => (isInstanceOf(MeetingType, node.data) ? Option.some(node.data) : Option.none())),
-            Option.map((meeting) => {
-              const client = context.getCapability(ClientCapabilities.Client);
-              const state = context.getCapability(TranscriptionCapabilities.MeetingTranscriptionState);
-              const settingsStore = get(context.capabilities(Capabilities.SettingsStore))[0];
-              const settings = get(
-                rxFromSignal(() => settingsStore?.getStore<TranscriptionSettingsProps>(TRANSCRIPTION_PLUGIN)!.value),
-              );
-
-              // TODO(dmaretskyi): Request via capability.
-              const aiClient: AIServiceClient | undefined = new AIServiceEdgeClient({
-                // TODO(burdon): Get from config.
-                endpoint: AI_SERVICE_ENDPOINT.REMOTE,
+      filter: (node): node is Node<MeetingType> =>
+        isInstanceOf(MeetingType, node.data) && node.type !== PLANK_COMPANION_TYPE,
+      actions: ({ node }) => {
+        const meeting = node.data;
+        const state = context.requestCapability(TranscriptionCapabilities.MeetingTranscriptionState);
+        return [
+          {
+            id: `${fullyQualifiedId(meeting)}/action/start-stop-transcription`,
+            data: async () => {
+              const call = context.requestCapability(MeetingCapabilities.CallManager);
+              invariant(state.transcriptionManager);
+              state.enabled = !state.enabled;
+              void state.transcriptionManager.setEnabled(state.enabled);
+              const transcript = await getMeetingTranscript(context, meeting);
+              invariant(transcript, 'Failed to create transcript');
+              call.setActivity(getSchemaTypename(TranscriptType)!, {
+                value: codec.encode({
+                  enabled: state.enabled,
+                  queueDxn: transcript.queue.dxn.toString(),
+                }),
               });
-
-              return [
-                {
-                  id: `${fullyQualifiedId(meeting)}${ATTENDABLE_PATH_SEPARATOR}${getSchemaTypename(TranscriptType)}`,
-                  type: PLANK_COMPANION_TYPE,
-                  data: get(rxFromSignal(() => meeting.artifacts[getSchemaTypename(TranscriptType)!]?.target)),
-                  properties: {
-                    label: ['transcript companion label', { ns: TRANSCRIPTION_PLUGIN }],
-                    icon: 'ph--subtitles--regular',
-                    position: 'hoist',
-                    disposition: 'hidden',
-                    schema: TranscriptType,
-                    getIntent: ({ space }: { space: Space }) =>
-                      createIntent(TranscriptionAction.Create, { spaceId: space.id }),
-
-                    // TODO(burdon): Is this the right place to inisitalize the state? Change to a capability?
-                    onJoin: async ({ meeting }: { meeting?: MeetingType }) => {
-                      const identity = client.halo.identity.get();
-                      invariant(identity);
-                      const space = getSpace(meeting);
-                      invariant(space);
-
-                      let messageEnricher;
-                      if (aiClient && settings.entityExtraction) {
-                        messageEnricher = createEntityExtractionEnricher({
-                          aiClient,
-                          // TODO(dmaretskyi): Have those be discovered from the schema graph or contributed by capabilities?
-                          //  This forced me to add a dependency on markdown plugin.
-                          //  This will be replaced with a vector search index anyway, so its not a big deal.
-                          contextTypes: [DocumentType, DataType.Person, DataType.Organization],
-                          space,
-                        });
-                      }
-
-                      // TODO(burdon): The TranscriptionManager singleton is part of the state and should just be updated here.
-                      state.transcriptionManager = await new TranscriptionManager({
-                        edgeClient: client.edge,
-                        messageEnricher,
-                      })
-                        .setIdentityDid(identity.did)
-                        .open();
-                    },
-                    onLeave: async () => {
-                      await state.transcriptionManager?.close();
-                      state.transcriptionManager = undefined;
-                      state.enabled = false;
-                    },
-                    onCallStateUpdated: async (callState: CallState) => {
-                      const typename = getSchemaTypename(TranscriptType);
-                      const transcription = typename ? callState.activities?.[typename] : undefined;
-                      if (!transcription?.payload) {
-                        return;
-                      }
-
-                      const payload: TranscriptionPayload = transcription.payload;
-                      state.enabled = !!payload.enabled;
-                      if (payload.queueDxn) {
-                        // NOTE: Must set queue before enabling transcription.
-                        state.transcriptionManager?.setQueue(DXN.parse(payload.queueDxn));
-                      }
-                      await state.transcriptionManager?.setEnabled(payload.enabled);
-                    },
-                    onMediaStateUpdated: async ([mediaState, isSpeaking]: [MediaState, boolean]) => {
-                      void state.transcriptionManager?.setAudioTrack(mediaState.audioTrack);
-                      void state.transcriptionManager?.setRecording(isSpeaking);
-                    },
-                  } satisfies MeetingCallProperties,
-                },
-              ];
-            }),
-            Option.getOrElse(() => []),
-          ),
-        ),
+            },
+            properties: {
+              label: state.enabled
+                ? ['stop transcription label', { ns: TRANSCRIPTION_PLUGIN }]
+                : ['start transcription label', { ns: TRANSCRIPTION_PLUGIN }],
+              icon: 'ph--record--regular',
+              disposition: 'toolbar',
+              classNames: state.enabled ? 'text-activeInCall' : '',
+            },
+          },
+        ];
+      },
+      connector: ({ node }) => {
+        const meeting = node.data;
+        const client = context.requestCapability(ClientCapabilities.Client);
+        const state = context.requestCapability(TranscriptionCapabilities.MeetingTranscriptionState);
+
+        return [
+          {
+            id: `${fullyQualifiedId(meeting)}${ATTENDABLE_PATH_SEPARATOR}${getSchemaTypename(TranscriptType)}`,
+            type: PLANK_COMPANION_TYPE,
+            data: meeting.artifacts[getSchemaTypename(TranscriptType)!]?.target,
+            properties: {
+              label: ['transcript companion label', { ns: TRANSCRIPTION_PLUGIN }],
+              icon: 'ph--subtitles--regular',
+              disposition: 'hidden',
+              schema: TranscriptType,
+              getIntent: ({ space }: { space: Space }) =>
+                createIntent(TranscriptionAction.Create, { spaceId: space.id }),
+              onJoin: () => {
+                const transcriptionManager = new TranscriptionManager(client.edge);
+                const identity = client.halo.identity.get();
+                invariant(identity);
+                transcriptionManager.setName(
+                  identity.profile?.displayName ?? generateName(identity.identityKey.toHex()),
+                );
+                const fallbackValue = keyToFallback(identity!.identityKey);
+                const userHue = identity!.profile?.data?.hue || fallbackValue.hue;
+                transcriptionManager.setHue(userHue);
+                void transcriptionManager.open();
+                state.transcriptionManager = transcriptionManager;
+              },
+              onLeave: () => {
+                void state.transcriptionManager?.close();
+                state.transcriptionManager = undefined;
+                state.enabled = false;
+              },
+              onCallStateUpdated: (callState: CallState) => {
+                const typename = getSchemaTypename(TranscriptType);
+                const transcription = typename ? callState.activities?.[typename] : undefined;
+                if (!transcription?.payload?.value) {
+                  return;
+                }
+
+                const payload = codec.decode(transcription.payload.value);
+                state.enabled = payload.enabled;
+                void state.transcriptionManager?.setEnabled(payload.enabled);
+                void state.transcriptionManager?.setQueue(payload.queueDxn);
+              },
+              onMediaStateUpdated: ([mediaState, isSpeaking]: [MediaState, boolean]) => {
+                void state.transcriptionManager?.setAudioTrack(mediaState.audioTrack);
+                void state.transcriptionManager?.setRecording(isSpeaking);
+              },
+            },
+          },
+        ];
+      },
     }),
   ]);
-
-type EntityExtractionEnricherFactoryOptions = {
-  aiClient: AIServiceClient;
-  contextTypes: Schema.Schema.AnyNoContext[];
-  space: Space;
-};
-
-const createEntityExtractionEnricher = ({ aiClient, contextTypes, space }: EntityExtractionEnricherFactoryOptions) => {
-  return async (message: DataType.Message) => {
-    const { objects } = await space.db
-      .query(Query.select(Filter.or(...contextTypes.map((s) => Filter.type(s as Schema.Schema<BaseEchoObject>)))))
-      .run();
-    log.info('context', { objects });
-
-    const { message: enhancedMessage, timeElapsed } = await processTranscriptMessage({
-      aiService: aiClient,
-      message,
-      context: {
-        objects: await Promise.all(objects.map((o) => processContextObject(o))),
-      },
-      options: { timeout: ENTITY_EXTRACTOR_TIMEOUT, fallbackToRaw: true },
-    });
-
-    log.info('entity extraction time', { timeElapsed });
-    return enhancedMessage;
-  };
-};
-
-// TODO(dmaretskyi): Use Type.Any
-const processContextObject = async (object: BaseEchoObject): Promise<any> => {
-  // TODO(dmaretskyi): Documents need special processing is the content is behind a ref.
-  // TODO(dmaretskyi): Think about a way to handle this serialization with a decorator.
-  // if (isInstanceOf(DocumentType, object)) {
-  //   return {
-  //     ...object,
-  //     content: await object.content.load(),
-  //   };
-  // }
-
-  return object;
-};
-
-const ENTITY_EXTRACTOR_TIMEOUT = 25_000;

package/src/capabilities/capabilities.ts

@@ -3,7 +3,7 @@
 //
 
 import { defineCapability } from '@dxos/app-framework';
-import { type Live } from '@dxos/live-object';
+import { type ReactiveObject } from '@dxos/live-object';
 
 import { TRANSCRIPTION_PLUGIN } from '../meta';
 import { type TranscriberParams, type Transcriber, type TranscriptionManager } from '../transcriber';
@@ -12,14 +12,13 @@ export namespace TranscriptionCapabilities {
   export type GetTranscriberProps = {
     audioStreamTrack: MediaStreamTrack;
     onSegments: TranscriberParams['onSegments'];
-    config?: TranscriberParams['config'];
   };
   export type GetTranscriber = (props: GetTranscriberProps) => Transcriber;
   export const Transcriber = defineCapability<GetTranscriber>(`${TRANSCRIPTION_PLUGIN}/capability/transcriber`);
 
-  export type MeetingTranscriptionState = Live<{
+  export type MeetingTranscriptionState = ReactiveObject<{
+    enabled: boolean;
     transcriptionManager?: TranscriptionManager;
-    enabled: boolean; // TODO(burdon): Move state into TranscriptionManager.
   }>;
   export const MeetingTranscriptionState = defineCapability<MeetingTranscriptionState>(
     `${TRANSCRIPTION_PLUGIN}/capability/meeting-transcription-state`,

package/src/capabilities/index.ts

@@ -9,6 +9,5 @@ export const IntentResolver = lazy(() => import('./intent-resolver'));
 export const MeetingTranscriptionState = lazy(() => import('./meeting-transcription-state'));
 export const ReactSurface = lazy(() => import('./react-surface'));
 export const Transcriber = lazy(() => import('./transcriber'));
-export const Settings = lazy(() => import('./settings'));
 
 export * from './capabilities';

package/src/capabilities/intent-resolver.ts

@@ -2,21 +2,20 @@
 // Copyright 2025 DXOS.org
 //
 
-import { Capabilities, contributes, createResolver, type PluginContext } from '@dxos/app-framework';
-import { createQueueDxn } from '@dxos/echo-schema';
-import { type SpaceId } from '@dxos/keys';
-import { live, refFromDXN } from '@dxos/live-object';
+import { Capabilities, contributes, createResolver, type PluginsContext } from '@dxos/app-framework';
+import { create, refFromDXN } from '@dxos/live-object';
+import { type SpaceId } from '@dxos/react-client/echo';
 
 import { TranscriptionAction, TranscriptType } from '../types';
-import { getTimeStr } from '../util';
+import { getTimeStr, randomQueueDxn } from '../util';
 
-export default (context: PluginContext) =>
+export default (context: PluginsContext) =>
   contributes(Capabilities.IntentResolver, [
     createResolver({
       intent: TranscriptionAction.Create,
       resolve: ({ name, spaceId }) => {
-        const transcript = live(TranscriptType, {
-          queue: refFromDXN(createQueueDxn(spaceId as SpaceId)),
+        const transcript = create(TranscriptType, {
+          queue: refFromDXN(randomQueueDxn(spaceId as SpaceId)),
           name: name ?? `Transcript ${getTimeStr(Date.now())}`,
         });
 

package/src/capabilities/meeting-transcription-state.ts

@@ -3,12 +3,12 @@
 //
 
 import { contributes } from '@dxos/app-framework';
-import { live } from '@dxos/live-object';
+import { create } from '@dxos/live-object';
 
 import { TranscriptionCapabilities } from './capabilities';
 
 export default () => {
-  const state = live<TranscriptionCapabilities.MeetingTranscriptionState>({
+  const state = create<TranscriptionCapabilities.MeetingTranscriptionState>({
     enabled: false,
   });
 

package/src/capabilities/react-surface.tsx

@@ -5,23 +5,15 @@
 import React from 'react';
 
 import { Capabilities, contributes, createSurface } from '@dxos/app-framework';
-import { SettingsStore } from '@dxos/local-storage';
 
-import { TranscriptionContainer, TranscriptionSettings } from '../components';
-import { meta } from '../meta';
-import { isTranscript, type TranscriptionSettingsProps, type TranscriptType } from '../types';
+import { TranscriptionContainer } from '../components';
+import { TRANSCRIPTION_PLUGIN } from '../meta';
+import { isTranscript, type TranscriptType } from '../types';
 
 export default () =>
   contributes(Capabilities.ReactSurface, [
     createSurface({
-      id: `${meta.id}/plugin-settings`,
-      role: 'article',
-      filter: (data): data is { subject: SettingsStore<TranscriptionSettingsProps> } =>
-        data.subject instanceof SettingsStore && data.subject.prefix === meta.id,
-      component: ({ data: { subject } }) => <TranscriptionSettings settings={subject.value} />,
-    }),
-    createSurface({
-      id: `${meta.id}/article/transcript`,
+      id: `${TRANSCRIPTION_PLUGIN}/article/transcript`,
       role: ['article', 'section'],
       filter: (data): data is { subject: TranscriptType } => isTranscript(data.subject),
       component: ({ data, role }) => <TranscriptionContainer transcript={data.subject} role={role} />,

package/src/capabilities/transcriber.ts

@@ -29,13 +29,12 @@ const TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
  * Records audio while user is speaking and transcribes it after user is done speaking.
  */
 export default () => {
-  const getTranscriber: TranscriptionCapabilities.GetTranscriber = ({ audioStreamTrack, onSegments, config }) => {
+  const getTranscriber: TranscriptionCapabilities.GetTranscriber = ({ audioStreamTrack, onSegments }) => {
     // Initialize audio transcription.
     return new Transcriber({
       config: {
         transcribeAfterChunksAmount: TRANSCRIBE_AFTER_CHUNKS_AMOUNT,
         prefixBufferChunksAmount: PREFIXED_CHUNKS_AMOUNT,
-        ...config,
       },
       recorder: new MediaStreamRecorder({
         mediaStreamTrack: audioStreamTrack,