@dxos/plugin-transcription 0.8.1 → 0.8.2-main.2f9c567

package/dist/types/src/types/schema.d.ts

@@ -1,57 +1,43 @@
-import { Expando, S } from '@dxos/echo-schema';
+import { Schema } from 'effect';
+import { Expando } from '@dxos/echo-schema';
 /**
  * Root transcript object created when the user starts a transcription.
  */
-export declare const TranscriptSchema: S.Struct<{
-    name: typeof S.String;
-    started: S.optional<typeof S.Date>;
-    ended: S.optional<typeof S.Date>;
+export declare const TranscriptSchema: Schema.Struct<{
+    name: typeof Schema.String;
+    started: Schema.optional<typeof Schema.String>;
+    ended: Schema.optional<typeof Schema.String>;
     /**
      * Queue containing TranscriptBlock objects.
      */
     queue: import("@dxos/echo-schema").Ref$<Expando>;
 }>;
-declare const TranscriptType_base: import("@dxos/echo-schema").TypedObjectPrototype<import("@dxos/echo-schema").TypedObjectFields<{
-    readonly name: typeof S.String;
-    readonly started: S.optional<typeof S.Date>;
-    readonly ended: S.optional<typeof S.Date>;
-    readonly queue: import("@dxos/echo-schema").Ref$<Expando>;
-}, import("@dxos/echo-schema").TypedObjectOptions>, S.Struct.Encoded<{
-    readonly name: typeof S.String;
-    readonly started: S.optional<typeof S.Date>;
-    readonly ended: S.optional<typeof S.Date>;
-    readonly queue: import("@dxos/echo-schema").Ref$<Expando>;
-}>>;
+declare const TranscriptType_base: import("@dxos/echo-schema").TypedObjectPrototype<import("@dxos/echo-schema").TypedObjectFields<Readonly<{
+    name: typeof Schema.String;
+    started: Schema.optional<typeof Schema.String>;
+    ended: Schema.optional<typeof Schema.String>;
+    /**
+     * Queue containing TranscriptBlock objects.
+     */
+    queue: import("@dxos/echo-schema").Ref$<Expando>;
+}>, import("@dxos/echo-schema").TypedObjectOptions>, Schema.Struct.Encoded<Readonly<{
+    name: typeof Schema.String;
+    started: Schema.optional<typeof Schema.String>;
+    ended: Schema.optional<typeof Schema.String>;
+    /**
+     * Queue containing TranscriptBlock objects.
+     */
+    queue: import("@dxos/echo-schema").Ref$<Expando>;
+}>>>;
 export declare class TranscriptType extends TranscriptType_base {
 }
 /**
  * First message in queue.
  * Contains metadata for the recording and transcript.
  */
-declare const TranscriptHeader: S.Struct<{
-    started: S.optional<typeof S.Date>;
-}>;
-export type TranscriptHeader = S.Schema.Type<typeof TranscriptHeader>;
-/**
- * Transcription fragment.
- */
-declare const TranscriptSegment: S.Struct<{
-    started: typeof S.Date;
-    text: typeof S.String;
+declare const TranscriptHeader: Schema.Struct<{
+    started: Schema.optional<typeof Schema.String>;
 }>;
-export type TranscriptSegment = S.Schema.Type<typeof TranscriptSegment>;
-/**
- * Transcription block (from single speaker).
- */
-export declare const TranscriptBlock: import("@dxos/echo-schema").EchoObjectSchema<S.Struct<{
-    id: typeof S.String;
-    authorName: S.optional<typeof S.String>;
-    authorHue: S.optional<typeof S.String>;
-    segments: S.Array$<S.Struct<{
-        started: typeof S.Date;
-        text: typeof S.String;
-    }>>;
-}>>;
-export type TranscriptBlock = S.Schema.Type<typeof TranscriptBlock>;
+export type TranscriptHeader = Schema.Schema.Type<typeof TranscriptHeader>;
 export {};
 //# sourceMappingURL=schema.d.ts.map

package/dist/types/src/types/schema.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../../src/types/schema.ts"],"names":[],"mappings":"AAIA,OAAO,EAAc,OAAO,EAAO,CAAC,EAAe,MAAM,mBAAmB,CAAC;AAE7E;;GAEG;AACH,eAAO,MAAM,gBAAgB;;;;IAO3B;;OAEG;;EAGH,CAAC;;;;;;;;;;;;AAEH,qBAAa,cAAe,SAAQ,mBAGT;CAAG;AAI9B;;;GAGG;AACH,QAAA,MAAM,gBAAgB;;EAEpB,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAEtE;;GAEG;AACH,QAAA,MAAM,iBAAiB;;;EAIrB,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAExE;;GAEG;AACH,eAAO,MAAM,eAAe;;;;;;;;GAKwD,CAAC;AAErF,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,eAAe,CAAC,CAAC"}
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../../src/types/schema.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAEhC,OAAO,EAAE,OAAO,EAAoB,MAAM,mBAAmB,CAAC;AAE9D;;GAEG;AACH,eAAO,MAAM,gBAAgB;;;;IAO3B;;OAEG;;EAGH,CAAC;;;;;IALD;;OAEG;;;;;;IAFH;;OAEG;;;AAKL,qBAAa,cAAe,SAAQ,mBAGT;CAAG;AAI9B;;;GAGG;AACH,QAAA,MAAM,gBAAgB;;EAEpB,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,gBAAgB,CAAC,CAAC"}

package/dist/types/src/types/types.d.ts

@@ -1,18 +1,18 @@
-import { S } from '@dxos/echo-schema';
+import { Schema } from 'effect';
 import { TranscriptType } from './schema';
 /**
  * Endpoint to the calls service.
  */
 export declare const TRANSCRIPTION_URL = "https://calls-service.dxos.workers.dev";
 export declare namespace TranscriptionAction {
-    const Create_base: S.TaggedClass<Create, "dxos.org/plugin/transcription/action/create", {
-        readonly _tag: S.tag<"dxos.org/plugin/transcription/action/create">;
+    const Create_base: Schema.TaggedClass<Create, "dxos.org/plugin/transcription/action/create", {
+        readonly _tag: Schema.tag<"dxos.org/plugin/transcription/action/create">;
     } & {
-        input: S.Struct<{
-            name: S.optional<typeof S.String>;
-            spaceId: typeof S.String;
+        input: Schema.Struct<{
+            name: Schema.optional<typeof Schema.String>;
+            spaceId: Schema.Schema<import("@dxos/keys").SpaceId, string, never>;
         }>;
-        output: S.Struct<{
+        output: Schema.Struct<{
             object: typeof TranscriptType;
         }>;
     }>;
@@ -21,4 +21,8 @@ export declare namespace TranscriptionAction {
     export {};
 }
 export declare const isTranscript: (object: unknown) => object is typeof TranscriptType;
+export declare const TranscriptionSettingsSchema: Schema.mutable<Schema.Struct<{
+    entityExtraction: Schema.PropertySignature<"?:", boolean | undefined, never, "?:", boolean | undefined, true, never>;
+}>>;
+export type TranscriptionSettingsProps = Schema.Schema.Type<typeof TranscriptionSettingsSchema>;
 //# sourceMappingURL=types.d.ts.map

package/dist/types/src/types/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../src/types/types.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,CAAC,EAAgB,MAAM,mBAAmB,CAAC;AAGpD,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAK1C;;GAEG;AAEH,eAAO,MAAM,iBAAiB,2CAA2C,CAAC;AAE1E,yBAAiB,mBAAmB,CAAC;;;;;;;;;;;;IAGnC,MAAM,OAAO,MAAO,SAAQ,WAS1B;KAAG;;CACN;AAED,eAAO,MAAM,YAAY,WAAY,OAAO,KAAG,MAAM,IAAI,OAAO,cAE/D,CAAC"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../src/types/types.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAKhC,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAK1C;;GAEG;AAEH,eAAO,MAAM,iBAAiB,2CAA2C,CAAC;AAE1E,yBAAiB,mBAAmB,CAAC;;;;;;;;;;;;IAGnC,MAAM,OAAO,MAAO,SAAQ,WAQ1B;KAAG;;CACN;AAED,eAAO,MAAM,YAAY,WAAY,OAAO,KAAG,MAAM,IAAI,OAAO,cAE/D,CAAC;AAGF,eAAO,MAAM,2BAA2B;;GAIvC,CAAC;AAEF,MAAM,MAAM,0BAA0B,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,2BAA2B,CAAC,CAAC"}

package/dist/types/src/util/index.d.ts

@@ -1,5 +1,4 @@
 export * from './array';
 export * from './get-time-str';
 export * from './monitor-audio-level';
-export * from './random-queue-dxn';
 //# sourceMappingURL=index.d.ts.map

package/dist/types/src/util/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/util/index.ts"],"names":[],"mappings":"AAIA,cAAc,SAAS,CAAC;AACxB,cAAc,gBAAgB,CAAC;AAC/B,cAAc,uBAAuB,CAAC;AACtC,cAAc,oBAAoB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/util/index.ts"],"names":[],"mappings":"AAIA,cAAc,SAAS,CAAC;AACxB,cAAc,gBAAgB,CAAC;AAC/B,cAAc,uBAAuB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dxos/plugin-transcription",
-  "version": "0.8.1",
+  "version": "0.8.2-main.2f9c567",
   "description": "Voice transcription.",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
@@ -32,9 +32,11 @@
     "src"
   ],
   "dependencies": {
+    "@codemirror/state": "^6.4.1",
+    "@codemirror/view": "^6.34.1",
     "@preact/signals-core": "^1.6.0",
     "date-fns": "^3.3.1",
-    "effect": "3.13.3",
+    "effect": "3.14.21",
     "extendable-media-recorder": "^9.2.19",
     "extendable-media-recorder-wav-encoder": "^7.0.123",
     "main-thread-scheduling": "^14.1.1",
@@ -42,54 +44,65 @@
     "react-router-dom": "^6.4.0",
     "react-use": "^17.5.1",
     "wavefile": "^11.0.0",
-    "@dxos/app-framework": "0.8.1",
-    "@dxos/artifact": "0.8.1",
-    "@dxos/assistant": "0.8.1",
-    "@dxos/async": "0.8.1",
-    "@dxos/client": "0.8.1",
-    "@dxos/context": "0.8.1",
-    "@dxos/codec-protobuf": "0.8.1",
-    "@dxos/debug": "0.8.1",
-    "@dxos/display-name": "0.8.1",
-    "@dxos/echo-db": "0.8.1",
-    "@dxos/echo-schema": "0.8.1",
-    "@dxos/invariant": "0.8.1",
-    "@dxos/keys": "0.8.1",
-    "@dxos/live-object": "0.8.1",
-    "@dxos/log": "0.8.1",
-    "@dxos/plugin-client": "0.8.1",
-    "@dxos/plugin-graph": "0.8.1",
-    "@dxos/plugin-deck": "0.8.1",
-    "@dxos/plugin-meeting": "0.8.1",
-    "@dxos/react-client": "0.8.1",
-    "@dxos/protocols": "0.8.1",
-    "@dxos/plugin-space": "0.8.1",
-    "@dxos/react-ui-components": "0.8.1",
-    "@dxos/react-ui-sfx": "0.8.1",
-    "@dxos/react-ui-attention": "0.8.1",
-    "@dxos/react-ui-grid": "0.8.1",
-    "@dxos/react-ui-stack": "0.8.1",
-    "@dxos/schema": "0.8.1",
-    "@dxos/react-edge-client": "0.8.1",
-    "@dxos/tracing": "0.8.1",
-    "@dxos/util": "0.8.1"
+    "@dxos/app-framework": "0.8.2-main.2f9c567",
+    "@dxos/assistant": "0.8.2-main.2f9c567",
+    "@dxos/client": "0.8.2-main.2f9c567",
+    "@dxos/async": "0.8.2-main.2f9c567",
+    "@dxos/artifact": "0.8.2-main.2f9c567",
+    "@dxos/codec-protobuf": "0.8.2-main.2f9c567",
+    "@dxos/context": "0.8.2-main.2f9c567",
+    "@dxos/display-name": "0.8.2-main.2f9c567",
+    "@dxos/debug": "0.8.2-main.2f9c567",
+    "@dxos/echo": "0.8.2-main.2f9c567",
+    "@dxos/echo-db": "0.8.2-main.2f9c567",
+    "@dxos/echo-schema": "0.8.2-main.2f9c567",
+    "@dxos/local-storage": "0.8.2-main.2f9c567",
+    "@dxos/invariant": "0.8.2-main.2f9c567",
+    "@dxos/keys": "0.8.2-main.2f9c567",
+    "@dxos/live-object": "0.8.2-main.2f9c567",
+    "@dxos/log": "0.8.2-main.2f9c567",
+    "@dxos/plugin-client": "0.8.2-main.2f9c567",
+    "@dxos/plugin-deck": "0.8.2-main.2f9c567",
+    "@dxos/plugin-graph": "0.8.2-main.2f9c567",
+    "@dxos/plugin-markdown": "0.8.2-main.2f9c567",
+    "@dxos/plugin-space": "0.8.2-main.2f9c567",
+    "@dxos/plugin-meeting": "0.8.2-main.2f9c567",
+    "@dxos/protocols": "0.8.2-main.2f9c567",
+    "@dxos/react-client": "0.8.2-main.2f9c567",
+    "@dxos/react-edge-client": "0.8.2-main.2f9c567",
+    "@dxos/react-ui-components": "0.8.2-main.2f9c567",
+    "@dxos/react-ui-form": "0.8.2-main.2f9c567",
+    "@dxos/react-ui-attention": "0.8.2-main.2f9c567",
+    "@dxos/react-ui-editor": "0.8.2-main.2f9c567",
+    "@dxos/react-ui-grid": "0.8.2-main.2f9c567",
+    "@dxos/react-ui-sfx": "0.8.2-main.2f9c567",
+    "@dxos/react-ui-stack": "0.8.2-main.2f9c567",
+    "@dxos/schema": "0.8.2-main.2f9c567",
+    "@dxos/tracing": "0.8.2-main.2f9c567",
+    "@dxos/util": "0.8.2-main.2f9c567"
   },
   "devDependencies": {
     "@types/react": "~18.2.0",
     "react": "~18.2.0",
     "react-dom": "~18.2.0",
     "tailwindcss": "~3.4.1",
-    "@dxos/random": "0.8.1",
-    "@dxos/react-ui": "0.8.1",
-    "@dxos/react-ui-theme": "0.8.1",
-    "@dxos/storybook-utils": "0.8.1",
-    "@dxos/test-utils": "0.8.1"
+    "@dxos/plugin-preview": "0.8.2-main.2f9c567",
+    "@dxos/plugin-storybook-layout": "0.8.2-main.2f9c567",
+    "@dxos/plugin-theme": "0.8.2-main.2f9c567",
+    "@dxos/react-ui": "0.8.2-main.2f9c567",
+    "@dxos/random": "0.8.2-main.2f9c567",
+    "@dxos/react-ui-syntax-highlighter": "0.8.2-main.2f9c567",
+    "@dxos/react-ui-theme": "0.8.2-main.2f9c567",
+    "@dxos/react-ui-form": "0.8.2-main.2f9c567",
+    "@dxos/test-utils": "0.8.2-main.2f9c567",
+    "@dxos/storybook-utils": "0.8.2-main.2f9c567"
   },
   "optionalDependencies": {
+    "effect": "^3.13.3",
     "react": "~18.2.0",
     "react-dom": "~18.2.0",
-    "@dxos/react-ui": "0.8.1",
-    "@dxos/react-ui-theme": "0.8.1"
+    "@dxos/react-ui-theme": "0.8.2-main.2f9c567",
+    "@dxos/react-ui": "0.8.2-main.2f9c567"
   },
   "publishConfig": {
     "access": "public"

package/src/TranscriptionPlugin.tsx

@@ -6,11 +6,21 @@ import { Capabilities, Events, contributes, defineModule, definePlugin } from '@
 import { QueueImpl } from '@dxos/echo-db';
 import { isInstanceOf } from '@dxos/echo-schema';
 import { ClientCapabilities, ClientEvents } from '@dxos/plugin-client';
+import { getSpace } from '@dxos/react-client/echo';
+import { DataType } from '@dxos/schema';
 
-import { AppGraphBuilder, IntentResolver, MeetingTranscriptionState, ReactSurface, Transcriber } from './capabilities';
+import {
+  AppGraphBuilder,
+  IntentResolver,
+  MeetingTranscriptionState,
+  ReactSurface,
+  Transcriber,
+  Settings,
+} from './capabilities';
+import { renderMarkdown } from './components';
 import { meta } from './meta';
 import translations from './translations';
-import { TranscriptBlock, TranscriptType } from './types';
+import { TranscriptType } from './types';
 
 export const TranscriptionPlugin = () =>
   definePlugin(meta, [
@@ -20,14 +30,9 @@ export const TranscriptionPlugin = () =>
       activate: () => contributes(Capabilities.Translations, translations),
     }),
     defineModule({
-      id: `${meta.id}/module/transcription`,
-      activatesOn: Events.SetupAppGraph,
-      activate: Transcriber,
-    }),
-    defineModule({
-      id: `${meta.id}/module/meeting-transcription-state`,
-      activatesOn: Events.SetupAppGraph,
-      activate: MeetingTranscriptionState,
+      id: `${meta.id}/module/settings`,
+      activatesOn: Events.SetupSettings,
+      activate: Settings,
     }),
     defineModule({
       id: `${meta.id}/module/metadata`,
@@ -40,11 +45,13 @@ export const TranscriptionPlugin = () =>
             // TODO(wittjosiah): Factor out. Artifact? Separate capability?
             getTextContent: async (transcript: TranscriptType) => {
               const client = context.requestCapability(ClientCapabilities.Client);
+              const space = getSpace(transcript);
+              const members = space?.members.get().map((member) => member.identity) ?? [];
               const queue = new QueueImpl(client.edge, transcript.queue.dxn);
               await queue.refresh();
               const content = queue.items
-                .filter((block) => isInstanceOf(TranscriptBlock, block))
-                .map((block) => `${block.authorName}: ${block.segments.map((segment) => segment.text).join('\n')}`)
+                .filter((message) => isInstanceOf(DataType.Message, message))
+                .flatMap((message, index) => renderMarkdown(members)(message, index))
                 .join('\n\n');
               return content;
             },
@@ -71,4 +78,14 @@ export const TranscriptionPlugin = () =>
       activatesOn: Events.SetupAppGraph,
       activate: AppGraphBuilder,
     }),
+    defineModule({
+      id: `${meta.id}/module/transcription`,
+      activatesOn: Events.SetupAppGraph,
+      activate: Transcriber,
+    }),
+    defineModule({
+      id: `${meta.id}/module/meeting-transcription-state`,
+      activatesOn: Events.SetupAppGraph,
+      activate: MeetingTranscriptionState,
+    }),
   ]);

package/src/capabilities/app-graph-builder.ts

@@ -2,39 +2,35 @@
 // Copyright 2025 DXOS.org
 //
 
+import type { Schema } from 'effect';
+
 import { Capabilities, contributes, createIntent, type PluginsContext } from '@dxos/app-framework';
-import { fullyQualifiedId, getSpace, makeRef, type Space } from '@dxos/client/echo';
-import { generateName } from '@dxos/display-name';
-import { getSchemaTypename, isInstanceOf } from '@dxos/echo-schema';
+import { AIServiceEdgeClient, type AIServiceClient } from '@dxos/assistant';
+import { AI_SERVICE_ENDPOINT } from '@dxos/assistant/testing';
+import { Filter, fullyQualifiedId, getSpace, makeRef, type Space } from '@dxos/client/echo';
+import { getSchemaTypename, isInstanceOf, type BaseEchoObject } from '@dxos/echo-schema';
 import { invariant } from '@dxos/invariant';
+import { DXN } from '@dxos/keys';
+import { log } from '@dxos/log';
 import { ClientCapabilities } from '@dxos/plugin-client';
 import { PLANK_COMPANION_TYPE, ATTENDABLE_PATH_SEPARATOR } from '@dxos/plugin-deck/types';
 import { createExtension, type Node } from '@dxos/plugin-graph';
+import { DocumentType } from '@dxos/plugin-markdown/types';
 import { MeetingCapabilities, type CallState, type MediaState } from '@dxos/plugin-meeting';
-import { MeetingType } from '@dxos/plugin-meeting/types';
-import { buf } from '@dxos/protocols/buf';
-import {
-  TranscriptionPayloadSchema,
-  type TranscriptionPayload as TranscriptionPayloadProto,
-} from '@dxos/protocols/buf/dxos/edge/calls_pb';
-import { keyToFallback } from '@dxos/util';
+import { MeetingType, type MeetingCallProperties } from '@dxos/plugin-meeting/types';
+import { type buf } from '@dxos/protocols/buf';
+import { type TranscriptionPayloadSchema } from '@dxos/protocols/buf/dxos/edge/calls_pb';
+import { DataType } from '@dxos/schema';
 
 import { TranscriptionCapabilities } from './capabilities';
+import { processTranscriptMessage } from '../entity-extraction';
 import { TRANSCRIPTION_PLUGIN } from '../meta';
 import { TranscriptionManager } from '../transcriber';
-import { TranscriptionAction, TranscriptType } from '../types';
+import { TranscriptionAction, type TranscriptionSettingsProps, TranscriptType } from '../types';
 
 // TODO(wittjosiah): Factor out.
 // TODO(wittjosiah): Can we stop using protobuf for this?
 type TranscriptionPayload = buf.MessageInitShape<typeof TranscriptionPayloadSchema>;
-const codec = {
-  encode: (message: TranscriptionPayload): Uint8Array => {
-    return buf.toBinary(TranscriptionPayloadSchema, buf.create(TranscriptionPayloadSchema, message));
-  },
-  decode: (message: Uint8Array): TranscriptionPayloadProto => {
-    return buf.fromBinary(TranscriptionPayloadSchema, message);
-  },
-};
 
 const getMeetingTranscript = async (
   context: PluginsContext,
@@ -60,8 +56,7 @@ export default (context: PluginsContext) =>
   contributes(Capabilities.AppGraphBuilder, [
     createExtension({
       id: `${TRANSCRIPTION_PLUGIN}/meeting-transcript`,
-      filter: (node): node is Node<MeetingType> =>
-        isInstanceOf(MeetingType, node.data) && node.type !== PLANK_COMPANION_TYPE,
+      filter: (node): node is Node<MeetingType> => isInstanceOf(MeetingType, node.data),
       actions: ({ node }) => {
         const meeting = node.data;
         const state = context.requestCapability(TranscriptionCapabilities.MeetingTranscriptionState);
@@ -69,26 +64,25 @@ export default (context: PluginsContext) =>
           {
             id: `${fullyQualifiedId(meeting)}/action/start-stop-transcription`,
             data: async () => {
-              const call = context.requestCapability(MeetingCapabilities.CallManager);
-              invariant(state.transcriptionManager);
-              state.enabled = !state.enabled;
-              void state.transcriptionManager.setEnabled(state.enabled);
+              // NOTE: We are not saving the state of the transcription manager here.
+              // We expect the state to be updated through `onCallStateUpdated` once it is propagated through Swarm.
+              // This is done to avoid race conditions and to not handle optimistic updates.
               const transcript = await getMeetingTranscript(context, meeting);
               invariant(transcript, 'Failed to create transcript');
-              call.setActivity(getSchemaTypename(TranscriptType)!, {
-                value: codec.encode({
-                  enabled: state.enabled,
-                  queueDxn: transcript.queue.dxn.toString(),
-                }),
+
+              const callManager = context.requestCapability(MeetingCapabilities.CallManager);
+              callManager.setActivity(getSchemaTypename(TranscriptType)!, {
+                queueDxn: transcript.queue.dxn.toString(),
+                enabled: !state.enabled,
               });
             },
             properties: {
               label: state.enabled
                 ? ['stop transcription label', { ns: TRANSCRIPTION_PLUGIN }]
                 : ['start transcription label', { ns: TRANSCRIPTION_PLUGIN }],
-              icon: 'ph--record--regular',
+              icon: 'ph--subtitles--regular',
               disposition: 'toolbar',
-              classNames: state.enabled ? 'text-activeInCall' : '',
+              classNames: state.enabled ? 'bg-callAlert' : '',
             },
           },
         ];
@@ -97,6 +91,14 @@ export default (context: PluginsContext) =>
         const meeting = node.data;
         const client = context.requestCapability(ClientCapabilities.Client);
         const state = context.requestCapability(TranscriptionCapabilities.MeetingTranscriptionState);
+        const settings = context
+          .requestCapability(Capabilities.SettingsStore)
+          .getStore<TranscriptionSettingsProps>(TRANSCRIPTION_PLUGIN)!.value;
+
+        // TODO(dmaretskyi): Request via capability.
+        const aiClient: AIServiceClient | undefined = new AIServiceEdgeClient({
+          endpoint: AI_SERVICE_ENDPOINT.REMOTE,
+        });
 
         return [
           {
@@ -106,47 +108,104 @@ export default (context: PluginsContext) =>
             properties: {
               label: ['transcript companion label', { ns: TRANSCRIPTION_PLUGIN }],
               icon: 'ph--subtitles--regular',
+              position: 'hoist',
               disposition: 'hidden',
               schema: TranscriptType,
               getIntent: ({ space }: { space: Space }) =>
                 createIntent(TranscriptionAction.Create, { spaceId: space.id }),
-              onJoin: () => {
-                const transcriptionManager = new TranscriptionManager(client.edge);
+
+              // TODO(burdon): Is this the right place to inisitalize the state? Change to a capability?
+              onJoin: async ({ meeting }: { meeting: MeetingType; roomId: string }) => {
                 const identity = client.halo.identity.get();
                 invariant(identity);
-                transcriptionManager.setName(
-                  identity.profile?.displayName ?? generateName(identity.identityKey.toHex()),
-                );
-                const fallbackValue = keyToFallback(identity!.identityKey);
-                const userHue = identity!.profile?.data?.hue || fallbackValue.hue;
-                transcriptionManager.setHue(userHue);
-                void transcriptionManager.open();
-                state.transcriptionManager = transcriptionManager;
+                const space = getSpace(meeting);
+                invariant(space);
+
+                let messageEnricher;
+                if (aiClient && settings.entityExtraction) {
+                  messageEnricher = createEntityExtractionEnricher({
+                    aiClient,
+                    // TODO(dmaretskyi): Have those be discovered from the schema graph or contributed by capabilities?
+                    //  This forced me to add a dependency on markdown plugin.
+                    //  This will be replaced with a vector search index anyway, so its not a big deal.
+                    contextTypes: [DocumentType, DataType.Person, DataType.Organization],
+                    space,
+                  });
+                }
+
+                // TODO(burdon): The TranscriptionManager singleton is part of the state and should just be updated here.
+                state.transcriptionManager = await new TranscriptionManager({
+                  edgeClient: client.edge,
+                  messageEnricher,
+                })
+                  .setIdentityDid(identity.did)
+                  .open();
               },
-              onLeave: () => {
-                void state.transcriptionManager?.close();
+              onLeave: async () => {
+                await state.transcriptionManager?.close();
                 state.transcriptionManager = undefined;
                 state.enabled = false;
               },
-              onCallStateUpdated: (callState: CallState) => {
+              onCallStateUpdated: async (callState: CallState) => {
                 const typename = getSchemaTypename(TranscriptType);
                 const transcription = typename ? callState.activities?.[typename] : undefined;
-                if (!transcription?.payload?.value) {
+                if (!transcription?.payload) {
                   return;
                 }
 
-                const payload = codec.decode(transcription.payload.value);
-                state.enabled = payload.enabled;
-                void state.transcriptionManager?.setEnabled(payload.enabled);
-                void state.transcriptionManager?.setQueue(payload.queueDxn);
+                const payload: TranscriptionPayload = transcription.payload;
+                state.enabled = !!payload.enabled;
+                if (payload.queueDxn) {
+                  // NOTE: Must set queue before enabling transcription.
+                  state.transcriptionManager?.setQueue(DXN.parse(payload.queueDxn));
+                }
+                await state.transcriptionManager?.setEnabled(payload.enabled);
               },
-              onMediaStateUpdated: ([mediaState, isSpeaking]: [MediaState, boolean]) => {
+              onMediaStateUpdated: async ([mediaState, isSpeaking]: [MediaState, boolean]) => {
                 void state.transcriptionManager?.setAudioTrack(mediaState.audioTrack);
                 void state.transcriptionManager?.setRecording(isSpeaking);
               },
-            },
+            } satisfies MeetingCallProperties,
           },
         ];
       },
     }),
   ]);
+
+type EntityExtractionEnricherFactoryOptions = {
+  aiClient: AIServiceClient;
+  contextTypes: Schema.Schema.AnyNoContext[];
+  space: Space;
+};
+
+const createEntityExtractionEnricher = ({ aiClient, contextTypes, space }: EntityExtractionEnricherFactoryOptions) => {
+  return async (message: DataType.Message) => {
+    const { objects } = await space.db.query(Filter.or(...contextTypes.map((s) => Filter.schema(s)))).run();
+    log.info('context', { objects });
+
+    const { message: enhancedMessage, timeElapsed } = await processTranscriptMessage({
+      aiService: aiClient,
+      message,
+      context: { objects: await Promise.all(objects.map(processContextObject)) },
+      options: { timeout: ENTITY_EXTRACTOR_TIMEOUT, fallbackToRaw: true },
+    });
+
+    log.info('entity extraction time', { timeElapsed });
+    return enhancedMessage;
+  };
+};
+
+const processContextObject = async (object: BaseEchoObject): Promise<any> => {
+  // TODO(dmaretskyi): Documents need special processing is the content is behind a ref.
+  // TODO(dmaretskyi): Think about a way to handle this serialization with a decorator.
+  // if (isInstanceOf(DocumentType, object)) {
+  //   return {
+  //     ...object,
+  //     content: await object.content.load(),
+  //   };
+  // }
+
+  return object;
+};
+
+const ENTITY_EXTRACTOR_TIMEOUT = 25_000;

package/src/capabilities/capabilities.ts

@@ -3,7 +3,7 @@
 //
 
 import { defineCapability } from '@dxos/app-framework';
-import { type ReactiveObject } from '@dxos/live-object';
+import { type Live } from '@dxos/live-object';
 
 import { TRANSCRIPTION_PLUGIN } from '../meta';
 import { type TranscriberParams, type Transcriber, type TranscriptionManager } from '../transcriber';
@@ -12,13 +12,14 @@ export namespace TranscriptionCapabilities {
   export type GetTranscriberProps = {
     audioStreamTrack: MediaStreamTrack;
     onSegments: TranscriberParams['onSegments'];
+    config?: TranscriberParams['config'];
   };
   export type GetTranscriber = (props: GetTranscriberProps) => Transcriber;
   export const Transcriber = defineCapability<GetTranscriber>(`${TRANSCRIPTION_PLUGIN}/capability/transcriber`);
 
-  export type MeetingTranscriptionState = ReactiveObject<{
-    enabled: boolean;
+  export type MeetingTranscriptionState = Live<{
     transcriptionManager?: TranscriptionManager;
+    enabled: boolean; // TODO(burdon): Move state into TranscriptionManager.
   }>;
   export const MeetingTranscriptionState = defineCapability<MeetingTranscriptionState>(
     `${TRANSCRIPTION_PLUGIN}/capability/meeting-transcription-state`,

package/src/capabilities/index.ts

@@ -9,5 +9,6 @@ export const IntentResolver = lazy(() => import('./intent-resolver'));
 export const MeetingTranscriptionState = lazy(() => import('./meeting-transcription-state'));
 export const ReactSurface = lazy(() => import('./react-surface'));
 export const Transcriber = lazy(() => import('./transcriber'));
+export const Settings = lazy(() => import('./settings'));
 
 export * from './capabilities';

package/src/capabilities/intent-resolver.ts

@@ -3,19 +3,19 @@
 //
 
 import { Capabilities, contributes, createResolver, type PluginsContext } from '@dxos/app-framework';
-import { create, refFromDXN } from '@dxos/live-object';
-import { type SpaceId } from '@dxos/react-client/echo';
+import { live, refFromDXN } from '@dxos/live-object';
+import { type SpaceId, createQueueDxn } from '@dxos/react-client/echo';
 
 import { TranscriptionAction, TranscriptType } from '../types';
-import { getTimeStr, randomQueueDxn } from '../util';
+import { getTimeStr } from '../util';
 
 export default (context: PluginsContext) =>
   contributes(Capabilities.IntentResolver, [
     createResolver({
       intent: TranscriptionAction.Create,
       resolve: ({ name, spaceId }) => {
-        const transcript = create(TranscriptType, {
-          queue: refFromDXN(randomQueueDxn(spaceId as SpaceId)),
+        const transcript = live(TranscriptType, {
+          queue: refFromDXN(createQueueDxn(spaceId as SpaceId)),
           name: name ?? `Transcript ${getTimeStr(Date.now())}`,
         });