@dxos/plugin-transcription 0.8.2-main.f11618f → 0.8.2-main.fbd8ed0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. package/dist/lib/browser/{TranscriptContainer-D3EH2SLO.mjs → TranscriptContainer-CXA4ZBHW.mjs} +2 -2
  2. package/dist/lib/browser/{app-graph-builder-7HN3M24G.mjs → app-graph-builder-REQZ5SC5.mjs} +37 -35
  3. package/dist/lib/browser/app-graph-builder-REQZ5SC5.mjs.map +7 -0
  4. package/dist/lib/browser/{chunk-KSUYWWIU.mjs → chunk-3TKXI6TQ.mjs} +6 -6
  5. package/dist/lib/browser/chunk-3TKXI6TQ.mjs.map +7 -0
  6. package/dist/lib/browser/{chunk-YRWBY6UK.mjs → chunk-JHWEZRWC.mjs} +19 -17
  7. package/dist/lib/browser/chunk-JHWEZRWC.mjs.map +7 -0
  8. package/dist/lib/browser/{chunk-BSZSM6KA.mjs → chunk-K2LS2QOM.mjs} +6 -6
  9. package/dist/lib/browser/chunk-K2LS2QOM.mjs.map +7 -0
  10. package/dist/lib/browser/{chunk-IWZ7C3OA.mjs → chunk-LCTMCTYY.mjs} +2 -2
  11. package/dist/lib/browser/index.mjs +7 -7
  12. package/dist/lib/browser/index.mjs.map +3 -3
  13. package/dist/lib/browser/{intent-resolver-ZWKYRFE7.mjs → intent-resolver-DZ62CJPC.mjs} +3 -3
  14. package/dist/lib/browser/intent-resolver-DZ62CJPC.mjs.map +7 -0
  15. package/dist/lib/browser/meta.json +1 -1
  16. package/dist/lib/browser/{react-surface-F7DBDGHS.mjs → react-surface-PUXZNAZL.mjs} +4 -4
  17. package/dist/lib/browser/{settings-6YAHKVKX.mjs → settings-JWDS4J3F.mjs} +2 -2
  18. package/dist/lib/browser/{transcriber-MSBS4VBI.mjs → transcriber-YWWIV6KP.mjs} +3 -3
  19. package/dist/lib/browser/types/index.mjs +1 -1
  20. package/dist/lib/node/{TranscriptContainer-WCBZMA5K.cjs → TranscriptContainer-SSJVLRM6.cjs} +7 -7
  21. package/dist/lib/node/{TranscriptContainer-WCBZMA5K.cjs.map → TranscriptContainer-SSJVLRM6.cjs.map} +2 -2
  22. package/dist/lib/node/{app-graph-builder-BVWYUGUO.cjs → app-graph-builder-OAPU4GDH.cjs} +50 -48
  23. package/dist/lib/node/app-graph-builder-OAPU4GDH.cjs.map +7 -0
  24. package/dist/lib/node/{chunk-IXU57SCA.cjs → chunk-EJWB53WK.cjs} +19 -17
  25. package/dist/lib/node/chunk-EJWB53WK.cjs.map +7 -0
  26. package/dist/lib/node/{chunk-JSCC2AQW.cjs → chunk-EVE3H46F.cjs} +9 -9
  27. package/dist/lib/node/chunk-EVE3H46F.cjs.map +7 -0
  28. package/dist/lib/node/{chunk-OCHTKDL7.cjs → chunk-L27AK6FQ.cjs} +5 -5
  29. package/dist/lib/node/{chunk-7XIIUVN3.cjs → chunk-WKZLGVEJ.cjs} +9 -9
  30. package/dist/lib/node/chunk-WKZLGVEJ.cjs.map +7 -0
  31. package/dist/lib/node/index.cjs +28 -28
  32. package/dist/lib/node/index.cjs.map +3 -3
  33. package/dist/lib/node/{intent-resolver-RM4JOCDV.cjs → intent-resolver-TRBKI24D.cjs} +9 -9
  34. package/dist/lib/node/intent-resolver-TRBKI24D.cjs.map +7 -0
  35. package/dist/lib/node/meta.json +1 -1
  36. package/dist/lib/node/{react-surface-7F37EYZA.cjs → react-surface-SZAMHCX7.cjs} +10 -10
  37. package/dist/lib/node/{settings-ZMB4XRWS.cjs → settings-AWWMOSCV.cjs} +7 -7
  38. package/dist/lib/node/{transcriber-55OE4X6I.cjs → transcriber-AFUXFHCR.cjs} +8 -8
  39. package/dist/lib/node/types/index.cjs +7 -7
  40. package/dist/lib/node/types/index.cjs.map +1 -1
  41. package/dist/lib/node-esm/{TranscriptContainer-GBPY6KKA.mjs → TranscriptContainer-M53LI2TX.mjs} +2 -2
  42. package/dist/lib/node-esm/{app-graph-builder-QK25O5K5.mjs → app-graph-builder-UBP6EOBR.mjs} +37 -35
  43. package/dist/lib/node-esm/app-graph-builder-UBP6EOBR.mjs.map +7 -0
  44. package/dist/lib/node-esm/{chunk-B2BD7S4X.mjs → chunk-4KC3E5T4.mjs} +6 -6
  45. package/dist/lib/node-esm/chunk-4KC3E5T4.mjs.map +7 -0
  46. package/dist/lib/node-esm/{chunk-DXJN2JWV.mjs → chunk-MHQP2PAF.mjs} +2 -2
  47. package/dist/lib/node-esm/{chunk-OS2ITZUL.mjs → chunk-QYG2FOXA.mjs} +19 -17
  48. package/dist/lib/node-esm/chunk-QYG2FOXA.mjs.map +7 -0
  49. package/dist/lib/node-esm/{chunk-UWF6LM27.mjs → chunk-V5IWBPYO.mjs} +6 -6
  50. package/dist/lib/node-esm/chunk-V5IWBPYO.mjs.map +7 -0
  51. package/dist/lib/node-esm/index.mjs +7 -7
  52. package/dist/lib/node-esm/index.mjs.map +3 -3
  53. package/dist/lib/node-esm/{intent-resolver-HDRFXMKP.mjs → intent-resolver-HFVAL274.mjs} +3 -3
  54. package/dist/lib/node-esm/intent-resolver-HFVAL274.mjs.map +7 -0
  55. package/dist/lib/node-esm/meta.json +1 -1
  56. package/dist/lib/node-esm/{react-surface-D3L6VBEV.mjs → react-surface-UEJELL7I.mjs} +4 -4
  57. package/dist/lib/node-esm/{settings-I35TVDUG.mjs → settings-QFNFGFJ5.mjs} +2 -2
  58. package/dist/lib/node-esm/{transcriber-HH7EMCGH.mjs → transcriber-7RJEFPDI.mjs} +3 -3
  59. package/dist/lib/node-esm/types/index.mjs +1 -1
  60. package/dist/types/src/capabilities/app-graph-builder.d.ts +2 -179
  61. package/dist/types/src/capabilities/app-graph-builder.d.ts.map +1 -1
  62. package/dist/types/src/capabilities/index.d.ts +2 -178
  63. package/dist/types/src/capabilities/index.d.ts.map +1 -1
  64. package/dist/types/src/capabilities/intent-resolver.d.ts +2 -2
  65. package/dist/types/src/capabilities/intent-resolver.d.ts.map +1 -1
  66. package/dist/types/src/components/Transcript/Transcript.d.ts +3 -3
  67. package/dist/types/src/components/Transcript/Transcript.d.ts.map +1 -1
  68. package/dist/types/src/components/Transcript/Transcript.stories.d.ts +2 -2
  69. package/dist/types/src/components/Transcript/Transcript.stories.d.ts.map +1 -1
  70. package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts +10 -10
  71. package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts.map +1 -1
  72. package/dist/types/src/components/Transcript/transcript-extension.d.ts +2 -2
  73. package/dist/types/src/components/Transcript/transcript-extension.d.ts.map +1 -1
  74. package/dist/types/src/components/TranscriptionSettings/TranscriptionSettings.d.ts.map +1 -1
  75. package/dist/types/src/components/transcription.stories.d.ts.map +1 -1
  76. package/dist/types/src/entity-extraction/entity-extraction.d.ts +3 -3
  77. package/dist/types/src/entity-extraction/entity-extraction.d.ts.map +1 -1
  78. package/dist/types/src/hooks/useAudioFile.d.ts.map +1 -1
  79. package/dist/types/src/hooks/useAudioTrack.d.ts.map +1 -1
  80. package/dist/types/src/hooks/useQueueModelAdapter.d.ts.map +1 -1
  81. package/dist/types/src/hooks/useTranscriber.d.ts.map +1 -1
  82. package/dist/types/src/hooks/useVoiceInput.d.ts.map +1 -1
  83. package/dist/types/src/testing/testing.d.ts +11 -10
  84. package/dist/types/src/testing/testing.d.ts.map +1 -1
  85. package/dist/types/src/transcriber/transcriber.d.ts +2 -2
  86. package/dist/types/src/transcriber/transcriber.d.ts.map +1 -1
  87. package/dist/types/src/transcriber/transcription-manager.d.ts +2 -2
  88. package/dist/types/src/transcriber/transcription-manager.d.ts.map +1 -1
  89. package/dist/types/src/translations.d.ts +2 -20
  90. package/dist/types/src/translations.d.ts.map +1 -1
  91. package/dist/types/src/types/schema.d.ts +26 -19
  92. package/dist/types/src/types/schema.d.ts.map +1 -1
  93. package/dist/types/src/types/types.d.ts +17 -10
  94. package/dist/types/src/types/types.d.ts.map +1 -1
  95. package/dist/types/src/util/array.d.ts.map +1 -1
  96. package/dist/types/src/util/get-time-str.d.ts.map +1 -1
  97. package/dist/types/src/util/monitor-audio-level.d.ts.map +1 -1
  98. package/dist/types/tsconfig.tsbuildinfo +1 -1
  99. package/package.json +55 -49
  100. package/src/TranscriptionPlugin.tsx +3 -3
  101. package/src/capabilities/app-graph-builder.ts +145 -123
  102. package/src/capabilities/intent-resolver.ts +4 -3
  103. package/src/components/Transcript/Transcript.stories.tsx +6 -6
  104. package/src/components/Transcript/Transcript.tsx +3 -3
  105. package/src/components/Transcript/deprecated/OldTranscript.tsx +19 -13
  106. package/src/components/Transcript/transcript-extension.ts +3 -3
  107. package/src/components/TranscriptContainer.tsx +2 -2
  108. package/src/components/transcription.stories.tsx +17 -15
  109. package/src/entity-extraction/entity-extraction.ts +4 -4
  110. package/src/model/model.test.ts +14 -10
  111. package/src/testing/testing.ts +31 -23
  112. package/src/transcriber/transcriber.browser.test.ts +3 -3
  113. package/src/transcriber/transcriber.ts +6 -3
  114. package/src/transcriber/transcription-manager.ts +8 -8
  115. package/src/types/schema.ts +10 -8
  116. package/src/types/types.ts +14 -12
  117. package/dist/lib/browser/app-graph-builder-7HN3M24G.mjs.map +0 -7
  118. package/dist/lib/browser/chunk-BSZSM6KA.mjs.map +0 -7
  119. package/dist/lib/browser/chunk-KSUYWWIU.mjs.map +0 -7
  120. package/dist/lib/browser/chunk-YRWBY6UK.mjs.map +0 -7
  121. package/dist/lib/browser/intent-resolver-ZWKYRFE7.mjs.map +0 -7
  122. package/dist/lib/node/app-graph-builder-BVWYUGUO.cjs.map +0 -7
  123. package/dist/lib/node/chunk-7XIIUVN3.cjs.map +0 -7
  124. package/dist/lib/node/chunk-IXU57SCA.cjs.map +0 -7
  125. package/dist/lib/node/chunk-JSCC2AQW.cjs.map +0 -7
  126. package/dist/lib/node/intent-resolver-RM4JOCDV.cjs.map +0 -7
  127. package/dist/lib/node-esm/app-graph-builder-QK25O5K5.mjs.map +0 -7
  128. package/dist/lib/node-esm/chunk-B2BD7S4X.mjs.map +0 -7
  129. package/dist/lib/node-esm/chunk-OS2ITZUL.mjs.map +0 -7
  130. package/dist/lib/node-esm/chunk-UWF6LM27.mjs.map +0 -7
  131. package/dist/lib/node-esm/intent-resolver-HDRFXMKP.mjs.map +0 -7
  132. /package/dist/lib/browser/{TranscriptContainer-D3EH2SLO.mjs.map → TranscriptContainer-CXA4ZBHW.mjs.map} +0 -0
  133. /package/dist/lib/browser/{chunk-IWZ7C3OA.mjs.map → chunk-LCTMCTYY.mjs.map} +0 -0
  134. /package/dist/lib/browser/{react-surface-F7DBDGHS.mjs.map → react-surface-PUXZNAZL.mjs.map} +0 -0
  135. /package/dist/lib/browser/{settings-6YAHKVKX.mjs.map → settings-JWDS4J3F.mjs.map} +0 -0
  136. /package/dist/lib/browser/{transcriber-MSBS4VBI.mjs.map → transcriber-YWWIV6KP.mjs.map} +0 -0
  137. /package/dist/lib/node/{chunk-OCHTKDL7.cjs.map → chunk-L27AK6FQ.cjs.map} +0 -0
  138. /package/dist/lib/node/{react-surface-7F37EYZA.cjs.map → react-surface-SZAMHCX7.cjs.map} +0 -0
  139. /package/dist/lib/node/{settings-ZMB4XRWS.cjs.map → settings-AWWMOSCV.cjs.map} +0 -0
  140. /package/dist/lib/node/{transcriber-55OE4X6I.cjs.map → transcriber-AFUXFHCR.cjs.map} +0 -0
  141. /package/dist/lib/node-esm/{TranscriptContainer-GBPY6KKA.mjs.map → TranscriptContainer-M53LI2TX.mjs.map} +0 -0
  142. /package/dist/lib/node-esm/{chunk-DXJN2JWV.mjs.map → chunk-MHQP2PAF.mjs.map} +0 -0
  143. /package/dist/lib/node-esm/{react-surface-D3L6VBEV.mjs.map → react-surface-UEJELL7I.mjs.map} +0 -0
  144. /package/dist/lib/node-esm/{settings-I35TVDUG.mjs.map → settings-QFNFGFJ5.mjs.map} +0 -0
  145. /package/dist/lib/node-esm/{transcriber-HH7EMCGH.mjs.map → transcriber-7RJEFPDI.mjs.map} +0 -0
@@ -2,25 +2,26 @@
2
2
  // Copyright 2025 DXOS.org
3
3
  //
4
4
 
5
- import type { Schema } from 'effect';
5
+ import { Rx } from '@effect-rx/rx-react';
6
+ import { Option, pipe, type Schema } from 'effect';
6
7
 
7
- import { Capabilities, contributes, createIntent, type PluginsContext } from '@dxos/app-framework';
8
+ import { Capabilities, contributes, createIntent, type PluginContext } from '@dxos/app-framework';
8
9
  import { AIServiceEdgeClient, type AIServiceClient } from '@dxos/assistant';
9
10
  import { AI_SERVICE_ENDPOINT } from '@dxos/assistant/testing';
10
- import { Filter, fullyQualifiedId, getSpace, makeRef, type Space } from '@dxos/client/echo';
11
+ import { Filter, fullyQualifiedId, getSpace, makeRef, Query, type Space } from '@dxos/client/echo';
11
12
  import { getSchemaTypename, isInstanceOf, type BaseEchoObject } from '@dxos/echo-schema';
12
13
  import { invariant } from '@dxos/invariant';
13
14
  import { DXN } from '@dxos/keys';
14
15
  import { log } from '@dxos/log';
15
16
  import { ClientCapabilities } from '@dxos/plugin-client';
16
17
  import { PLANK_COMPANION_TYPE, ATTENDABLE_PATH_SEPARATOR } from '@dxos/plugin-deck/types';
17
- import { createExtension, type Node } from '@dxos/plugin-graph';
18
+ import { createExtension, rxFromSignal } from '@dxos/plugin-graph';
18
19
  import { DocumentType } from '@dxos/plugin-markdown/types';
19
20
  import { MeetingCapabilities, type CallState, type MediaState } from '@dxos/plugin-meeting';
20
21
  import { MeetingType, type MeetingCallProperties } from '@dxos/plugin-meeting/types';
21
22
  import { type buf } from '@dxos/protocols/buf';
22
23
  import { type TranscriptionPayloadSchema } from '@dxos/protocols/buf/dxos/edge/calls_pb';
23
- import { type MessageType, Contact, Organization } from '@dxos/schema';
24
+ import { DataType } from '@dxos/schema';
24
25
 
25
26
  import { TranscriptionCapabilities } from './capabilities';
26
27
  import { processTranscriptMessage } from '../entity-extraction';
@@ -33,10 +34,10 @@ import { TranscriptionAction, type TranscriptionSettingsProps, TranscriptType }
33
34
  type TranscriptionPayload = buf.MessageInitShape<typeof TranscriptionPayloadSchema>;
34
35
 
35
36
  const getMeetingTranscript = async (
36
- context: PluginsContext,
37
+ context: PluginContext,
37
38
  meeting: MeetingType,
38
39
  ): Promise<TranscriptType | undefined> => {
39
- const { dispatchPromise: dispatch } = context.requestCapability(Capabilities.IntentDispatcher);
40
+ const { dispatchPromise: dispatch } = context.getCapability(Capabilities.IntentDispatcher);
40
41
 
41
42
  const typename = getSchemaTypename(TranscriptType)!;
42
43
  const existing = await meeting.artifacts[typename]?.load();
@@ -52,123 +53,139 @@ const getMeetingTranscript = async (
52
53
  };
53
54
 
54
55
  // TODO(wittjosiah): Introduce a sound that plays when transcription is activated/deactivated.
55
- export default (context: PluginsContext) =>
56
+ export default (context: PluginContext) =>
56
57
  contributes(Capabilities.AppGraphBuilder, [
57
58
  createExtension({
58
59
  id: `${TRANSCRIPTION_PLUGIN}/meeting-transcript`,
59
- filter: (node): node is Node<MeetingType> => isInstanceOf(MeetingType, node.data),
60
- actions: ({ node }) => {
61
- const meeting = node.data;
62
- const state = context.requestCapability(TranscriptionCapabilities.MeetingTranscriptionState);
63
- return [
64
- {
65
- id: `${fullyQualifiedId(meeting)}/action/start-stop-transcription`,
66
- data: async () => {
67
- // NOTE: We are not saving the state of the transcription manager here.
68
- // We expect the state to be updated through `onCallStateUpdated` once it is propagated through Swarm.
69
- // This is done to avoid race conditions and to not handle optimistic updates.
70
- const transcript = await getMeetingTranscript(context, meeting);
71
- invariant(transcript, 'Failed to create transcript');
72
-
73
- const callManager = context.requestCapability(MeetingCapabilities.CallManager);
74
- callManager.setActivity(getSchemaTypename(TranscriptType)!, {
75
- queueDxn: transcript.queue.dxn.toString(),
76
- enabled: !state.enabled,
60
+ actions: (node) =>
61
+ Rx.make((get) =>
62
+ pipe(
63
+ get(node),
64
+ Option.flatMap((node) => (isInstanceOf(MeetingType, node.data) ? Option.some(node.data) : Option.none())),
65
+ Option.map((meeting) => {
66
+ const state = context.getCapability(TranscriptionCapabilities.MeetingTranscriptionState);
67
+ const enabled = get(rxFromSignal(() => state.enabled));
68
+ return [
69
+ {
70
+ id: `${fullyQualifiedId(meeting)}/action/start-stop-transcription`,
71
+ data: async () => {
72
+ // NOTE: We are not saving the state of the transcription manager here.
73
+ // We expect the state to be updated through `onCallStateUpdated` once it is propagated through Swarm.
74
+ // This is done to avoid race conditions and to not handle optimistic updates.
75
+ const transcript = await getMeetingTranscript(context, meeting);
76
+ invariant(transcript, 'Failed to create transcript');
77
+
78
+ const callManager = context.getCapability(MeetingCapabilities.CallManager);
79
+ callManager.setActivity(getSchemaTypename(TranscriptType)!, {
80
+ queueDxn: transcript.queue.dxn.toString(),
81
+ enabled: !enabled,
82
+ });
83
+ },
84
+ properties: {
85
+ label: enabled
86
+ ? ['stop transcription label', { ns: TRANSCRIPTION_PLUGIN }]
87
+ : ['start transcription label', { ns: TRANSCRIPTION_PLUGIN }],
88
+ icon: 'ph--subtitles--regular',
89
+ disposition: 'toolbar',
90
+ classNames: enabled ? 'bg-callAlert' : '',
91
+ },
92
+ },
93
+ ];
94
+ }),
95
+ Option.getOrElse(() => []),
96
+ ),
97
+ ),
98
+ connector: (node) =>
99
+ Rx.make((get) =>
100
+ pipe(
101
+ get(node),
102
+ Option.flatMap((node) => (isInstanceOf(MeetingType, node.data) ? Option.some(node.data) : Option.none())),
103
+ Option.map((meeting) => {
104
+ const client = context.getCapability(ClientCapabilities.Client);
105
+ const state = context.getCapability(TranscriptionCapabilities.MeetingTranscriptionState);
106
+ const settingsStore = get(context.capabilities(Capabilities.SettingsStore))[0];
107
+ const settings = get(
108
+ rxFromSignal(() => settingsStore?.getStore<TranscriptionSettingsProps>(TRANSCRIPTION_PLUGIN)!.value),
109
+ );
110
+
111
+ // TODO(dmaretskyi): Request via capability.
112
+ const aiClient: AIServiceClient | undefined = new AIServiceEdgeClient({
113
+ // TODO(burdon): Get from config.
114
+ endpoint: AI_SERVICE_ENDPOINT.REMOTE,
77
115
  });
78
- },
79
- properties: {
80
- label: state.enabled
81
- ? ['stop transcription label', { ns: TRANSCRIPTION_PLUGIN }]
82
- : ['start transcription label', { ns: TRANSCRIPTION_PLUGIN }],
83
- icon: 'ph--subtitles--regular',
84
- disposition: 'toolbar',
85
- classNames: state.enabled ? 'bg-callAlert' : '',
86
- },
87
- },
88
- ];
89
- },
90
- connector: ({ node }) => {
91
- const meeting = node.data;
92
- const client = context.requestCapability(ClientCapabilities.Client);
93
- const state = context.requestCapability(TranscriptionCapabilities.MeetingTranscriptionState);
94
- const settings = context
95
- .requestCapability(Capabilities.SettingsStore)
96
- .getStore<TranscriptionSettingsProps>(TRANSCRIPTION_PLUGIN)!.value;
97
-
98
- // TODO(dmaretskyi): Request via capability.
99
- const aiClient: AIServiceClient | undefined = new AIServiceEdgeClient({
100
- endpoint: AI_SERVICE_ENDPOINT.REMOTE,
101
- });
102
-
103
- return [
104
- {
105
- id: `${fullyQualifiedId(meeting)}${ATTENDABLE_PATH_SEPARATOR}${getSchemaTypename(TranscriptType)}`,
106
- type: PLANK_COMPANION_TYPE,
107
- data: meeting.artifacts[getSchemaTypename(TranscriptType)!]?.target,
108
- properties: {
109
- label: ['transcript companion label', { ns: TRANSCRIPTION_PLUGIN }],
110
- icon: 'ph--subtitles--regular',
111
- position: 'hoist',
112
- disposition: 'hidden',
113
- schema: TranscriptType,
114
- getIntent: ({ space }: { space: Space }) =>
115
- createIntent(TranscriptionAction.Create, { spaceId: space.id }),
116
-
117
- // TODO(burdon): Is this the right place to inisitalize the state? Change to a capability?
118
- onJoin: async ({ meeting }: { meeting: MeetingType; roomId: string }) => {
119
- const identity = client.halo.identity.get();
120
- invariant(identity);
121
- const space = getSpace(meeting);
122
- invariant(space);
123
-
124
- let messageEnricher;
125
- if (aiClient && settings.entityExtraction) {
126
- messageEnricher = createEntityExtractionEnricher({
127
- aiClient,
128
- // TODO(dmaretskyi): Have those be discovered from the schema graph or contributed by capabilities?
129
- // This forced me to add a dependency on markdown plugin.
130
- // This will be replaced with a vector search index anyway, so its not a big deal.
131
- contextTypes: [DocumentType, Contact, Organization],
132
- space,
133
- });
134
- }
135
-
136
- // TODO(burdon): The TranscriptionManager singleton is part of the state and should just be updated here.
137
- state.transcriptionManager = await new TranscriptionManager({
138
- edgeClient: client.edge,
139
- messageEnricher,
140
- })
141
- .setIdentityDid(identity.did)
142
- .open();
143
- },
144
- onLeave: async () => {
145
- await state.transcriptionManager?.close();
146
- state.transcriptionManager = undefined;
147
- state.enabled = false;
148
- },
149
- onCallStateUpdated: async (callState: CallState) => {
150
- const typename = getSchemaTypename(TranscriptType);
151
- const transcription = typename ? callState.activities?.[typename] : undefined;
152
- if (!transcription?.payload) {
153
- return;
154
- }
155
-
156
- const payload: TranscriptionPayload = transcription.payload;
157
- state.enabled = !!payload.enabled;
158
- if (payload.queueDxn) {
159
- // NOTE: Must set queue before enabling transcription.
160
- state.transcriptionManager?.setQueue(DXN.parse(payload.queueDxn));
161
- }
162
- await state.transcriptionManager?.setEnabled(payload.enabled);
163
- },
164
- onMediaStateUpdated: async ([mediaState, isSpeaking]: [MediaState, boolean]) => {
165
- void state.transcriptionManager?.setAudioTrack(mediaState.audioTrack);
166
- void state.transcriptionManager?.setRecording(isSpeaking);
167
- },
168
- } satisfies MeetingCallProperties,
169
- },
170
- ];
171
- },
116
+
117
+ return [
118
+ {
119
+ id: `${fullyQualifiedId(meeting)}${ATTENDABLE_PATH_SEPARATOR}${getSchemaTypename(TranscriptType)}`,
120
+ type: PLANK_COMPANION_TYPE,
121
+ data: get(rxFromSignal(() => meeting.artifacts[getSchemaTypename(TranscriptType)!]?.target)),
122
+ properties: {
123
+ label: ['transcript companion label', { ns: TRANSCRIPTION_PLUGIN }],
124
+ icon: 'ph--subtitles--regular',
125
+ position: 'hoist',
126
+ disposition: 'hidden',
127
+ schema: TranscriptType,
128
+ getIntent: ({ space }: { space: Space }) =>
129
+ createIntent(TranscriptionAction.Create, { spaceId: space.id }),
130
+
131
+ // TODO(burdon): Is this the right place to inisitalize the state? Change to a capability?
132
+ onJoin: async ({ meeting }: { meeting?: MeetingType }) => {
133
+ const identity = client.halo.identity.get();
134
+ invariant(identity);
135
+ const space = getSpace(meeting);
136
+ invariant(space);
137
+
138
+ let messageEnricher;
139
+ if (aiClient && settings.entityExtraction) {
140
+ messageEnricher = createEntityExtractionEnricher({
141
+ aiClient,
142
+ // TODO(dmaretskyi): Have those be discovered from the schema graph or contributed by capabilities?
143
+ // This forced me to add a dependency on markdown plugin.
144
+ // This will be replaced with a vector search index anyway, so its not a big deal.
145
+ contextTypes: [DocumentType, DataType.Person, DataType.Organization],
146
+ space,
147
+ });
148
+ }
149
+
150
+ // TODO(burdon): The TranscriptionManager singleton is part of the state and should just be updated here.
151
+ state.transcriptionManager = await new TranscriptionManager({
152
+ edgeClient: client.edge,
153
+ messageEnricher,
154
+ })
155
+ .setIdentityDid(identity.did)
156
+ .open();
157
+ },
158
+ onLeave: async () => {
159
+ await state.transcriptionManager?.close();
160
+ state.transcriptionManager = undefined;
161
+ state.enabled = false;
162
+ },
163
+ onCallStateUpdated: async (callState: CallState) => {
164
+ const typename = getSchemaTypename(TranscriptType);
165
+ const transcription = typename ? callState.activities?.[typename] : undefined;
166
+ if (!transcription?.payload) {
167
+ return;
168
+ }
169
+
170
+ const payload: TranscriptionPayload = transcription.payload;
171
+ state.enabled = !!payload.enabled;
172
+ if (payload.queueDxn) {
173
+ // NOTE: Must set queue before enabling transcription.
174
+ state.transcriptionManager?.setQueue(DXN.parse(payload.queueDxn));
175
+ }
176
+ await state.transcriptionManager?.setEnabled(payload.enabled);
177
+ },
178
+ onMediaStateUpdated: async ([mediaState, isSpeaking]: [MediaState, boolean]) => {
179
+ void state.transcriptionManager?.setAudioTrack(mediaState.audioTrack);
180
+ void state.transcriptionManager?.setRecording(isSpeaking);
181
+ },
182
+ } satisfies MeetingCallProperties,
183
+ },
184
+ ];
185
+ }),
186
+ Option.getOrElse(() => []),
187
+ ),
188
+ ),
172
189
  }),
173
190
  ]);
174
191
 
@@ -179,14 +196,18 @@ type EntityExtractionEnricherFactoryOptions = {
179
196
  };
180
197
 
181
198
  const createEntityExtractionEnricher = ({ aiClient, contextTypes, space }: EntityExtractionEnricherFactoryOptions) => {
182
- return async (message: MessageType) => {
183
- const { objects } = await space.db.query(Filter.or(...contextTypes.map((s) => Filter.schema(s)))).run();
199
+ return async (message: DataType.Message) => {
200
+ const { objects } = await space.db
201
+ .query(Query.select(Filter.or(...contextTypes.map((s) => Filter.type(s as Schema.Schema<BaseEchoObject>)))))
202
+ .run();
184
203
  log.info('context', { objects });
185
204
 
186
205
  const { message: enhancedMessage, timeElapsed } = await processTranscriptMessage({
187
206
  aiService: aiClient,
188
207
  message,
189
- context: { objects: await Promise.all(objects.map(processContextObject)) },
208
+ context: {
209
+ objects: await Promise.all(objects.map((o) => processContextObject(o))),
210
+ },
190
211
  options: { timeout: ENTITY_EXTRACTOR_TIMEOUT, fallbackToRaw: true },
191
212
  });
192
213
 
@@ -195,6 +216,7 @@ const createEntityExtractionEnricher = ({ aiClient, contextTypes, space }: Entit
195
216
  };
196
217
  };
197
218
 
219
+ // TODO(dmaretskyi): Use Type.Any
198
220
  const processContextObject = async (object: BaseEchoObject): Promise<any> => {
199
221
  // TODO(dmaretskyi): Documents need special processing is the content is behind a ref.
200
222
  // TODO(dmaretskyi): Think about a way to handle this serialization with a decorator.
@@ -2,14 +2,15 @@
2
2
  // Copyright 2025 DXOS.org
3
3
  //
4
4
 
5
- import { Capabilities, contributes, createResolver, type PluginsContext } from '@dxos/app-framework';
5
+ import { Capabilities, contributes, createResolver, type PluginContext } from '@dxos/app-framework';
6
+ import { createQueueDxn } from '@dxos/echo-schema';
7
+ import { type SpaceId } from '@dxos/keys';
6
8
  import { live, refFromDXN } from '@dxos/live-object';
7
- import { type SpaceId, createQueueDxn } from '@dxos/react-client/echo';
8
9
 
9
10
  import { TranscriptionAction, TranscriptType } from '../types';
10
11
  import { getTimeStr } from '../util';
11
12
 
12
- export default (context: PluginsContext) =>
13
+ export default (context: PluginContext) =>
13
14
  contributes(Capabilities.IntentResolver, [
14
15
  createResolver({
15
16
  intent: TranscriptionAction.Create,
@@ -20,7 +20,7 @@ import { useMembers, useSpace } from '@dxos/react-client/echo';
20
20
  import { IconButton, Toolbar } from '@dxos/react-ui';
21
21
  import { SyntaxHighlighter } from '@dxos/react-ui-syntax-highlighter';
22
22
  import { defaultTx } from '@dxos/react-ui-theme';
23
- import { Contact, Organization, type MessageType } from '@dxos/schema';
23
+ import { DataType } from '@dxos/schema';
24
24
  import { Testing } from '@dxos/schema/testing';
25
25
  import { withLayout } from '@dxos/storybook-utils';
26
26
 
@@ -69,7 +69,7 @@ const TranscriptContainer: FC<
69
69
  );
70
70
  };
71
71
 
72
- type StoryProps = { messages?: MessageType[] } & Pick<TranscriptProps, 'ignoreAttention' | 'attendableId'>;
72
+ type StoryProps = { messages?: DataType.Message[] } & Pick<TranscriptProps, 'ignoreAttention' | 'attendableId'>;
73
73
 
74
74
  /**
75
75
  * Basic story mutates array of messages.
@@ -78,11 +78,11 @@ const BasicStory = ({ messages: initialMessages = [], ...props }: StoryProps) =>
78
78
  const [reset, setReset] = useState({});
79
79
  const builder = useMemo(() => new MessageBuilder(), []);
80
80
  const model = useMemo(
81
- () => new SerializationModel<MessageType>(renderMarkdown([]), initialMessages),
81
+ () => new SerializationModel<DataType.Message>(renderMarkdown([]), initialMessages),
82
82
  [initialMessages, reset],
83
83
  );
84
84
  const [running, setRunning] = useState(true);
85
- const [currentMessage, setCurrentMessage] = useState<MessageType | null>(null);
85
+ const [currentMessage, setCurrentMessage] = useState<DataType.Message | null>(null);
86
86
  useEffect(() => {
87
87
  if (!running) {
88
88
  return;
@@ -186,7 +186,7 @@ const meta: Meta<typeof QueueStory> = {
186
186
  ThemePlugin({ tx: defaultTx }),
187
187
  StorybookLayoutPlugin(),
188
188
  ClientPlugin({
189
- types: [TestItem, Testing.DocumentType, Contact, Organization],
189
+ types: [TestItem, Testing.DocumentType, DataType.Person, DataType.Organization],
190
190
  onClientInitialized: async (_, client) => {
191
191
  await client.halo.createIdentity();
192
192
  },
@@ -197,7 +197,7 @@ const meta: Meta<typeof QueueStory> = {
197
197
  IntentPlugin(),
198
198
  ],
199
199
  }),
200
- withLayout({ tooltips: true, fullscreen: true }),
200
+ withLayout({ fullscreen: true }),
201
201
  ],
202
202
  parameters: {
203
203
  translations,
@@ -18,7 +18,7 @@ import {
18
18
  useTextEditor,
19
19
  } from '@dxos/react-ui-editor';
20
20
  import { mx } from '@dxos/react-ui-theme';
21
- import { type MessageType } from '@dxos/schema';
21
+ import { type DataType } from '@dxos/schema';
22
22
  import { isNotFalsy } from '@dxos/util';
23
23
 
24
24
  import { transcript } from './transcript-extension';
@@ -27,7 +27,7 @@ import { type TranscriptType } from '../../types';
27
27
 
28
28
  export const renderMarkdown =
29
29
  (identities: Identity[]) =>
30
- (message: MessageType, index: number, debug = false): string[] => {
30
+ (message: DataType.Message, index: number, debug = false): string[] => {
31
31
  if (message.sender.role === 'assistant') {
32
32
  // Start/stop block.
33
33
  return [message.blocks.find((block) => block.type === 'transcription')?.text ?? '', ''];
@@ -53,7 +53,7 @@ export const renderMarkdown =
53
53
  export type TranscriptProps = ThemedClassName<{
54
54
  transcript?: TranscriptType;
55
55
  space?: Space;
56
- model: SerializationModel<MessageType>;
56
+ model: SerializationModel<DataType.Message>;
57
57
  // TODO(wittjosiah): Move to container.
58
58
  attendableId?: string;
59
59
  ignoreAttention?: boolean;
@@ -3,11 +3,12 @@
3
3
  //
4
4
 
5
5
  import { intervalToDuration } from 'date-fns/intervalToDuration';
6
+ import { Schema } from 'effect';
6
7
  import { yieldOrContinue } from 'main-thread-scheduling';
7
8
  import React, { type FC, useCallback, useEffect, useMemo, useRef, useState, type WheelEvent } from 'react';
8
9
  import { type OnResizeCallback, useResizeDetector } from 'react-resize-detector';
9
10
 
10
- import { EchoObject, S } from '@dxos/echo-schema';
11
+ import { Type } from '@dxos/echo';
11
12
  import { IconButton, type ThemedClassName, useTranslation } from '@dxos/react-ui';
12
13
  import { useAttention } from '@dxos/react-ui-attention';
13
14
  import {
@@ -25,25 +26,30 @@ import { TRANSCRIPTION_PLUGIN } from '../../../meta';
25
26
  /**
26
27
  * Transcription fragment.
27
28
  */
28
- const TranscriptSegment = S.Struct({
29
+ const TranscriptSegment = Schema.Struct({
29
30
  // TODO(burdon): TS from service is not Unix TS (x1000).
30
- started: S.String,
31
- text: S.String,
31
+ started: Schema.String,
32
+ text: Schema.String,
32
33
  });
33
34
 
34
- type TranscriptSegment = S.Schema.Type<typeof TranscriptSegment>;
35
+ type TranscriptSegment = Schema.Schema.Type<typeof TranscriptSegment>;
35
36
 
36
37
  /**
37
38
  * Transcription block (from single speaker).
38
39
  */
39
- const TranscriptBlock = S.Struct({
40
- id: S.String,
41
- authorName: S.optional(S.String), // TODO(burdon): Replace with identityDid.
42
- authorHue: S.optional(S.String), // TOOD(burdon): Remove.
43
- segments: S.Array(TranscriptSegment),
44
- }).pipe(EchoObject({ typename: 'dxos.org/type/TranscriptBlock', version: '0.1.0' }));
45
-
46
- type TranscriptBlock = S.Schema.Type<typeof TranscriptBlock>;
40
+ const TranscriptBlock = Schema.Struct({
41
+ id: Schema.String,
42
+ authorName: Schema.optional(Schema.String), // TODO(burdon): Replace with identityDid.
43
+ authorHue: Schema.optional(Schema.String), // TOOD(burdon): Remove.
44
+ segments: Schema.Array(TranscriptSegment),
45
+ }).pipe(
46
+ Type.def({
47
+ typename: 'dxos.org/type/TranscriptBlock',
48
+ version: '0.1.0',
49
+ }),
50
+ );
51
+
52
+ type TranscriptBlock = Schema.Schema.Type<typeof TranscriptBlock>;
47
53
 
48
54
  // TODO(burdon): Actions (e.g., mark, summarize, translate, label, delete).
49
55
 
@@ -9,7 +9,7 @@ import { intervalToDuration } from 'date-fns/intervalToDuration';
9
9
 
10
10
  import { type CleanupFn, addEventListener, combine } from '@dxos/async';
11
11
  import { type RenderCallback } from '@dxos/react-ui-editor';
12
- import { type MessageType } from '@dxos/schema';
12
+ import { type DataType } from '@dxos/schema';
13
13
 
14
14
  import { DocumentAdapter, type SerializationModel } from '../../model';
15
15
 
@@ -17,7 +17,7 @@ import { DocumentAdapter, type SerializationModel } from '../../model';
17
17
  * Data structure that maps Chunks queue to lines with transcript state.
18
18
  */
19
19
  export type TranscriptOptions = {
20
- model: SerializationModel<MessageType>;
20
+ model: SerializationModel<DataType.Message>;
21
21
  started?: Date;
22
22
  renderButton?: RenderCallback<{ onClick: () => void }>;
23
23
  };
@@ -213,7 +213,7 @@ class TimestampMarker extends GutterMarker {
213
213
  }
214
214
  }
215
215
 
216
- const getStartTime = (started?: Date, message?: MessageType): Date | undefined => {
216
+ const getStartTime = (started?: Date, message?: DataType.Message): Date | undefined => {
217
217
  if (started) {
218
218
  return started;
219
219
  }
@@ -7,7 +7,7 @@ import React, { type FC } from 'react';
7
7
  import { fullyQualifiedId, getSpace } from '@dxos/client/echo';
8
8
  import { useMembers, useQueue } from '@dxos/react-client/echo';
9
9
  import { StackItem } from '@dxos/react-ui-stack';
10
- import { type MessageType } from '@dxos/schema';
10
+ import { type DataType } from '@dxos/schema';
11
11
 
12
12
  import { Transcript, renderMarkdown } from './Transcript';
13
13
  import { useQueueModelAdapter } from '../hooks';
@@ -17,7 +17,7 @@ export const TranscriptionContainer: FC<{ role: string; transcript: TranscriptTy
17
17
  const attendableId = fullyQualifiedId(transcript);
18
18
  const space = getSpace(transcript);
19
19
  const members = useMembers(space?.key).map((member) => member.identity);
20
- const queue = useQueue<MessageType>(transcript.queue.dxn, { pollInterval: 1_000 });
20
+ const queue = useQueue<DataType.Message>(transcript.queue.dxn, { pollInterval: 1_000 });
21
21
  const model = useQueueModelAdapter(renderMarkdown(members), queue);
22
22
 
23
23
  return (
@@ -21,7 +21,7 @@ import { withPluginManager } from '@dxos/app-framework/testing';
21
21
  import { AIServiceEdgeClient } from '@dxos/assistant';
22
22
  import { AI_SERVICE_ENDPOINT } from '@dxos/assistant/testing';
23
23
  import { Filter, MemoryQueue } from '@dxos/echo-db';
24
- import { create } from '@dxos/echo-schema';
24
+ import { create, createQueueDxn } from '@dxos/echo-schema';
25
25
  import { type DXN } from '@dxos/keys';
26
26
  import { log } from '@dxos/log';
27
27
  import { ClientPlugin } from '@dxos/plugin-client';
@@ -29,11 +29,11 @@ import { PreviewPlugin } from '@dxos/plugin-preview';
29
29
  import { SpacePlugin } from '@dxos/plugin-space';
30
30
  import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
31
31
  import { ThemePlugin } from '@dxos/plugin-theme';
32
- import { createQueueDxn, useQueue, useSpace } from '@dxos/react-client/echo';
32
+ import { useQueue, useSpace } from '@dxos/react-client/echo';
33
33
  import { IconButton, Toolbar } from '@dxos/react-ui';
34
34
  import { ScrollContainer } from '@dxos/react-ui-components';
35
35
  import { defaultTx } from '@dxos/react-ui-theme';
36
- import { Contact, MessageType, Organization } from '@dxos/schema';
36
+ import { DataType } from '@dxos/schema';
37
37
  import { seedTestData, Testing } from '@dxos/schema/testing';
38
38
  import { withLayout, withTheme } from '@dxos/storybook-utils';
39
39
 
@@ -46,7 +46,7 @@ import { TestItem } from '../testing';
46
46
  import { type TranscriberParams } from '../transcriber';
47
47
 
48
48
  const TranscriptionStory: FC<{
49
- model: SerializationModel<MessageType>;
49
+ model: SerializationModel<DataType.Message>;
50
50
  running: boolean;
51
51
  onRunningChange: Dispatch<SetStateAction<boolean>>;
52
52
  }> = ({ model, running, onRunningChange }) => {
@@ -81,14 +81,14 @@ const Microphone = ({ entityExtraction }: { entityExtraction?: boolean }) => {
81
81
 
82
82
  // Queue.
83
83
  const queueDxn = useMemo(() => createQueueDxn(), []);
84
- const queue = useMemo(() => new MemoryQueue<MessageType>(queueDxn), [queueDxn]);
84
+ const queue = useMemo(() => new MemoryQueue<DataType.Message>(queueDxn), [queueDxn]);
85
85
  const model = useQueueModelAdapter(renderMarkdown([]), queue);
86
86
  const space = useSpace();
87
87
 
88
88
  // Transcriber.
89
89
  const handleSegments = useCallback<TranscriberParams['onSegments']>(
90
90
  async (blocks) => {
91
- const message = create(MessageType, { sender: { name: 'You' }, created: new Date().toISOString(), blocks });
91
+ const message = create(DataType.Message, { sender: { name: 'You' }, created: new Date().toISOString(), blocks });
92
92
 
93
93
  if (entityExtraction) {
94
94
  if (!space) {
@@ -97,7 +97,13 @@ const Microphone = ({ entityExtraction }: { entityExtraction?: boolean }) => {
97
97
  }
98
98
  // TODO(dmaretskyi): Move to vector search index.
99
99
  const { objects } = await space.db
100
- .query(Filter.or(Filter.schema(Contact), Filter.schema(Organization), Filter.schema(Testing.DocumentType)))
100
+ .query(
101
+ Filter.or(
102
+ Filter.type(DataType.Person),
103
+ Filter.type(DataType.Organization),
104
+ Filter.type(Testing.DocumentType),
105
+ ),
106
+ )
101
107
  .run();
102
108
 
103
109
  log.info('context', { objects });
@@ -176,11 +182,11 @@ const AudioFile = ({ queueDxn, audioUrl }: { queueDxn: DXN; audioUrl: string; tr
176
182
  }, [audio, running]);
177
183
 
178
184
  // Transcriber.
179
- const queue = useQueue<MessageType>(queueDxn, { pollInterval: 500 });
185
+ const queue = useQueue<DataType.Message>(queueDxn, { pollInterval: 500 });
180
186
  const model = useQueueModelAdapter(renderMarkdown([]), queue);
181
187
  const handleSegments = useCallback<TranscriberParams['onSegments']>(
182
188
  async (blocks) => {
183
- const message = create(MessageType, { sender: { name: 'You' }, created: new Date().toISOString(), blocks });
189
+ const message = create(DataType.Message, { sender: { name: 'You' }, created: new Date().toISOString(), blocks });
184
190
  void queue?.append([message]);
185
191
  },
186
192
  [queue],
@@ -224,7 +230,7 @@ const meta: Meta<typeof AudioFile> = {
224
230
  ThemePlugin({ tx: defaultTx }),
225
231
  StorybookLayoutPlugin(),
226
232
  ClientPlugin({
227
- types: [TestItem, Contact, Organization, Testing.DocumentType],
233
+ types: [TestItem, DataType.Person, DataType.Organization, Testing.DocumentType],
228
234
  onClientInitialized: async (_, client) => {
229
235
  await client.halo.createIdentity();
230
236
  await client.spaces.waitUntilReady();
@@ -258,11 +264,7 @@ const meta: Meta<typeof AudioFile> = {
258
264
  // ],
259
265
  }),
260
266
  withTheme,
261
- withLayout({
262
- tooltips: true,
263
- fullscreen: true,
264
- classNames: 'justify-center',
265
- }),
267
+ withLayout({ fullscreen: true, classNames: 'justify-center' }),
266
268
  ],
267
269
  };
268
270