@dxos/plugin-transcription 0.8.2-main.2f9c567 → 0.8.2-main.36232bc

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. package/dist/lib/browser/{TranscriptContainer-3LU3XBEB.mjs → TranscriptContainer-2VFHUHLE.mjs} +2 -2
  2. package/dist/lib/browser/{app-graph-builder-YXELPN6F.mjs → app-graph-builder-TDP6PYV6.mjs} +20 -19
  3. package/dist/lib/browser/app-graph-builder-TDP6PYV6.mjs.map +7 -0
  4. package/dist/lib/browser/{chunk-GWY2Q2NE.mjs → chunk-KSUYWWIU.mjs} +6 -6
  5. package/dist/lib/browser/chunk-KSUYWWIU.mjs.map +7 -0
  6. package/dist/lib/browser/{chunk-HNIKXVCG.mjs → chunk-NFVDICSJ.mjs} +7 -8
  7. package/dist/lib/browser/chunk-NFVDICSJ.mjs.map +7 -0
  8. package/dist/lib/browser/{chunk-HRDAILQX.mjs → chunk-O6RY7FNK.mjs} +2 -2
  9. package/dist/lib/browser/{chunk-TGOKX32G.mjs → chunk-YRWBY6UK.mjs} +15 -17
  10. package/dist/lib/browser/chunk-YRWBY6UK.mjs.map +7 -0
  11. package/dist/lib/browser/index.mjs +6 -6
  12. package/dist/lib/browser/index.mjs.map +3 -3
  13. package/dist/lib/browser/{intent-resolver-BBB3DWV4.mjs → intent-resolver-ZWKYRFE7.mjs} +2 -2
  14. package/dist/lib/browser/meta.json +1 -1
  15. package/dist/lib/browser/{react-surface-R277DACJ.mjs → react-surface-F32UNCKZ.mjs} +4 -4
  16. package/dist/lib/browser/{settings-I226DZ27.mjs → settings-6YAHKVKX.mjs} +2 -2
  17. package/dist/lib/browser/{transcriber-QRSS6X6S.mjs → transcriber-MSBS4VBI.mjs} +3 -3
  18. package/dist/lib/browser/types/index.mjs +1 -1
  19. package/dist/lib/node/{TranscriptContainer-DICOMZLM.cjs → TranscriptContainer-E2D7PKO3.cjs} +7 -7
  20. package/dist/lib/node/{TranscriptContainer-DICOMZLM.cjs.map → TranscriptContainer-E2D7PKO3.cjs.map} +2 -2
  21. package/dist/lib/node/{app-graph-builder-I6LYVIYY.cjs → app-graph-builder-W4VDMVGV.cjs} +29 -28
  22. package/dist/lib/node/app-graph-builder-W4VDMVGV.cjs.map +7 -0
  23. package/dist/lib/node/{chunk-RK4BEN5E.cjs → chunk-7XIIUVN3.cjs} +9 -9
  24. package/dist/lib/node/chunk-7XIIUVN3.cjs.map +7 -0
  25. package/dist/lib/node/{chunk-34WFY7M7.cjs → chunk-IXU57SCA.cjs} +16 -18
  26. package/dist/lib/node/chunk-IXU57SCA.cjs.map +7 -0
  27. package/dist/lib/node/{chunk-4JOJCZPJ.cjs → chunk-NBC6V3M2.cjs} +10 -11
  28. package/dist/lib/node/chunk-NBC6V3M2.cjs.map +7 -0
  29. package/dist/lib/node/{chunk-K553L7CZ.cjs → chunk-TJMZ26GS.cjs} +5 -5
  30. package/dist/lib/node/index.cjs +27 -27
  31. package/dist/lib/node/index.cjs.map +3 -3
  32. package/dist/lib/node/{intent-resolver-7XY3WITN.cjs → intent-resolver-RM4JOCDV.cjs} +7 -7
  33. package/dist/lib/node/meta.json +1 -1
  34. package/dist/lib/node/{react-surface-HUSWAORU.cjs → react-surface-YQHBGPOC.cjs} +10 -10
  35. package/dist/lib/node/{settings-NDMU3EWQ.cjs → settings-ZMB4XRWS.cjs} +7 -7
  36. package/dist/lib/node/{transcriber-UZJMG44C.cjs → transcriber-55OE4X6I.cjs} +8 -8
  37. package/dist/lib/node/types/index.cjs +7 -7
  38. package/dist/lib/node/types/index.cjs.map +1 -1
  39. package/dist/lib/node-esm/{TranscriptContainer-K4E4P5T6.mjs → TranscriptContainer-5N3BSK2S.mjs} +2 -2
  40. package/dist/lib/node-esm/{app-graph-builder-6FY2UQOH.mjs → app-graph-builder-DDDVKUHS.mjs} +20 -19
  41. package/dist/lib/node-esm/app-graph-builder-DDDVKUHS.mjs.map +7 -0
  42. package/dist/lib/node-esm/{chunk-6C34FO4C.mjs → chunk-4RMJO6QQ.mjs} +7 -8
  43. package/dist/lib/node-esm/chunk-4RMJO6QQ.mjs.map +7 -0
  44. package/dist/lib/node-esm/{chunk-3I7AC3D3.mjs → chunk-HZBSWUJH.mjs} +2 -2
  45. package/dist/lib/node-esm/{chunk-KUQ3ZTRX.mjs → chunk-OS2ITZUL.mjs} +15 -17
  46. package/dist/lib/node-esm/chunk-OS2ITZUL.mjs.map +7 -0
  47. package/dist/lib/node-esm/{chunk-CKANLRS4.mjs → chunk-UWF6LM27.mjs} +6 -6
  48. package/dist/lib/node-esm/chunk-UWF6LM27.mjs.map +7 -0
  49. package/dist/lib/node-esm/index.mjs +6 -6
  50. package/dist/lib/node-esm/index.mjs.map +3 -3
  51. package/dist/lib/node-esm/{intent-resolver-ACGAPLDZ.mjs → intent-resolver-HDRFXMKP.mjs} +2 -2
  52. package/dist/lib/node-esm/meta.json +1 -1
  53. package/dist/lib/node-esm/{react-surface-G2OY4MWL.mjs → react-surface-DRDNXTFZ.mjs} +4 -4
  54. package/dist/lib/node-esm/{settings-O7X2UQQG.mjs → settings-I35TVDUG.mjs} +2 -2
  55. package/dist/lib/node-esm/{transcriber-NVVFH7IF.mjs → transcriber-HH7EMCGH.mjs} +3 -3
  56. package/dist/lib/node-esm/types/index.mjs +1 -1
  57. package/dist/types/src/capabilities/app-graph-builder.d.ts.map +1 -1
  58. package/dist/types/src/capabilities/index.d.ts.map +1 -1
  59. package/dist/types/src/components/Transcript/Transcript.d.ts +3 -3
  60. package/dist/types/src/components/Transcript/Transcript.d.ts.map +1 -1
  61. package/dist/types/src/components/Transcript/Transcript.stories.d.ts +2 -2
  62. package/dist/types/src/components/Transcript/Transcript.stories.d.ts.map +1 -1
  63. package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts +10 -10
  64. package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts.map +1 -1
  65. package/dist/types/src/components/Transcript/transcript-extension.d.ts +2 -2
  66. package/dist/types/src/components/Transcript/transcript-extension.d.ts.map +1 -1
  67. package/dist/types/src/components/transcription.stories.d.ts.map +1 -1
  68. package/dist/types/src/entity-extraction/entity-extraction.d.ts +3 -3
  69. package/dist/types/src/entity-extraction/entity-extraction.d.ts.map +1 -1
  70. package/dist/types/src/testing/testing.d.ts +10 -11
  71. package/dist/types/src/testing/testing.d.ts.map +1 -1
  72. package/dist/types/src/transcriber/transcriber.d.ts +2 -2
  73. package/dist/types/src/transcriber/transcriber.d.ts.map +1 -1
  74. package/dist/types/src/transcriber/transcription-manager.d.ts +2 -2
  75. package/dist/types/src/transcriber/transcription-manager.d.ts.map +1 -1
  76. package/dist/types/src/types/schema.d.ts +19 -26
  77. package/dist/types/src/types/schema.d.ts.map +1 -1
  78. package/dist/types/src/types/types.d.ts +10 -10
  79. package/dist/types/src/types/types.d.ts.map +1 -1
  80. package/package.json +49 -51
  81. package/src/TranscriptionPlugin.tsx +2 -2
  82. package/src/capabilities/app-graph-builder.ts +10 -10
  83. package/src/components/Transcript/Transcript.stories.tsx +5 -5
  84. package/src/components/Transcript/Transcript.tsx +4 -10
  85. package/src/components/Transcript/deprecated/OldTranscript.tsx +13 -19
  86. package/src/components/Transcript/transcript-extension.ts +3 -3
  87. package/src/components/TranscriptContainer.tsx +2 -2
  88. package/src/components/transcription.stories.tsx +8 -14
  89. package/src/entity-extraction/entity-extraction.test.ts +4 -23
  90. package/src/entity-extraction/entity-extraction.ts +6 -11
  91. package/src/model/model.test.ts +10 -14
  92. package/src/testing/testing.ts +21 -34
  93. package/src/transcriber/transcriber.browser.test.ts +3 -3
  94. package/src/transcriber/transcriber.ts +3 -6
  95. package/src/transcriber/transcription-manager.ts +8 -8
  96. package/src/types/schema.ts +8 -10
  97. package/src/types/types.ts +10 -12
  98. package/dist/lib/browser/app-graph-builder-YXELPN6F.mjs.map +0 -7
  99. package/dist/lib/browser/chunk-GWY2Q2NE.mjs.map +0 -7
  100. package/dist/lib/browser/chunk-HNIKXVCG.mjs.map +0 -7
  101. package/dist/lib/browser/chunk-TGOKX32G.mjs.map +0 -7
  102. package/dist/lib/node/app-graph-builder-I6LYVIYY.cjs.map +0 -7
  103. package/dist/lib/node/chunk-34WFY7M7.cjs.map +0 -7
  104. package/dist/lib/node/chunk-4JOJCZPJ.cjs.map +0 -7
  105. package/dist/lib/node/chunk-RK4BEN5E.cjs.map +0 -7
  106. package/dist/lib/node-esm/app-graph-builder-6FY2UQOH.mjs.map +0 -7
  107. package/dist/lib/node-esm/chunk-6C34FO4C.mjs.map +0 -7
  108. package/dist/lib/node-esm/chunk-CKANLRS4.mjs.map +0 -7
  109. package/dist/lib/node-esm/chunk-KUQ3ZTRX.mjs.map +0 -7
  110. /package/dist/lib/browser/{TranscriptContainer-3LU3XBEB.mjs.map → TranscriptContainer-2VFHUHLE.mjs.map} +0 -0
  111. /package/dist/lib/browser/{chunk-HRDAILQX.mjs.map → chunk-O6RY7FNK.mjs.map} +0 -0
  112. /package/dist/lib/browser/{intent-resolver-BBB3DWV4.mjs.map → intent-resolver-ZWKYRFE7.mjs.map} +0 -0
  113. /package/dist/lib/browser/{react-surface-R277DACJ.mjs.map → react-surface-F32UNCKZ.mjs.map} +0 -0
  114. /package/dist/lib/browser/{settings-I226DZ27.mjs.map → settings-6YAHKVKX.mjs.map} +0 -0
  115. /package/dist/lib/browser/{transcriber-QRSS6X6S.mjs.map → transcriber-MSBS4VBI.mjs.map} +0 -0
  116. /package/dist/lib/node/{chunk-K553L7CZ.cjs.map → chunk-TJMZ26GS.cjs.map} +0 -0
  117. /package/dist/lib/node/{intent-resolver-7XY3WITN.cjs.map → intent-resolver-RM4JOCDV.cjs.map} +0 -0
  118. /package/dist/lib/node/{react-surface-HUSWAORU.cjs.map → react-surface-YQHBGPOC.cjs.map} +0 -0
  119. /package/dist/lib/node/{settings-NDMU3EWQ.cjs.map → settings-ZMB4XRWS.cjs.map} +0 -0
  120. /package/dist/lib/node/{transcriber-UZJMG44C.cjs.map → transcriber-55OE4X6I.cjs.map} +0 -0
  121. /package/dist/lib/node-esm/{TranscriptContainer-K4E4P5T6.mjs.map → TranscriptContainer-5N3BSK2S.mjs.map} +0 -0
  122. /package/dist/lib/node-esm/{chunk-3I7AC3D3.mjs.map → chunk-HZBSWUJH.mjs.map} +0 -0
  123. /package/dist/lib/node-esm/{intent-resolver-ACGAPLDZ.mjs.map → intent-resolver-HDRFXMKP.mjs.map} +0 -0
  124. /package/dist/lib/node-esm/{react-surface-G2OY4MWL.mjs.map → react-surface-DRDNXTFZ.mjs.map} +0 -0
  125. /package/dist/lib/node-esm/{settings-O7X2UQQG.mjs.map → settings-I35TVDUG.mjs.map} +0 -0
  126. /package/dist/lib/node-esm/{transcriber-NVVFH7IF.mjs.map → transcriber-HH7EMCGH.mjs.map} +0 -0
@@ -3,12 +3,11 @@
3
3
  //
4
4
 
5
5
  import { intervalToDuration } from 'date-fns/intervalToDuration';
6
- import { Schema } from 'effect';
7
6
  import { yieldOrContinue } from 'main-thread-scheduling';
8
7
  import React, { type FC, useCallback, useEffect, useMemo, useRef, useState, type WheelEvent } from 'react';
9
8
  import { type OnResizeCallback, useResizeDetector } from 'react-resize-detector';
10
9
 
11
- import { Type } from '@dxos/echo';
10
+ import { EchoObject, S } from '@dxos/echo-schema';
12
11
  import { IconButton, type ThemedClassName, useTranslation } from '@dxos/react-ui';
13
12
  import { useAttention } from '@dxos/react-ui-attention';
14
13
  import {
@@ -26,30 +25,25 @@ import { TRANSCRIPTION_PLUGIN } from '../../../meta';
26
25
  /**
27
26
  * Transcription fragment.
28
27
  */
29
- const TranscriptSegment = Schema.Struct({
28
+ const TranscriptSegment = S.Struct({
30
29
  // TODO(burdon): TS from service is not Unix TS (x1000).
31
- started: Schema.String,
32
- text: Schema.String,
30
+ started: S.String,
31
+ text: S.String,
33
32
  });
34
33
 
35
- type TranscriptSegment = Schema.Schema.Type<typeof TranscriptSegment>;
34
+ type TranscriptSegment = S.Schema.Type<typeof TranscriptSegment>;
36
35
 
37
36
  /**
38
37
  * Transcription block (from single speaker).
39
38
  */
40
- const TranscriptBlock = Schema.Struct({
41
- id: Schema.String,
42
- authorName: Schema.optional(Schema.String), // TODO(burdon): Replace with identityDid.
43
- authorHue: Schema.optional(Schema.String), // TOOD(burdon): Remove.
44
- segments: Schema.Array(TranscriptSegment),
45
- }).pipe(
46
- Type.def({
47
- typename: 'dxos.org/type/TranscriptBlock',
48
- version: '0.1.0',
49
- }),
50
- );
51
-
52
- type TranscriptBlock = Schema.Schema.Type<typeof TranscriptBlock>;
39
+ const TranscriptBlock = S.Struct({
40
+ id: S.String,
41
+ authorName: S.optional(S.String), // TODO(burdon): Replace with identityDid.
42
+ authorHue: S.optional(S.String), // TOOD(burdon): Remove.
43
+ segments: S.Array(TranscriptSegment),
44
+ }).pipe(EchoObject({ typename: 'dxos.org/type/TranscriptBlock', version: '0.1.0' }));
45
+
46
+ type TranscriptBlock = S.Schema.Type<typeof TranscriptBlock>;
53
47
 
54
48
  // TODO(burdon): Actions (e.g., mark, summarize, translate, label, delete).
55
49
 
@@ -9,7 +9,7 @@ import { intervalToDuration } from 'date-fns/intervalToDuration';
9
9
 
10
10
  import { type CleanupFn, addEventListener, combine } from '@dxos/async';
11
11
  import { type RenderCallback } from '@dxos/react-ui-editor';
12
- import { type DataType } from '@dxos/schema';
12
+ import { type MessageType } from '@dxos/schema';
13
13
 
14
14
  import { DocumentAdapter, type SerializationModel } from '../../model';
15
15
 
@@ -17,7 +17,7 @@ import { DocumentAdapter, type SerializationModel } from '../../model';
17
17
  * Data structure that maps Chunks queue to lines with transcript state.
18
18
  */
19
19
  export type TranscriptOptions = {
20
- model: SerializationModel<DataType.Message>;
20
+ model: SerializationModel<MessageType>;
21
21
  started?: Date;
22
22
  renderButton?: RenderCallback<{ onClick: () => void }>;
23
23
  };
@@ -213,7 +213,7 @@ class TimestampMarker extends GutterMarker {
213
213
  }
214
214
  }
215
215
 
216
- const getStartTime = (started?: Date, message?: DataType.Message): Date | undefined => {
216
+ const getStartTime = (started?: Date, message?: MessageType): Date | undefined => {
217
217
  if (started) {
218
218
  return started;
219
219
  }
@@ -7,7 +7,7 @@ import React, { type FC } from 'react';
7
7
  import { fullyQualifiedId, getSpace } from '@dxos/client/echo';
8
8
  import { useMembers, useQueue } from '@dxos/react-client/echo';
9
9
  import { StackItem } from '@dxos/react-ui-stack';
10
- import { type DataType } from '@dxos/schema';
10
+ import { type MessageType } from '@dxos/schema';
11
11
 
12
12
  import { Transcript, renderMarkdown } from './Transcript';
13
13
  import { useQueueModelAdapter } from '../hooks';
@@ -17,7 +17,7 @@ export const TranscriptionContainer: FC<{ role: string; transcript: TranscriptTy
17
17
  const attendableId = fullyQualifiedId(transcript);
18
18
  const space = getSpace(transcript);
19
19
  const members = useMembers(space?.key).map((member) => member.identity);
20
- const queue = useQueue<DataType.Message>(transcript.queue.dxn, { pollInterval: 1_000 });
20
+ const queue = useQueue<MessageType>(transcript.queue.dxn, { pollInterval: 1_000 });
21
21
  const model = useQueueModelAdapter(renderMarkdown(members), queue);
22
22
 
23
23
  return (
@@ -33,7 +33,7 @@ import { createQueueDxn, useQueue, useSpace } from '@dxos/react-client/echo';
33
33
  import { IconButton, Toolbar } from '@dxos/react-ui';
34
34
  import { ScrollContainer } from '@dxos/react-ui-components';
35
35
  import { defaultTx } from '@dxos/react-ui-theme';
36
- import { DataType } from '@dxos/schema';
36
+ import { Contact, MessageType, Organization } from '@dxos/schema';
37
37
  import { seedTestData, Testing } from '@dxos/schema/testing';
38
38
  import { withLayout, withTheme } from '@dxos/storybook-utils';
39
39
 
@@ -46,7 +46,7 @@ import { TestItem } from '../testing';
46
46
  import { type TranscriberParams } from '../transcriber';
47
47
 
48
48
  const TranscriptionStory: FC<{
49
- model: SerializationModel<DataType.Message>;
49
+ model: SerializationModel<MessageType>;
50
50
  running: boolean;
51
51
  onRunningChange: Dispatch<SetStateAction<boolean>>;
52
52
  }> = ({ model, running, onRunningChange }) => {
@@ -81,14 +81,14 @@ const Microphone = ({ entityExtraction }: { entityExtraction?: boolean }) => {
81
81
 
82
82
  // Queue.
83
83
  const queueDxn = useMemo(() => createQueueDxn(), []);
84
- const queue = useMemo(() => new MemoryQueue<DataType.Message>(queueDxn), [queueDxn]);
84
+ const queue = useMemo(() => new MemoryQueue<MessageType>(queueDxn), [queueDxn]);
85
85
  const model = useQueueModelAdapter(renderMarkdown([]), queue);
86
86
  const space = useSpace();
87
87
 
88
88
  // Transcriber.
89
89
  const handleSegments = useCallback<TranscriberParams['onSegments']>(
90
90
  async (blocks) => {
91
- const message = create(DataType.Message, { sender: { name: 'You' }, created: new Date().toISOString(), blocks });
91
+ const message = create(MessageType, { sender: { name: 'You' }, created: new Date().toISOString(), blocks });
92
92
 
93
93
  if (entityExtraction) {
94
94
  if (!space) {
@@ -97,13 +97,7 @@ const Microphone = ({ entityExtraction }: { entityExtraction?: boolean }) => {
97
97
  }
98
98
  // TODO(dmaretskyi): Move to vector search index.
99
99
  const { objects } = await space.db
100
- .query(
101
- Filter.or(
102
- Filter.schema(DataType.Person),
103
- Filter.schema(DataType.Organization),
104
- Filter.schema(Testing.DocumentType),
105
- ),
106
- )
100
+ .query(Filter.or(Filter.schema(Contact), Filter.schema(Organization), Filter.schema(Testing.DocumentType)))
107
101
  .run();
108
102
 
109
103
  log.info('context', { objects });
@@ -182,11 +176,11 @@ const AudioFile = ({ queueDxn, audioUrl }: { queueDxn: DXN; audioUrl: string; tr
182
176
  }, [audio, running]);
183
177
 
184
178
  // Transcriber.
185
- const queue = useQueue<DataType.Message>(queueDxn, { pollInterval: 500 });
179
+ const queue = useQueue<MessageType>(queueDxn, { pollInterval: 500 });
186
180
  const model = useQueueModelAdapter(renderMarkdown([]), queue);
187
181
  const handleSegments = useCallback<TranscriberParams['onSegments']>(
188
182
  async (blocks) => {
189
- const message = create(DataType.Message, { sender: { name: 'You' }, created: new Date().toISOString(), blocks });
183
+ const message = create(MessageType, { sender: { name: 'You' }, created: new Date().toISOString(), blocks });
190
184
  void queue?.append([message]);
191
185
  },
192
186
  [queue],
@@ -230,7 +224,7 @@ const meta: Meta<typeof AudioFile> = {
230
224
  ThemePlugin({ tx: defaultTx }),
231
225
  StorybookLayoutPlugin(),
232
226
  ClientPlugin({
233
- types: [TestItem, DataType.Person, DataType.Organization, Testing.DocumentType],
227
+ types: [TestItem, Contact, Organization, Testing.DocumentType],
234
228
  onClientInitialized: async (_, client) => {
235
229
  await client.halo.createIdentity();
236
230
  await client.spaces.waitUntilReady();
@@ -17,7 +17,7 @@ const aiService = new AIServiceEdgeClient({
17
17
  });
18
18
 
19
19
  describe.skip('EntityExtraction', { timeout: 180_000 }, () => {
20
- test.skip('should process a transcript block', async () => {
20
+ test('should process a transcript block', async () => {
21
21
  const { transcriptMessages, documents, contacts } = await createTestData();
22
22
 
23
23
  log.info('context', { documents, contacts });
@@ -35,11 +35,11 @@ describe.skip('EntityExtraction', { timeout: 180_000 }, () => {
35
35
  }
36
36
  });
37
37
 
38
- test.only('computational irreducibility', async () => {
39
- const { transcriptWoflram, documents, contacts } = await createTestData();
38
+ test.skip('computational irreducibility', async () => {
39
+ const { transcriptMessages, documents, contacts } = await createTestData();
40
40
 
41
41
  log.info('context', { documents, contacts });
42
- const message = transcriptWoflram[0];
42
+ const message = transcriptMessages[0];
43
43
  log.info('input', message);
44
44
 
45
45
  await Promise.all(
@@ -55,25 +55,6 @@ describe.skip('EntityExtraction', { timeout: 180_000 }, () => {
55
55
  }),
56
56
  );
57
57
  });
58
-
59
- test('org and document linking', async () => {
60
- const { transcriptJosiah, documents, contacts, organizations } = await createTestData();
61
-
62
- log.info('context', { contacts, organizations, documents });
63
-
64
- for (const message of transcriptJosiah) {
65
- log.info('input', message);
66
-
67
- const { message: enhancedMessage } = await processTranscriptMessage({
68
- message,
69
- aiService,
70
- context: {
71
- objects: [...documents, ...Object.values(contacts), ...Object.values(organizations)],
72
- },
73
- });
74
- log.info('output', enhancedMessage);
75
- }
76
- });
77
58
  });
78
59
 
79
60
  describe('postprocessText', () => {
@@ -8,15 +8,14 @@ import { createTemplate, Message, structuredOutputParser } from '@dxos/artifact'
8
8
  import type { AIServiceClient } from '@dxos/assistant';
9
9
  import { MixedStreamParser } from '@dxos/assistant';
10
10
  import { asyncTimeout } from '@dxos/async';
11
- import { type BaseEchoObject, create, ObjectId } from '@dxos/echo-schema';
12
- import { DXN } from '@dxos/keys';
11
+ import { type BaseEchoObject, create } from '@dxos/echo-schema';
13
12
  import { log } from '@dxos/log';
14
- import { DataType } from '@dxos/schema';
13
+ import { MessageType } from '@dxos/schema';
15
14
 
16
15
  import SYSTEM_PROMPT from './system-prompt.tpl?raw';
17
16
 
18
17
  type ProcessTranscriptMessageParams = {
19
- message: DataType.Message;
18
+ message: MessageType;
20
19
  aiService: AIServiceClient;
21
20
  context: {
22
21
  objects?: BaseEchoObject[];
@@ -38,7 +37,7 @@ type ProcessTranscriptMessageParams = {
38
37
  };
39
38
 
40
39
  type ProcessTranscriptMessageResult = {
41
- message: DataType.Message;
40
+ message: MessageType;
42
41
  timeElapsed: number;
43
42
  };
44
43
 
@@ -98,7 +97,7 @@ export const processTranscriptMessage = async (
98
97
  log.info('entity extraction result', { refs: result.references });
99
98
 
100
99
  return {
101
- message: create(DataType.Message, {
100
+ message: create(MessageType, {
102
101
  ...params.message,
103
102
  blocks: params.message.blocks.map((block, i) =>
104
103
  block.type !== 'transcription'
@@ -137,13 +136,9 @@ export const processTranscriptMessage = async (
137
136
  // TODO(dmaretskyi): Lookup and verifiy ids from provided context.
138
137
  export const postprocessText = (text: string, quotes: ReferencedQuotes) => {
139
138
  for (const quote of quotes.references) {
140
- if (!ObjectId.isValid(quote.id)) {
141
- continue;
142
- }
143
-
144
139
  // Use a case-insensitive regular expression to replace the quote
145
140
  const regex = new RegExp(quote.quote.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'gi');
146
- text = text.replace(regex, `[${quote.quote}][${DXN.fromLocalObjectId(quote.id).toString()}]`);
141
+ text = text.replace(regex, `[${quote.quote}][dxn:echo:@:${quote.id}]`);
147
142
  }
148
143
  return text;
149
144
  };
@@ -5,15 +5,11 @@
5
5
  import { EditorView } from '@codemirror/view';
6
6
  import { describe, test } from 'vitest';
7
7
 
8
- import { type DataType } from '@dxos/schema';
8
+ import { type MessageType } from '@dxos/schema';
9
9
 
10
10
  import { SerializationModel, DocumentAdapter, type ChunkRenderer } from './model';
11
11
 
12
- const blockToMarkdown: ChunkRenderer<DataType.Message> = (
13
- message: DataType.Message,
14
- index: number,
15
- debug = true,
16
- ): string[] => {
12
+ const blockToMarkdown: ChunkRenderer<MessageType> = (message: MessageType, index: number, debug = true): string[] => {
17
13
  return [
18
14
  `###### ${message.sender.name}`,
19
15
  ...message.blocks.filter((block) => block.type === 'transcription').map((block) => block.text),
@@ -25,12 +21,12 @@ const createDate = () => new Date().toISOString();
25
21
 
26
22
  describe('SerializationModel', () => {
27
23
  test('basic', ({ expect }) => {
28
- const model = new SerializationModel<DataType.Message>(blockToMarkdown);
24
+ const model = new SerializationModel<MessageType>(blockToMarkdown);
29
25
  expect(model.chunks.length).to.eq(0);
30
26
  expect(model.doc.toString()).to.eq('');
31
27
 
32
28
  // Create message.
33
- const message: DataType.Message = {
29
+ const message: MessageType = {
34
30
  id: '1',
35
31
  created: createDate(),
36
32
  sender: { name: 'Alice' },
@@ -61,14 +57,14 @@ describe('SerializationModel', () => {
61
57
 
62
58
  test('sync - append', ({ expect }) => {
63
59
  const view = new EditorView({ extensions: [], doc: '' });
64
- const model = new SerializationModel<DataType.Message>(blockToMarkdown);
60
+ const model = new SerializationModel<MessageType>(blockToMarkdown);
65
61
  const adapter = new DocumentAdapter(view);
66
62
  expect(adapter.lineCount()).to.eq(0);
67
63
  expect(view.state.doc.toString()).to.eq('');
68
64
 
69
65
  // Append message.
70
66
  {
71
- const message: DataType.Message = {
67
+ const message: MessageType = {
72
68
  id: '1',
73
69
  created: createDate(),
74
70
  sender: { name: 'Alice' },
@@ -87,7 +83,7 @@ describe('SerializationModel', () => {
87
83
 
88
84
  // Append message.
89
85
  {
90
- const message: DataType.Message = {
86
+ const message: MessageType = {
91
87
  id: '2',
92
88
  created: createDate(),
93
89
  sender: { name: 'Bob' },
@@ -107,13 +103,13 @@ describe('SerializationModel', () => {
107
103
 
108
104
  test('sync - append, update, delete', ({ expect }) => {
109
105
  const view = new EditorView({ extensions: [], doc: '' });
110
- const model = new SerializationModel<DataType.Message>(blockToMarkdown);
106
+ const model = new SerializationModel<MessageType>(blockToMarkdown);
111
107
  const adapter = new DocumentAdapter(view);
112
108
  expect(adapter.lineCount()).to.eq(0);
113
109
 
114
110
  // Append message.
115
111
  {
116
- const message: DataType.Message = {
112
+ const message: MessageType = {
117
113
  id: '1',
118
114
  created: createDate(),
119
115
  sender: { name: 'Alice' },
@@ -146,7 +142,7 @@ describe('SerializationModel', () => {
146
142
 
147
143
  // Append message.
148
144
  {
149
- const message: DataType.Message = {
145
+ const message: MessageType = {
150
146
  id: '2',
151
147
  created: createDate(),
152
148
  sender: { name: 'Bob' },
@@ -2,7 +2,6 @@
2
2
  // Copyright 2025 DXOS.org
3
3
  //
4
4
 
5
- import { Schema, SchemaAST } from 'effect';
6
5
  import { useEffect, useMemo, useState } from 'react';
7
6
 
8
7
  import { AIServiceEdgeClient } from '@dxos/assistant';
@@ -10,38 +9,32 @@ import { AI_SERVICE_ENDPOINT } from '@dxos/assistant/testing';
10
9
  import { scheduleTaskInterval } from '@dxos/async';
11
10
  import { createQueueDxn, Filter, type Queue } from '@dxos/client/echo';
12
11
  import { Context } from '@dxos/context';
13
- import { Type } from '@dxos/echo';
14
- import { create, ObjectId } from '@dxos/echo-schema';
12
+ import { AST, create, EchoObject, ObjectId, S } from '@dxos/echo-schema';
15
13
  import { IdentityDid } from '@dxos/keys';
16
14
  import { log } from '@dxos/log';
17
15
  import { faker } from '@dxos/random';
18
16
  import { live, makeRef, useQueue, type Space } from '@dxos/react-client/echo';
19
- import { DataType } from '@dxos/schema';
17
+ import { Contact, MessageType, Organization, type TranscriptionContentBlock } from '@dxos/schema';
20
18
  import { Testing, seedTestData } from '@dxos/schema/testing';
21
19
 
22
20
  import { processTranscriptMessage } from '../entity-extraction';
23
21
 
24
22
  // TODO(burdon): Reconcile with plugin-markdown. Move to @dxos/schema/testing.
25
23
 
26
- export const TestItem = Schema.Struct({
27
- title: Schema.String.annotations({
28
- [SchemaAST.TitleAnnotationId]: 'Title',
29
- [SchemaAST.DescriptionAnnotationId]: 'Product title',
24
+ export const TestItem = S.Struct({
25
+ title: S.String.annotations({
26
+ [AST.TitleAnnotationId]: 'Title',
27
+ [AST.DescriptionAnnotationId]: 'Product title',
30
28
  }),
31
- description: Schema.String.annotations({
32
- [SchemaAST.TitleAnnotationId]: 'Description',
33
- [SchemaAST.DescriptionAnnotationId]: 'Product description',
29
+ description: S.String.annotations({
30
+ [AST.TitleAnnotationId]: 'Description',
31
+ [AST.DescriptionAnnotationId]: 'Product description',
34
32
  }),
35
- }).pipe(
36
- Type.def({
37
- typename: 'dxos.org/type/Test',
38
- version: '0.1.0',
39
- }),
40
- );
33
+ }).pipe(EchoObject({ typename: 'dxos.org/type/Test', version: '0.1.0' }));
41
34
 
42
35
  // TODO(wittjosiah): Make builder generic and reuse for all message types.
43
36
  abstract class AbstractMessageBuilder {
44
- abstract createMessage(numSegments?: number): Promise<DataType.Message>;
37
+ abstract createMessage(numSegments?: number): Promise<MessageType>;
45
38
  }
46
39
 
47
40
  /**
@@ -62,7 +55,7 @@ export class MessageBuilder extends AbstractMessageBuilder {
62
55
  super();
63
56
  }
64
57
 
65
- override async createMessage(numSegments = 1): Promise<DataType.Message> {
58
+ override async createMessage(numSegments = 1): Promise<MessageType> {
66
59
  return {
67
60
  id: ObjectId.random().toString(),
68
61
  created: this.next().toISOString(),
@@ -71,7 +64,7 @@ export class MessageBuilder extends AbstractMessageBuilder {
71
64
  };
72
65
  }
73
66
 
74
- createBlock(): DataType.MessageBlock.Transcription {
67
+ createBlock(): TranscriptionContentBlock {
75
68
  let text = faker.lorem.paragraph();
76
69
  if (this._space) {
77
70
  const label = faker.commerce.productName();
@@ -103,7 +96,7 @@ class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
103
96
 
104
97
  space: Space | undefined;
105
98
  currentMessage: number = 0;
106
- transcriptMessages: DataType.Message[] = [];
99
+ transcriptMessages: MessageType[] = [];
107
100
 
108
101
  async connect(space: Space) {
109
102
  this.space = space;
@@ -111,19 +104,13 @@ class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
111
104
  this.transcriptMessages = transcriptMessages;
112
105
  }
113
106
 
114
- override async createMessage(): Promise<DataType.Message> {
107
+ override async createMessage(): Promise<MessageType> {
115
108
  if (!this.space) {
116
109
  throw new Error('Space not connected');
117
110
  }
118
111
 
119
112
  const { objects } = await this.space.db
120
- .query(
121
- Filter.or(
122
- Filter.schema(DataType.Person),
123
- Filter.schema(DataType.Organization),
124
- Filter.schema(Testing.DocumentType),
125
- ),
126
- )
113
+ .query(Filter.or(Filter.schema(Contact), Filter.schema(Organization), Filter.schema(Testing.DocumentType)))
127
114
  .run();
128
115
 
129
116
  log.info('context', { objects });
@@ -148,7 +135,7 @@ type UseTestTranscriptionQueue = (
148
135
  queueId?: ObjectId,
149
136
  running?: boolean,
150
137
  interval?: number,
151
- ) => Queue<DataType.Message> | undefined;
138
+ ) => Queue<MessageType> | undefined;
152
139
 
153
140
  /**
154
141
  * Test transcriptionqueue.
@@ -160,7 +147,7 @@ export const useTestTranscriptionQueue: UseTestTranscriptionQueue = (
160
147
  interval = 1_000,
161
148
  ) => {
162
149
  const queueDxn = useMemo(() => (space ? createQueueDxn(space.id, queueId) : undefined), [space, queueId]);
163
- const queue = useQueue<DataType.Message>(queueDxn);
150
+ const queue = useQueue<MessageType>(queueDxn);
164
151
  const builder = useMemo(() => new MessageBuilder(space), [space]);
165
152
 
166
153
  useEffect(() => {
@@ -170,7 +157,7 @@ export const useTestTranscriptionQueue: UseTestTranscriptionQueue = (
170
157
 
171
158
  const i = setInterval(() => {
172
159
  void builder.createMessage(Math.ceil(Math.random() * 3)).then((message) => {
173
- queue.append([create(DataType.Message, message)]);
160
+ queue.append([create(MessageType, message)]);
174
161
  });
175
162
  }, interval);
176
163
  return () => clearInterval(i);
@@ -190,7 +177,7 @@ export const useTestTranscriptionQueueWithEntityExtraction: UseTestTranscription
190
177
  interval = 1_000,
191
178
  ) => {
192
179
  const queueDxn = useMemo(() => (space ? createQueueDxn(space.id, queueId) : undefined), [space, queueId]);
193
- const queue = useQueue<DataType.Message>(queueDxn);
180
+ const queue = useQueue<MessageType>(queueDxn);
194
181
  const [builder] = useState(() => new EntityExtractionMessageBuilder());
195
182
 
196
183
  useEffect(() => {
@@ -207,7 +194,7 @@ export const useTestTranscriptionQueueWithEntityExtraction: UseTestTranscription
207
194
  ctx,
208
195
  async () => {
209
196
  const message = await builder.createMessage();
210
- queue.append([create(DataType.Message, message)]);
197
+ queue.append([create(MessageType, message)]);
211
198
  },
212
199
  interval,
213
200
  );
@@ -9,7 +9,7 @@ import { WaveFile } from 'wavefile';
9
9
 
10
10
  import { Trigger } from '@dxos/async';
11
11
  import { log } from '@dxos/log';
12
- import { type DataType } from '@dxos/schema';
12
+ import { type TranscriptionContentBlock } from '@dxos/schema';
13
13
  import { openAndClose } from '@dxos/test-utils';
14
14
  import { trace, TRACE_PROCESSOR } from '@dxos/tracing';
15
15
 
@@ -93,7 +93,7 @@ describe.skip('Transcriber', () => {
93
93
  });
94
94
 
95
95
  test.skip('transcription of audio recording', { timeout: 10_000 }, async () => {
96
- const trigger = new Trigger<DataType.MessageBlock.Transcription[]>({ autoReset: true });
96
+ const trigger = new Trigger<TranscriptionContentBlock[]>({ autoReset: true });
97
97
  const recorder = new MockAudioRecorder({
98
98
  buffer: await readFile('test.wav'),
99
99
  chunkDuration: 3_000,
@@ -127,7 +127,7 @@ describe.skip('Transcriber', () => {
127
127
  });
128
128
 
129
129
  test.skip('transcription of audio recording with overlapping chunks', { timeout: 20_000 }, async () => {
130
- const trigger = new Trigger<DataType.MessageBlock.Transcription[]>({ autoReset: true });
130
+ const trigger = new Trigger<TranscriptionContentBlock[]>({ autoReset: true });
131
131
  const recorder = new MockAudioRecorder({
132
132
  buffer: await readFile('test.wav'),
133
133
  chunkDuration: 3_000,
@@ -7,7 +7,7 @@ import { WaveFile } from 'wavefile';
7
7
  import { DeferredTask, Trigger } from '@dxos/async';
8
8
  import { type Context, LifecycleState, Resource } from '@dxos/context';
9
9
  import { log } from '@dxos/log';
10
- import { type DataType } from '@dxos/schema';
10
+ import { type TranscriptionContentBlock } from '@dxos/schema';
11
11
  import { trace } from '@dxos/tracing';
12
12
 
13
13
  import { type AudioRecorder, type AudioChunk } from './audio-recorder';
@@ -65,7 +65,7 @@ export type TranscribeConfig = {
65
65
  export type TranscriberParams = {
66
66
  config: TranscribeConfig;
67
67
  recorder: AudioRecorder;
68
- onSegments: (segments: DataType.MessageBlock.Transcription[]) => Promise<void>;
68
+ onSegments: (segments: TranscriptionContentBlock[]) => Promise<void>;
69
69
  };
70
70
 
71
71
  /**
@@ -216,10 +216,7 @@ export class Transcriber extends Resource {
216
216
  return segments;
217
217
  }
218
218
 
219
- private _alignSegments(
220
- segments: WhisperSegment[],
221
- originalChunks: AudioChunk[],
222
- ): DataType.MessageBlock.Transcription[] {
219
+ private _alignSegments(segments: WhisperSegment[], originalChunks: AudioChunk[]): TranscriptionContentBlock[] {
223
220
  // Absolute zero for all relative timestamps in the segments.
224
221
  const zeroTimestamp = originalChunks.at(0)!.timestamp;
225
222
 
@@ -9,7 +9,7 @@ import { create } from '@dxos/echo-schema';
9
9
  import { type DXN } from '@dxos/keys';
10
10
  import { log } from '@dxos/log';
11
11
  import { type EdgeHttpClient } from '@dxos/react-edge-client';
12
- import { DataType } from '@dxos/schema';
12
+ import { MessageType, type TranscriptionContentBlock } from '@dxos/schema';
13
13
 
14
14
  import { MediaStreamRecorder } from './media-stream-recorder';
15
15
  import { Transcriber } from './transcriber';
@@ -30,7 +30,7 @@ const PREFIXED_CHUNKS_AMOUNT = 10;
30
30
  */
31
31
  const TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
32
32
 
33
- export type TranscriptMessageEnricher = (message: DataType.Message) => Promise<DataType.Message>;
33
+ export type TranscriptMessageEnricher = (message: MessageType) => Promise<MessageType>;
34
34
 
35
35
  export type TranscriptionManagerOptions = {
36
36
  edgeClient: EdgeHttpClient;
@@ -51,7 +51,7 @@ export class TranscriptionManager extends Resource {
51
51
  private _identityDid?: string = undefined;
52
52
  private _mediaRecorder?: MediaStreamRecorder = undefined;
53
53
  private _transcriber?: Transcriber = undefined;
54
- private _queue?: Queue<DataType.Message> = undefined;
54
+ private _queue?: Queue<MessageType> = undefined;
55
55
  private _enabled = false;
56
56
 
57
57
  constructor(options: TranscriptionManagerOptions) {
@@ -71,7 +71,7 @@ export class TranscriptionManager extends Resource {
71
71
  setQueue(queueDxn: DXN): TranscriptionManager {
72
72
  if (this._queue?.dxn.toString() !== queueDxn.toString()) {
73
73
  log.info('setQueue', { queueDxn: queueDxn.toString() });
74
- this._queue = new QueueImpl<DataType.Message>(this._edgeClient, queueDxn);
74
+ this._queue = new QueueImpl<MessageType>(this._edgeClient, queueDxn);
75
75
  }
76
76
  return this;
77
77
  }
@@ -129,7 +129,7 @@ export class TranscriptionManager extends Resource {
129
129
  if (this._enabled) {
130
130
  await this._transcriber?.open();
131
131
  // TODO(burdon): Started and stopped blocks appear twice.
132
- const block = create(DataType.Message, {
132
+ const block = create(MessageType, {
133
133
  created: new Date().toISOString(),
134
134
  blocks: [{ type: 'transcription', text: 'Started', started: new Date().toISOString() }],
135
135
  sender: { role: 'assistant' },
@@ -137,7 +137,7 @@ export class TranscriptionManager extends Resource {
137
137
  this._queue?.append([block]);
138
138
  } else {
139
139
  await this._transcriber?.close();
140
- const block = create(DataType.Message, {
140
+ const block = create(MessageType, {
141
141
  created: new Date().toISOString(),
142
142
  blocks: [{ type: 'transcription', text: 'Stopped', started: new Date().toISOString() }],
143
143
  sender: { role: 'assistant' },
@@ -173,12 +173,12 @@ export class TranscriptionManager extends Resource {
173
173
  }
174
174
  }
175
175
 
176
- private async _onSegments(segments: DataType.MessageBlock.Transcription[]) {
176
+ private async _onSegments(segments: TranscriptionContentBlock[]) {
177
177
  if (!this.isOpen || !this._queue) {
178
178
  return;
179
179
  }
180
180
 
181
- let block = create(DataType.Message, {
181
+ let block = create(MessageType, {
182
182
  created: new Date().toISOString(),
183
183
  blocks: segments,
184
184
  sender: { identityDid: this._identityDid },