@dxos/plugin-transcription 0.8.2-main.f11618f → 0.8.2-main.fbd8ed0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. package/dist/lib/browser/{TranscriptContainer-D3EH2SLO.mjs → TranscriptContainer-CXA4ZBHW.mjs} +2 -2
  2. package/dist/lib/browser/{app-graph-builder-7HN3M24G.mjs → app-graph-builder-REQZ5SC5.mjs} +37 -35
  3. package/dist/lib/browser/app-graph-builder-REQZ5SC5.mjs.map +7 -0
  4. package/dist/lib/browser/{chunk-KSUYWWIU.mjs → chunk-3TKXI6TQ.mjs} +6 -6
  5. package/dist/lib/browser/chunk-3TKXI6TQ.mjs.map +7 -0
  6. package/dist/lib/browser/{chunk-YRWBY6UK.mjs → chunk-JHWEZRWC.mjs} +19 -17
  7. package/dist/lib/browser/chunk-JHWEZRWC.mjs.map +7 -0
  8. package/dist/lib/browser/{chunk-BSZSM6KA.mjs → chunk-K2LS2QOM.mjs} +6 -6
  9. package/dist/lib/browser/chunk-K2LS2QOM.mjs.map +7 -0
  10. package/dist/lib/browser/{chunk-IWZ7C3OA.mjs → chunk-LCTMCTYY.mjs} +2 -2
  11. package/dist/lib/browser/index.mjs +7 -7
  12. package/dist/lib/browser/index.mjs.map +3 -3
  13. package/dist/lib/browser/{intent-resolver-ZWKYRFE7.mjs → intent-resolver-DZ62CJPC.mjs} +3 -3
  14. package/dist/lib/browser/intent-resolver-DZ62CJPC.mjs.map +7 -0
  15. package/dist/lib/browser/meta.json +1 -1
  16. package/dist/lib/browser/{react-surface-F7DBDGHS.mjs → react-surface-PUXZNAZL.mjs} +4 -4
  17. package/dist/lib/browser/{settings-6YAHKVKX.mjs → settings-JWDS4J3F.mjs} +2 -2
  18. package/dist/lib/browser/{transcriber-MSBS4VBI.mjs → transcriber-YWWIV6KP.mjs} +3 -3
  19. package/dist/lib/browser/types/index.mjs +1 -1
  20. package/dist/lib/node/{TranscriptContainer-WCBZMA5K.cjs → TranscriptContainer-SSJVLRM6.cjs} +7 -7
  21. package/dist/lib/node/{TranscriptContainer-WCBZMA5K.cjs.map → TranscriptContainer-SSJVLRM6.cjs.map} +2 -2
  22. package/dist/lib/node/{app-graph-builder-BVWYUGUO.cjs → app-graph-builder-OAPU4GDH.cjs} +50 -48
  23. package/dist/lib/node/app-graph-builder-OAPU4GDH.cjs.map +7 -0
  24. package/dist/lib/node/{chunk-IXU57SCA.cjs → chunk-EJWB53WK.cjs} +19 -17
  25. package/dist/lib/node/chunk-EJWB53WK.cjs.map +7 -0
  26. package/dist/lib/node/{chunk-JSCC2AQW.cjs → chunk-EVE3H46F.cjs} +9 -9
  27. package/dist/lib/node/chunk-EVE3H46F.cjs.map +7 -0
  28. package/dist/lib/node/{chunk-OCHTKDL7.cjs → chunk-L27AK6FQ.cjs} +5 -5
  29. package/dist/lib/node/{chunk-7XIIUVN3.cjs → chunk-WKZLGVEJ.cjs} +9 -9
  30. package/dist/lib/node/chunk-WKZLGVEJ.cjs.map +7 -0
  31. package/dist/lib/node/index.cjs +28 -28
  32. package/dist/lib/node/index.cjs.map +3 -3
  33. package/dist/lib/node/{intent-resolver-RM4JOCDV.cjs → intent-resolver-TRBKI24D.cjs} +9 -9
  34. package/dist/lib/node/intent-resolver-TRBKI24D.cjs.map +7 -0
  35. package/dist/lib/node/meta.json +1 -1
  36. package/dist/lib/node/{react-surface-7F37EYZA.cjs → react-surface-SZAMHCX7.cjs} +10 -10
  37. package/dist/lib/node/{settings-ZMB4XRWS.cjs → settings-AWWMOSCV.cjs} +7 -7
  38. package/dist/lib/node/{transcriber-55OE4X6I.cjs → transcriber-AFUXFHCR.cjs} +8 -8
  39. package/dist/lib/node/types/index.cjs +7 -7
  40. package/dist/lib/node/types/index.cjs.map +1 -1
  41. package/dist/lib/node-esm/{TranscriptContainer-GBPY6KKA.mjs → TranscriptContainer-M53LI2TX.mjs} +2 -2
  42. package/dist/lib/node-esm/{app-graph-builder-QK25O5K5.mjs → app-graph-builder-UBP6EOBR.mjs} +37 -35
  43. package/dist/lib/node-esm/app-graph-builder-UBP6EOBR.mjs.map +7 -0
  44. package/dist/lib/node-esm/{chunk-B2BD7S4X.mjs → chunk-4KC3E5T4.mjs} +6 -6
  45. package/dist/lib/node-esm/chunk-4KC3E5T4.mjs.map +7 -0
  46. package/dist/lib/node-esm/{chunk-DXJN2JWV.mjs → chunk-MHQP2PAF.mjs} +2 -2
  47. package/dist/lib/node-esm/{chunk-OS2ITZUL.mjs → chunk-QYG2FOXA.mjs} +19 -17
  48. package/dist/lib/node-esm/chunk-QYG2FOXA.mjs.map +7 -0
  49. package/dist/lib/node-esm/{chunk-UWF6LM27.mjs → chunk-V5IWBPYO.mjs} +6 -6
  50. package/dist/lib/node-esm/chunk-V5IWBPYO.mjs.map +7 -0
  51. package/dist/lib/node-esm/index.mjs +7 -7
  52. package/dist/lib/node-esm/index.mjs.map +3 -3
  53. package/dist/lib/node-esm/{intent-resolver-HDRFXMKP.mjs → intent-resolver-HFVAL274.mjs} +3 -3
  54. package/dist/lib/node-esm/intent-resolver-HFVAL274.mjs.map +7 -0
  55. package/dist/lib/node-esm/meta.json +1 -1
  56. package/dist/lib/node-esm/{react-surface-D3L6VBEV.mjs → react-surface-UEJELL7I.mjs} +4 -4
  57. package/dist/lib/node-esm/{settings-I35TVDUG.mjs → settings-QFNFGFJ5.mjs} +2 -2
  58. package/dist/lib/node-esm/{transcriber-HH7EMCGH.mjs → transcriber-7RJEFPDI.mjs} +3 -3
  59. package/dist/lib/node-esm/types/index.mjs +1 -1
  60. package/dist/types/src/capabilities/app-graph-builder.d.ts +2 -179
  61. package/dist/types/src/capabilities/app-graph-builder.d.ts.map +1 -1
  62. package/dist/types/src/capabilities/index.d.ts +2 -178
  63. package/dist/types/src/capabilities/index.d.ts.map +1 -1
  64. package/dist/types/src/capabilities/intent-resolver.d.ts +2 -2
  65. package/dist/types/src/capabilities/intent-resolver.d.ts.map +1 -1
  66. package/dist/types/src/components/Transcript/Transcript.d.ts +3 -3
  67. package/dist/types/src/components/Transcript/Transcript.d.ts.map +1 -1
  68. package/dist/types/src/components/Transcript/Transcript.stories.d.ts +2 -2
  69. package/dist/types/src/components/Transcript/Transcript.stories.d.ts.map +1 -1
  70. package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts +10 -10
  71. package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts.map +1 -1
  72. package/dist/types/src/components/Transcript/transcript-extension.d.ts +2 -2
  73. package/dist/types/src/components/Transcript/transcript-extension.d.ts.map +1 -1
  74. package/dist/types/src/components/TranscriptionSettings/TranscriptionSettings.d.ts.map +1 -1
  75. package/dist/types/src/components/transcription.stories.d.ts.map +1 -1
  76. package/dist/types/src/entity-extraction/entity-extraction.d.ts +3 -3
  77. package/dist/types/src/entity-extraction/entity-extraction.d.ts.map +1 -1
  78. package/dist/types/src/hooks/useAudioFile.d.ts.map +1 -1
  79. package/dist/types/src/hooks/useAudioTrack.d.ts.map +1 -1
  80. package/dist/types/src/hooks/useQueueModelAdapter.d.ts.map +1 -1
  81. package/dist/types/src/hooks/useTranscriber.d.ts.map +1 -1
  82. package/dist/types/src/hooks/useVoiceInput.d.ts.map +1 -1
  83. package/dist/types/src/testing/testing.d.ts +11 -10
  84. package/dist/types/src/testing/testing.d.ts.map +1 -1
  85. package/dist/types/src/transcriber/transcriber.d.ts +2 -2
  86. package/dist/types/src/transcriber/transcriber.d.ts.map +1 -1
  87. package/dist/types/src/transcriber/transcription-manager.d.ts +2 -2
  88. package/dist/types/src/transcriber/transcription-manager.d.ts.map +1 -1
  89. package/dist/types/src/translations.d.ts +2 -20
  90. package/dist/types/src/translations.d.ts.map +1 -1
  91. package/dist/types/src/types/schema.d.ts +26 -19
  92. package/dist/types/src/types/schema.d.ts.map +1 -1
  93. package/dist/types/src/types/types.d.ts +17 -10
  94. package/dist/types/src/types/types.d.ts.map +1 -1
  95. package/dist/types/src/util/array.d.ts.map +1 -1
  96. package/dist/types/src/util/get-time-str.d.ts.map +1 -1
  97. package/dist/types/src/util/monitor-audio-level.d.ts.map +1 -1
  98. package/dist/types/tsconfig.tsbuildinfo +1 -1
  99. package/package.json +55 -49
  100. package/src/TranscriptionPlugin.tsx +3 -3
  101. package/src/capabilities/app-graph-builder.ts +145 -123
  102. package/src/capabilities/intent-resolver.ts +4 -3
  103. package/src/components/Transcript/Transcript.stories.tsx +6 -6
  104. package/src/components/Transcript/Transcript.tsx +3 -3
  105. package/src/components/Transcript/deprecated/OldTranscript.tsx +19 -13
  106. package/src/components/Transcript/transcript-extension.ts +3 -3
  107. package/src/components/TranscriptContainer.tsx +2 -2
  108. package/src/components/transcription.stories.tsx +17 -15
  109. package/src/entity-extraction/entity-extraction.ts +4 -4
  110. package/src/model/model.test.ts +14 -10
  111. package/src/testing/testing.ts +31 -23
  112. package/src/transcriber/transcriber.browser.test.ts +3 -3
  113. package/src/transcriber/transcriber.ts +6 -3
  114. package/src/transcriber/transcription-manager.ts +8 -8
  115. package/src/types/schema.ts +10 -8
  116. package/src/types/types.ts +14 -12
  117. package/dist/lib/browser/app-graph-builder-7HN3M24G.mjs.map +0 -7
  118. package/dist/lib/browser/chunk-BSZSM6KA.mjs.map +0 -7
  119. package/dist/lib/browser/chunk-KSUYWWIU.mjs.map +0 -7
  120. package/dist/lib/browser/chunk-YRWBY6UK.mjs.map +0 -7
  121. package/dist/lib/browser/intent-resolver-ZWKYRFE7.mjs.map +0 -7
  122. package/dist/lib/node/app-graph-builder-BVWYUGUO.cjs.map +0 -7
  123. package/dist/lib/node/chunk-7XIIUVN3.cjs.map +0 -7
  124. package/dist/lib/node/chunk-IXU57SCA.cjs.map +0 -7
  125. package/dist/lib/node/chunk-JSCC2AQW.cjs.map +0 -7
  126. package/dist/lib/node/intent-resolver-RM4JOCDV.cjs.map +0 -7
  127. package/dist/lib/node-esm/app-graph-builder-QK25O5K5.mjs.map +0 -7
  128. package/dist/lib/node-esm/chunk-B2BD7S4X.mjs.map +0 -7
  129. package/dist/lib/node-esm/chunk-OS2ITZUL.mjs.map +0 -7
  130. package/dist/lib/node-esm/chunk-UWF6LM27.mjs.map +0 -7
  131. package/dist/lib/node-esm/intent-resolver-HDRFXMKP.mjs.map +0 -7
  132. /package/dist/lib/browser/{TranscriptContainer-D3EH2SLO.mjs.map → TranscriptContainer-CXA4ZBHW.mjs.map} +0 -0
  133. /package/dist/lib/browser/{chunk-IWZ7C3OA.mjs.map → chunk-LCTMCTYY.mjs.map} +0 -0
  134. /package/dist/lib/browser/{react-surface-F7DBDGHS.mjs.map → react-surface-PUXZNAZL.mjs.map} +0 -0
  135. /package/dist/lib/browser/{settings-6YAHKVKX.mjs.map → settings-JWDS4J3F.mjs.map} +0 -0
  136. /package/dist/lib/browser/{transcriber-MSBS4VBI.mjs.map → transcriber-YWWIV6KP.mjs.map} +0 -0
  137. /package/dist/lib/node/{chunk-OCHTKDL7.cjs.map → chunk-L27AK6FQ.cjs.map} +0 -0
  138. /package/dist/lib/node/{react-surface-7F37EYZA.cjs.map → react-surface-SZAMHCX7.cjs.map} +0 -0
  139. /package/dist/lib/node/{settings-ZMB4XRWS.cjs.map → settings-AWWMOSCV.cjs.map} +0 -0
  140. /package/dist/lib/node/{transcriber-55OE4X6I.cjs.map → transcriber-AFUXFHCR.cjs.map} +0 -0
  141. /package/dist/lib/node-esm/{TranscriptContainer-GBPY6KKA.mjs.map → TranscriptContainer-M53LI2TX.mjs.map} +0 -0
  142. /package/dist/lib/node-esm/{chunk-DXJN2JWV.mjs.map → chunk-MHQP2PAF.mjs.map} +0 -0
  143. /package/dist/lib/node-esm/{react-surface-D3L6VBEV.mjs.map → react-surface-UEJELL7I.mjs.map} +0 -0
  144. /package/dist/lib/node-esm/{settings-I35TVDUG.mjs.map → settings-QFNFGFJ5.mjs.map} +0 -0
  145. /package/dist/lib/node-esm/{transcriber-HH7EMCGH.mjs.map → transcriber-7RJEFPDI.mjs.map} +0 -0
@@ -11,12 +11,12 @@ import { asyncTimeout } from '@dxos/async';
11
11
  import { type BaseEchoObject, create, ObjectId } from '@dxos/echo-schema';
12
12
  import { DXN } from '@dxos/keys';
13
13
  import { log } from '@dxos/log';
14
- import { MessageType } from '@dxos/schema';
14
+ import { DataType } from '@dxos/schema';
15
15
 
16
16
  import SYSTEM_PROMPT from './system-prompt.tpl?raw';
17
17
 
18
18
  type ProcessTranscriptMessageParams = {
19
- message: MessageType;
19
+ message: DataType.Message;
20
20
  aiService: AIServiceClient;
21
21
  context: {
22
22
  objects?: BaseEchoObject[];
@@ -38,7 +38,7 @@ type ProcessTranscriptMessageParams = {
38
38
  };
39
39
 
40
40
  type ProcessTranscriptMessageResult = {
41
- message: MessageType;
41
+ message: DataType.Message;
42
42
  timeElapsed: number;
43
43
  };
44
44
 
@@ -98,7 +98,7 @@ export const processTranscriptMessage = async (
98
98
  log.info('entity extraction result', { refs: result.references });
99
99
 
100
100
  return {
101
- message: create(MessageType, {
101
+ message: create(DataType.Message, {
102
102
  ...params.message,
103
103
  blocks: params.message.blocks.map((block, i) =>
104
104
  block.type !== 'transcription'
@@ -5,11 +5,15 @@
5
5
  import { EditorView } from '@codemirror/view';
6
6
  import { describe, test } from 'vitest';
7
7
 
8
- import { type MessageType } from '@dxos/schema';
8
+ import { type DataType } from '@dxos/schema';
9
9
 
10
10
  import { SerializationModel, DocumentAdapter, type ChunkRenderer } from './model';
11
11
 
12
- const blockToMarkdown: ChunkRenderer<MessageType> = (message: MessageType, index: number, debug = true): string[] => {
12
+ const blockToMarkdown: ChunkRenderer<DataType.Message> = (
13
+ message: DataType.Message,
14
+ index: number,
15
+ debug = true,
16
+ ): string[] => {
13
17
  return [
14
18
  `###### ${message.sender.name}`,
15
19
  ...message.blocks.filter((block) => block.type === 'transcription').map((block) => block.text),
@@ -21,12 +25,12 @@ const createDate = () => new Date().toISOString();
21
25
 
22
26
  describe('SerializationModel', () => {
23
27
  test('basic', ({ expect }) => {
24
- const model = new SerializationModel<MessageType>(blockToMarkdown);
28
+ const model = new SerializationModel<DataType.Message>(blockToMarkdown);
25
29
  expect(model.chunks.length).to.eq(0);
26
30
  expect(model.doc.toString()).to.eq('');
27
31
 
28
32
  // Create message.
29
- const message: MessageType = {
33
+ const message: DataType.Message = {
30
34
  id: '1',
31
35
  created: createDate(),
32
36
  sender: { name: 'Alice' },
@@ -57,14 +61,14 @@ describe('SerializationModel', () => {
57
61
 
58
62
  test('sync - append', ({ expect }) => {
59
63
  const view = new EditorView({ extensions: [], doc: '' });
60
- const model = new SerializationModel<MessageType>(blockToMarkdown);
64
+ const model = new SerializationModel<DataType.Message>(blockToMarkdown);
61
65
  const adapter = new DocumentAdapter(view);
62
66
  expect(adapter.lineCount()).to.eq(0);
63
67
  expect(view.state.doc.toString()).to.eq('');
64
68
 
65
69
  // Append message.
66
70
  {
67
- const message: MessageType = {
71
+ const message: DataType.Message = {
68
72
  id: '1',
69
73
  created: createDate(),
70
74
  sender: { name: 'Alice' },
@@ -83,7 +87,7 @@ describe('SerializationModel', () => {
83
87
 
84
88
  // Append message.
85
89
  {
86
- const message: MessageType = {
90
+ const message: DataType.Message = {
87
91
  id: '2',
88
92
  created: createDate(),
89
93
  sender: { name: 'Bob' },
@@ -103,13 +107,13 @@ describe('SerializationModel', () => {
103
107
 
104
108
  test('sync - append, update, delete', ({ expect }) => {
105
109
  const view = new EditorView({ extensions: [], doc: '' });
106
- const model = new SerializationModel<MessageType>(blockToMarkdown);
110
+ const model = new SerializationModel<DataType.Message>(blockToMarkdown);
107
111
  const adapter = new DocumentAdapter(view);
108
112
  expect(adapter.lineCount()).to.eq(0);
109
113
 
110
114
  // Append message.
111
115
  {
112
- const message: MessageType = {
116
+ const message: DataType.Message = {
113
117
  id: '1',
114
118
  created: createDate(),
115
119
  sender: { name: 'Alice' },
@@ -142,7 +146,7 @@ describe('SerializationModel', () => {
142
146
 
143
147
  // Append message.
144
148
  {
145
- const message: MessageType = {
149
+ const message: DataType.Message = {
146
150
  id: '2',
147
151
  created: createDate(),
148
152
  sender: { name: 'Bob' },
@@ -2,39 +2,45 @@
2
2
  // Copyright 2025 DXOS.org
3
3
  //
4
4
 
5
+ import { Schema, SchemaAST } from 'effect';
5
6
  import { useEffect, useMemo, useState } from 'react';
6
7
 
7
8
  import { AIServiceEdgeClient } from '@dxos/assistant';
8
9
  import { AI_SERVICE_ENDPOINT } from '@dxos/assistant/testing';
9
10
  import { scheduleTaskInterval } from '@dxos/async';
10
- import { createQueueDxn, Filter, type Queue } from '@dxos/client/echo';
11
+ import { Filter, type Queue } from '@dxos/client/echo';
11
12
  import { Context } from '@dxos/context';
12
- import { AST, create, EchoObject, ObjectId, S } from '@dxos/echo-schema';
13
+ import { Type } from '@dxos/echo';
14
+ import { create, createQueueDxn, ObjectId } from '@dxos/echo-schema';
13
15
  import { IdentityDid } from '@dxos/keys';
14
16
  import { log } from '@dxos/log';
15
17
  import { faker } from '@dxos/random';
16
18
  import { live, makeRef, useQueue, type Space } from '@dxos/react-client/echo';
17
- import { Contact, MessageType, Organization, type TranscriptionContentBlock } from '@dxos/schema';
19
+ import { DataType } from '@dxos/schema';
18
20
  import { Testing, seedTestData } from '@dxos/schema/testing';
19
21
 
20
22
  import { processTranscriptMessage } from '../entity-extraction';
21
23
 
22
24
  // TODO(burdon): Reconcile with plugin-markdown. Move to @dxos/schema/testing.
23
-
24
- export const TestItem = S.Struct({
25
- title: S.String.annotations({
26
- [AST.TitleAnnotationId]: 'Title',
27
- [AST.DescriptionAnnotationId]: 'Product title',
25
+ export const TestItem = Schema.Struct({
26
+ title: Schema.String.annotations({
27
+ [SchemaAST.TitleAnnotationId]: 'Title',
28
+ [SchemaAST.DescriptionAnnotationId]: 'Product title',
29
+ }),
30
+ description: Schema.String.annotations({
31
+ [SchemaAST.TitleAnnotationId]: 'Description',
32
+ [SchemaAST.DescriptionAnnotationId]: 'Product description',
28
33
  }),
29
- description: S.String.annotations({
30
- [AST.TitleAnnotationId]: 'Description',
31
- [AST.DescriptionAnnotationId]: 'Product description',
34
+ }).pipe(
35
+ Type.def({
36
+ typename: 'dxos.org/type/Test',
37
+ version: '0.1.0',
32
38
  }),
33
- }).pipe(EchoObject({ typename: 'dxos.org/type/Test', version: '0.1.0' }));
39
+ );
34
40
 
35
41
  // TODO(wittjosiah): Make builder generic and reuse for all message types.
36
42
  abstract class AbstractMessageBuilder {
37
- abstract createMessage(numSegments?: number): Promise<MessageType>;
43
+ abstract createMessage(numSegments?: number): Promise<DataType.Message>;
38
44
  }
39
45
 
40
46
  /**
@@ -55,7 +61,7 @@ export class MessageBuilder extends AbstractMessageBuilder {
55
61
  super();
56
62
  }
57
63
 
58
- override async createMessage(numSegments = 1): Promise<MessageType> {
64
+ override async createMessage(numSegments = 1): Promise<DataType.Message> {
59
65
  return {
60
66
  id: ObjectId.random().toString(),
61
67
  created: this.next().toISOString(),
@@ -64,7 +70,7 @@ export class MessageBuilder extends AbstractMessageBuilder {
64
70
  };
65
71
  }
66
72
 
67
- createBlock(): TranscriptionContentBlock {
73
+ createBlock(): DataType.MessageBlock.Transcription {
68
74
  let text = faker.lorem.paragraph();
69
75
  if (this._space) {
70
76
  const label = faker.commerce.productName();
@@ -96,7 +102,7 @@ class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
96
102
 
97
103
  space: Space | undefined;
98
104
  currentMessage: number = 0;
99
- transcriptMessages: MessageType[] = [];
105
+ transcriptMessages: DataType.Message[] = [];
100
106
 
101
107
  async connect(space: Space) {
102
108
  this.space = space;
@@ -104,13 +110,15 @@ class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
104
110
  this.transcriptMessages = transcriptMessages;
105
111
  }
106
112
 
107
- override async createMessage(): Promise<MessageType> {
113
+ override async createMessage(): Promise<DataType.Message> {
108
114
  if (!this.space) {
109
115
  throw new Error('Space not connected');
110
116
  }
111
117
 
112
118
  const { objects } = await this.space.db
113
- .query(Filter.or(Filter.schema(Contact), Filter.schema(Organization), Filter.schema(Testing.DocumentType)))
119
+ .query(
120
+ Filter.or(Filter.type(DataType.Person), Filter.type(DataType.Organization), Filter.type(Testing.DocumentType)),
121
+ )
114
122
  .run();
115
123
 
116
124
  log.info('context', { objects });
@@ -135,7 +143,7 @@ type UseTestTranscriptionQueue = (
135
143
  queueId?: ObjectId,
136
144
  running?: boolean,
137
145
  interval?: number,
138
- ) => Queue<MessageType> | undefined;
146
+ ) => Queue<DataType.Message> | undefined;
139
147
 
140
148
  /**
141
149
  * Test transcriptionqueue.
@@ -147,7 +155,7 @@ export const useTestTranscriptionQueue: UseTestTranscriptionQueue = (
147
155
  interval = 1_000,
148
156
  ) => {
149
157
  const queueDxn = useMemo(() => (space ? createQueueDxn(space.id, queueId) : undefined), [space, queueId]);
150
- const queue = useQueue<MessageType>(queueDxn);
158
+ const queue = useQueue<DataType.Message>(queueDxn);
151
159
  const builder = useMemo(() => new MessageBuilder(space), [space]);
152
160
 
153
161
  useEffect(() => {
@@ -157,7 +165,7 @@ export const useTestTranscriptionQueue: UseTestTranscriptionQueue = (
157
165
 
158
166
  const i = setInterval(() => {
159
167
  void builder.createMessage(Math.ceil(Math.random() * 3)).then((message) => {
160
- queue.append([create(MessageType, message)]);
168
+ queue.append([create(DataType.Message, message)]);
161
169
  });
162
170
  }, interval);
163
171
  return () => clearInterval(i);
@@ -177,7 +185,7 @@ export const useTestTranscriptionQueueWithEntityExtraction: UseTestTranscription
177
185
  interval = 1_000,
178
186
  ) => {
179
187
  const queueDxn = useMemo(() => (space ? createQueueDxn(space.id, queueId) : undefined), [space, queueId]);
180
- const queue = useQueue<MessageType>(queueDxn);
188
+ const queue = useQueue<DataType.Message>(queueDxn);
181
189
  const [builder] = useState(() => new EntityExtractionMessageBuilder());
182
190
 
183
191
  useEffect(() => {
@@ -194,7 +202,7 @@ export const useTestTranscriptionQueueWithEntityExtraction: UseTestTranscription
194
202
  ctx,
195
203
  async () => {
196
204
  const message = await builder.createMessage();
197
- queue.append([create(MessageType, message)]);
205
+ queue.append([create(DataType.Message, message)]);
198
206
  },
199
207
  interval,
200
208
  );
@@ -9,7 +9,7 @@ import { WaveFile } from 'wavefile';
9
9
 
10
10
  import { Trigger } from '@dxos/async';
11
11
  import { log } from '@dxos/log';
12
- import { type TranscriptionContentBlock } from '@dxos/schema';
12
+ import { type DataType } from '@dxos/schema';
13
13
  import { openAndClose } from '@dxos/test-utils';
14
14
  import { trace, TRACE_PROCESSOR } from '@dxos/tracing';
15
15
 
@@ -93,7 +93,7 @@ describe.skip('Transcriber', () => {
93
93
  });
94
94
 
95
95
  test.skip('transcription of audio recording', { timeout: 10_000 }, async () => {
96
- const trigger = new Trigger<TranscriptionContentBlock[]>({ autoReset: true });
96
+ const trigger = new Trigger<DataType.MessageBlock.Transcription[]>({ autoReset: true });
97
97
  const recorder = new MockAudioRecorder({
98
98
  buffer: await readFile('test.wav'),
99
99
  chunkDuration: 3_000,
@@ -127,7 +127,7 @@ describe.skip('Transcriber', () => {
127
127
  });
128
128
 
129
129
  test.skip('transcription of audio recording with overlapping chunks', { timeout: 20_000 }, async () => {
130
- const trigger = new Trigger<TranscriptionContentBlock[]>({ autoReset: true });
130
+ const trigger = new Trigger<DataType.MessageBlock.Transcription[]>({ autoReset: true });
131
131
  const recorder = new MockAudioRecorder({
132
132
  buffer: await readFile('test.wav'),
133
133
  chunkDuration: 3_000,
@@ -7,7 +7,7 @@ import { WaveFile } from 'wavefile';
7
7
  import { DeferredTask, Trigger } from '@dxos/async';
8
8
  import { type Context, LifecycleState, Resource } from '@dxos/context';
9
9
  import { log } from '@dxos/log';
10
- import { type TranscriptionContentBlock } from '@dxos/schema';
10
+ import { type DataType } from '@dxos/schema';
11
11
  import { trace } from '@dxos/tracing';
12
12
 
13
13
  import { type AudioRecorder, type AudioChunk } from './audio-recorder';
@@ -65,7 +65,7 @@ export type TranscribeConfig = {
65
65
  export type TranscriberParams = {
66
66
  config: TranscribeConfig;
67
67
  recorder: AudioRecorder;
68
- onSegments: (segments: TranscriptionContentBlock[]) => Promise<void>;
68
+ onSegments: (segments: DataType.MessageBlock.Transcription[]) => Promise<void>;
69
69
  };
70
70
 
71
71
  /**
@@ -216,7 +216,10 @@ export class Transcriber extends Resource {
216
216
  return segments;
217
217
  }
218
218
 
219
- private _alignSegments(segments: WhisperSegment[], originalChunks: AudioChunk[]): TranscriptionContentBlock[] {
219
+ private _alignSegments(
220
+ segments: WhisperSegment[],
221
+ originalChunks: AudioChunk[],
222
+ ): DataType.MessageBlock.Transcription[] {
220
223
  // Absolute zero for all relative timestamps in the segments.
221
224
  const zeroTimestamp = originalChunks.at(0)!.timestamp;
222
225
 
@@ -9,7 +9,7 @@ import { create } from '@dxos/echo-schema';
9
9
  import { type DXN } from '@dxos/keys';
10
10
  import { log } from '@dxos/log';
11
11
  import { type EdgeHttpClient } from '@dxos/react-edge-client';
12
- import { MessageType, type TranscriptionContentBlock } from '@dxos/schema';
12
+ import { DataType } from '@dxos/schema';
13
13
 
14
14
  import { MediaStreamRecorder } from './media-stream-recorder';
15
15
  import { Transcriber } from './transcriber';
@@ -30,7 +30,7 @@ const PREFIXED_CHUNKS_AMOUNT = 10;
30
30
  */
31
31
  const TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
32
32
 
33
- export type TranscriptMessageEnricher = (message: MessageType) => Promise<MessageType>;
33
+ export type TranscriptMessageEnricher = (message: DataType.Message) => Promise<DataType.Message>;
34
34
 
35
35
  export type TranscriptionManagerOptions = {
36
36
  edgeClient: EdgeHttpClient;
@@ -51,7 +51,7 @@ export class TranscriptionManager extends Resource {
51
51
  private _identityDid?: string = undefined;
52
52
  private _mediaRecorder?: MediaStreamRecorder = undefined;
53
53
  private _transcriber?: Transcriber = undefined;
54
- private _queue?: Queue<MessageType> = undefined;
54
+ private _queue?: Queue<DataType.Message> = undefined;
55
55
  private _enabled = false;
56
56
 
57
57
  constructor(options: TranscriptionManagerOptions) {
@@ -71,7 +71,7 @@ export class TranscriptionManager extends Resource {
71
71
  setQueue(queueDxn: DXN): TranscriptionManager {
72
72
  if (this._queue?.dxn.toString() !== queueDxn.toString()) {
73
73
  log.info('setQueue', { queueDxn: queueDxn.toString() });
74
- this._queue = new QueueImpl<MessageType>(this._edgeClient, queueDxn);
74
+ this._queue = new QueueImpl<DataType.Message>(this._edgeClient, queueDxn);
75
75
  }
76
76
  return this;
77
77
  }
@@ -129,7 +129,7 @@ export class TranscriptionManager extends Resource {
129
129
  if (this._enabled) {
130
130
  await this._transcriber?.open();
131
131
  // TODO(burdon): Started and stopped blocks appear twice.
132
- const block = create(MessageType, {
132
+ const block = create(DataType.Message, {
133
133
  created: new Date().toISOString(),
134
134
  blocks: [{ type: 'transcription', text: 'Started', started: new Date().toISOString() }],
135
135
  sender: { role: 'assistant' },
@@ -137,7 +137,7 @@ export class TranscriptionManager extends Resource {
137
137
  this._queue?.append([block]);
138
138
  } else {
139
139
  await this._transcriber?.close();
140
- const block = create(MessageType, {
140
+ const block = create(DataType.Message, {
141
141
  created: new Date().toISOString(),
142
142
  blocks: [{ type: 'transcription', text: 'Stopped', started: new Date().toISOString() }],
143
143
  sender: { role: 'assistant' },
@@ -173,12 +173,12 @@ export class TranscriptionManager extends Resource {
173
173
  }
174
174
  }
175
175
 
176
- private async _onSegments(segments: TranscriptionContentBlock[]) {
176
+ private async _onSegments(segments: DataType.MessageBlock.Transcription[]) {
177
177
  if (!this.isOpen || !this._queue) {
178
178
  return;
179
179
  }
180
180
 
181
- let block = create(MessageType, {
181
+ let block = create(DataType.Message, {
182
182
  created: new Date().toISOString(),
183
183
  blocks: segments,
184
184
  sender: { identityDid: this._identityDid },
@@ -2,17 +2,19 @@
2
2
  // Copyright 2025 DXOS.org
3
3
  //
4
4
 
5
- import { Expando, Ref, S, TypedObject } from '@dxos/echo-schema';
5
+ import { Schema } from 'effect';
6
+
7
+ import { Expando, Ref, TypedObject } from '@dxos/echo-schema';
6
8
 
7
9
  /**
8
10
  * Root transcript object created when the user starts a transcription.
9
11
  */
10
- export const TranscriptSchema = S.Struct({
11
- name: S.String,
12
+ export const TranscriptSchema = Schema.Struct({
13
+ name: Schema.String,
12
14
 
13
15
  // TODO(burdon): Use Date or string?
14
- started: S.optional(S.String),
15
- ended: S.optional(S.String),
16
+ started: Schema.optional(Schema.String),
17
+ ended: Schema.optional(Schema.String),
16
18
 
17
19
  /**
18
20
  * Queue containing TranscriptBlock objects.
@@ -32,8 +34,8 @@ export class TranscriptType extends TypedObject({
32
34
  * First message in queue.
33
35
  * Contains metadata for the recording and transcript.
34
36
  */
35
- const TranscriptHeader = S.Struct({
36
- started: S.optional(S.String),
37
+ const TranscriptHeader = Schema.Struct({
38
+ started: Schema.optional(Schema.String),
37
39
  });
38
40
 
39
- export type TranscriptHeader = S.Schema.Type<typeof TranscriptHeader>;
41
+ export type TranscriptHeader = Schema.Schema.Type<typeof TranscriptHeader>;
@@ -2,8 +2,10 @@
2
2
  // Copyright 2025 DXOS.org
3
3
  //
4
4
 
5
- import { SpaceIdSchema, S, isInstanceOf } from '@dxos/echo-schema';
6
- import { isLiveObject } from '@dxos/react-client/echo';
5
+ import { Schema } from 'effect';
6
+
7
+ import { isInstanceOf } from '@dxos/echo-schema';
8
+ import { isLiveObject, SpaceId } from '@dxos/react-client/echo';
7
9
 
8
10
  import { TranscriptType } from './schema';
9
11
  import { TRANSCRIPTION_PLUGIN } from '../meta';
@@ -19,12 +21,12 @@ export const TRANSCRIPTION_URL = 'https://calls-service.dxos.workers.dev';
19
21
  export namespace TranscriptionAction {
20
22
  const TRANSCRIPTION_ACTION = `${TRANSCRIPTION_PLUGIN}/action`;
21
23
 
22
- export class Create extends S.TaggedClass<Create>()(`${TRANSCRIPTION_ACTION}/create`, {
23
- input: S.Struct({
24
- name: S.optional(S.String),
25
- spaceId: SpaceIdSchema,
24
+ export class Create extends Schema.TaggedClass<Create>()(`${TRANSCRIPTION_ACTION}/create`, {
25
+ input: Schema.Struct({
26
+ name: Schema.optional(Schema.String),
27
+ spaceId: SpaceId,
26
28
  }),
27
- output: S.Struct({
29
+ output: Schema.Struct({
28
30
  object: TranscriptType,
29
31
  }),
30
32
  }) {}
@@ -34,11 +36,11 @@ export const isTranscript = (object: unknown): object is typeof TranscriptType =
34
36
  return isLiveObject(object) && isInstanceOf(TranscriptType, object);
35
37
  };
36
38
 
37
- // TODO(burdon): Create with decode consistently: S.decodeSync(TranscriptionSettingsSchema)({}))
38
- export const TranscriptionSettingsSchema = S.mutable(
39
- S.Struct({
40
- entityExtraction: S.optional(S.Boolean).pipe(S.withConstructorDefault(() => true)),
39
+ // TODO(burdon): Create with decode consistently: Schema.decodeSync(TranscriptionSettingsSchema)({}))
40
+ export const TranscriptionSettingsSchema = Schema.mutable(
41
+ Schema.Struct({
42
+ entityExtraction: Schema.optional(Schema.Boolean).pipe(Schema.withConstructorDefault(() => true)),
41
43
  }),
42
44
  );
43
45
 
44
- export type TranscriptionSettingsProps = S.Schema.Type<typeof TranscriptionSettingsSchema>;
46
+ export type TranscriptionSettingsProps = Schema.Schema.Type<typeof TranscriptionSettingsSchema>;
@@ -1,7 +0,0 @@
1
- {
2
- "version": 3,
3
- "sources": ["../../../src/capabilities/app-graph-builder.ts", "../../../src/entity-extraction/entity-extraction.ts", "raw-loader:./system-prompt.tpl?raw"],
4
- "sourcesContent": ["//\n// Copyright 2025 DXOS.org\n//\n\nimport type { Schema } from 'effect';\n\nimport { Capabilities, contributes, createIntent, type PluginsContext } from '@dxos/app-framework';\nimport { AIServiceEdgeClient, type AIServiceClient } from '@dxos/assistant';\nimport { AI_SERVICE_ENDPOINT } from '@dxos/assistant/testing';\nimport { Filter, fullyQualifiedId, getSpace, makeRef, type Space } from '@dxos/client/echo';\nimport { getSchemaTypename, isInstanceOf, type BaseEchoObject } from '@dxos/echo-schema';\nimport { invariant } from '@dxos/invariant';\nimport { DXN } from '@dxos/keys';\nimport { log } from '@dxos/log';\nimport { ClientCapabilities } from '@dxos/plugin-client';\nimport { PLANK_COMPANION_TYPE, ATTENDABLE_PATH_SEPARATOR } from '@dxos/plugin-deck/types';\nimport { createExtension, type Node } from '@dxos/plugin-graph';\nimport { DocumentType } from '@dxos/plugin-markdown/types';\nimport { MeetingCapabilities, type CallState, type MediaState } from '@dxos/plugin-meeting';\nimport { MeetingType, type MeetingCallProperties } from '@dxos/plugin-meeting/types';\nimport { type buf } from '@dxos/protocols/buf';\nimport { type TranscriptionPayloadSchema } from '@dxos/protocols/buf/dxos/edge/calls_pb';\nimport { type MessageType, Contact, Organization } from '@dxos/schema';\n\nimport { TranscriptionCapabilities } from './capabilities';\nimport { processTranscriptMessage } from '../entity-extraction';\nimport { TRANSCRIPTION_PLUGIN } from '../meta';\nimport { TranscriptionManager } from '../transcriber';\nimport { TranscriptionAction, type TranscriptionSettingsProps, TranscriptType } from '../types';\n\n// TODO(wittjosiah): Factor out.\n// TODO(wittjosiah): Can we stop using protobuf for this?\ntype TranscriptionPayload = buf.MessageInitShape<typeof TranscriptionPayloadSchema>;\n\nconst getMeetingTranscript = async (\n context: PluginsContext,\n meeting: MeetingType,\n): Promise<TranscriptType | undefined> => {\n const { dispatchPromise: dispatch } = context.requestCapability(Capabilities.IntentDispatcher);\n\n const typename = getSchemaTypename(TranscriptType)!;\n const existing = await meeting.artifacts[typename]?.load();\n if (existing instanceof TranscriptType) {\n return existing;\n }\n\n const space = getSpace(meeting);\n invariant(space);\n const { data } = await dispatch(createIntent(TranscriptionAction.Create, { spaceId: space.id }));\n meeting.artifacts[typename] = makeRef(data!.object);\n return data?.object;\n};\n\n// TODO(wittjosiah): Introduce a sound that plays when transcription is activated/deactivated.\nexport default (context: PluginsContext) =>\n contributes(Capabilities.AppGraphBuilder, [\n createExtension({\n id: `${TRANSCRIPTION_PLUGIN}/meeting-transcript`,\n filter: (node): node is Node<MeetingType> => isInstanceOf(MeetingType, node.data),\n actions: ({ node }) => {\n const meeting = node.data;\n const state = context.requestCapability(TranscriptionCapabilities.MeetingTranscriptionState);\n return [\n {\n id: `${fullyQualifiedId(meeting)}/action/start-stop-transcription`,\n data: async () => {\n // NOTE: We are not saving the state of the transcription manager here.\n // We expect the state to be updated through `onCallStateUpdated` once it is propagated through Swarm.\n // This is done to avoid race conditions and to not handle optimistic updates.\n const transcript = await getMeetingTranscript(context, meeting);\n invariant(transcript, 'Failed to create transcript');\n\n const callManager = context.requestCapability(MeetingCapabilities.CallManager);\n callManager.setActivity(getSchemaTypename(TranscriptType)!, {\n queueDxn: transcript.queue.dxn.toString(),\n enabled: !state.enabled,\n });\n },\n properties: {\n label: state.enabled\n ? ['stop transcription label', { ns: TRANSCRIPTION_PLUGIN }]\n : ['start transcription label', { ns: TRANSCRIPTION_PLUGIN }],\n icon: 'ph--subtitles--regular',\n disposition: 'toolbar',\n classNames: state.enabled ? 'bg-callAlert' : '',\n },\n },\n ];\n },\n connector: ({ node }) => {\n const meeting = node.data;\n const client = context.requestCapability(ClientCapabilities.Client);\n const state = context.requestCapability(TranscriptionCapabilities.MeetingTranscriptionState);\n const settings = context\n .requestCapability(Capabilities.SettingsStore)\n .getStore<TranscriptionSettingsProps>(TRANSCRIPTION_PLUGIN)!.value;\n\n // TODO(dmaretskyi): Request via capability.\n const aiClient: AIServiceClient | undefined = new AIServiceEdgeClient({\n endpoint: AI_SERVICE_ENDPOINT.REMOTE,\n });\n\n return [\n {\n id: `${fullyQualifiedId(meeting)}${ATTENDABLE_PATH_SEPARATOR}${getSchemaTypename(TranscriptType)}`,\n type: PLANK_COMPANION_TYPE,\n data: meeting.artifacts[getSchemaTypename(TranscriptType)!]?.target,\n properties: {\n label: ['transcript companion label', { ns: TRANSCRIPTION_PLUGIN }],\n icon: 'ph--subtitles--regular',\n position: 'hoist',\n disposition: 'hidden',\n schema: TranscriptType,\n getIntent: ({ space }: { space: Space }) =>\n createIntent(TranscriptionAction.Create, { spaceId: space.id }),\n\n // TODO(burdon): Is this the right place to inisitalize the state? Change to a capability?\n onJoin: async ({ meeting }: { meeting: MeetingType; roomId: string }) => {\n const identity = client.halo.identity.get();\n invariant(identity);\n const space = getSpace(meeting);\n invariant(space);\n\n let messageEnricher;\n if (aiClient && settings.entityExtraction) {\n messageEnricher = createEntityExtractionEnricher({\n aiClient,\n // TODO(dmaretskyi): Have those be discovered from the schema graph or contributed by capabilities?\n // This forced me to add a dependency on markdown plugin.\n // This will be replaced with a vector search index anyway, so its not a big deal.\n contextTypes: [DocumentType, Contact, Organization],\n space,\n });\n }\n\n // TODO(burdon): The TranscriptionManager singleton is part of the state and should just be updated here.\n state.transcriptionManager = await new TranscriptionManager({\n edgeClient: client.edge,\n messageEnricher,\n })\n .setIdentityDid(identity.did)\n .open();\n },\n onLeave: async () => {\n await state.transcriptionManager?.close();\n state.transcriptionManager = undefined;\n state.enabled = false;\n },\n onCallStateUpdated: async (callState: CallState) => {\n const typename = getSchemaTypename(TranscriptType);\n const transcription = typename ? callState.activities?.[typename] : undefined;\n if (!transcription?.payload) {\n return;\n }\n\n const payload: TranscriptionPayload = transcription.payload;\n state.enabled = !!payload.enabled;\n if (payload.queueDxn) {\n // NOTE: Must set queue before enabling transcription.\n state.transcriptionManager?.setQueue(DXN.parse(payload.queueDxn));\n }\n await state.transcriptionManager?.setEnabled(payload.enabled);\n },\n onMediaStateUpdated: async ([mediaState, isSpeaking]: [MediaState, boolean]) => {\n void state.transcriptionManager?.setAudioTrack(mediaState.audioTrack);\n void state.transcriptionManager?.setRecording(isSpeaking);\n },\n } satisfies MeetingCallProperties,\n },\n ];\n },\n }),\n ]);\n\ntype EntityExtractionEnricherFactoryOptions = {\n aiClient: AIServiceClient;\n contextTypes: Schema.Schema.AnyNoContext[];\n space: Space;\n};\n\nconst createEntityExtractionEnricher = ({ aiClient, contextTypes, space }: EntityExtractionEnricherFactoryOptions) => {\n return async (message: MessageType) => {\n const { objects } = await space.db.query(Filter.or(...contextTypes.map((s) => Filter.schema(s)))).run();\n log.info('context', { objects });\n\n const { message: enhancedMessage, timeElapsed } = await processTranscriptMessage({\n aiService: aiClient,\n message,\n context: { objects: await Promise.all(objects.map(processContextObject)) },\n options: { timeout: ENTITY_EXTRACTOR_TIMEOUT, fallbackToRaw: true },\n });\n\n log.info('entity extraction time', { timeElapsed });\n return enhancedMessage;\n };\n};\n\nconst processContextObject = async (object: BaseEchoObject): Promise<any> => {\n // TODO(dmaretskyi): Documents need special processing is the content is behind a ref.\n // TODO(dmaretskyi): Think about a way to handle this serialization with a decorator.\n // if (isInstanceOf(DocumentType, object)) {\n // return {\n // ...object,\n // content: await object.content.load(),\n // };\n // }\n\n return object;\n};\n\nconst ENTITY_EXTRACTOR_TIMEOUT = 25_000;\n", "//\n// Copyright 2025 DXOS.org\n//\n\nimport { Schema } from 'effect';\n\nimport { createTemplate, Message, structuredOutputParser } from '@dxos/artifact';\nimport type { AIServiceClient } from '@dxos/assistant';\nimport { MixedStreamParser } from '@dxos/assistant';\nimport { asyncTimeout } from '@dxos/async';\nimport { type BaseEchoObject, create, ObjectId } from '@dxos/echo-schema';\nimport { DXN } from '@dxos/keys';\nimport { log } from '@dxos/log';\nimport { MessageType } from '@dxos/schema';\n\nimport SYSTEM_PROMPT from './system-prompt.tpl?raw';\n\ntype ProcessTranscriptMessageParams = {\n message: MessageType;\n aiService: AIServiceClient;\n context: {\n objects?: BaseEchoObject[];\n };\n\n options?: {\n /**\n * Timeout for the entity extraction.\n */\n timeout?: number;\n\n /**\n * Fallback to raw text if the entity extraction fails.\n * Otherwise the function will throw an error.\n * @default false\n */\n fallbackToRaw?: boolean;\n };\n};\n\ntype ProcessTranscriptMessageResult = {\n message: MessageType;\n timeElapsed: number;\n};\n\n/**\n * Process Handlebars template.\n */\nconst createSystemPrompt = (): string => {\n const template = createTemplate(SYSTEM_PROMPT);\n return template({});\n};\n\nconst ReferencedQuotes = Schema.Struct({\n references: Schema.Array(\n Schema.Struct({\n quote: Schema.String,\n id: Schema.String,\n }),\n ).annotations({\n description:\n 'The references to the context objects that are mentioned in the transcript. quote should match the original transcript text exactly, while id is the id of the context object.',\n }),\n});\ninterface ReferencedQuotes extends Schema.Schema.Type<typeof ReferencedQuotes> {}\n\n// TODO(dmaretskyi): Move context to a vector search index.\nexport const processTranscriptMessage = async (\n params: ProcessTranscriptMessageParams,\n): Promise<ProcessTranscriptMessageResult> => {\n try {\n const outputParser = structuredOutputParser(ReferencedQuotes);\n const runParser = async (): Promise<ProcessTranscriptMessageResult> => {\n const startTime = performance.now();\n const result = outputParser.getResult(\n await new MixedStreamParser().parse(\n await params.aiService.execStream({\n model: '@anthropic/claude-3-5-haiku-20241022',\n systemPrompt: createSystemPrompt(),\n history: [\n create(Message, {\n role: 'user',\n content: [\n {\n type: 'text',\n text: `<context>${JSON.stringify(params.context.objects)}</context>`,\n },\n {\n type: 'text',\n text: `<transcript>${JSON.stringify(params.message.blocks)}</transcript>`,\n },\n ],\n }),\n ],\n tools: [outputParser.tool],\n }),\n ),\n );\n log.info('entity extraction result', { refs: result.references });\n\n return {\n message: create(MessageType, {\n ...params.message,\n blocks: params.message.blocks.map((block, i) =>\n block.type !== 'transcription'\n ? block\n : {\n ...block,\n text: postprocessText(block.text, result),\n },\n ),\n }),\n timeElapsed: performance.now() - startTime,\n };\n };\n\n if (params.options?.timeout && params.options.timeout > 0) {\n return await asyncTimeout(runParser(), params.options.timeout);\n } else {\n return await runParser();\n }\n } catch (error) {\n if (params.options?.fallbackToRaw) {\n log.warn('failed to process transcript message, falling back to raw text', { error });\n return {\n message: params.message,\n timeElapsed: 0,\n };\n } else {\n throw error;\n }\n }\n};\n\n/**\n * Finds and replaces all quotes with DXNs references.\n */\n// TODO(dmaretskyi): Lookup and verifiy ids from provided context.\nexport const postprocessText = (text: string, quotes: ReferencedQuotes) => {\n for (const quote of quotes.references) {\n if (!ObjectId.isValid(quote.id)) {\n continue;\n }\n\n // Use a case-insensitive regular expression to replace the quote\n const regex = new RegExp(quote.quote.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&'), 'gi');\n text = text.replace(regex, `[${quote.quote}][${DXN.fromLocalObjectId(quote.id).toString()}]`);\n }\n return text;\n};\n", "You are an AI assistant specialized in processing and enhancing transcripts with contextual information.\nYour task is to analyze a given transcript and extract references to the context objects.\n\nThe context and transcript are provided in the user message in `context` and `transcript` XML tags.\n\nHere are your instructions:\n\n1. Read the transcript and context information provided above.\n\n2. Process each segment of the transcript separately.\n\n3. For each segment:\n a. Analyze the content and identify relevant information from the context.\n b. Extract a number of references to the context objects that are mentioned in the segment.\n c. Output the references in the format `{ \"quote\": \"earnings report\", \"id\": \"01JT0JP9AX0XKGZX4MV4B69VT6\" }`.\n d. If there are no references, just output an empty array.\n\n4. Maintain the exact order and structure of the original transcript segments.\n\nOutput Format:\nThe final output should be a JSON array of references.\n\nExample output structure:\n```json\n{\n \"references\": [\n { \"quote\": \"earnings report\", \"id\": \"01JT0JP9AX0XKGZX4MV4B69VT6\" },\n { \"quote\": \"John\", \"id\": \"01JT0VM0VN3ZKN7GDBM8CV9BA7\" }\n ]\n}\n```\n\nRemember:\n- Only quote the text that is in the transcript.\n- Analyze each segment carefully before processing it.\n- Only reference objects that appear in the context and exactly by their id, If the object is not in the context, do not insert a reference. \n\nNow, process the transcript and context provided below according to these instructions."],
5
- "mappings": ";;;;;;;;;;;;;;;;AAMA,SAASA,cAAcC,aAAaC,oBAAyC;AAC7E,SAASC,2BAAiD;AAC1D,SAASC,2BAA2B;AACpC,SAASC,QAAQC,kBAAkBC,UAAUC,eAA2B;AACxE,SAASC,mBAAmBC,oBAAyC;AACrE,SAASC,iBAAiB;AAC1B,SAASC,OAAAA,YAAW;AACpB,SAASC,OAAAA,YAAW;AACpB,SAASC,0BAA0B;AACnC,SAASC,sBAAsBC,iCAAiC;AAChE,SAASC,uBAAkC;AAC3C,SAASC,oBAAoB;AAC7B,SAASC,2BAA4D;AACrE,SAASC,mBAA+C;AAGxD,SAA2BC,SAASC,oBAAoB;;;AClBxD,SAASC,cAAc;AAEvB,SAASC,gBAAgBC,SAASC,8BAA8B;AAEhE,SAASC,yBAAyB;AAClC,SAASC,oBAAoB;AAC7B,SAA8BC,QAAQC,gBAAgB;AACtD,SAASC,WAAW;AACpB,SAASC,WAAW;AACpB,SAASC,mBAAmB;;;ACb5B;;;;AD+CA,IAAMC,qBAAqB,MAAA;AACzB,QAAMC,WAAWC,eAAeC,qBAAAA;AAChC,SAAOF,SAAS,CAAC,CAAA;AACnB;AAEA,IAAMG,mBAAmBC,OAAOC,OAAO;EACrCC,YAAYF,OAAOG,MACjBH,OAAOC,OAAO;IACZG,OAAOJ,OAAOK;IACdC,IAAIN,OAAOK;EACb,CAAA,CAAA,EACAE,YAAY;IACZC,aACE;EACJ,CAAA;AACF,CAAA;AAIO,IAAMC,2BAA2B,OACtCC,WAAAA;AAEA,MAAI;AACF,UAAMC,eAAeC,uBAAuBb,gBAAAA;AAC5C,UAAMc,YAAY,YAAA;AAChB,YAAMC,YAAYC,YAAYC,IAAG;AACjC,YAAMC,SAASN,aAAaO,UAC1B,MAAM,IAAIC,kBAAAA,EAAoBC,MAC5B,MAAMV,OAAOW,UAAUC,WAAW;QAChCC,OAAO;QACPC,cAAc7B,mBAAAA;QACd8B,SAAS;UACPC,OAAOC,SAAS;YACdC,MAAM;YACNC,SAAS;cACP;gBACEC,MAAM;gBACNC,MAAM,YAAYC,KAAKC,UAAUvB,OAAOwB,QAAQC,OAAO,CAAA;cACzD;cACA;gBACEL,MAAM;gBACNC,MAAM,eAAeC,KAAKC,UAAUvB,OAAO0B,QAAQC,MAAM,CAAA;cAC3D;;UAEJ,CAAA;;QAEFC,OAAO;UAAC3B,aAAa4B;;MACvB,CAAA,CAAA,CAAA;AAGJC,UAAIC,KAAK,4BAA4B;QAAEC,MAAMzB,OAAOf;MAAW,GAAA;;;;;;AAE/D,aAAO;QACLkC,SAASV,OAAOiB,aAAa;UAC3B,GAAGjC,OAAO0B;UACVC,QAAQ3B,OAAO0B,QAAQC,OAAOO,IAAI,CAACC,OAAOC,MACxCD,MAAMf,SAAS,kBACXe,QACA;YACE,GAAGA;YACHd,MAAMgB,gBAAgBF,MAAMd,MAAMd,MAAAA;UACpC,CAAA;QAER,CAAA;QACA+B,aAAajC,YAAYC,IAAG,IAAKF;MACnC;IACF;AAEA,QAAIJ,OAAOuC,SAASC,WAAWxC,OAAOuC,QAAQC,UAAU,GAAG;AACzD,aAAO,MAAMC,aAAatC,UAAAA,GAAaH,OAAOuC,QAAQC,OAAO;IAC/D,OAAO;AACL,aAAO,MAAMrC,UAAAA;IACf;EACF,SAASuC,OAAO;AACd,QAAI1C,OAAOuC,SAASI,eAAe;AACjCb,UAAIc,KAAK,kEAAkE;QAAEF;MAAM,GAAA;;;;;;AACnF,aAAO;QACLhB,SAAS1B,OAAO0B;QAChBY,aAAa;MACf;IACF,OAAO;AACL,YAAMI;IACR;EACF;AACF;AAMO,IAAML,kBAAkB,CAAChB,MAAcwB,WAAAA;AAC5C,aAAWnD,SAASmD,OAAOrD,YAAY;AACrC,QAAI,CAACsD,SAASC,QAAQrD,MAAME,EAAE,GAAG;AAC/B;IACF;AAGA,UAAMoD,QAAQ,IAAIC,OAAOvD,MAAMA,MAAMwD,QAAQ,uBAAuB,MAAA,GAAS,IAAA;AAC7E7B,WAAOA,KAAK6B,QAAQF,OAAO,IAAItD,MAAMA,KAAK,KAAKyD,IAAIC,kBAAkB1D,MAAME,EAAE,EAAEyD,SAAQ,CAAA,GAAK;EAC9F;AACA,SAAOhC;AACT;;;;ADlHA,IAAMiC,uBAAuB,OAC3BC,SACAC,YAAAA;AAEA,QAAM,EAAEC,iBAAiBC,SAAQ,IAAKH,QAAQI,kBAAkBC,aAAaC,gBAAgB;AAE7F,QAAMC,WAAWC,kBAAkBC,cAAAA;AACnC,QAAMC,WAAW,MAAMT,QAAQU,UAAUJ,QAAAA,GAAWK,KAAAA;AACpD,MAAIF,oBAAoBD,gBAAgB;AACtC,WAAOC;EACT;AAEA,QAAMG,QAAQC,SAASb,OAAAA;AACvBc,YAAUF,OAAAA,QAAAA;;;;;;;;;AACV,QAAM,EAAEG,KAAI,IAAK,MAAMb,SAASc,aAAaC,oBAAoBC,QAAQ;IAAEC,SAASP,MAAMQ;EAAG,CAAA,CAAA;AAC7FpB,UAAQU,UAAUJ,QAAAA,IAAYe,QAAQN,KAAMO,MAAM;AAClD,SAAOP,MAAMO;AACf;AAGA,IAAA,4BAAe,CAACvB,YACdwB,YAAYnB,aAAaoB,iBAAiB;EACxCC,gBAAgB;IACdL,IAAI,GAAGM,oBAAAA;IACPC,QAAQ,CAACC,SAAoCC,aAAaC,aAAaF,KAAKb,IAAI;IAChFgB,SAAS,CAAC,EAAEH,KAAI,MAAE;AAChB,YAAM5B,UAAU4B,KAAKb;AACrB,YAAMiB,QAAQjC,QAAQI,kBAAkB8B,0BAA0BC,yBAAyB;AAC3F,aAAO;QACL;UACEd,IAAI,GAAGe,iBAAiBnC,OAAAA,CAAAA;UACxBe,MAAM,YAAA;AAIJ,kBAAMqB,aAAa,MAAMtC,qBAAqBC,SAASC,OAAAA;AACvDc,sBAAUsB,YAAY,+BAAA;;;;;;;;;AAEtB,kBAAMC,cAActC,QAAQI,kBAAkBmC,oBAAoBC,WAAW;AAC7EF,wBAAYG,YAAYjC,kBAAkBC,cAAAA,GAAkB;cAC1DiC,UAAUL,WAAWM,MAAMC,IAAIC,SAAQ;cACvCC,SAAS,CAACb,MAAMa;YAClB,CAAA;UACF;UACAC,YAAY;YACVC,OAAOf,MAAMa,UACT;cAAC;cAA4B;gBAAEG,IAAItB;cAAqB;gBACxD;cAAC;cAA6B;gBAAEsB,IAAItB;cAAqB;;YAC7DuB,MAAM;YACNC,aAAa;YACbC,YAAYnB,MAAMa,UAAU,iBAAiB;UAC/C;QACF;;IAEJ;IACAO,WAAW,CAAC,EAAExB,KAAI,MAAE;AAClB,YAAM5B,UAAU4B,KAAKb;AACrB,YAAMsC,SAAStD,QAAQI,kBAAkBmD,mBAAmBC,MAAM;AAClE,YAAMvB,QAAQjC,QAAQI,kBAAkB8B,0BAA0BC,yBAAyB;AAC3F,YAAMsB,WAAWzD,QACdI,kBAAkBC,aAAaqD,aAAa,EAC5CC,SAAqChC,oBAAAA,EAAuBiC;AAG/D,YAAMC,WAAwC,IAAIC,oBAAoB;QACpEC,UAAUC,oBAAoBC;MAChC,CAAA;AAEA,aAAO;QACL;UACE5C,IAAI,GAAGe,iBAAiBnC,OAAAA,CAAAA,GAAWiE,yBAAAA,GAA4B1D,kBAAkBC,cAAAA,CAAAA;UACjF0D,MAAMC;UACNpD,MAAMf,QAAQU,UAAUH,kBAAkBC,cAAAA,CAAAA,GAAmB4D;UAC7DtB,YAAY;YACVC,OAAO;cAAC;cAA8B;gBAAEC,IAAItB;cAAqB;;YACjEuB,MAAM;YACNoB,UAAU;YACVnB,aAAa;YACboB,QAAQ9D;YACR+D,WAAW,CAAC,EAAE3D,MAAK,MACjBI,aAAaC,oBAAoBC,QAAQ;cAAEC,SAASP,MAAMQ;YAAG,CAAA;;YAG/DoD,QAAQ,OAAO,EAAExE,SAAAA,SAAO,MAA4C;AAClE,oBAAMyE,WAAWpB,OAAOqB,KAAKD,SAASE,IAAG;AACzC7D,wBAAU2D,UAAAA,QAAAA;;;;;;;;;AACV,oBAAM7D,QAAQC,SAASb,QAAAA;AACvBc,wBAAUF,OAAAA,QAAAA;;;;;;;;;AAEV,kBAAIgE;AACJ,kBAAIhB,YAAYJ,SAASqB,kBAAkB;AACzCD,kCAAkBE,+BAA+B;kBAC/ClB;;;;kBAIAmB,cAAc;oBAACC;oBAAcC;oBAASC;;kBACtCtE;gBACF,CAAA;cACF;AAGAoB,oBAAMmD,uBAAuB,MAAM,IAAIC,qBAAqB;gBAC1DC,YAAYhC,OAAOiC;gBACnBV;cACF,CAAA,EACGW,eAAed,SAASe,GAAG,EAC3BC,KAAI;YACT;YACAC,SAAS,YAAA;AACP,oBAAM1D,MAAMmD,sBAAsBQ,MAAAA;AAClC3D,oBAAMmD,uBAAuBS;AAC7B5D,oBAAMa,UAAU;YAClB;YACAgD,oBAAoB,OAAOC,cAAAA;AACzB,oBAAMxF,WAAWC,kBAAkBC,cAAAA;AACnC,oBAAMuF,gBAAgBzF,WAAWwF,UAAUE,aAAa1F,QAAAA,IAAYsF;AACpE,kBAAI,CAACG,eAAeE,SAAS;AAC3B;cACF;AAEA,oBAAMA,UAAgCF,cAAcE;AACpDjE,oBAAMa,UAAU,CAAC,CAACoD,QAAQpD;AAC1B,kBAAIoD,QAAQxD,UAAU;AAEpBT,sBAAMmD,sBAAsBe,SAASC,KAAIC,MAAMH,QAAQxD,QAAQ,CAAA;cACjE;AACA,oBAAMT,MAAMmD,sBAAsBkB,WAAWJ,QAAQpD,OAAO;YAC9D;YACAyD,qBAAqB,OAAO,CAACC,YAAYC,UAAAA,MAAkC;AACzE,mBAAKxE,MAAMmD,sBAAsBsB,cAAcF,WAAWG,UAAU;AACpE,mBAAK1E,MAAMmD,sBAAsBwB,aAAaH,UAAAA;YAChD;UACF;QACF;;IAEJ;EACF,CAAA;CACD;AAQH,IAAM1B,iCAAiC,CAAC,EAAElB,UAAUmB,cAAcnE,MAAK,MAA0C;AAC/G,SAAO,OAAOgG,YAAAA;AACZ,UAAM,EAAEC,QAAO,IAAK,MAAMjG,MAAMkG,GAAGC,MAAMC,OAAOC,GAAE,GAAIlC,aAAamC,IAAI,CAACC,MAAMH,OAAO1C,OAAO6C,CAAAA,CAAAA,CAAAA,CAAAA,EAAMC,IAAG;AACrGC,IAAAA,KAAIC,KAAK,WAAW;MAAET;IAAQ,GAAA;;;;;;AAE9B,UAAM,EAAED,SAASW,iBAAiBC,YAAW,IAAK,MAAMC,yBAAyB;MAC/EC,WAAW9D;MACXgD;MACA7G,SAAS;QAAE8G,SAAS,MAAMc,QAAQC,IAAIf,QAAQK,IAAIW,oBAAAA,CAAAA;MAAuB;MACzEC,SAAS;QAAEC,SAASC;QAA0BC,eAAe;MAAK;IACpE,CAAA;AAEAZ,IAAAA,KAAIC,KAAK,0BAA0B;MAAEE;IAAY,GAAA;;;;;;AACjD,WAAOD;EACT;AACF;AAEA,IAAMM,uBAAuB,OAAOvG,WAAAA;AAUlC,SAAOA;AACT;AAEA,IAAM0G,2BAA2B;",
6
- "names": ["Capabilities", "contributes", "createIntent", "AIServiceEdgeClient", "AI_SERVICE_ENDPOINT", "Filter", "fullyQualifiedId", "getSpace", "makeRef", "getSchemaTypename", "isInstanceOf", "invariant", "DXN", "log", "ClientCapabilities", "PLANK_COMPANION_TYPE", "ATTENDABLE_PATH_SEPARATOR", "createExtension", "DocumentType", "MeetingCapabilities", "MeetingType", "Contact", "Organization", "Schema", "createTemplate", "Message", "structuredOutputParser", "MixedStreamParser", "asyncTimeout", "create", "ObjectId", "DXN", "log", "MessageType", "createSystemPrompt", "template", "createTemplate", "SYSTEM_PROMPT", "ReferencedQuotes", "Schema", "Struct", "references", "Array", "quote", "String", "id", "annotations", "description", "processTranscriptMessage", "params", "outputParser", "structuredOutputParser", "runParser", "startTime", "performance", "now", "result", "getResult", "MixedStreamParser", "parse", "aiService", "execStream", "model", "systemPrompt", "history", "create", "Message", "role", "content", "type", "text", "JSON", "stringify", "context", "objects", "message", "blocks", "tools", "tool", "log", "info", "refs", "MessageType", "map", "block", "i", "postprocessText", "timeElapsed", "options", "timeout", "asyncTimeout", "error", "fallbackToRaw", "warn", "quotes", "ObjectId", "isValid", "regex", "RegExp", "replace", "DXN", "fromLocalObjectId", "toString", "getMeetingTranscript", "context", "meeting", "dispatchPromise", "dispatch", "requestCapability", "Capabilities", "IntentDispatcher", "typename", "getSchemaTypename", "TranscriptType", "existing", "artifacts", "load", "space", "getSpace", "invariant", "data", "createIntent", "TranscriptionAction", "Create", "spaceId", "id", "makeRef", "object", "contributes", "AppGraphBuilder", "createExtension", "TRANSCRIPTION_PLUGIN", "filter", "node", "isInstanceOf", "MeetingType", "actions", "state", "TranscriptionCapabilities", "MeetingTranscriptionState", "fullyQualifiedId", "transcript", "callManager", "MeetingCapabilities", "CallManager", "setActivity", "queueDxn", "queue", "dxn", "toString", "enabled", "properties", "label", "ns", "icon", "disposition", "classNames", "connector", "client", "ClientCapabilities", "Client", "settings", "SettingsStore", "getStore", "value", "aiClient", "AIServiceEdgeClient", "endpoint", "AI_SERVICE_ENDPOINT", "REMOTE", "ATTENDABLE_PATH_SEPARATOR", "type", "PLANK_COMPANION_TYPE", "target", "position", "schema", "getIntent", "onJoin", "identity", "halo", "get", "messageEnricher", "entityExtraction", "createEntityExtractionEnricher", "contextTypes", "DocumentType", "Contact", "Organization", "transcriptionManager", "TranscriptionManager", "edgeClient", "edge", "setIdentityDid", "did", "open", "onLeave", "close", "undefined", "onCallStateUpdated", "callState", "transcription", "activities", "payload", "setQueue", "DXN", "parse", "setEnabled", "onMediaStateUpdated", "mediaState", "isSpeaking", "setAudioTrack", "audioTrack", "setRecording", "message", "objects", "db", "query", "Filter", "or", "map", "s", "run", "log", "info", "enhancedMessage", "timeElapsed", "processTranscriptMessage", "aiService", "Promise", "all", "processContextObject", "options", "timeout", "ENTITY_EXTRACTOR_TIMEOUT", "fallbackToRaw"]
7
- }