@dxos/plugin-transcription 0.8.4-main.2e9d522 → 0.8.4-main.5ea62a8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. package/dist/lib/browser/TranscriptContainer-OX7OOODC.mjs +11 -0
  2. package/dist/lib/browser/blueprint-definition-QQSGTX5F.mjs +142 -0
  3. package/dist/lib/browser/blueprint-definition-QQSGTX5F.mjs.map +7 -0
  4. package/dist/lib/browser/{chunk-4FJIYH7T.mjs → chunk-26XDG6GQ.mjs} +2 -2
  5. package/dist/lib/browser/{chunk-5WH45KBY.mjs → chunk-3X7HP44M.mjs} +8 -1
  6. package/dist/lib/browser/{chunk-5WH45KBY.mjs.map → chunk-3X7HP44M.mjs.map} +1 -1
  7. package/dist/lib/browser/{chunk-5KPYIC3N.mjs → chunk-5I4BMOGE.mjs} +422 -402
  8. package/dist/lib/browser/chunk-5I4BMOGE.mjs.map +7 -0
  9. package/dist/lib/browser/{chunk-FFEPEPCU.mjs → chunk-BA64VLLR.mjs} +85 -94
  10. package/dist/lib/browser/chunk-BA64VLLR.mjs.map +7 -0
  11. package/dist/lib/browser/chunk-HXVGV7T6.mjs +63 -0
  12. package/dist/lib/browser/chunk-HXVGV7T6.mjs.map +7 -0
  13. package/dist/lib/browser/{chunk-U6QBOGTN.mjs → chunk-TNEQFHIU.mjs} +2 -2
  14. package/dist/lib/browser/{chunk-U6QBOGTN.mjs.map → chunk-TNEQFHIU.mjs.map} +1 -1
  15. package/dist/lib/browser/index.mjs +18 -12
  16. package/dist/lib/browser/index.mjs.map +3 -3
  17. package/dist/lib/browser/intent-resolver-T3EGLS3L.mjs +25 -0
  18. package/dist/lib/browser/intent-resolver-T3EGLS3L.mjs.map +7 -0
  19. package/dist/lib/browser/meta.json +1 -1
  20. package/dist/lib/browser/{react-surface-E5D5AE3F.mjs → react-surface-QYX74JNG.mjs} +8 -8
  21. package/dist/lib/browser/react-surface-QYX74JNG.mjs.map +7 -0
  22. package/dist/lib/browser/{transcriber-PXT4QBFY.mjs → transcriber-JOOAQVYV.mjs} +5 -5
  23. package/dist/lib/browser/transcriber-JOOAQVYV.mjs.map +7 -0
  24. package/dist/lib/browser/types/index.mjs +6 -8
  25. package/dist/lib/node-esm/{TranscriptContainer-VBAOM5DG.mjs → TranscriptContainer-BUWK63SI.mjs} +4 -4
  26. package/dist/lib/node-esm/blueprint-definition-4PLCTH5Q.mjs +143 -0
  27. package/dist/lib/node-esm/blueprint-definition-4PLCTH5Q.mjs.map +7 -0
  28. package/dist/lib/node-esm/{chunk-RHF2ALML.mjs → chunk-3QAIZGMW.mjs} +2 -2
  29. package/dist/lib/node-esm/{chunk-RHF2ALML.mjs.map → chunk-3QAIZGMW.mjs.map} +1 -1
  30. package/dist/lib/node-esm/{chunk-TJKAO7VR.mjs → chunk-DT3AOC3E.mjs} +422 -402
  31. package/dist/lib/node-esm/chunk-DT3AOC3E.mjs.map +7 -0
  32. package/dist/lib/node-esm/{chunk-HT3W7LDP.mjs → chunk-J5GQ3KGX.mjs} +85 -94
  33. package/dist/lib/node-esm/chunk-J5GQ3KGX.mjs.map +7 -0
  34. package/dist/lib/node-esm/{chunk-Y2B4HWSS.mjs → chunk-NWD757KH.mjs} +2 -2
  35. package/dist/lib/node-esm/{chunk-XMUCZBP3.mjs → chunk-QARUDLHN.mjs} +7 -1
  36. package/dist/lib/node-esm/{chunk-XMUCZBP3.mjs.map → chunk-QARUDLHN.mjs.map} +1 -1
  37. package/dist/lib/node-esm/chunk-W72HVQMD.mjs +64 -0
  38. package/dist/lib/node-esm/chunk-W72HVQMD.mjs.map +7 -0
  39. package/dist/lib/node-esm/index.mjs +18 -12
  40. package/dist/lib/node-esm/index.mjs.map +3 -3
  41. package/dist/lib/node-esm/intent-resolver-VDHCUAIO.mjs +26 -0
  42. package/dist/lib/node-esm/intent-resolver-VDHCUAIO.mjs.map +7 -0
  43. package/dist/lib/node-esm/meta.json +1 -1
  44. package/dist/lib/node-esm/{react-surface-QRDJOHN5.mjs → react-surface-TM7XOQBR.mjs} +8 -8
  45. package/dist/lib/node-esm/react-surface-TM7XOQBR.mjs.map +7 -0
  46. package/dist/lib/node-esm/{transcriber-YUTEVANE.mjs → transcriber-QPS55YP2.mjs} +5 -5
  47. package/dist/lib/node-esm/transcriber-QPS55YP2.mjs.map +7 -0
  48. package/dist/lib/node-esm/types/index.mjs +6 -8
  49. package/dist/types/src/TranscriptionPlugin.d.ts.map +1 -1
  50. package/dist/types/src/capabilities/blueprint-definition.d.ts +5 -0
  51. package/dist/types/src/capabilities/blueprint-definition.d.ts.map +1 -0
  52. package/dist/types/src/capabilities/capabilities.d.ts +1 -1
  53. package/dist/types/src/capabilities/capabilities.d.ts.map +1 -1
  54. package/dist/types/src/capabilities/index.d.ts +3 -2
  55. package/dist/types/src/capabilities/index.d.ts.map +1 -1
  56. package/dist/types/src/capabilities/intent-resolver.d.ts +1 -1
  57. package/dist/types/src/capabilities/intent-resolver.d.ts.map +1 -1
  58. package/dist/types/src/capabilities/react-surface.d.ts +1 -1
  59. package/dist/types/src/capabilities/transcriber.d.ts +1 -1
  60. package/dist/types/src/capabilities/transcriber.d.ts.map +1 -1
  61. package/dist/types/src/components/Transcript/Transcript.d.ts +4 -4
  62. package/dist/types/src/components/Transcript/Transcript.d.ts.map +1 -1
  63. package/dist/types/src/components/Transcript/Transcript.stories.d.ts +32 -15
  64. package/dist/types/src/components/Transcript/Transcript.stories.d.ts.map +1 -1
  65. package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts +2 -2
  66. package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts.map +1 -1
  67. package/dist/types/src/components/TranscriptContainer.d.ts +2 -2
  68. package/dist/types/src/components/TranscriptContainer.d.ts.map +1 -1
  69. package/dist/types/src/components/stories/FileTranscription.stories.d.ts +5 -2
  70. package/dist/types/src/components/stories/FileTranscription.stories.d.ts.map +1 -1
  71. package/dist/types/src/components/stories/KeyWordRecognition.stories.d.ts +10 -5
  72. package/dist/types/src/components/stories/KeyWordRecognition.stories.d.ts.map +1 -1
  73. package/dist/types/src/components/stories/MicrophoneTranscription.stories.d.ts +7 -4
  74. package/dist/types/src/components/stories/MicrophoneTranscription.stories.d.ts.map +1 -1
  75. package/dist/types/src/components/stories/TranscriptionStory.d.ts +1 -0
  76. package/dist/types/src/components/stories/TranscriptionStory.d.ts.map +1 -1
  77. package/dist/types/src/functions/index.d.ts +3 -0
  78. package/dist/types/src/functions/index.d.ts.map +1 -0
  79. package/dist/types/src/functions/open.d.ts +7 -0
  80. package/dist/types/src/functions/open.d.ts.map +1 -0
  81. package/dist/types/src/functions/summarize.d.ts +11 -0
  82. package/dist/types/src/functions/summarize.d.ts.map +1 -0
  83. package/dist/types/src/hooks/useAudioFile.d.ts.map +1 -1
  84. package/dist/types/src/hooks/useVoiceInput.d.ts.map +1 -1
  85. package/dist/types/src/segments-normalization/normalization.d.ts +366 -321
  86. package/dist/types/src/segments-normalization/normalization.d.ts.map +1 -1
  87. package/dist/types/src/testing/testing.d.ts +1 -1
  88. package/dist/types/src/testing/testing.d.ts.map +1 -1
  89. package/dist/types/src/transcriber/media-stream-recorder.d.ts +1 -1
  90. package/dist/types/src/transcriber/media-stream-recorder.d.ts.map +1 -1
  91. package/dist/types/src/transcriber/transcriber.d.ts.map +1 -1
  92. package/dist/types/src/transcriber/transcription-manager.d.ts +3 -7
  93. package/dist/types/src/transcriber/transcription-manager.d.ts.map +1 -1
  94. package/dist/types/src/types/Transcript.d.ts +30 -0
  95. package/dist/types/src/types/Transcript.d.ts.map +1 -0
  96. package/dist/types/src/types/TranscriptAction.d.ts +23 -0
  97. package/dist/types/src/types/TranscriptAction.d.ts.map +1 -0
  98. package/dist/types/src/types/index.d.ts +2 -1
  99. package/dist/types/src/types/index.d.ts.map +1 -1
  100. package/dist/types/src/types/types.d.ts +0 -26
  101. package/dist/types/src/types/types.d.ts.map +1 -1
  102. package/dist/types/tsconfig.tsbuildinfo +1 -1
  103. package/package.json +55 -53
  104. package/src/TranscriptionPlugin.tsx +10 -5
  105. package/src/capabilities/blueprint-definition.ts +31 -0
  106. package/src/capabilities/capabilities.ts +3 -3
  107. package/src/capabilities/index.ts +1 -0
  108. package/src/capabilities/intent-resolver.ts +5 -11
  109. package/src/capabilities/react-surface.tsx +2 -2
  110. package/src/capabilities/transcriber.ts +3 -2
  111. package/src/components/Transcript/Transcript.stories.tsx +12 -13
  112. package/src/components/Transcript/Transcript.tsx +8 -7
  113. package/src/components/Transcript/deprecated/OldTranscript.tsx +4 -4
  114. package/src/components/TranscriptContainer.tsx +5 -4
  115. package/src/components/stories/FileTranscription.stories.tsx +22 -12
  116. package/src/components/stories/KeyWordRecognition.stories.tsx +12 -6
  117. package/src/components/stories/MicrophoneTranscription.stories.tsx +10 -9
  118. package/src/components/stories/TranscriptionStory.tsx +9 -5
  119. package/src/functions/index.ts +6 -0
  120. package/src/functions/open.ts +37 -0
  121. package/src/functions/summarize.ts +97 -0
  122. package/src/hooks/useAudioFile.ts +1 -10
  123. package/src/hooks/useVoiceInput.ts +3 -2
  124. package/src/model/model.test.ts +1 -1
  125. package/src/segments-normalization/message-normalizer.ts +1 -1
  126. package/src/segments-normalization/normalization.test.ts +4 -4
  127. package/src/segments-normalization/normalization.ts +37 -36
  128. package/src/testing/testing.ts +3 -3
  129. package/src/transcriber/media-stream-recorder.ts +1 -1
  130. package/src/transcriber/transcriber.browser.test.ts +2 -1
  131. package/src/transcriber/transcriber.ts +2 -1
  132. package/src/transcriber/transcription-manager.ts +17 -38
  133. package/src/types/{schema.ts → Transcript.ts} +11 -8
  134. package/src/types/TranscriptAction.ts +24 -0
  135. package/src/types/index.ts +2 -1
  136. package/src/types/types.ts +0 -23
  137. package/dist/lib/browser/TranscriptContainer-6WCKCXQ6.mjs +0 -11
  138. package/dist/lib/browser/chunk-5KPYIC3N.mjs.map +0 -7
  139. package/dist/lib/browser/chunk-FFEPEPCU.mjs.map +0 -7
  140. package/dist/lib/browser/chunk-R7INNNUQ.mjs +0 -55
  141. package/dist/lib/browser/chunk-R7INNNUQ.mjs.map +0 -7
  142. package/dist/lib/browser/intent-resolver-3O33MMP5.mjs +0 -30
  143. package/dist/lib/browser/intent-resolver-3O33MMP5.mjs.map +0 -7
  144. package/dist/lib/browser/react-surface-E5D5AE3F.mjs.map +0 -7
  145. package/dist/lib/browser/transcriber-PXT4QBFY.mjs.map +0 -7
  146. package/dist/lib/node-esm/chunk-HT3W7LDP.mjs.map +0 -7
  147. package/dist/lib/node-esm/chunk-TJKAO7VR.mjs.map +0 -7
  148. package/dist/lib/node-esm/chunk-U65ZWK3L.mjs +0 -56
  149. package/dist/lib/node-esm/chunk-U65ZWK3L.mjs.map +0 -7
  150. package/dist/lib/node-esm/intent-resolver-YMN5SXNC.mjs +0 -31
  151. package/dist/lib/node-esm/intent-resolver-YMN5SXNC.mjs.map +0 -7
  152. package/dist/lib/node-esm/react-surface-QRDJOHN5.mjs.map +0 -7
  153. package/dist/lib/node-esm/transcriber-YUTEVANE.mjs.map +0 -7
  154. package/dist/types/src/types/schema.d.ts +0 -41
  155. package/dist/types/src/types/schema.d.ts.map +0 -1
  156. /package/dist/lib/browser/{TranscriptContainer-6WCKCXQ6.mjs.map → TranscriptContainer-OX7OOODC.mjs.map} +0 -0
  157. /package/dist/lib/browser/{chunk-4FJIYH7T.mjs.map → chunk-26XDG6GQ.mjs.map} +0 -0
  158. /package/dist/lib/node-esm/{TranscriptContainer-VBAOM5DG.mjs.map → TranscriptContainer-BUWK63SI.mjs.map} +0 -0
  159. /package/dist/lib/node-esm/{chunk-Y2B4HWSS.mjs.map → chunk-NWD757KH.mjs.map} +0 -0
@@ -4,7 +4,7 @@
4
4
 
5
5
  import '@dxos-theme';
6
6
 
7
- import { type StoryObj, type Meta } from '@storybook/react-vite';
7
+ import { type Meta, type StoryObj } from '@storybook/react-vite';
8
8
  import React, { useEffect, useMemo, useState } from 'react';
9
9
 
10
10
  import { log } from '@dxos/log';
@@ -17,7 +17,11 @@ type Word = {
17
17
  matched: boolean;
18
18
  };
19
19
 
20
- const SpeechRecognition = ({ keywords }: { keywords: string[] }) => {
20
+ type StoryProps = {
21
+ keywords: string[];
22
+ };
23
+
24
+ const DefaultStory = ({ keywords }: StoryProps) => {
21
25
  const [running, setRunning] = useState(false);
22
26
  const [matchingWords, setMatchingWords] = useState<Word[]>(keywords.map((word) => ({ text: word, matched: false })));
23
27
 
@@ -113,15 +117,17 @@ const SpeechRecognition = ({ keywords }: { keywords: string[] }) => {
113
117
  );
114
118
  };
115
119
 
116
- const meta: Meta<typeof SpeechRecognition> = {
120
+ const meta = {
117
121
  title: 'plugins/plugin-transcription/KeyWordDetection',
122
+ render: DefaultStory,
118
123
  decorators: [withTheme, withLayout({ fullscreen: true, classNames: 'justify-center' })],
119
- };
124
+ } satisfies Meta<typeof DefaultStory>;
120
125
 
121
126
  export default meta;
122
127
 
123
- export const SpeechRecognitionAPI: StoryObj<typeof SpeechRecognition> = {
124
- render: SpeechRecognition,
128
+ type Story = StoryObj<typeof meta>;
129
+
130
+ export const SpeechRecognitionAPI: Story = {
125
131
  args: {
126
132
  keywords: ['kai', 'computer', 'hello'],
127
133
  },
@@ -15,7 +15,7 @@ import {
15
15
  extractionNerFn,
16
16
  getNer,
17
17
  processTranscriptMessage,
18
- } from '@dxos/assistant';
18
+ } from '@dxos/assistant/extraction';
19
19
  import { Filter, Obj, type Type } from '@dxos/echo';
20
20
  import { MemoryQueue } from '@dxos/echo-db';
21
21
  import { createQueueDXN } from '@dxos/echo-schema';
@@ -30,17 +30,18 @@ import { ThemePlugin } from '@dxos/plugin-theme';
30
30
  import { IndexKind, useSpace } from '@dxos/react-client/echo';
31
31
  import { defaultTx } from '@dxos/react-ui-theme';
32
32
  import { DataType } from '@dxos/schema';
33
- import { seedTestData, Testing } from '@dxos/schema/testing';
33
+ import { Testing, seedTestData } from '@dxos/schema/testing';
34
34
  import { withLayout, withTheme } from '@dxos/storybook-utils';
35
35
 
36
- import { TranscriptionStory } from './TranscriptionStory';
37
- import { useIsSpeaking } from './useIsSpeaking';
38
- import { TranscriptionPlugin } from '../../TranscriptionPlugin';
39
36
  import { useAudioTrack, useQueueModelAdapter, useTranscriber } from '../../hooks';
40
37
  import { TestItem } from '../../testing';
41
38
  import { type MediaStreamRecorderParams, type TranscriberParams } from '../../transcriber';
39
+ import { TranscriptionPlugin } from '../../TranscriptionPlugin';
42
40
  import { renderMarkdown } from '../Transcript';
43
41
 
42
+ import { TranscriptionStory } from './TranscriptionStory';
43
+ import { useIsSpeaking } from './useIsSpeaking';
44
+
44
45
  const TRANSCRIBER_CONFIG = {
45
46
  transcribeAfterChunksAmount: 50,
46
47
  prefixBufferChunksAmount: 10,
@@ -192,7 +193,7 @@ const DefaultStory = ({
192
193
  return <TranscriptionStory model={model} running={running} onRunningChange={setRunning} />;
193
194
  };
194
195
 
195
- const meta: Meta<typeof DefaultStory> = {
196
+ const meta = {
196
197
  title: 'plugins/plugin-transcription/MicrophoneTranscription',
197
198
  render: DefaultStory,
198
199
  decorators: [
@@ -202,7 +203,7 @@ const meta: Meta<typeof DefaultStory> = {
202
203
  StorybookLayoutPlugin(),
203
204
  ClientPlugin({
204
205
  types: [TestItem, DataType.Person, DataType.Organization, Testing.DocumentType],
205
- onClientInitialized: async (_, client) => {
206
+ onClientInitialized: async ({ client }) => {
206
207
  await client.halo.createIdentity();
207
208
  await client.spaces.waitUntilReady();
208
209
  await client.spaces.default.waitUntilReady();
@@ -233,11 +234,11 @@ const meta: Meta<typeof DefaultStory> = {
233
234
  withTheme,
234
235
  withLayout({ fullscreen: true, classNames: 'justify-center' }),
235
236
  ],
236
- };
237
+ } satisfies Meta<typeof DefaultStory>;
237
238
 
238
239
  export default meta;
239
240
 
240
- type Story = StoryObj<typeof DefaultStory>;
241
+ type Story = StoryObj<typeof meta>;
241
242
 
242
243
  export const Default: Story = {
243
244
  args: {
@@ -12,14 +12,15 @@ import { ScrollContainer } from '@dxos/react-ui-components';
12
12
  import { type DataType } from '@dxos/schema';
13
13
 
14
14
  import { type SerializationModel } from '../../model';
15
- import { Transcript } from '../Transcript';
15
+ import { TranscriptView } from '../Transcript';
16
16
 
17
17
  export const TranscriptionStory: FC<{
18
18
  model: SerializationModel<DataType.Message>;
19
+ disabled?: boolean;
19
20
  running: boolean;
20
21
  onRunningChange: Dispatch<SetStateAction<boolean>>;
21
22
  audioRef?: RefObject<HTMLAudioElement>;
22
- }> = ({ model, running, onRunningChange, audioRef }) => {
23
+ }> = ({ model, running, onRunningChange, audioRef, disabled }) => {
23
24
  const space = useSpace();
24
25
 
25
26
  return (
@@ -28,14 +29,17 @@ export const TranscriptionStory: FC<{
28
29
  <Toolbar.Root>
29
30
  <IconButton
30
31
  iconOnly
32
+ disabled={disabled}
31
33
  icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
32
34
  label={running ? 'Pause' : 'Play'}
33
35
  onClick={() => onRunningChange((running) => !running)}
34
36
  />
35
37
  </Toolbar.Root>
36
- <ScrollContainer>
37
- <Transcript space={space} model={model} attendableId='story' />
38
- </ScrollContainer>
38
+ <ScrollContainer.Root pin>
39
+ <ScrollContainer.Content>
40
+ <TranscriptView space={space} model={model} attendableId='story' />
41
+ </ScrollContainer.Content>
42
+ </ScrollContainer.Root>
39
43
  </div>
40
44
  );
41
45
  };
@@ -0,0 +1,6 @@
1
+ //
2
+ // Copyright 2024 DXOS.org
3
+ //
4
+
5
+ export { default as open } from './open';
6
+ export { default as summarize } from './summarize';
@@ -0,0 +1,37 @@
1
+ //
2
+ // Copyright 2025 DXOS.org
3
+ //
4
+
5
+ import { Effect, Schema } from 'effect';
6
+
7
+ import { ArtifactId } from '@dxos/assistant';
8
+ import { Obj } from '@dxos/echo';
9
+ import { DatabaseService, QueueService, defineFunction } from '@dxos/functions';
10
+ import { DataType } from '@dxos/schema';
11
+
12
+ import { renderMarkdown } from '../components';
13
+ import { Transcript } from '../types';
14
+
15
+ export default defineFunction({
16
+ name: 'dxos.org/function/transcription/open',
17
+ description: 'Opens and reads the contents of a transcription object.',
18
+ inputSchema: Schema.Struct({
19
+ id: ArtifactId.annotations({
20
+ description: 'The ID of the transcription object.',
21
+ }),
22
+ }),
23
+ outputSchema: Schema.Struct({
24
+ content: Schema.String,
25
+ }),
26
+ handler: Effect.fn(function* ({ data: { id } }) {
27
+ const transcript = yield* DatabaseService.resolve(ArtifactId.toDXN(id), Transcript.Transcript);
28
+ const { dxn } = yield* Effect.promise(() => transcript.queue.load());
29
+ const queue = yield* QueueService.getQueue(dxn);
30
+ yield* Effect.promise(() => queue?.queryObjects());
31
+ const content = queue?.objects
32
+ .filter((message) => Obj.instanceOf(DataType.Message, message))
33
+ .flatMap((message, index) => renderMarkdown([])(message, index))
34
+ .join('\n\n');
35
+ return { content };
36
+ }),
37
+ });
@@ -0,0 +1,97 @@
1
+ //
2
+ // Copyright 2025 DXOS.org
3
+ //
4
+
5
+ import { Array, Effect, Layer, Option, Schema, pipe } from 'effect';
6
+
7
+ import { AiService, ConsolePrinter, ToolExecutionService, ToolResolverService } from '@dxos/ai';
8
+ import { AiSession, GenerationObserver } from '@dxos/assistant';
9
+ import { TracingService, defineFunction } from '@dxos/functions';
10
+ import { trim } from '@dxos/util';
11
+
12
+ /**
13
+ * Summarize a transcript of a meeting.
14
+ */
15
+ export default defineFunction({
16
+ name: 'dxos.org/function/transcription/summarize',
17
+ description: 'Summarize a transcript of a meeting.',
18
+ inputSchema: Schema.Struct({
19
+ transcript: Schema.String.annotations({
20
+ description: 'The transcript of the meeting.',
21
+ }),
22
+ notes: Schema.optional(Schema.String).annotations({
23
+ description: 'Additional notes from the participants.',
24
+ }),
25
+ }),
26
+ outputSchema: Schema.Struct({
27
+ summary: Schema.String.annotations({
28
+ description: 'The summary of the transcript.',
29
+ }),
30
+ }),
31
+ handler: Effect.fnUntraced(
32
+ function* ({ data: { transcript, notes } }) {
33
+ const result = yield* new AiSession().run({
34
+ prompt: `Transcript: ${transcript}\n\nNotes: ${notes}`,
35
+ history: [],
36
+ system: systemPrompt,
37
+ observer: GenerationObserver.fromPrinter(new ConsolePrinter({ tag: 'summarize' })),
38
+ });
39
+
40
+ const summary = pipe(
41
+ result,
42
+ Array.findLast((msg) => msg.sender.role === 'assistant' && msg.blocks.some((block) => block._tag === 'text')),
43
+ Option.flatMap((msg) =>
44
+ pipe(
45
+ msg.blocks,
46
+ Array.findLast((block) => block._tag === 'text'),
47
+ Option.map((block) => block.text),
48
+ ),
49
+ ),
50
+ Option.getOrThrowWith(() => new Error('No summary found')),
51
+ );
52
+
53
+ return { summary };
54
+ },
55
+ Effect.provide(
56
+ Layer.mergeAll(
57
+ AiService.model('@anthropic/claude-sonnet-4-0'),
58
+ ToolResolverService.layerEmpty,
59
+ ToolExecutionService.layerEmpty,
60
+ TracingService.layerNoop,
61
+ ),
62
+ ),
63
+ ),
64
+ });
65
+
66
+ const systemPrompt = trim`
67
+ You are a helpful assistant that summarizes transcripts of meetings.
68
+
69
+ # Goal
70
+ Create a markdown summary of the meeting transcript with text notes provided.
71
+ Notes are very important so make sure to include them in the summary if they contain meaningful information.
72
+
73
+ # Formatting
74
+ - Format the summary as a markdown document without extra comments like "Here is the summary of the transcript:".
75
+ - Use markdown formatting for headings and bullet points.
76
+ - Format the summary as a list of key points and takeaways.
77
+ - All names of people should be in bold.
78
+
79
+ # Note Taking
80
+ - Correlate items in the summary with the person of origin to build a coherent narrative.
81
+ - Include short quotes verbatim where appropriate. Especially when concerned with design decisions and problem descriptions.
82
+
83
+ # Tasks
84
+ At the end of the summary include tasks.
85
+ Extract only the tasks that are:
86
+ - Directly actionable.
87
+ - Clearly assigned to a person or team (or can easily be inferred).
88
+ - Strongly implied by the conversation and/or user note (no speculative tasks).
89
+ - Specific enough that someone reading them would know exactly what to do next.
90
+
91
+ Format all tasks as markdown checkboxes using the syntax:
92
+ - [ ] Task description.
93
+
94
+ Additional information can be included (indented).
95
+
96
+ If no actionable tasks are found, omit this tasks section.
97
+ `;
@@ -42,16 +42,7 @@ export const useAudioFile = (audioUrl: string, constraints?: MediaTrackConstrain
42
42
  audio.addEventListener(
43
43
  'canplay',
44
44
  async () => {
45
- log.info('starting...');
46
- try {
47
- // Try to play the audio
48
- await audio.play();
49
- resolve();
50
- } catch (playError) {
51
- log.error('Play failed', { playError });
52
- // Still resolve as the audio is ready, even if autoplay failed.
53
- resolve();
54
- }
45
+ resolve();
55
46
  },
56
47
  { once: true },
57
48
  );
@@ -2,16 +2,17 @@
2
2
  // Copyright 2025 DXOS.org
3
3
  //
4
4
 
5
- import { useState, useEffect, useCallback } from 'react';
5
+ import { useCallback, useEffect, useState } from 'react';
6
6
 
7
7
  import { scheduleMicroTask } from '@dxos/async';
8
8
  import { Context } from '@dxos/context';
9
9
  import { log } from '@dxos/log';
10
10
  import { useSoundEffect } from '@dxos/react-ui-sfx';
11
11
 
12
+ import { type TranscriberParams } from '../transcriber';
13
+
12
14
  import { useAudioTrack } from './useAudioTrack';
13
15
  import { useTranscriber } from './useTranscriber';
14
- import { type TranscriberParams } from '../transcriber';
15
16
 
16
17
  export type UseVoiceInputProps = {
17
18
  active?: boolean;
@@ -9,7 +9,7 @@ import { Obj } from '@dxos/echo';
9
9
  import type { ObjectId } from '@dxos/keys';
10
10
  import { DataType } from '@dxos/schema';
11
11
 
12
- import { SerializationModel, DocumentAdapter, type ChunkRenderer } from './model';
12
+ import { type ChunkRenderer, DocumentAdapter, SerializationModel } from './model';
13
13
 
14
14
  const blockToMarkdown: ChunkRenderer<DataType.Message> = (
15
15
  message: DataType.Message,
@@ -8,7 +8,7 @@
8
8
 
9
9
  import { effect } from '@preact/signals-core';
10
10
 
11
- import { asyncTimeout, DeferredTask } from '@dxos/async';
11
+ import { DeferredTask, asyncTimeout } from '@dxos/async';
12
12
  import { LifecycleState, Resource } from '@dxos/context';
13
13
  import { type Queue } from '@dxos/echo-db';
14
14
  import { type FunctionExecutor } from '@dxos/functions';
@@ -49,14 +49,13 @@ const messages: MessageWithRangeId[] = [
49
49
  'in classical physics objects have well-defined properties such as position speed and momentum',
50
50
  ].map((string, index) =>
51
51
  Obj.make(DataType.Message, {
52
- created: new Date(Date.now() + 1000 * index).toISOString(),
52
+ created: new Date(Date.now() + 1_000 * index).toISOString(),
53
53
  sender,
54
- blocks: [{ _tag: 'transcript', started: new Date(Date.now() + 1000 * index).toISOString(), text: string }],
54
+ blocks: [{ _tag: 'transcript', started: new Date(Date.now() + 1_000 * index).toISOString(), text: string }],
55
55
  }),
56
56
  );
57
57
 
58
- // eslint-disable-next-line no-unused-vars
59
- const _REMOTE_AI = true;
58
+ // const REMOTE_AI = true;
60
59
 
61
60
  describe.skip('SentenceNormalization', () => {
62
61
  const getExecutor = () => {
@@ -98,6 +97,7 @@ describe.skip('SentenceNormalization', () => {
98
97
  buffer = sentences.slice(activeSentenceIndex);
99
98
  }
100
99
  }
100
+
101
101
  log.info('sentences', {
102
102
  originalMessages: JSON.stringify(messages, null, 2),
103
103
  sentences: JSON.stringify(sentences, null, 2),
@@ -2,18 +2,19 @@
2
2
  // Copyright 2025 DXOS.org
3
3
  //
4
4
 
5
- // TODO(burdon): Needs refactoring.
5
+ // ISSUE(burdon): defineFunction
6
6
  // @ts-nocheck
7
7
 
8
8
  import { Schema } from 'effect';
9
9
 
10
- import { DEFAULT_EDGE_MODEL, AiService } from '@dxos/ai';
10
+ import { AiService, DEFAULT_EDGE_MODEL } from '@dxos/ai';
11
11
  import { AiSession } from '@dxos/assistant';
12
12
  import { Obj } from '@dxos/echo';
13
13
  import { defineFunction } from '@dxos/functions';
14
14
  import { ObjectId } from '@dxos/keys';
15
15
  import { log } from '@dxos/log';
16
16
  import { DataType } from '@dxos/schema';
17
+ import { trim } from '@dxos/util';
17
18
 
18
19
  const MessageWithRangeId = Schema.extend(
19
20
  DataType.Message,
@@ -42,40 +43,6 @@ export const NormalizationOutput = Schema.Struct({
42
43
  });
43
44
  export interface NormalizationOutput extends Schema.Schema.Type<typeof NormalizationOutput> {}
44
45
 
45
- const prompt = `
46
- You are observing a real-time transcript of a single person speaking.
47
- The transcription is delivered in chunks of 10 seconds or less. As a result, individual sentences may be split across multiple messages, or multiple sentences may appear within a single message. Additionally, because this is real-time transcription, punctuation and capitalization may be incorrect or missing.
48
-
49
- # Task Description:
50
- - Your task is to detect and reconstruct broken or incomplete sentences by merging segments into coherent, grammatically correct sentences where appropriate.
51
-
52
- # Input Format:
53
- - The input is an array of messages that contain transcription blocks.
54
- - Each message is a DataType.Message, as described in the output format.
55
-
56
- # Output Format:
57
- - You have been provided with the tool that defines the output format; make sure to query it.
58
- - Do not output anything other than the expected format.
59
-
60
- # Segment Handling Rules:
61
- - Sort messages by timestamp.
62
- - Leave ID of the first message in a sentence.
63
- - Each divided sentence should be added to the output as a separate message or block within existing messages, preserving the original order.
64
- - If one message contains multiple sentences, you should not split them.
65
- - The last sentence may be incomplete; it should still be output as a separate block or message.
66
- - Keep track of the original timestamps of the messages and blocks and, use the timestamp of the first message/block of a sentence as the 'started' field of the merged message.
67
- - The 'rangeId' field of the merged message should include 'messageId'-s and 'rangeId'-s of all messages that have been used to construct this message.
68
-
69
- # Punctuation and Capitalization:
70
- - Do not rely on the original punctuation and capitalization—they may be incorrect.
71
- - Use logical reasoning to apply appropriate punctuation (e.g., period, comma, question mark, exclamation mark) and capitalization.
72
-
73
- # Restrictions:
74
- - Do not alter the order of the original messages; maintain the natural flow of speech.
75
- - Do not interpret or infer meaning beyond reconstructing sentences.
76
- - Do not add or remove any words or phrases.
77
- `;
78
-
79
46
  export const sentenceNormalization = defineFunction<NormalizationInput, NormalizationOutput>({
80
47
  description: 'Post process of transcription for sentence normalization',
81
48
  inputSchema: NormalizationInput,
@@ -112,3 +79,37 @@ export const sentenceNormalization = defineFunction<NormalizationInput, Normaliz
112
79
  return response;
113
80
  },
114
81
  });
82
+
83
+ const prompt = trim`
84
+ You are observing a real-time transcript of a single person speaking.
85
+ The transcription is delivered in chunks of 10 seconds or less. As a result, individual sentences may be split across multiple messages, or multiple sentences may appear within a single message. Additionally, because this is real-time transcription, punctuation and capitalization may be incorrect or missing.
86
+
87
+ # Task Description:
88
+ - Your task is to detect and reconstruct broken or incomplete sentences by merging segments into coherent, grammatically correct sentences where appropriate.
89
+
90
+ # Input Format:
91
+ - The input is an array of messages that contain transcription blocks.
92
+ - Each message is a DataType.Message, as described in the output format.
93
+
94
+ # Output Format:
95
+ - You have been provided with the tool that defines the output format; make sure to query it.
96
+ - Do not output anything other than the expected format.
97
+
98
+ # Segment Handling Rules:
99
+ - Sort messages by timestamp.
100
+ - Leave ID of the first message in a sentence.
101
+ - Each divided sentence should be added to the output as a separate message or block within existing messages, preserving the original order.
102
+ - If one message contains multiple sentences, you should not split them.
103
+ - The last sentence may be incomplete; it should still be output as a separate block or message.
104
+ - Keep track of the original timestamps of the messages and blocks and, use the timestamp of the first message/block of a sentence as the 'started' field of the merged message.
105
+ - The 'rangeId' field of the merged message should include 'messageId'-s and 'rangeId'-s of all messages that have been used to construct this message.
106
+
107
+ # Punctuation and Capitalization:
108
+ - Do not rely on the original punctuation and capitalization—they may be incorrect.
109
+ - Use logical reasoning to apply appropriate punctuation (e.g., period, comma, question mark, exclamation mark) and capitalization.
110
+
111
+ # Restrictions:
112
+ - Do not alter the order of the original messages; maintain the natural flow of speech.
113
+ - Do not interpret or infer meaning beyond reconstructing sentences.
114
+ - Do not add or remove any words or phrases.
115
+ `;
@@ -5,17 +5,17 @@
5
5
  import { Schema } from 'effect';
6
6
  import { useEffect, useMemo, useState } from 'react';
7
7
 
8
- import { extractionAnthropicFn, processTranscriptMessage } from '@dxos/assistant';
8
+ import { extractionAnthropicFn, processTranscriptMessage } from '@dxos/assistant/extraction';
9
9
  import { scheduleTaskInterval } from '@dxos/async';
10
10
  import { Filter, type Queue } from '@dxos/client/echo';
11
11
  import { Context } from '@dxos/context';
12
- import { Obj, Ref, Type, type Key } from '@dxos/echo';
12
+ import { type Key, Obj, Ref, Type } from '@dxos/echo';
13
13
  import { createQueueDXN } from '@dxos/echo-schema';
14
14
  import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
15
15
  import { IdentityDid } from '@dxos/keys';
16
16
  import { log } from '@dxos/log';
17
17
  import { faker } from '@dxos/random';
18
- import { useQueue, type Space } from '@dxos/react-client/echo';
18
+ import { type Space, useQueue } from '@dxos/react-client/echo';
19
19
  import { DataType } from '@dxos/schema';
20
20
  import { Testing, seedTestData } from '@dxos/schema/testing';
21
21
 
@@ -10,7 +10,7 @@ import { invariant } from '@dxos/invariant';
10
10
  import { log } from '@dxos/log';
11
11
  import { trace } from '@dxos/tracing';
12
12
 
13
- import { type WavConfig, type AudioChunk, type AudioRecorder } from './audio-recorder';
13
+ import { type AudioChunk, type AudioRecorder, type WavConfig } from './audio-recorder';
14
14
 
15
15
  let initializingPromise: Promise<void> | undefined;
16
16
 
@@ -4,6 +4,7 @@
4
4
 
5
5
  import fs from 'node:fs';
6
6
  import path from 'node:path';
7
+
7
8
  import { describe, expect, test } from 'vitest';
8
9
  import { WaveFile } from 'wavefile';
9
10
 
@@ -11,7 +12,7 @@ import { Trigger } from '@dxos/async';
11
12
  import { log } from '@dxos/log';
12
13
  import { type DataType } from '@dxos/schema';
13
14
  import { openAndClose } from '@dxos/test-utils';
14
- import { trace, TRACE_PROCESSOR } from '@dxos/tracing';
15
+ import { TRACE_PROCESSOR, trace } from '@dxos/tracing';
15
16
 
16
17
  import { type AudioChunk, type AudioRecorder, Transcriber } from '../transcriber';
17
18
  import { mergeFloat64Arrays } from '../util';
@@ -10,10 +10,11 @@ import { log } from '@dxos/log';
10
10
  import { type DataType } from '@dxos/schema';
11
11
  import { trace } from '@dxos/tracing';
12
12
 
13
- import { type AudioRecorder, type AudioChunk } from './audio-recorder';
14
13
  import { TRANSCRIPTION_URL } from '../types';
15
14
  import { mergeFloat64Arrays } from '../util';
16
15
 
16
+ import { type AudioChunk, type AudioRecorder } from './audio-recorder';
17
+
17
18
  type WhisperWord = {
18
19
  word: string;
19
20
 
@@ -90,19 +90,17 @@ export class TranscriptionManager extends Resource {
90
90
  return this;
91
91
  }
92
92
 
93
- /**
94
- * Enable or disable transcription.
95
- * @param enabled - Whether to enable transcription.
96
- */
97
- @synchronized
98
- async setEnabled(enabled?: boolean): Promise<void> {
93
+ async setEnabled(enabled: boolean): Promise<void> {
99
94
  if (this._enabled.value === enabled) {
100
95
  return;
101
96
  }
102
97
 
103
98
  this._enabled.value = enabled ?? false;
104
- // TODO(burdon): Why is toggle called here?
105
- this.isOpen && (await this._toggleTranscriber());
99
+ if (enabled) {
100
+ await this._maybeRestartTranscriber();
101
+ } else {
102
+ await this._stopTranscriber();
103
+ }
106
104
  }
107
105
 
108
106
  @synchronized
@@ -112,44 +110,19 @@ export class TranscriptionManager extends Resource {
112
110
  }
113
111
 
114
112
  this._audioStreamTrack = track;
115
- this.isOpen && (await this._toggleTranscriber());
113
+ await this._maybeRestartTranscriber();
116
114
  }
117
115
 
118
116
  protected override async _open(): Promise<void> {
119
- await this._toggleTranscriber();
117
+ await this._maybeRestartTranscriber();
120
118
  }
121
119
 
122
120
  protected override async _close(): Promise<void> {
123
- void this._transcriber?.close();
124
- }
125
-
126
- // TODO(burdon): Change this to setEnables (explicit), not toggle.
127
- private async _toggleTranscriber(): Promise<void> {
128
- await this._maybeReinitTranscriber();
129
-
130
- // Open or close transcriber if transcription is enabled or disabled.
131
- if (this._enabled.value) {
132
- await this._transcriber?.open();
133
- // TODO(burdon): Started and stopped blocks appear twice.
134
- const block = Obj.make(DataType.Message, {
135
- created: new Date().toISOString(),
136
- blocks: [{ _tag: 'transcript', text: 'Started', started: new Date().toISOString() }],
137
- sender: { role: 'assistant' },
138
- });
139
- await this._queue?.append([block]);
140
- } else {
141
- await this._transcriber?.close();
142
- const block = Obj.make(DataType.Message, {
143
- created: new Date().toISOString(),
144
- blocks: [{ _tag: 'transcript', text: 'Stopped', started: new Date().toISOString() }],
145
- sender: { role: 'assistant' },
146
- });
147
- await this._queue?.append([block]);
148
- }
121
+ await this._stopTranscriber();
149
122
  }
150
123
 
151
- private async _maybeReinitTranscriber(): Promise<void> {
152
- if (!this._audioStreamTrack || !this._enabled.value) {
124
+ private async _maybeRestartTranscriber(): Promise<void> {
125
+ if (!this._audioStreamTrack || !this._enabled.value || !this.isOpen) {
153
126
  return;
154
127
  }
155
128
 
@@ -172,9 +145,15 @@ export class TranscriptionManager extends Resource {
172
145
  recorder: this._mediaRecorder,
173
146
  onSegments: (segments) => this._onSegments(segments),
174
147
  });
148
+
149
+ await this._transcriber?.open();
175
150
  }
176
151
  }
177
152
 
153
+ private async _stopTranscriber(): Promise<void> {
154
+ await this._transcriber?.close();
155
+ }
156
+
178
157
  private async _onSegments(segments: DataType.MessageBlock.Transcript[]): Promise<void> {
179
158
  if (!this.isOpen || !this._queue) {
180
159
  return;