@dxos/plugin-transcription 0.8.4-main.2e9d522 → 0.8.4-main.5ea62a8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/TranscriptContainer-OX7OOODC.mjs +11 -0
- package/dist/lib/browser/blueprint-definition-QQSGTX5F.mjs +142 -0
- package/dist/lib/browser/blueprint-definition-QQSGTX5F.mjs.map +7 -0
- package/dist/lib/browser/{chunk-4FJIYH7T.mjs → chunk-26XDG6GQ.mjs} +2 -2
- package/dist/lib/browser/{chunk-5WH45KBY.mjs → chunk-3X7HP44M.mjs} +8 -1
- package/dist/lib/browser/{chunk-5WH45KBY.mjs.map → chunk-3X7HP44M.mjs.map} +1 -1
- package/dist/lib/browser/{chunk-5KPYIC3N.mjs → chunk-5I4BMOGE.mjs} +422 -402
- package/dist/lib/browser/chunk-5I4BMOGE.mjs.map +7 -0
- package/dist/lib/browser/{chunk-FFEPEPCU.mjs → chunk-BA64VLLR.mjs} +85 -94
- package/dist/lib/browser/chunk-BA64VLLR.mjs.map +7 -0
- package/dist/lib/browser/chunk-HXVGV7T6.mjs +63 -0
- package/dist/lib/browser/chunk-HXVGV7T6.mjs.map +7 -0
- package/dist/lib/browser/{chunk-U6QBOGTN.mjs → chunk-TNEQFHIU.mjs} +2 -2
- package/dist/lib/browser/{chunk-U6QBOGTN.mjs.map → chunk-TNEQFHIU.mjs.map} +1 -1
- package/dist/lib/browser/index.mjs +18 -12
- package/dist/lib/browser/index.mjs.map +3 -3
- package/dist/lib/browser/intent-resolver-T3EGLS3L.mjs +25 -0
- package/dist/lib/browser/intent-resolver-T3EGLS3L.mjs.map +7 -0
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/{react-surface-E5D5AE3F.mjs → react-surface-QYX74JNG.mjs} +8 -8
- package/dist/lib/browser/react-surface-QYX74JNG.mjs.map +7 -0
- package/dist/lib/browser/{transcriber-PXT4QBFY.mjs → transcriber-JOOAQVYV.mjs} +5 -5
- package/dist/lib/browser/transcriber-JOOAQVYV.mjs.map +7 -0
- package/dist/lib/browser/types/index.mjs +6 -8
- package/dist/lib/node-esm/{TranscriptContainer-VBAOM5DG.mjs → TranscriptContainer-BUWK63SI.mjs} +4 -4
- package/dist/lib/node-esm/blueprint-definition-4PLCTH5Q.mjs +143 -0
- package/dist/lib/node-esm/blueprint-definition-4PLCTH5Q.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-RHF2ALML.mjs → chunk-3QAIZGMW.mjs} +2 -2
- package/dist/lib/node-esm/{chunk-RHF2ALML.mjs.map → chunk-3QAIZGMW.mjs.map} +1 -1
- package/dist/lib/node-esm/{chunk-TJKAO7VR.mjs → chunk-DT3AOC3E.mjs} +422 -402
- package/dist/lib/node-esm/chunk-DT3AOC3E.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-HT3W7LDP.mjs → chunk-J5GQ3KGX.mjs} +85 -94
- package/dist/lib/node-esm/chunk-J5GQ3KGX.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-Y2B4HWSS.mjs → chunk-NWD757KH.mjs} +2 -2
- package/dist/lib/node-esm/{chunk-XMUCZBP3.mjs → chunk-QARUDLHN.mjs} +7 -1
- package/dist/lib/node-esm/{chunk-XMUCZBP3.mjs.map → chunk-QARUDLHN.mjs.map} +1 -1
- package/dist/lib/node-esm/chunk-W72HVQMD.mjs +64 -0
- package/dist/lib/node-esm/chunk-W72HVQMD.mjs.map +7 -0
- package/dist/lib/node-esm/index.mjs +18 -12
- package/dist/lib/node-esm/index.mjs.map +3 -3
- package/dist/lib/node-esm/intent-resolver-VDHCUAIO.mjs +26 -0
- package/dist/lib/node-esm/intent-resolver-VDHCUAIO.mjs.map +7 -0
- package/dist/lib/node-esm/meta.json +1 -1
- package/dist/lib/node-esm/{react-surface-QRDJOHN5.mjs → react-surface-TM7XOQBR.mjs} +8 -8
- package/dist/lib/node-esm/react-surface-TM7XOQBR.mjs.map +7 -0
- package/dist/lib/node-esm/{transcriber-YUTEVANE.mjs → transcriber-QPS55YP2.mjs} +5 -5
- package/dist/lib/node-esm/transcriber-QPS55YP2.mjs.map +7 -0
- package/dist/lib/node-esm/types/index.mjs +6 -8
- package/dist/types/src/TranscriptionPlugin.d.ts.map +1 -1
- package/dist/types/src/capabilities/blueprint-definition.d.ts +5 -0
- package/dist/types/src/capabilities/blueprint-definition.d.ts.map +1 -0
- package/dist/types/src/capabilities/capabilities.d.ts +1 -1
- package/dist/types/src/capabilities/capabilities.d.ts.map +1 -1
- package/dist/types/src/capabilities/index.d.ts +3 -2
- package/dist/types/src/capabilities/index.d.ts.map +1 -1
- package/dist/types/src/capabilities/intent-resolver.d.ts +1 -1
- package/dist/types/src/capabilities/intent-resolver.d.ts.map +1 -1
- package/dist/types/src/capabilities/react-surface.d.ts +1 -1
- package/dist/types/src/capabilities/transcriber.d.ts +1 -1
- package/dist/types/src/capabilities/transcriber.d.ts.map +1 -1
- package/dist/types/src/components/Transcript/Transcript.d.ts +4 -4
- package/dist/types/src/components/Transcript/Transcript.d.ts.map +1 -1
- package/dist/types/src/components/Transcript/Transcript.stories.d.ts +32 -15
- package/dist/types/src/components/Transcript/Transcript.stories.d.ts.map +1 -1
- package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts +2 -2
- package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts.map +1 -1
- package/dist/types/src/components/TranscriptContainer.d.ts +2 -2
- package/dist/types/src/components/TranscriptContainer.d.ts.map +1 -1
- package/dist/types/src/components/stories/FileTranscription.stories.d.ts +5 -2
- package/dist/types/src/components/stories/FileTranscription.stories.d.ts.map +1 -1
- package/dist/types/src/components/stories/KeyWordRecognition.stories.d.ts +10 -5
- package/dist/types/src/components/stories/KeyWordRecognition.stories.d.ts.map +1 -1
- package/dist/types/src/components/stories/MicrophoneTranscription.stories.d.ts +7 -4
- package/dist/types/src/components/stories/MicrophoneTranscription.stories.d.ts.map +1 -1
- package/dist/types/src/components/stories/TranscriptionStory.d.ts +1 -0
- package/dist/types/src/components/stories/TranscriptionStory.d.ts.map +1 -1
- package/dist/types/src/functions/index.d.ts +3 -0
- package/dist/types/src/functions/index.d.ts.map +1 -0
- package/dist/types/src/functions/open.d.ts +7 -0
- package/dist/types/src/functions/open.d.ts.map +1 -0
- package/dist/types/src/functions/summarize.d.ts +11 -0
- package/dist/types/src/functions/summarize.d.ts.map +1 -0
- package/dist/types/src/hooks/useAudioFile.d.ts.map +1 -1
- package/dist/types/src/hooks/useVoiceInput.d.ts.map +1 -1
- package/dist/types/src/segments-normalization/normalization.d.ts +366 -321
- package/dist/types/src/segments-normalization/normalization.d.ts.map +1 -1
- package/dist/types/src/testing/testing.d.ts +1 -1
- package/dist/types/src/testing/testing.d.ts.map +1 -1
- package/dist/types/src/transcriber/media-stream-recorder.d.ts +1 -1
- package/dist/types/src/transcriber/media-stream-recorder.d.ts.map +1 -1
- package/dist/types/src/transcriber/transcriber.d.ts.map +1 -1
- package/dist/types/src/transcriber/transcription-manager.d.ts +3 -7
- package/dist/types/src/transcriber/transcription-manager.d.ts.map +1 -1
- package/dist/types/src/types/Transcript.d.ts +30 -0
- package/dist/types/src/types/Transcript.d.ts.map +1 -0
- package/dist/types/src/types/TranscriptAction.d.ts +23 -0
- package/dist/types/src/types/TranscriptAction.d.ts.map +1 -0
- package/dist/types/src/types/index.d.ts +2 -1
- package/dist/types/src/types/index.d.ts.map +1 -1
- package/dist/types/src/types/types.d.ts +0 -26
- package/dist/types/src/types/types.d.ts.map +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +55 -53
- package/src/TranscriptionPlugin.tsx +10 -5
- package/src/capabilities/blueprint-definition.ts +31 -0
- package/src/capabilities/capabilities.ts +3 -3
- package/src/capabilities/index.ts +1 -0
- package/src/capabilities/intent-resolver.ts +5 -11
- package/src/capabilities/react-surface.tsx +2 -2
- package/src/capabilities/transcriber.ts +3 -2
- package/src/components/Transcript/Transcript.stories.tsx +12 -13
- package/src/components/Transcript/Transcript.tsx +8 -7
- package/src/components/Transcript/deprecated/OldTranscript.tsx +4 -4
- package/src/components/TranscriptContainer.tsx +5 -4
- package/src/components/stories/FileTranscription.stories.tsx +22 -12
- package/src/components/stories/KeyWordRecognition.stories.tsx +12 -6
- package/src/components/stories/MicrophoneTranscription.stories.tsx +10 -9
- package/src/components/stories/TranscriptionStory.tsx +9 -5
- package/src/functions/index.ts +6 -0
- package/src/functions/open.ts +37 -0
- package/src/functions/summarize.ts +97 -0
- package/src/hooks/useAudioFile.ts +1 -10
- package/src/hooks/useVoiceInput.ts +3 -2
- package/src/model/model.test.ts +1 -1
- package/src/segments-normalization/message-normalizer.ts +1 -1
- package/src/segments-normalization/normalization.test.ts +4 -4
- package/src/segments-normalization/normalization.ts +37 -36
- package/src/testing/testing.ts +3 -3
- package/src/transcriber/media-stream-recorder.ts +1 -1
- package/src/transcriber/transcriber.browser.test.ts +2 -1
- package/src/transcriber/transcriber.ts +2 -1
- package/src/transcriber/transcription-manager.ts +17 -38
- package/src/types/{schema.ts → Transcript.ts} +11 -8
- package/src/types/TranscriptAction.ts +24 -0
- package/src/types/index.ts +2 -1
- package/src/types/types.ts +0 -23
- package/dist/lib/browser/TranscriptContainer-6WCKCXQ6.mjs +0 -11
- package/dist/lib/browser/chunk-5KPYIC3N.mjs.map +0 -7
- package/dist/lib/browser/chunk-FFEPEPCU.mjs.map +0 -7
- package/dist/lib/browser/chunk-R7INNNUQ.mjs +0 -55
- package/dist/lib/browser/chunk-R7INNNUQ.mjs.map +0 -7
- package/dist/lib/browser/intent-resolver-3O33MMP5.mjs +0 -30
- package/dist/lib/browser/intent-resolver-3O33MMP5.mjs.map +0 -7
- package/dist/lib/browser/react-surface-E5D5AE3F.mjs.map +0 -7
- package/dist/lib/browser/transcriber-PXT4QBFY.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-HT3W7LDP.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-TJKAO7VR.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-U65ZWK3L.mjs +0 -56
- package/dist/lib/node-esm/chunk-U65ZWK3L.mjs.map +0 -7
- package/dist/lib/node-esm/intent-resolver-YMN5SXNC.mjs +0 -31
- package/dist/lib/node-esm/intent-resolver-YMN5SXNC.mjs.map +0 -7
- package/dist/lib/node-esm/react-surface-QRDJOHN5.mjs.map +0 -7
- package/dist/lib/node-esm/transcriber-YUTEVANE.mjs.map +0 -7
- package/dist/types/src/types/schema.d.ts +0 -41
- package/dist/types/src/types/schema.d.ts.map +0 -1
- /package/dist/lib/browser/{TranscriptContainer-6WCKCXQ6.mjs.map → TranscriptContainer-OX7OOODC.mjs.map} +0 -0
- /package/dist/lib/browser/{chunk-4FJIYH7T.mjs.map → chunk-26XDG6GQ.mjs.map} +0 -0
- /package/dist/lib/node-esm/{TranscriptContainer-VBAOM5DG.mjs.map → TranscriptContainer-BUWK63SI.mjs.map} +0 -0
- /package/dist/lib/node-esm/{chunk-Y2B4HWSS.mjs.map → chunk-NWD757KH.mjs.map} +0 -0
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
import '@dxos-theme';
|
|
6
6
|
|
|
7
|
-
import { type
|
|
7
|
+
import { type Meta, type StoryObj } from '@storybook/react-vite';
|
|
8
8
|
import React, { useEffect, useMemo, useState } from 'react';
|
|
9
9
|
|
|
10
10
|
import { log } from '@dxos/log';
|
|
@@ -17,7 +17,11 @@ type Word = {
|
|
|
17
17
|
matched: boolean;
|
|
18
18
|
};
|
|
19
19
|
|
|
20
|
-
|
|
20
|
+
type StoryProps = {
|
|
21
|
+
keywords: string[];
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
const DefaultStory = ({ keywords }: StoryProps) => {
|
|
21
25
|
const [running, setRunning] = useState(false);
|
|
22
26
|
const [matchingWords, setMatchingWords] = useState<Word[]>(keywords.map((word) => ({ text: word, matched: false })));
|
|
23
27
|
|
|
@@ -113,15 +117,17 @@ const SpeechRecognition = ({ keywords }: { keywords: string[] }) => {
|
|
|
113
117
|
);
|
|
114
118
|
};
|
|
115
119
|
|
|
116
|
-
const meta
|
|
120
|
+
const meta = {
|
|
117
121
|
title: 'plugins/plugin-transcription/KeyWordDetection',
|
|
122
|
+
render: DefaultStory,
|
|
118
123
|
decorators: [withTheme, withLayout({ fullscreen: true, classNames: 'justify-center' })],
|
|
119
|
-
}
|
|
124
|
+
} satisfies Meta<typeof DefaultStory>;
|
|
120
125
|
|
|
121
126
|
export default meta;
|
|
122
127
|
|
|
123
|
-
|
|
124
|
-
|
|
128
|
+
type Story = StoryObj<typeof meta>;
|
|
129
|
+
|
|
130
|
+
export const SpeechRecognitionAPI: Story = {
|
|
125
131
|
args: {
|
|
126
132
|
keywords: ['kai', 'computer', 'hello'],
|
|
127
133
|
},
|
|
@@ -15,7 +15,7 @@ import {
|
|
|
15
15
|
extractionNerFn,
|
|
16
16
|
getNer,
|
|
17
17
|
processTranscriptMessage,
|
|
18
|
-
} from '@dxos/assistant';
|
|
18
|
+
} from '@dxos/assistant/extraction';
|
|
19
19
|
import { Filter, Obj, type Type } from '@dxos/echo';
|
|
20
20
|
import { MemoryQueue } from '@dxos/echo-db';
|
|
21
21
|
import { createQueueDXN } from '@dxos/echo-schema';
|
|
@@ -30,17 +30,18 @@ import { ThemePlugin } from '@dxos/plugin-theme';
|
|
|
30
30
|
import { IndexKind, useSpace } from '@dxos/react-client/echo';
|
|
31
31
|
import { defaultTx } from '@dxos/react-ui-theme';
|
|
32
32
|
import { DataType } from '@dxos/schema';
|
|
33
|
-
import {
|
|
33
|
+
import { Testing, seedTestData } from '@dxos/schema/testing';
|
|
34
34
|
import { withLayout, withTheme } from '@dxos/storybook-utils';
|
|
35
35
|
|
|
36
|
-
import { TranscriptionStory } from './TranscriptionStory';
|
|
37
|
-
import { useIsSpeaking } from './useIsSpeaking';
|
|
38
|
-
import { TranscriptionPlugin } from '../../TranscriptionPlugin';
|
|
39
36
|
import { useAudioTrack, useQueueModelAdapter, useTranscriber } from '../../hooks';
|
|
40
37
|
import { TestItem } from '../../testing';
|
|
41
38
|
import { type MediaStreamRecorderParams, type TranscriberParams } from '../../transcriber';
|
|
39
|
+
import { TranscriptionPlugin } from '../../TranscriptionPlugin';
|
|
42
40
|
import { renderMarkdown } from '../Transcript';
|
|
43
41
|
|
|
42
|
+
import { TranscriptionStory } from './TranscriptionStory';
|
|
43
|
+
import { useIsSpeaking } from './useIsSpeaking';
|
|
44
|
+
|
|
44
45
|
const TRANSCRIBER_CONFIG = {
|
|
45
46
|
transcribeAfterChunksAmount: 50,
|
|
46
47
|
prefixBufferChunksAmount: 10,
|
|
@@ -192,7 +193,7 @@ const DefaultStory = ({
|
|
|
192
193
|
return <TranscriptionStory model={model} running={running} onRunningChange={setRunning} />;
|
|
193
194
|
};
|
|
194
195
|
|
|
195
|
-
const meta
|
|
196
|
+
const meta = {
|
|
196
197
|
title: 'plugins/plugin-transcription/MicrophoneTranscription',
|
|
197
198
|
render: DefaultStory,
|
|
198
199
|
decorators: [
|
|
@@ -202,7 +203,7 @@ const meta: Meta<typeof DefaultStory> = {
|
|
|
202
203
|
StorybookLayoutPlugin(),
|
|
203
204
|
ClientPlugin({
|
|
204
205
|
types: [TestItem, DataType.Person, DataType.Organization, Testing.DocumentType],
|
|
205
|
-
onClientInitialized: async (
|
|
206
|
+
onClientInitialized: async ({ client }) => {
|
|
206
207
|
await client.halo.createIdentity();
|
|
207
208
|
await client.spaces.waitUntilReady();
|
|
208
209
|
await client.spaces.default.waitUntilReady();
|
|
@@ -233,11 +234,11 @@ const meta: Meta<typeof DefaultStory> = {
|
|
|
233
234
|
withTheme,
|
|
234
235
|
withLayout({ fullscreen: true, classNames: 'justify-center' }),
|
|
235
236
|
],
|
|
236
|
-
}
|
|
237
|
+
} satisfies Meta<typeof DefaultStory>;
|
|
237
238
|
|
|
238
239
|
export default meta;
|
|
239
240
|
|
|
240
|
-
type Story = StoryObj<typeof
|
|
241
|
+
type Story = StoryObj<typeof meta>;
|
|
241
242
|
|
|
242
243
|
export const Default: Story = {
|
|
243
244
|
args: {
|
|
@@ -12,14 +12,15 @@ import { ScrollContainer } from '@dxos/react-ui-components';
|
|
|
12
12
|
import { type DataType } from '@dxos/schema';
|
|
13
13
|
|
|
14
14
|
import { type SerializationModel } from '../../model';
|
|
15
|
-
import {
|
|
15
|
+
import { TranscriptView } from '../Transcript';
|
|
16
16
|
|
|
17
17
|
export const TranscriptionStory: FC<{
|
|
18
18
|
model: SerializationModel<DataType.Message>;
|
|
19
|
+
disabled?: boolean;
|
|
19
20
|
running: boolean;
|
|
20
21
|
onRunningChange: Dispatch<SetStateAction<boolean>>;
|
|
21
22
|
audioRef?: RefObject<HTMLAudioElement>;
|
|
22
|
-
}> = ({ model, running, onRunningChange, audioRef }) => {
|
|
23
|
+
}> = ({ model, running, onRunningChange, audioRef, disabled }) => {
|
|
23
24
|
const space = useSpace();
|
|
24
25
|
|
|
25
26
|
return (
|
|
@@ -28,14 +29,17 @@ export const TranscriptionStory: FC<{
|
|
|
28
29
|
<Toolbar.Root>
|
|
29
30
|
<IconButton
|
|
30
31
|
iconOnly
|
|
32
|
+
disabled={disabled}
|
|
31
33
|
icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
|
|
32
34
|
label={running ? 'Pause' : 'Play'}
|
|
33
35
|
onClick={() => onRunningChange((running) => !running)}
|
|
34
36
|
/>
|
|
35
37
|
</Toolbar.Root>
|
|
36
|
-
<ScrollContainer>
|
|
37
|
-
<
|
|
38
|
-
|
|
38
|
+
<ScrollContainer.Root pin>
|
|
39
|
+
<ScrollContainer.Content>
|
|
40
|
+
<TranscriptView space={space} model={model} attendableId='story' />
|
|
41
|
+
</ScrollContainer.Content>
|
|
42
|
+
</ScrollContainer.Root>
|
|
39
43
|
</div>
|
|
40
44
|
);
|
|
41
45
|
};
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2025 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import { Effect, Schema } from 'effect';
|
|
6
|
+
|
|
7
|
+
import { ArtifactId } from '@dxos/assistant';
|
|
8
|
+
import { Obj } from '@dxos/echo';
|
|
9
|
+
import { DatabaseService, QueueService, defineFunction } from '@dxos/functions';
|
|
10
|
+
import { DataType } from '@dxos/schema';
|
|
11
|
+
|
|
12
|
+
import { renderMarkdown } from '../components';
|
|
13
|
+
import { Transcript } from '../types';
|
|
14
|
+
|
|
15
|
+
export default defineFunction({
|
|
16
|
+
name: 'dxos.org/function/transcription/open',
|
|
17
|
+
description: 'Opens and reads the contents of a transcription object.',
|
|
18
|
+
inputSchema: Schema.Struct({
|
|
19
|
+
id: ArtifactId.annotations({
|
|
20
|
+
description: 'The ID of the transcription object.',
|
|
21
|
+
}),
|
|
22
|
+
}),
|
|
23
|
+
outputSchema: Schema.Struct({
|
|
24
|
+
content: Schema.String,
|
|
25
|
+
}),
|
|
26
|
+
handler: Effect.fn(function* ({ data: { id } }) {
|
|
27
|
+
const transcript = yield* DatabaseService.resolve(ArtifactId.toDXN(id), Transcript.Transcript);
|
|
28
|
+
const { dxn } = yield* Effect.promise(() => transcript.queue.load());
|
|
29
|
+
const queue = yield* QueueService.getQueue(dxn);
|
|
30
|
+
yield* Effect.promise(() => queue?.queryObjects());
|
|
31
|
+
const content = queue?.objects
|
|
32
|
+
.filter((message) => Obj.instanceOf(DataType.Message, message))
|
|
33
|
+
.flatMap((message, index) => renderMarkdown([])(message, index))
|
|
34
|
+
.join('\n\n');
|
|
35
|
+
return { content };
|
|
36
|
+
}),
|
|
37
|
+
});
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2025 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import { Array, Effect, Layer, Option, Schema, pipe } from 'effect';
|
|
6
|
+
|
|
7
|
+
import { AiService, ConsolePrinter, ToolExecutionService, ToolResolverService } from '@dxos/ai';
|
|
8
|
+
import { AiSession, GenerationObserver } from '@dxos/assistant';
|
|
9
|
+
import { TracingService, defineFunction } from '@dxos/functions';
|
|
10
|
+
import { trim } from '@dxos/util';
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Summarize a transcript of a meeting.
|
|
14
|
+
*/
|
|
15
|
+
export default defineFunction({
|
|
16
|
+
name: 'dxos.org/function/transcription/summarize',
|
|
17
|
+
description: 'Summarize a transcript of a meeting.',
|
|
18
|
+
inputSchema: Schema.Struct({
|
|
19
|
+
transcript: Schema.String.annotations({
|
|
20
|
+
description: 'The transcript of the meeting.',
|
|
21
|
+
}),
|
|
22
|
+
notes: Schema.optional(Schema.String).annotations({
|
|
23
|
+
description: 'Additional notes from the participants.',
|
|
24
|
+
}),
|
|
25
|
+
}),
|
|
26
|
+
outputSchema: Schema.Struct({
|
|
27
|
+
summary: Schema.String.annotations({
|
|
28
|
+
description: 'The summary of the transcript.',
|
|
29
|
+
}),
|
|
30
|
+
}),
|
|
31
|
+
handler: Effect.fnUntraced(
|
|
32
|
+
function* ({ data: { transcript, notes } }) {
|
|
33
|
+
const result = yield* new AiSession().run({
|
|
34
|
+
prompt: `Transcript: ${transcript}\n\nNotes: ${notes}`,
|
|
35
|
+
history: [],
|
|
36
|
+
system: systemPrompt,
|
|
37
|
+
observer: GenerationObserver.fromPrinter(new ConsolePrinter({ tag: 'summarize' })),
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
const summary = pipe(
|
|
41
|
+
result,
|
|
42
|
+
Array.findLast((msg) => msg.sender.role === 'assistant' && msg.blocks.some((block) => block._tag === 'text')),
|
|
43
|
+
Option.flatMap((msg) =>
|
|
44
|
+
pipe(
|
|
45
|
+
msg.blocks,
|
|
46
|
+
Array.findLast((block) => block._tag === 'text'),
|
|
47
|
+
Option.map((block) => block.text),
|
|
48
|
+
),
|
|
49
|
+
),
|
|
50
|
+
Option.getOrThrowWith(() => new Error('No summary found')),
|
|
51
|
+
);
|
|
52
|
+
|
|
53
|
+
return { summary };
|
|
54
|
+
},
|
|
55
|
+
Effect.provide(
|
|
56
|
+
Layer.mergeAll(
|
|
57
|
+
AiService.model('@anthropic/claude-sonnet-4-0'),
|
|
58
|
+
ToolResolverService.layerEmpty,
|
|
59
|
+
ToolExecutionService.layerEmpty,
|
|
60
|
+
TracingService.layerNoop,
|
|
61
|
+
),
|
|
62
|
+
),
|
|
63
|
+
),
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
const systemPrompt = trim`
|
|
67
|
+
You are a helpful assistant that summarizes transcripts of meetings.
|
|
68
|
+
|
|
69
|
+
# Goal
|
|
70
|
+
Create a markdown summary of the meeting transcript with text notes provided.
|
|
71
|
+
Notes are very important so make sure to include them in the summary if they contain meaningful information.
|
|
72
|
+
|
|
73
|
+
# Formatting
|
|
74
|
+
- Format the summary as a markdown document without extra comments like "Here is the summary of the transcript:".
|
|
75
|
+
- Use markdown formatting for headings and bullet points.
|
|
76
|
+
- Format the summary as a list of key points and takeaways.
|
|
77
|
+
- All names of people should be in bold.
|
|
78
|
+
|
|
79
|
+
# Note Taking
|
|
80
|
+
- Correlate items in the summary with the person of origin to build a coherent narrative.
|
|
81
|
+
- Include short quotes verbatim where appropriate. Especially when concerned with design decisions and problem descriptions.
|
|
82
|
+
|
|
83
|
+
# Tasks
|
|
84
|
+
At the end of the summary include tasks.
|
|
85
|
+
Extract only the tasks that are:
|
|
86
|
+
- Directly actionable.
|
|
87
|
+
- Clearly assigned to a person or team (or can easily be inferred).
|
|
88
|
+
- Strongly implied by the conversation and/or user note (no speculative tasks).
|
|
89
|
+
- Specific enough that someone reading them would know exactly what to do next.
|
|
90
|
+
|
|
91
|
+
Format all tasks as markdown checkboxes using the syntax:
|
|
92
|
+
- [ ] Task description.
|
|
93
|
+
|
|
94
|
+
Additional information can be included (indented).
|
|
95
|
+
|
|
96
|
+
If no actionable tasks are found, omit this tasks section.
|
|
97
|
+
`;
|
|
@@ -42,16 +42,7 @@ export const useAudioFile = (audioUrl: string, constraints?: MediaTrackConstrain
|
|
|
42
42
|
audio.addEventListener(
|
|
43
43
|
'canplay',
|
|
44
44
|
async () => {
|
|
45
|
-
|
|
46
|
-
try {
|
|
47
|
-
// Try to play the audio
|
|
48
|
-
await audio.play();
|
|
49
|
-
resolve();
|
|
50
|
-
} catch (playError) {
|
|
51
|
-
log.error('Play failed', { playError });
|
|
52
|
-
// Still resolve as the audio is ready, even if autoplay failed.
|
|
53
|
-
resolve();
|
|
54
|
-
}
|
|
45
|
+
resolve();
|
|
55
46
|
},
|
|
56
47
|
{ once: true },
|
|
57
48
|
);
|
|
@@ -2,16 +2,17 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import {
|
|
5
|
+
import { useCallback, useEffect, useState } from 'react';
|
|
6
6
|
|
|
7
7
|
import { scheduleMicroTask } from '@dxos/async';
|
|
8
8
|
import { Context } from '@dxos/context';
|
|
9
9
|
import { log } from '@dxos/log';
|
|
10
10
|
import { useSoundEffect } from '@dxos/react-ui-sfx';
|
|
11
11
|
|
|
12
|
+
import { type TranscriberParams } from '../transcriber';
|
|
13
|
+
|
|
12
14
|
import { useAudioTrack } from './useAudioTrack';
|
|
13
15
|
import { useTranscriber } from './useTranscriber';
|
|
14
|
-
import { type TranscriberParams } from '../transcriber';
|
|
15
16
|
|
|
16
17
|
export type UseVoiceInputProps = {
|
|
17
18
|
active?: boolean;
|
package/src/model/model.test.ts
CHANGED
|
@@ -9,7 +9,7 @@ import { Obj } from '@dxos/echo';
|
|
|
9
9
|
import type { ObjectId } from '@dxos/keys';
|
|
10
10
|
import { DataType } from '@dxos/schema';
|
|
11
11
|
|
|
12
|
-
import {
|
|
12
|
+
import { type ChunkRenderer, DocumentAdapter, SerializationModel } from './model';
|
|
13
13
|
|
|
14
14
|
const blockToMarkdown: ChunkRenderer<DataType.Message> = (
|
|
15
15
|
message: DataType.Message,
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
|
|
9
9
|
import { effect } from '@preact/signals-core';
|
|
10
10
|
|
|
11
|
-
import {
|
|
11
|
+
import { DeferredTask, asyncTimeout } from '@dxos/async';
|
|
12
12
|
import { LifecycleState, Resource } from '@dxos/context';
|
|
13
13
|
import { type Queue } from '@dxos/echo-db';
|
|
14
14
|
import { type FunctionExecutor } from '@dxos/functions';
|
|
@@ -49,14 +49,13 @@ const messages: MessageWithRangeId[] = [
|
|
|
49
49
|
'in classical physics objects have well-defined properties such as position speed and momentum',
|
|
50
50
|
].map((string, index) =>
|
|
51
51
|
Obj.make(DataType.Message, {
|
|
52
|
-
created: new Date(Date.now() +
|
|
52
|
+
created: new Date(Date.now() + 1_000 * index).toISOString(),
|
|
53
53
|
sender,
|
|
54
|
-
blocks: [{ _tag: 'transcript', started: new Date(Date.now() +
|
|
54
|
+
blocks: [{ _tag: 'transcript', started: new Date(Date.now() + 1_000 * index).toISOString(), text: string }],
|
|
55
55
|
}),
|
|
56
56
|
);
|
|
57
57
|
|
|
58
|
-
//
|
|
59
|
-
const _REMOTE_AI = true;
|
|
58
|
+
// const REMOTE_AI = true;
|
|
60
59
|
|
|
61
60
|
describe.skip('SentenceNormalization', () => {
|
|
62
61
|
const getExecutor = () => {
|
|
@@ -98,6 +97,7 @@ describe.skip('SentenceNormalization', () => {
|
|
|
98
97
|
buffer = sentences.slice(activeSentenceIndex);
|
|
99
98
|
}
|
|
100
99
|
}
|
|
100
|
+
|
|
101
101
|
log.info('sentences', {
|
|
102
102
|
originalMessages: JSON.stringify(messages, null, 2),
|
|
103
103
|
sentences: JSON.stringify(sentences, null, 2),
|
|
@@ -2,18 +2,19 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
//
|
|
5
|
+
// ISSUE(burdon): defineFunction
|
|
6
6
|
// @ts-nocheck
|
|
7
7
|
|
|
8
8
|
import { Schema } from 'effect';
|
|
9
9
|
|
|
10
|
-
import {
|
|
10
|
+
import { AiService, DEFAULT_EDGE_MODEL } from '@dxos/ai';
|
|
11
11
|
import { AiSession } from '@dxos/assistant';
|
|
12
12
|
import { Obj } from '@dxos/echo';
|
|
13
13
|
import { defineFunction } from '@dxos/functions';
|
|
14
14
|
import { ObjectId } from '@dxos/keys';
|
|
15
15
|
import { log } from '@dxos/log';
|
|
16
16
|
import { DataType } from '@dxos/schema';
|
|
17
|
+
import { trim } from '@dxos/util';
|
|
17
18
|
|
|
18
19
|
const MessageWithRangeId = Schema.extend(
|
|
19
20
|
DataType.Message,
|
|
@@ -42,40 +43,6 @@ export const NormalizationOutput = Schema.Struct({
|
|
|
42
43
|
});
|
|
43
44
|
export interface NormalizationOutput extends Schema.Schema.Type<typeof NormalizationOutput> {}
|
|
44
45
|
|
|
45
|
-
const prompt = `
|
|
46
|
-
You are observing a real-time transcript of a single person speaking.
|
|
47
|
-
The transcription is delivered in chunks of 10 seconds or less. As a result, individual sentences may be split across multiple messages, or multiple sentences may appear within a single message. Additionally, because this is real-time transcription, punctuation and capitalization may be incorrect or missing.
|
|
48
|
-
|
|
49
|
-
# Task Description:
|
|
50
|
-
- Your task is to detect and reconstruct broken or incomplete sentences by merging segments into coherent, grammatically correct sentences where appropriate.
|
|
51
|
-
|
|
52
|
-
# Input Format:
|
|
53
|
-
- The input is an array of messages that contain transcription blocks.
|
|
54
|
-
- Each message is a DataType.Message, as described in the output format.
|
|
55
|
-
|
|
56
|
-
# Output Format:
|
|
57
|
-
- You have been provided with the tool that defines the output format; make sure to query it.
|
|
58
|
-
- Do not output anything other than the expected format.
|
|
59
|
-
|
|
60
|
-
# Segment Handling Rules:
|
|
61
|
-
- Sort messages by timestamp.
|
|
62
|
-
- Leave ID of the first message in a sentence.
|
|
63
|
-
- Each divided sentence should be added to the output as a separate message or block within existing messages, preserving the original order.
|
|
64
|
-
- If one message contains multiple sentences, you should not split them.
|
|
65
|
-
- The last sentence may be incomplete; it should still be output as a separate block or message.
|
|
66
|
-
- Keep track of the original timestamps of the messages and blocks and, use the timestamp of the first message/block of a sentence as the 'started' field of the merged message.
|
|
67
|
-
- The 'rangeId' field of the merged message should include 'messageId'-s and 'rangeId'-s of all messages that have been used to construct this message.
|
|
68
|
-
|
|
69
|
-
# Punctuation and Capitalization:
|
|
70
|
-
- Do not rely on the original punctuation and capitalization—they may be incorrect.
|
|
71
|
-
- Use logical reasoning to apply appropriate punctuation (e.g., period, comma, question mark, exclamation mark) and capitalization.
|
|
72
|
-
|
|
73
|
-
# Restrictions:
|
|
74
|
-
- Do not alter the order of the original messages; maintain the natural flow of speech.
|
|
75
|
-
- Do not interpret or infer meaning beyond reconstructing sentences.
|
|
76
|
-
- Do not add or remove any words or phrases.
|
|
77
|
-
`;
|
|
78
|
-
|
|
79
46
|
export const sentenceNormalization = defineFunction<NormalizationInput, NormalizationOutput>({
|
|
80
47
|
description: 'Post process of transcription for sentence normalization',
|
|
81
48
|
inputSchema: NormalizationInput,
|
|
@@ -112,3 +79,37 @@ export const sentenceNormalization = defineFunction<NormalizationInput, Normaliz
|
|
|
112
79
|
return response;
|
|
113
80
|
},
|
|
114
81
|
});
|
|
82
|
+
|
|
83
|
+
const prompt = trim`
|
|
84
|
+
You are observing a real-time transcript of a single person speaking.
|
|
85
|
+
The transcription is delivered in chunks of 10 seconds or less. As a result, individual sentences may be split across multiple messages, or multiple sentences may appear within a single message. Additionally, because this is real-time transcription, punctuation and capitalization may be incorrect or missing.
|
|
86
|
+
|
|
87
|
+
# Task Description:
|
|
88
|
+
- Your task is to detect and reconstruct broken or incomplete sentences by merging segments into coherent, grammatically correct sentences where appropriate.
|
|
89
|
+
|
|
90
|
+
# Input Format:
|
|
91
|
+
- The input is an array of messages that contain transcription blocks.
|
|
92
|
+
- Each message is a DataType.Message, as described in the output format.
|
|
93
|
+
|
|
94
|
+
# Output Format:
|
|
95
|
+
- You have been provided with the tool that defines the output format; make sure to query it.
|
|
96
|
+
- Do not output anything other than the expected format.
|
|
97
|
+
|
|
98
|
+
# Segment Handling Rules:
|
|
99
|
+
- Sort messages by timestamp.
|
|
100
|
+
- Leave ID of the first message in a sentence.
|
|
101
|
+
- Each divided sentence should be added to the output as a separate message or block within existing messages, preserving the original order.
|
|
102
|
+
- If one message contains multiple sentences, you should not split them.
|
|
103
|
+
- The last sentence may be incomplete; it should still be output as a separate block or message.
|
|
104
|
+
- Keep track of the original timestamps of the messages and blocks and, use the timestamp of the first message/block of a sentence as the 'started' field of the merged message.
|
|
105
|
+
- The 'rangeId' field of the merged message should include 'messageId'-s and 'rangeId'-s of all messages that have been used to construct this message.
|
|
106
|
+
|
|
107
|
+
# Punctuation and Capitalization:
|
|
108
|
+
- Do not rely on the original punctuation and capitalization—they may be incorrect.
|
|
109
|
+
- Use logical reasoning to apply appropriate punctuation (e.g., period, comma, question mark, exclamation mark) and capitalization.
|
|
110
|
+
|
|
111
|
+
# Restrictions:
|
|
112
|
+
- Do not alter the order of the original messages; maintain the natural flow of speech.
|
|
113
|
+
- Do not interpret or infer meaning beyond reconstructing sentences.
|
|
114
|
+
- Do not add or remove any words or phrases.
|
|
115
|
+
`;
|
package/src/testing/testing.ts
CHANGED
|
@@ -5,17 +5,17 @@
|
|
|
5
5
|
import { Schema } from 'effect';
|
|
6
6
|
import { useEffect, useMemo, useState } from 'react';
|
|
7
7
|
|
|
8
|
-
import { extractionAnthropicFn, processTranscriptMessage } from '@dxos/assistant';
|
|
8
|
+
import { extractionAnthropicFn, processTranscriptMessage } from '@dxos/assistant/extraction';
|
|
9
9
|
import { scheduleTaskInterval } from '@dxos/async';
|
|
10
10
|
import { Filter, type Queue } from '@dxos/client/echo';
|
|
11
11
|
import { Context } from '@dxos/context';
|
|
12
|
-
import { Obj, Ref, Type
|
|
12
|
+
import { type Key, Obj, Ref, Type } from '@dxos/echo';
|
|
13
13
|
import { createQueueDXN } from '@dxos/echo-schema';
|
|
14
14
|
import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
|
|
15
15
|
import { IdentityDid } from '@dxos/keys';
|
|
16
16
|
import { log } from '@dxos/log';
|
|
17
17
|
import { faker } from '@dxos/random';
|
|
18
|
-
import {
|
|
18
|
+
import { type Space, useQueue } from '@dxos/react-client/echo';
|
|
19
19
|
import { DataType } from '@dxos/schema';
|
|
20
20
|
import { Testing, seedTestData } from '@dxos/schema/testing';
|
|
21
21
|
|
|
@@ -10,7 +10,7 @@ import { invariant } from '@dxos/invariant';
|
|
|
10
10
|
import { log } from '@dxos/log';
|
|
11
11
|
import { trace } from '@dxos/tracing';
|
|
12
12
|
|
|
13
|
-
import { type
|
|
13
|
+
import { type AudioChunk, type AudioRecorder, type WavConfig } from './audio-recorder';
|
|
14
14
|
|
|
15
15
|
let initializingPromise: Promise<void> | undefined;
|
|
16
16
|
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
import fs from 'node:fs';
|
|
6
6
|
import path from 'node:path';
|
|
7
|
+
|
|
7
8
|
import { describe, expect, test } from 'vitest';
|
|
8
9
|
import { WaveFile } from 'wavefile';
|
|
9
10
|
|
|
@@ -11,7 +12,7 @@ import { Trigger } from '@dxos/async';
|
|
|
11
12
|
import { log } from '@dxos/log';
|
|
12
13
|
import { type DataType } from '@dxos/schema';
|
|
13
14
|
import { openAndClose } from '@dxos/test-utils';
|
|
14
|
-
import {
|
|
15
|
+
import { TRACE_PROCESSOR, trace } from '@dxos/tracing';
|
|
15
16
|
|
|
16
17
|
import { type AudioChunk, type AudioRecorder, Transcriber } from '../transcriber';
|
|
17
18
|
import { mergeFloat64Arrays } from '../util';
|
|
@@ -10,10 +10,11 @@ import { log } from '@dxos/log';
|
|
|
10
10
|
import { type DataType } from '@dxos/schema';
|
|
11
11
|
import { trace } from '@dxos/tracing';
|
|
12
12
|
|
|
13
|
-
import { type AudioRecorder, type AudioChunk } from './audio-recorder';
|
|
14
13
|
import { TRANSCRIPTION_URL } from '../types';
|
|
15
14
|
import { mergeFloat64Arrays } from '../util';
|
|
16
15
|
|
|
16
|
+
import { type AudioChunk, type AudioRecorder } from './audio-recorder';
|
|
17
|
+
|
|
17
18
|
type WhisperWord = {
|
|
18
19
|
word: string;
|
|
19
20
|
|
|
@@ -90,19 +90,17 @@ export class TranscriptionManager extends Resource {
|
|
|
90
90
|
return this;
|
|
91
91
|
}
|
|
92
92
|
|
|
93
|
-
|
|
94
|
-
* Enable or disable transcription.
|
|
95
|
-
* @param enabled - Whether to enable transcription.
|
|
96
|
-
*/
|
|
97
|
-
@synchronized
|
|
98
|
-
async setEnabled(enabled?: boolean): Promise<void> {
|
|
93
|
+
async setEnabled(enabled: boolean): Promise<void> {
|
|
99
94
|
if (this._enabled.value === enabled) {
|
|
100
95
|
return;
|
|
101
96
|
}
|
|
102
97
|
|
|
103
98
|
this._enabled.value = enabled ?? false;
|
|
104
|
-
|
|
105
|
-
|
|
99
|
+
if (enabled) {
|
|
100
|
+
await this._maybeRestartTranscriber();
|
|
101
|
+
} else {
|
|
102
|
+
await this._stopTranscriber();
|
|
103
|
+
}
|
|
106
104
|
}
|
|
107
105
|
|
|
108
106
|
@synchronized
|
|
@@ -112,44 +110,19 @@ export class TranscriptionManager extends Resource {
|
|
|
112
110
|
}
|
|
113
111
|
|
|
114
112
|
this._audioStreamTrack = track;
|
|
115
|
-
|
|
113
|
+
await this._maybeRestartTranscriber();
|
|
116
114
|
}
|
|
117
115
|
|
|
118
116
|
protected override async _open(): Promise<void> {
|
|
119
|
-
await this.
|
|
117
|
+
await this._maybeRestartTranscriber();
|
|
120
118
|
}
|
|
121
119
|
|
|
122
120
|
protected override async _close(): Promise<void> {
|
|
123
|
-
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
// TODO(burdon): Change this to setEnables (explicit), not toggle.
|
|
127
|
-
private async _toggleTranscriber(): Promise<void> {
|
|
128
|
-
await this._maybeReinitTranscriber();
|
|
129
|
-
|
|
130
|
-
// Open or close transcriber if transcription is enabled or disabled.
|
|
131
|
-
if (this._enabled.value) {
|
|
132
|
-
await this._transcriber?.open();
|
|
133
|
-
// TODO(burdon): Started and stopped blocks appear twice.
|
|
134
|
-
const block = Obj.make(DataType.Message, {
|
|
135
|
-
created: new Date().toISOString(),
|
|
136
|
-
blocks: [{ _tag: 'transcript', text: 'Started', started: new Date().toISOString() }],
|
|
137
|
-
sender: { role: 'assistant' },
|
|
138
|
-
});
|
|
139
|
-
await this._queue?.append([block]);
|
|
140
|
-
} else {
|
|
141
|
-
await this._transcriber?.close();
|
|
142
|
-
const block = Obj.make(DataType.Message, {
|
|
143
|
-
created: new Date().toISOString(),
|
|
144
|
-
blocks: [{ _tag: 'transcript', text: 'Stopped', started: new Date().toISOString() }],
|
|
145
|
-
sender: { role: 'assistant' },
|
|
146
|
-
});
|
|
147
|
-
await this._queue?.append([block]);
|
|
148
|
-
}
|
|
121
|
+
await this._stopTranscriber();
|
|
149
122
|
}
|
|
150
123
|
|
|
151
|
-
private async
|
|
152
|
-
if (!this._audioStreamTrack || !this._enabled.value) {
|
|
124
|
+
private async _maybeRestartTranscriber(): Promise<void> {
|
|
125
|
+
if (!this._audioStreamTrack || !this._enabled.value || !this.isOpen) {
|
|
153
126
|
return;
|
|
154
127
|
}
|
|
155
128
|
|
|
@@ -172,9 +145,15 @@ export class TranscriptionManager extends Resource {
|
|
|
172
145
|
recorder: this._mediaRecorder,
|
|
173
146
|
onSegments: (segments) => this._onSegments(segments),
|
|
174
147
|
});
|
|
148
|
+
|
|
149
|
+
await this._transcriber?.open();
|
|
175
150
|
}
|
|
176
151
|
}
|
|
177
152
|
|
|
153
|
+
private async _stopTranscriber(): Promise<void> {
|
|
154
|
+
await this._transcriber?.close();
|
|
155
|
+
}
|
|
156
|
+
|
|
178
157
|
private async _onSegments(segments: DataType.MessageBlock.Transcript[]): Promise<void> {
|
|
179
158
|
if (!this.isOpen || !this._queue) {
|
|
180
159
|
return;
|