@dxos/plugin-transcription 0.8.4-main.dedc0f3 → 0.8.4-main.e8ec1fe
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/TranscriptContainer-OQFBAGSP.mjs +11 -0
- package/dist/lib/browser/{blueprint-definition-TJABB43G.mjs → blueprint-definition-GZ7NOODS.mjs} +23 -18
- package/dist/lib/browser/blueprint-definition-GZ7NOODS.mjs.map +7 -0
- package/dist/lib/browser/{chunk-TNEQFHIU.mjs → chunk-3MKZ2MNQ.mjs} +5 -5
- package/dist/lib/browser/chunk-3MKZ2MNQ.mjs.map +7 -0
- package/dist/lib/browser/{chunk-BA64VLLR.mjs → chunk-7GNN7E3U.mjs} +46 -66
- package/dist/lib/browser/chunk-7GNN7E3U.mjs.map +7 -0
- package/dist/lib/browser/{chunk-VVXRHJCH.mjs → chunk-C5YV3JZF.mjs} +114 -122
- package/dist/lib/browser/chunk-C5YV3JZF.mjs.map +7 -0
- package/dist/lib/browser/{chunk-AGZYQGSP.mjs → chunk-O33THH53.mjs} +2 -2
- package/dist/lib/browser/chunk-QOSZIUE2.mjs +32 -0
- package/dist/lib/browser/chunk-QOSZIUE2.mjs.map +7 -0
- package/dist/lib/browser/{chunk-3X7HP44M.mjs → chunk-VBUIO677.mjs} +8 -5
- package/dist/lib/browser/chunk-VBUIO677.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +13 -16
- package/dist/lib/browser/index.mjs.map +3 -3
- package/dist/lib/browser/{intent-resolver-T3EGLS3L.mjs → intent-resolver-5CGCLOXB.mjs} +8 -8
- package/dist/lib/browser/intent-resolver-5CGCLOXB.mjs.map +7 -0
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/{react-surface-4Y3YISAD.mjs → react-surface-OK55WK7J.mjs} +9 -11
- package/dist/lib/browser/react-surface-OK55WK7J.mjs.map +7 -0
- package/dist/lib/browser/{transcriber-JOOAQVYV.mjs → transcriber-6YDM5ZDT.mjs} +7 -7
- package/dist/lib/browser/{transcriber-JOOAQVYV.mjs.map → transcriber-6YDM5ZDT.mjs.map} +1 -1
- package/dist/lib/browser/types/index.mjs +3 -5
- package/dist/lib/node-esm/{TranscriptContainer-EGAAYVOP.mjs → TranscriptContainer-GM47P6WH.mjs} +4 -4
- package/dist/lib/node-esm/{blueprint-definition-R4UF7YJZ.mjs → blueprint-definition-GAHQBYTF.mjs} +23 -18
- package/dist/lib/node-esm/blueprint-definition-GAHQBYTF.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-J5GQ3KGX.mjs → chunk-5JN2MHYA.mjs} +46 -66
- package/dist/lib/node-esm/chunk-5JN2MHYA.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-SFQNMFHX.mjs → chunk-IKGUNN3Z.mjs} +114 -122
- package/dist/lib/node-esm/chunk-IKGUNN3Z.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-CF5Z6FAB.mjs → chunk-SSYFSLB7.mjs} +2 -2
- package/dist/lib/node-esm/{chunk-3QAIZGMW.mjs → chunk-SWUVPFKR.mjs} +5 -5
- package/dist/lib/node-esm/chunk-SWUVPFKR.mjs.map +7 -0
- package/dist/lib/node-esm/chunk-U4YPUD2F.mjs +33 -0
- package/dist/lib/node-esm/chunk-U4YPUD2F.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-QARUDLHN.mjs → chunk-YVNQN2J6.mjs} +8 -5
- package/dist/lib/node-esm/chunk-YVNQN2J6.mjs.map +7 -0
- package/dist/lib/node-esm/index.mjs +13 -16
- package/dist/lib/node-esm/index.mjs.map +3 -3
- package/dist/lib/node-esm/{intent-resolver-VDHCUAIO.mjs → intent-resolver-4H4CXIAL.mjs} +8 -8
- package/dist/lib/node-esm/intent-resolver-4H4CXIAL.mjs.map +7 -0
- package/dist/lib/node-esm/meta.json +1 -1
- package/dist/lib/node-esm/{react-surface-ZIXDGJAN.mjs → react-surface-EK6K2MMD.mjs} +9 -11
- package/dist/lib/node-esm/react-surface-EK6K2MMD.mjs.map +7 -0
- package/dist/lib/node-esm/{transcriber-QPS55YP2.mjs → transcriber-CPD3FZ7M.mjs} +7 -7
- package/dist/lib/node-esm/{transcriber-QPS55YP2.mjs.map → transcriber-CPD3FZ7M.mjs.map} +1 -1
- package/dist/lib/node-esm/types/index.mjs +3 -5
- package/dist/types/src/TranscriptionPlugin.d.ts +1 -1
- package/dist/types/src/TranscriptionPlugin.d.ts.map +1 -1
- package/dist/types/src/capabilities/blueprint-definition.d.ts +1 -1
- package/dist/types/src/capabilities/capabilities.d.ts.map +1 -1
- package/dist/types/src/capabilities/intent-resolver.d.ts.map +1 -1
- package/dist/types/src/components/Transcript/Transcript.d.ts +3 -4
- package/dist/types/src/components/Transcript/Transcript.d.ts.map +1 -1
- package/dist/types/src/components/Transcript/Transcript.stories.d.ts +3 -3
- package/dist/types/src/components/Transcript/Transcript.stories.d.ts.map +1 -1
- package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts +1 -1
- package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts.map +1 -1
- package/dist/types/src/components/Transcript/transcript-extension.d.ts +2 -2
- package/dist/types/src/components/Transcript/transcript-extension.d.ts.map +1 -1
- package/dist/types/src/components/TranscriptContainer.d.ts +1 -1
- package/dist/types/src/components/TranscriptContainer.d.ts.map +1 -1
- package/dist/types/src/components/stories/FileTranscription.stories.d.ts +0 -1
- package/dist/types/src/components/stories/FileTranscription.stories.d.ts.map +1 -1
- package/dist/types/src/components/stories/KeyWordRecognition.stories.d.ts +0 -1
- package/dist/types/src/components/stories/KeyWordRecognition.stories.d.ts.map +1 -1
- package/dist/types/src/components/stories/MicrophoneTranscription.stories.d.ts +0 -1
- package/dist/types/src/components/stories/MicrophoneTranscription.stories.d.ts.map +1 -1
- package/dist/types/src/components/stories/TranscriptionStory.d.ts +3 -3
- package/dist/types/src/components/stories/TranscriptionStory.d.ts.map +1 -1
- package/dist/types/src/functions/open.d.ts +1 -1
- package/dist/types/src/functions/open.d.ts.map +1 -1
- package/dist/types/src/functions/summarize.d.ts +1 -1
- package/dist/types/src/functions/summarize.d.ts.map +1 -1
- package/dist/types/src/hooks/useAudioFile.d.ts.map +1 -1
- package/dist/types/src/hooks/useTranscriber.d.ts.map +1 -1
- package/dist/types/src/meta.d.ts +0 -1
- package/dist/types/src/meta.d.ts.map +1 -1
- package/dist/types/src/segments-normalization/message-normalizer.d.ts +3 -3
- package/dist/types/src/segments-normalization/message-normalizer.d.ts.map +1 -1
- package/dist/types/src/segments-normalization/normalization.d.ts +32 -17
- package/dist/types/src/segments-normalization/normalization.d.ts.map +1 -1
- package/dist/types/src/segments-normalization/utils.d.ts +2 -2
- package/dist/types/src/segments-normalization/utils.d.ts.map +1 -1
- package/dist/types/src/testing/testing.d.ts +6 -6
- package/dist/types/src/testing/testing.d.ts.map +1 -1
- package/dist/types/src/transcriber/transcriber.d.ts +2 -2
- package/dist/types/src/transcriber/transcriber.d.ts.map +1 -1
- package/dist/types/src/transcriber/transcription-manager.d.ts +3 -3
- package/dist/types/src/transcriber/transcription-manager.d.ts.map +1 -1
- package/dist/types/src/types/TranscriptAction.d.ts +1 -1
- package/dist/types/src/types/TranscriptAction.d.ts.map +1 -1
- package/dist/types/src/types/index.d.ts +0 -1
- package/dist/types/src/types/index.d.ts.map +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +63 -62
- package/src/TranscriptionPlugin.tsx +56 -57
- package/src/capabilities/capabilities.ts +3 -3
- package/src/capabilities/intent-resolver.ts +2 -1
- package/src/capabilities/react-surface.tsx +1 -1
- package/src/components/Transcript/Transcript.stories.tsx +15 -14
- package/src/components/Transcript/Transcript.tsx +7 -7
- package/src/components/Transcript/deprecated/OldTranscript.tsx +5 -5
- package/src/components/Transcript/transcript-extension.ts +10 -6
- package/src/components/TranscriptContainer.tsx +5 -5
- package/src/components/stories/FileTranscription.stories.tsx +18 -17
- package/src/components/stories/KeyWordRecognition.stories.tsx +4 -6
- package/src/components/stories/MicrophoneTranscription.stories.tsx +19 -18
- package/src/components/stories/TranscriptionStory.tsx +7 -8
- package/src/functions/open.ts +6 -5
- package/src/functions/summarize.ts +10 -4
- package/src/hooks/useAudioFile.ts +50 -45
- package/src/hooks/useTranscriber.ts +7 -2
- package/src/meta.ts +7 -4
- package/src/model/model.test.ts +11 -11
- package/src/segments-normalization/message-normalizer.ts +6 -6
- package/src/segments-normalization/normalization.test.ts +6 -6
- package/src/segments-normalization/normalization.ts +7 -7
- package/src/segments-normalization/utils.ts +2 -2
- package/src/testing/testing.ts +23 -18
- package/src/transcriber/transcriber.browser.test.ts +3 -3
- package/src/transcriber/transcriber.ts +3 -3
- package/src/transcriber/transcription-manager.ts +6 -6
- package/src/types/TranscriptAction.ts +3 -4
- package/src/types/index.ts +0 -1
- package/dist/lib/browser/TranscriptContainer-4RBK74JP.mjs +0 -11
- package/dist/lib/browser/blueprint-definition-TJABB43G.mjs.map +0 -7
- package/dist/lib/browser/chunk-3X7HP44M.mjs.map +0 -7
- package/dist/lib/browser/chunk-BA64VLLR.mjs.map +0 -7
- package/dist/lib/browser/chunk-HXVGV7T6.mjs +0 -63
- package/dist/lib/browser/chunk-HXVGV7T6.mjs.map +0 -7
- package/dist/lib/browser/chunk-TNEQFHIU.mjs.map +0 -7
- package/dist/lib/browser/chunk-VVXRHJCH.mjs.map +0 -7
- package/dist/lib/browser/intent-resolver-T3EGLS3L.mjs.map +0 -7
- package/dist/lib/browser/react-surface-4Y3YISAD.mjs.map +0 -7
- package/dist/lib/node-esm/blueprint-definition-R4UF7YJZ.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-3QAIZGMW.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-J5GQ3KGX.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-QARUDLHN.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-SFQNMFHX.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-W72HVQMD.mjs +0 -64
- package/dist/lib/node-esm/chunk-W72HVQMD.mjs.map +0 -7
- package/dist/lib/node-esm/intent-resolver-VDHCUAIO.mjs.map +0 -7
- package/dist/lib/node-esm/react-surface-ZIXDGJAN.mjs.map +0 -7
- package/dist/types/src/types/Transcript.d.ts +0 -30
- package/dist/types/src/types/Transcript.d.ts.map +0 -1
- package/src/types/Transcript.ts +0 -44
- /package/dist/lib/browser/{TranscriptContainer-4RBK74JP.mjs.map → TranscriptContainer-OQFBAGSP.mjs.map} +0 -0
- /package/dist/lib/browser/{chunk-AGZYQGSP.mjs.map → chunk-O33THH53.mjs.map} +0 -0
- /package/dist/lib/node-esm/{TranscriptContainer-EGAAYVOP.mjs.map → TranscriptContainer-GM47P6WH.mjs.map} +0 -0
- /package/dist/lib/node-esm/{chunk-CF5Z6FAB.mjs.map → chunk-SSYFSLB7.mjs.map} +0 -0
|
@@ -2,8 +2,6 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import '@dxos-theme';
|
|
6
|
-
|
|
7
5
|
import { type Meta, type StoryObj } from '@storybook/react-vite';
|
|
8
6
|
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
|
9
7
|
|
|
@@ -12,19 +10,20 @@ import { withPluginManager } from '@dxos/app-framework/testing';
|
|
|
12
10
|
import { scheduleTask } from '@dxos/async';
|
|
13
11
|
import { Context } from '@dxos/context';
|
|
14
12
|
import { Obj } from '@dxos/echo';
|
|
13
|
+
import { createQueueDXN } from '@dxos/echo/internal';
|
|
15
14
|
import { MemoryQueue } from '@dxos/echo-db';
|
|
16
|
-
import {
|
|
17
|
-
import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
|
|
15
|
+
import { FunctionExecutor, ServiceContainer } from '@dxos/functions-runtime';
|
|
18
16
|
import { log } from '@dxos/log';
|
|
19
17
|
import { ClientPlugin } from '@dxos/plugin-client';
|
|
20
18
|
import { PreviewPlugin } from '@dxos/plugin-preview';
|
|
21
19
|
import { SpacePlugin } from '@dxos/plugin-space';
|
|
22
20
|
import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
|
|
23
21
|
import { ThemePlugin } from '@dxos/plugin-theme';
|
|
22
|
+
import { withLayout, withTheme } from '@dxos/react-ui/testing';
|
|
24
23
|
import { defaultTx } from '@dxos/react-ui-theme';
|
|
25
|
-
import {
|
|
26
|
-
import {
|
|
27
|
-
import {
|
|
24
|
+
import { Testing } from '@dxos/schema/testing';
|
|
25
|
+
import { type Actor, Message, Organization, Person } from '@dxos/types';
|
|
26
|
+
import { seedTestData } from '@dxos/types/testing';
|
|
28
27
|
|
|
29
28
|
import { useAudioFile, useQueueModelAdapter, useTranscriber } from '../../hooks';
|
|
30
29
|
import { MessageNormalizer, getActorId } from '../../segments-normalization';
|
|
@@ -52,7 +51,7 @@ const AudioFile = ({
|
|
|
52
51
|
audioConstraints?: MediaTrackConstraints;
|
|
53
52
|
}) => {
|
|
54
53
|
const [running, setRunning] = useState(false);
|
|
55
|
-
const actor:
|
|
54
|
+
const actor: Actor.Actor = useMemo(() => ({ name: 'You' }), []);
|
|
56
55
|
|
|
57
56
|
// Audio.
|
|
58
57
|
const { audio, track, stream } = useAudioFile(audioUrl, audioConstraints);
|
|
@@ -81,13 +80,13 @@ const AudioFile = ({
|
|
|
81
80
|
// Transcriber.
|
|
82
81
|
// TODO(dmaretskyi): Use space.queues.create() instead.
|
|
83
82
|
const queueDxn = useMemo(() => createQueueDXN(), []);
|
|
84
|
-
const queue = useMemo(() => new MemoryQueue<
|
|
83
|
+
const queue = useMemo(() => new MemoryQueue<Message.Message>(queueDxn), [queueDxn]);
|
|
85
84
|
|
|
86
85
|
const model = useQueueModelAdapter(renderByline([]), queue);
|
|
87
86
|
const handleSegments = useCallback<TranscriberParams['onSegments']>(
|
|
88
87
|
async (blocks) => {
|
|
89
88
|
void queue?.append([
|
|
90
|
-
Obj.make(
|
|
89
|
+
Obj.make(Message.Message, {
|
|
91
90
|
created: new Date().toISOString(),
|
|
92
91
|
sender: actor,
|
|
93
92
|
blocks,
|
|
@@ -189,12 +188,12 @@ const AudioFile = ({
|
|
|
189
188
|
const meta = {
|
|
190
189
|
title: 'plugins/plugin-transcription/FileTranscription',
|
|
191
190
|
decorators: [
|
|
191
|
+
withTheme,
|
|
192
|
+
withLayout({ container: 'column' }),
|
|
192
193
|
withPluginManager({
|
|
193
194
|
plugins: [
|
|
194
|
-
ThemePlugin({ tx: defaultTx }),
|
|
195
|
-
StorybookLayoutPlugin(),
|
|
196
195
|
ClientPlugin({
|
|
197
|
-
types: [TestItem,
|
|
196
|
+
types: [TestItem, Person.Person, Organization.Organization, Testing.DocumentType],
|
|
198
197
|
onClientInitialized: async ({ client }) => {
|
|
199
198
|
await client.halo.createIdentity();
|
|
200
199
|
await client.spaces.waitUntilReady();
|
|
@@ -202,16 +201,18 @@ const meta = {
|
|
|
202
201
|
await seedTestData(client.spaces.default);
|
|
203
202
|
},
|
|
204
203
|
}),
|
|
205
|
-
SpacePlugin(),
|
|
204
|
+
SpacePlugin({}),
|
|
205
|
+
IntentPlugin(),
|
|
206
|
+
|
|
207
|
+
// UI
|
|
208
|
+
ThemePlugin({ tx: defaultTx }),
|
|
206
209
|
SettingsPlugin(),
|
|
207
210
|
PreviewPlugin(),
|
|
208
|
-
IntentPlugin(),
|
|
209
211
|
TranscriptionPlugin(),
|
|
212
|
+
StorybookLayoutPlugin({}),
|
|
210
213
|
],
|
|
211
214
|
fireEvents: [Events.SetupAppGraph],
|
|
212
215
|
}),
|
|
213
|
-
withTheme,
|
|
214
|
-
withLayout({ fullscreen: true, classNames: 'justify-center' }),
|
|
215
216
|
],
|
|
216
217
|
} satisfies Meta;
|
|
217
218
|
|
|
@@ -2,15 +2,13 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import '@dxos-theme';
|
|
6
|
-
|
|
7
5
|
import { type Meta, type StoryObj } from '@storybook/react-vite';
|
|
8
6
|
import React, { useEffect, useMemo, useState } from 'react';
|
|
9
7
|
|
|
10
8
|
import { log } from '@dxos/log';
|
|
11
9
|
import { Toolbar } from '@dxos/react-ui';
|
|
10
|
+
import { withLayout, withTheme } from '@dxos/react-ui/testing';
|
|
12
11
|
import { mx } from '@dxos/react-ui-theme';
|
|
13
|
-
import { withLayout, withTheme } from '@dxos/storybook-utils';
|
|
14
12
|
|
|
15
13
|
type Word = {
|
|
16
14
|
text: string;
|
|
@@ -88,7 +86,7 @@ const DefaultStory = ({ keywords }: StoryProps) => {
|
|
|
88
86
|
}, [running, recognition]);
|
|
89
87
|
|
|
90
88
|
return (
|
|
91
|
-
<div className={mx('flex flex-col gap-2
|
|
89
|
+
<div className={mx('flex flex-col gap-2 is-[30rem]')}>
|
|
92
90
|
<Toolbar.Root>
|
|
93
91
|
<Toolbar.IconButton
|
|
94
92
|
iconOnly
|
|
@@ -103,7 +101,7 @@ const DefaultStory = ({ keywords }: StoryProps) => {
|
|
|
103
101
|
<span
|
|
104
102
|
key={index}
|
|
105
103
|
className={mx(
|
|
106
|
-
'
|
|
104
|
+
'pli-3 plb-1 m-2 rounded border',
|
|
107
105
|
word.matched ? 'bg-red-200 text-red-800 border-red-300' : 'bg-gray-200 text-gray-600 border-gray-300',
|
|
108
106
|
)}
|
|
109
107
|
>
|
|
@@ -120,7 +118,7 @@ const DefaultStory = ({ keywords }: StoryProps) => {
|
|
|
120
118
|
const meta = {
|
|
121
119
|
title: 'plugins/plugin-transcription/KeyWordDetection',
|
|
122
120
|
render: DefaultStory,
|
|
123
|
-
decorators: [withTheme, withLayout({
|
|
121
|
+
decorators: [withTheme, withLayout({ container: 'column' })],
|
|
124
122
|
} satisfies Meta<typeof DefaultStory>;
|
|
125
123
|
|
|
126
124
|
export default meta;
|
|
@@ -2,8 +2,6 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import '@dxos-theme';
|
|
6
|
-
|
|
7
5
|
import { type Meta, type StoryObj } from '@storybook/react-vite';
|
|
8
6
|
import React, { useCallback, useEffect, useMemo, useState } from 'react';
|
|
9
7
|
|
|
@@ -17,9 +15,9 @@ import {
|
|
|
17
15
|
processTranscriptMessage,
|
|
18
16
|
} from '@dxos/assistant/extraction';
|
|
19
17
|
import { Filter, Obj, type Type } from '@dxos/echo';
|
|
18
|
+
import { createQueueDXN } from '@dxos/echo/internal';
|
|
20
19
|
import { MemoryQueue } from '@dxos/echo-db';
|
|
21
|
-
import {
|
|
22
|
-
import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
|
|
20
|
+
import { FunctionExecutor, ServiceContainer } from '@dxos/functions-runtime';
|
|
23
21
|
import { invariant } from '@dxos/invariant';
|
|
24
22
|
import { log } from '@dxos/log';
|
|
25
23
|
import { ClientPlugin } from '@dxos/plugin-client';
|
|
@@ -28,10 +26,11 @@ import { SpacePlugin } from '@dxos/plugin-space';
|
|
|
28
26
|
import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
|
|
29
27
|
import { ThemePlugin } from '@dxos/plugin-theme';
|
|
30
28
|
import { IndexKind, useSpace } from '@dxos/react-client/echo';
|
|
29
|
+
import { withLayout, withTheme } from '@dxos/react-ui/testing';
|
|
31
30
|
import { defaultTx } from '@dxos/react-ui-theme';
|
|
32
|
-
import {
|
|
33
|
-
import {
|
|
34
|
-
import {
|
|
31
|
+
import { Testing } from '@dxos/schema/testing';
|
|
32
|
+
import { Message, Organization, Person } from '@dxos/types';
|
|
33
|
+
import { seedTestData } from '@dxos/types/testing';
|
|
35
34
|
|
|
36
35
|
import { useAudioTrack, useQueueModelAdapter, useTranscriber } from '../../hooks';
|
|
37
36
|
import { TestItem } from '../../testing';
|
|
@@ -77,7 +76,7 @@ const DefaultStory = ({
|
|
|
77
76
|
// Queue.
|
|
78
77
|
// TODO(dmaretskyi): Use space.queues.create() instead.
|
|
79
78
|
const queueDxn = useMemo(() => createQueueDXN(), []);
|
|
80
|
-
const queue = useMemo(() => new MemoryQueue<
|
|
79
|
+
const queue = useMemo(() => new MemoryQueue<Message.Message>(queueDxn), [queueDxn]);
|
|
81
80
|
const model = useQueueModelAdapter(renderByline([]), queue);
|
|
82
81
|
const space = useSpace();
|
|
83
82
|
|
|
@@ -107,8 +106,8 @@ const DefaultStory = ({
|
|
|
107
106
|
objects = space.db
|
|
108
107
|
.query(
|
|
109
108
|
Filter.or(
|
|
110
|
-
Filter.type(
|
|
111
|
-
Filter.type(
|
|
109
|
+
Filter.type(Person.Person),
|
|
110
|
+
Filter.type(Organization.Organization),
|
|
112
111
|
Filter.type(Testing.DocumentType),
|
|
113
112
|
),
|
|
114
113
|
)
|
|
@@ -134,7 +133,7 @@ const DefaultStory = ({
|
|
|
134
133
|
// Transcriber.
|
|
135
134
|
const handleSegments = useCallback<TranscriberParams['onSegments']>(
|
|
136
135
|
async (blocks) => {
|
|
137
|
-
const message = Obj.make(
|
|
136
|
+
const message = Obj.make(Message.Message, {
|
|
138
137
|
sender: { name: 'You' },
|
|
139
138
|
created: new Date().toISOString(),
|
|
140
139
|
blocks,
|
|
@@ -197,12 +196,12 @@ const meta = {
|
|
|
197
196
|
title: 'plugins/plugin-transcription/MicrophoneTranscription',
|
|
198
197
|
render: DefaultStory,
|
|
199
198
|
decorators: [
|
|
199
|
+
withTheme,
|
|
200
|
+
withLayout({ container: 'column' }),
|
|
200
201
|
withPluginManager({
|
|
201
202
|
plugins: [
|
|
202
|
-
ThemePlugin({ tx: defaultTx }),
|
|
203
|
-
StorybookLayoutPlugin(),
|
|
204
203
|
ClientPlugin({
|
|
205
|
-
types: [TestItem,
|
|
204
|
+
types: [TestItem, Person.Person, Organization.Organization, Testing.DocumentType],
|
|
206
205
|
onClientInitialized: async ({ client }) => {
|
|
207
206
|
await client.halo.createIdentity();
|
|
208
207
|
await client.spaces.waitUntilReady();
|
|
@@ -223,16 +222,18 @@ const meta = {
|
|
|
223
222
|
await seedTestData(client.spaces.default);
|
|
224
223
|
},
|
|
225
224
|
}),
|
|
226
|
-
SpacePlugin(),
|
|
225
|
+
SpacePlugin({}),
|
|
226
|
+
IntentPlugin(),
|
|
227
227
|
SettingsPlugin(),
|
|
228
|
+
|
|
229
|
+
// UI
|
|
230
|
+
ThemePlugin({ tx: defaultTx }),
|
|
228
231
|
PreviewPlugin(),
|
|
229
|
-
IntentPlugin(),
|
|
230
232
|
TranscriptionPlugin(),
|
|
233
|
+
StorybookLayoutPlugin({}),
|
|
231
234
|
],
|
|
232
235
|
fireEvents: [Events.SetupAppGraph],
|
|
233
236
|
}),
|
|
234
|
-
withTheme,
|
|
235
|
-
withLayout({ fullscreen: true, classNames: 'justify-center' }),
|
|
236
237
|
],
|
|
237
238
|
} satisfies Meta<typeof DefaultStory>;
|
|
238
239
|
|
|
@@ -7,24 +7,23 @@ import '@dxos-theme';
|
|
|
7
7
|
import React, { type Dispatch, type FC, type RefObject, type SetStateAction } from 'react';
|
|
8
8
|
|
|
9
9
|
import { useSpace } from '@dxos/react-client/echo';
|
|
10
|
-
import { IconButton, Toolbar } from '@dxos/react-ui';
|
|
11
|
-
import {
|
|
12
|
-
import { type DataType } from '@dxos/schema';
|
|
10
|
+
import { IconButton, ScrollContainer, Toolbar } from '@dxos/react-ui';
|
|
11
|
+
import { type Message } from '@dxos/types';
|
|
13
12
|
|
|
14
13
|
import { type SerializationModel } from '../../model';
|
|
15
14
|
import { TranscriptView } from '../Transcript';
|
|
16
15
|
|
|
17
16
|
export const TranscriptionStory: FC<{
|
|
18
|
-
model: SerializationModel<
|
|
17
|
+
model: SerializationModel<Message.Message>;
|
|
19
18
|
disabled?: boolean;
|
|
20
19
|
running: boolean;
|
|
21
20
|
onRunningChange: Dispatch<SetStateAction<boolean>>;
|
|
22
|
-
audioRef?: RefObject<HTMLAudioElement>;
|
|
21
|
+
audioRef?: RefObject<HTMLAudioElement | null>;
|
|
23
22
|
}> = ({ model, running, onRunningChange, audioRef, disabled }) => {
|
|
24
23
|
const space = useSpace();
|
|
25
24
|
|
|
26
25
|
return (
|
|
27
|
-
<div className='flex flex-col
|
|
26
|
+
<div className='flex flex-col is-[30rem]'>
|
|
28
27
|
{audioRef && <audio ref={audioRef} autoPlay />}
|
|
29
28
|
<Toolbar.Root>
|
|
30
29
|
<IconButton
|
|
@@ -36,9 +35,9 @@ export const TranscriptionStory: FC<{
|
|
|
36
35
|
/>
|
|
37
36
|
</Toolbar.Root>
|
|
38
37
|
<ScrollContainer.Root pin>
|
|
39
|
-
<ScrollContainer.
|
|
38
|
+
<ScrollContainer.Viewport>
|
|
40
39
|
<TranscriptView space={space} model={model} />
|
|
41
|
-
</ScrollContainer.
|
|
40
|
+
</ScrollContainer.Viewport>
|
|
42
41
|
</ScrollContainer.Root>
|
|
43
42
|
</div>
|
|
44
43
|
);
|
package/src/functions/open.ts
CHANGED
|
@@ -2,18 +2,19 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import
|
|
5
|
+
import * as Effect from 'effect/Effect';
|
|
6
|
+
import * as Schema from 'effect/Schema';
|
|
6
7
|
|
|
7
8
|
import { ArtifactId } from '@dxos/assistant';
|
|
8
9
|
import { Obj } from '@dxos/echo';
|
|
9
10
|
import { DatabaseService, QueueService, defineFunction } from '@dxos/functions';
|
|
10
|
-
import {
|
|
11
|
+
import { Message, Transcript } from '@dxos/types';
|
|
11
12
|
|
|
12
13
|
import { renderByline } from '../components';
|
|
13
|
-
import { Transcript } from '../types';
|
|
14
14
|
|
|
15
15
|
export default defineFunction({
|
|
16
|
-
|
|
16
|
+
key: 'dxos.org/function/transcription/open',
|
|
17
|
+
name: 'Open',
|
|
17
18
|
description: 'Opens and reads the contents of a transcription object.',
|
|
18
19
|
inputSchema: Schema.Struct({
|
|
19
20
|
id: ArtifactId.annotations({
|
|
@@ -29,7 +30,7 @@ export default defineFunction({
|
|
|
29
30
|
const queue = yield* QueueService.getQueue(dxn);
|
|
30
31
|
yield* Effect.promise(() => queue?.queryObjects());
|
|
31
32
|
const content = queue?.objects
|
|
32
|
-
.filter((message) => Obj.instanceOf(
|
|
33
|
+
.filter((message) => Obj.instanceOf(Message.Message, message))
|
|
33
34
|
.flatMap((message, index) => renderByline([])(message, index))
|
|
34
35
|
.join('\n\n');
|
|
35
36
|
return { content };
|
|
@@ -2,7 +2,12 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import
|
|
5
|
+
import * as Array from 'effect/Array';
|
|
6
|
+
import * as Effect from 'effect/Effect';
|
|
7
|
+
import * as Function from 'effect/Function';
|
|
8
|
+
import * as Layer from 'effect/Layer';
|
|
9
|
+
import * as Option from 'effect/Option';
|
|
10
|
+
import * as Schema from 'effect/Schema';
|
|
6
11
|
|
|
7
12
|
import { AiService, ConsolePrinter, ToolExecutionService, ToolResolverService } from '@dxos/ai';
|
|
8
13
|
import { AiSession, GenerationObserver } from '@dxos/assistant';
|
|
@@ -13,7 +18,8 @@ import { trim } from '@dxos/util';
|
|
|
13
18
|
* Summarize a transcript of a meeting.
|
|
14
19
|
*/
|
|
15
20
|
export default defineFunction({
|
|
16
|
-
|
|
21
|
+
key: 'dxos.org/function/transcription/summarize',
|
|
22
|
+
name: 'Summarize',
|
|
17
23
|
description: 'Summarize a transcript of a meeting.',
|
|
18
24
|
inputSchema: Schema.Struct({
|
|
19
25
|
transcript: Schema.String.annotations({
|
|
@@ -37,11 +43,11 @@ export default defineFunction({
|
|
|
37
43
|
observer: GenerationObserver.fromPrinter(new ConsolePrinter({ tag: 'summarize' })),
|
|
38
44
|
});
|
|
39
45
|
|
|
40
|
-
const summary = pipe(
|
|
46
|
+
const summary = Function.pipe(
|
|
41
47
|
result,
|
|
42
48
|
Array.findLast((msg) => msg.sender.role === 'assistant' && msg.blocks.some((block) => block._tag === 'text')),
|
|
43
49
|
Option.flatMap((msg) =>
|
|
44
|
-
pipe(
|
|
50
|
+
Function.pipe(
|
|
45
51
|
msg.blocks,
|
|
46
52
|
Array.findLast((block) => block._tag === 'text'),
|
|
47
53
|
Option.map((block) => block.text),
|
|
@@ -20,58 +20,63 @@ export const useAudioFile = (audioUrl: string, constraints?: MediaTrackConstrain
|
|
|
20
20
|
const ctx = new Context();
|
|
21
21
|
|
|
22
22
|
scheduleTask(ctx, async () => {
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
URL.
|
|
28
|
-
|
|
23
|
+
// TODO(wittjosiah): Fetch to external url fails in headless storybook test.
|
|
24
|
+
try {
|
|
25
|
+
const response = await fetch(audioUrl);
|
|
26
|
+
const blob = await response.blob();
|
|
27
|
+
const objectUrl = URL.createObjectURL(blob);
|
|
28
|
+
ctx.onDispose(() => {
|
|
29
|
+
URL.revokeObjectURL(objectUrl);
|
|
30
|
+
});
|
|
29
31
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
+
const audio = new Audio();
|
|
33
|
+
audio.src = objectUrl;
|
|
32
34
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
35
|
+
await new Promise<void>((resolve, reject) => {
|
|
36
|
+
audio.addEventListener(
|
|
37
|
+
'error',
|
|
38
|
+
(err) => {
|
|
39
|
+
log.error('error', { err });
|
|
40
|
+
reject(err);
|
|
41
|
+
},
|
|
42
|
+
{ once: true },
|
|
43
|
+
);
|
|
44
|
+
audio.addEventListener(
|
|
45
|
+
'canplay',
|
|
46
|
+
async () => {
|
|
47
|
+
resolve();
|
|
48
|
+
},
|
|
49
|
+
{ once: true },
|
|
50
|
+
);
|
|
51
|
+
audio.load();
|
|
52
|
+
});
|
|
51
53
|
|
|
52
|
-
|
|
54
|
+
const audioCtx = new AudioContext();
|
|
53
55
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
56
|
+
// Resume AudioContext if it's suspended.
|
|
57
|
+
if (audioCtx.state === 'suspended') {
|
|
58
|
+
await audioCtx.resume();
|
|
59
|
+
}
|
|
58
60
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
+
const destination = audioCtx.createMediaStreamDestination();
|
|
62
|
+
destination.channelCount = 1;
|
|
61
63
|
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
64
|
+
const source = audioCtx.createMediaElementSource(audio);
|
|
65
|
+
source.connect(destination);
|
|
66
|
+
const track = destination.stream.getAudioTracks()[0];
|
|
67
|
+
if (constraints) {
|
|
68
|
+
await track.applyConstraints(constraints).catch((err) => log.catch(err));
|
|
69
|
+
}
|
|
68
70
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
71
|
+
// Also connect to speakers so audio is audible.
|
|
72
|
+
setStream({
|
|
73
|
+
audio,
|
|
74
|
+
stream: destination.stream,
|
|
75
|
+
track: destination.stream.getAudioTracks()[0],
|
|
76
|
+
});
|
|
77
|
+
} catch (err) {
|
|
78
|
+
log.catch(err);
|
|
79
|
+
}
|
|
75
80
|
});
|
|
76
81
|
|
|
77
82
|
return () => {
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
import { useEffect, useMemo } from 'react';
|
|
6
6
|
|
|
7
|
-
import { useCapabilities } from '@dxos/app-framework';
|
|
7
|
+
import { useCapabilities } from '@dxos/app-framework/react';
|
|
8
8
|
|
|
9
9
|
import { TranscriptionCapabilities } from '../capabilities';
|
|
10
10
|
import { type Transcriber } from '../transcriber';
|
|
@@ -26,7 +26,12 @@ export const useTranscriber = ({
|
|
|
26
26
|
return undefined;
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
-
return getTranscriber({
|
|
29
|
+
return getTranscriber({
|
|
30
|
+
audioStreamTrack,
|
|
31
|
+
recorderConfig,
|
|
32
|
+
transcriberConfig,
|
|
33
|
+
onSegments,
|
|
34
|
+
});
|
|
30
35
|
}, [getTranscriber, audioStreamTrack, recorderConfig, transcriberConfig, onSegments]);
|
|
31
36
|
|
|
32
37
|
useEffect(() => {
|
package/src/meta.ts
CHANGED
|
@@ -3,14 +3,17 @@
|
|
|
3
3
|
//
|
|
4
4
|
|
|
5
5
|
import { type PluginMeta } from '@dxos/app-framework';
|
|
6
|
-
|
|
7
|
-
export const TRANSCRIPTION_PLUGIN = 'dxos.org/plugin/transcription';
|
|
6
|
+
import { trim } from '@dxos/util';
|
|
8
7
|
|
|
9
8
|
export const meta: PluginMeta = {
|
|
10
|
-
id:
|
|
9
|
+
id: 'dxos.org/plugin/transcription',
|
|
11
10
|
name: 'Transcription',
|
|
12
|
-
description:
|
|
11
|
+
description: trim`
|
|
12
|
+
Real-time voice-to-text transcription service for capturing spoken content.
|
|
13
|
+
Convert audio input into searchable text that integrates seamlessly with notes and documents.
|
|
14
|
+
`,
|
|
13
15
|
icon: 'ph--microphone--regular',
|
|
16
|
+
iconHue: 'sky',
|
|
14
17
|
source: 'https://github.com/dxos/dxos/tree/main/packages/plugins/plugin-transcription',
|
|
15
18
|
tags: ['labs'],
|
|
16
19
|
screenshots: [],
|
package/src/model/model.test.ts
CHANGED
|
@@ -7,12 +7,12 @@ import { describe, test } from 'vitest';
|
|
|
7
7
|
|
|
8
8
|
import { Obj } from '@dxos/echo';
|
|
9
9
|
import type { ObjectId } from '@dxos/keys';
|
|
10
|
-
import {
|
|
10
|
+
import { Message } from '@dxos/types';
|
|
11
11
|
|
|
12
12
|
import { type ChunkRenderer, DocumentAdapter, SerializationModel } from './model';
|
|
13
13
|
|
|
14
|
-
const blockToMarkdown: ChunkRenderer<
|
|
15
|
-
message:
|
|
14
|
+
const blockToMarkdown: ChunkRenderer<Message.Message> = (
|
|
15
|
+
message: Message.Message,
|
|
16
16
|
index: number,
|
|
17
17
|
debug = true,
|
|
18
18
|
): string[] => {
|
|
@@ -27,12 +27,12 @@ const createDate = () => new Date().toISOString();
|
|
|
27
27
|
|
|
28
28
|
describe('SerializationModel', () => {
|
|
29
29
|
test('basic', ({ expect }) => {
|
|
30
|
-
const model = new SerializationModel<
|
|
30
|
+
const model = new SerializationModel<Message.Message>(blockToMarkdown);
|
|
31
31
|
expect(model.chunks.length).to.eq(0);
|
|
32
32
|
expect(model.doc.toString()).to.eq('');
|
|
33
33
|
|
|
34
34
|
// Create message.
|
|
35
|
-
const message = Obj.make(
|
|
35
|
+
const message = Obj.make(Message.Message, {
|
|
36
36
|
created: createDate(),
|
|
37
37
|
sender: { name: 'Alice' },
|
|
38
38
|
blocks: [
|
|
@@ -62,14 +62,14 @@ describe('SerializationModel', () => {
|
|
|
62
62
|
|
|
63
63
|
test('sync - append', ({ expect }) => {
|
|
64
64
|
const view = new EditorView({ extensions: [], doc: '' });
|
|
65
|
-
const model = new SerializationModel<
|
|
65
|
+
const model = new SerializationModel<Message.Message>(blockToMarkdown);
|
|
66
66
|
const adapter = new DocumentAdapter(view);
|
|
67
67
|
expect(adapter.lineCount()).to.eq(0);
|
|
68
68
|
expect(view.state.doc.toString()).to.eq('');
|
|
69
69
|
|
|
70
70
|
// Append message.
|
|
71
71
|
{
|
|
72
|
-
const message = Obj.make(
|
|
72
|
+
const message = Obj.make(Message.Message, {
|
|
73
73
|
created: createDate(),
|
|
74
74
|
sender: { name: 'Alice' },
|
|
75
75
|
blocks: [
|
|
@@ -87,7 +87,7 @@ describe('SerializationModel', () => {
|
|
|
87
87
|
|
|
88
88
|
// Append message.
|
|
89
89
|
{
|
|
90
|
-
const message = Obj.make(
|
|
90
|
+
const message = Obj.make(Message.Message, {
|
|
91
91
|
created: createDate(),
|
|
92
92
|
sender: { name: 'Bob' },
|
|
93
93
|
blocks: [
|
|
@@ -106,7 +106,7 @@ describe('SerializationModel', () => {
|
|
|
106
106
|
|
|
107
107
|
test('sync - append, update, delete', ({ expect }) => {
|
|
108
108
|
const view = new EditorView({ extensions: [], doc: '' });
|
|
109
|
-
const model = new SerializationModel<
|
|
109
|
+
const model = new SerializationModel<Message.Message>(blockToMarkdown);
|
|
110
110
|
const adapter = new DocumentAdapter(view);
|
|
111
111
|
expect(adapter.lineCount()).to.eq(0);
|
|
112
112
|
|
|
@@ -114,7 +114,7 @@ describe('SerializationModel', () => {
|
|
|
114
114
|
|
|
115
115
|
// Append message.
|
|
116
116
|
{
|
|
117
|
-
const message = Obj.make(
|
|
117
|
+
const message = Obj.make(Message.Message, {
|
|
118
118
|
created: createDate(),
|
|
119
119
|
sender: { name: 'Alice' },
|
|
120
120
|
blocks: [
|
|
@@ -147,7 +147,7 @@ describe('SerializationModel', () => {
|
|
|
147
147
|
|
|
148
148
|
// Append message.
|
|
149
149
|
{
|
|
150
|
-
const message = Obj.make(
|
|
150
|
+
const message = Obj.make(Message.Message, {
|
|
151
151
|
created: createDate(),
|
|
152
152
|
sender: { name: 'Bob' },
|
|
153
153
|
blocks: [{ _tag: 'transcript', started: createDate(), text: 'Hello again!' }],
|
|
@@ -11,11 +11,11 @@ import { effect } from '@preact/signals-core';
|
|
|
11
11
|
import { DeferredTask, asyncTimeout } from '@dxos/async';
|
|
12
12
|
import { LifecycleState, Resource } from '@dxos/context';
|
|
13
13
|
import { type Queue } from '@dxos/echo-db';
|
|
14
|
-
import { type FunctionExecutor } from '@dxos/functions';
|
|
14
|
+
import { type FunctionExecutor } from '@dxos/functions-runtime';
|
|
15
15
|
import { log } from '@dxos/log';
|
|
16
|
-
import { type
|
|
16
|
+
import { type Message } from '@dxos/types';
|
|
17
17
|
|
|
18
|
-
import { type MessageWithRangeId, sentenceNormalization } from './normalization';
|
|
18
|
+
import { type MessageWithRangeId, type NormalizationOutput, sentenceNormalization } from './normalization';
|
|
19
19
|
import { getActorId } from './utils';
|
|
20
20
|
|
|
21
21
|
const PROCESSING_TIMEOUT = 20_000; // ms
|
|
@@ -23,7 +23,7 @@ const MAX_RANGE_ID_COUNT = 10;
|
|
|
23
23
|
|
|
24
24
|
export type SegmentsNormalizerParams = {
|
|
25
25
|
functionExecutor: FunctionExecutor;
|
|
26
|
-
queue: Queue<
|
|
26
|
+
queue: Queue<Message.Message>;
|
|
27
27
|
startingCursor: QueueCursor;
|
|
28
28
|
};
|
|
29
29
|
|
|
@@ -36,7 +36,7 @@ export type QueueCursor = {
|
|
|
36
36
|
// TODO(mykola): .
|
|
37
37
|
export class MessageNormalizer extends Resource {
|
|
38
38
|
private readonly _functionExecutor: FunctionExecutor;
|
|
39
|
-
private _queue: Queue<
|
|
39
|
+
private _queue: Queue<Message.Message>;
|
|
40
40
|
private _cursor: QueueCursor;
|
|
41
41
|
private _messagesToProcess: MessageWithRangeId[] = [];
|
|
42
42
|
private _normalizationTask?: DeferredTask;
|
|
@@ -81,7 +81,7 @@ export class MessageNormalizer extends Resource {
|
|
|
81
81
|
|
|
82
82
|
try {
|
|
83
83
|
// TODO(mykola): Executor should support timeout.
|
|
84
|
-
const response = await asyncTimeout(
|
|
84
|
+
const response: NormalizationOutput = await asyncTimeout(
|
|
85
85
|
this._functionExecutor.invoke(sentenceNormalization, { messages }),
|
|
86
86
|
PROCESSING_TIMEOUT,
|
|
87
87
|
);
|