@dxos/plugin-transcription 0.8.4-main.5ea62a8 → 0.8.4-main.72ec0f3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/TranscriptContainer-EKSP5DQ6.mjs +11 -0
- package/dist/lib/browser/{blueprint-definition-QQSGTX5F.mjs → blueprint-definition-WBWHO4OH.mjs} +24 -16
- package/dist/lib/browser/blueprint-definition-WBWHO4OH.mjs.map +7 -0
- package/dist/lib/browser/{chunk-TNEQFHIU.mjs → chunk-3MKZ2MNQ.mjs} +5 -5
- package/dist/lib/browser/chunk-3MKZ2MNQ.mjs.map +7 -0
- package/dist/lib/browser/{chunk-5I4BMOGE.mjs → chunk-EHT3TE6U.mjs} +129 -174
- package/dist/lib/browser/chunk-EHT3TE6U.mjs.map +7 -0
- package/dist/lib/browser/{chunk-26XDG6GQ.mjs → chunk-OCI42DWB.mjs} +2 -2
- package/dist/lib/browser/{chunk-BA64VLLR.mjs → chunk-PVU4XCKI.mjs} +46 -66
- package/dist/lib/browser/chunk-PVU4XCKI.mjs.map +7 -0
- package/dist/lib/browser/{chunk-3X7HP44M.mjs → chunk-VBUIO677.mjs} +8 -5
- package/dist/lib/browser/chunk-VBUIO677.mjs.map +7 -0
- package/dist/lib/browser/{chunk-HXVGV7T6.mjs → chunk-W7WKZTM4.mjs} +4 -4
- package/dist/lib/browser/chunk-W7WKZTM4.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +13 -14
- package/dist/lib/browser/index.mjs.map +3 -3
- package/dist/lib/browser/{intent-resolver-T3EGLS3L.mjs → intent-resolver-ZTKTEAQX.mjs} +5 -5
- package/dist/lib/browser/{intent-resolver-T3EGLS3L.mjs.map → intent-resolver-ZTKTEAQX.mjs.map} +1 -1
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/{react-surface-QYX74JNG.mjs → react-surface-NARMYO2O.mjs} +8 -8
- package/dist/lib/browser/{react-surface-QYX74JNG.mjs.map → react-surface-NARMYO2O.mjs.map} +1 -1
- package/dist/lib/browser/{transcriber-JOOAQVYV.mjs → transcriber-E2KAOXPS.mjs} +7 -7
- package/dist/lib/browser/{transcriber-JOOAQVYV.mjs.map → transcriber-E2KAOXPS.mjs.map} +1 -1
- package/dist/lib/browser/types/index.mjs +2 -2
- package/dist/lib/node-esm/{TranscriptContainer-BUWK63SI.mjs → TranscriptContainer-WU3EDQR3.mjs} +4 -4
- package/dist/lib/node-esm/{blueprint-definition-4PLCTH5Q.mjs → blueprint-definition-WUHZFRWZ.mjs} +24 -16
- package/dist/lib/node-esm/blueprint-definition-WUHZFRWZ.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-DT3AOC3E.mjs → chunk-7XASA7DX.mjs} +129 -174
- package/dist/lib/node-esm/chunk-7XASA7DX.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-W72HVQMD.mjs → chunk-OLMSTF3M.mjs} +4 -4
- package/dist/lib/node-esm/chunk-OLMSTF3M.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-3QAIZGMW.mjs → chunk-SWUVPFKR.mjs} +5 -5
- package/dist/lib/node-esm/chunk-SWUVPFKR.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-J5GQ3KGX.mjs → chunk-UVOYXXDJ.mjs} +46 -66
- package/dist/lib/node-esm/chunk-UVOYXXDJ.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-NWD757KH.mjs → chunk-X5BKN32O.mjs} +2 -2
- package/dist/lib/node-esm/{chunk-QARUDLHN.mjs → chunk-YVNQN2J6.mjs} +8 -5
- package/dist/lib/node-esm/chunk-YVNQN2J6.mjs.map +7 -0
- package/dist/lib/node-esm/index.mjs +13 -14
- package/dist/lib/node-esm/index.mjs.map +3 -3
- package/dist/lib/node-esm/{intent-resolver-VDHCUAIO.mjs → intent-resolver-55WH2GY6.mjs} +5 -5
- package/dist/lib/node-esm/{intent-resolver-VDHCUAIO.mjs.map → intent-resolver-55WH2GY6.mjs.map} +1 -1
- package/dist/lib/node-esm/meta.json +1 -1
- package/dist/lib/node-esm/{react-surface-TM7XOQBR.mjs → react-surface-5MQA3IUS.mjs} +8 -8
- package/dist/lib/node-esm/{react-surface-TM7XOQBR.mjs.map → react-surface-5MQA3IUS.mjs.map} +1 -1
- package/dist/lib/node-esm/{transcriber-QPS55YP2.mjs → transcriber-ZPJM4U2W.mjs} +7 -7
- package/dist/lib/node-esm/{transcriber-QPS55YP2.mjs.map → transcriber-ZPJM4U2W.mjs.map} +1 -1
- package/dist/lib/node-esm/types/index.mjs +2 -2
- package/dist/types/src/TranscriptionPlugin.d.ts +1 -1
- package/dist/types/src/TranscriptionPlugin.d.ts.map +1 -1
- package/dist/types/src/capabilities/blueprint-definition.d.ts +1 -1
- package/dist/types/src/capabilities/capabilities.d.ts.map +1 -1
- package/dist/types/src/components/Transcript/Transcript.d.ts +5 -5
- package/dist/types/src/components/Transcript/Transcript.d.ts.map +1 -1
- package/dist/types/src/components/Transcript/Transcript.stories.d.ts +3 -5
- package/dist/types/src/components/Transcript/Transcript.stories.d.ts.map +1 -1
- package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts +1 -1
- package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts.map +1 -1
- package/dist/types/src/components/Transcript/transcript-extension.d.ts +3 -7
- package/dist/types/src/components/Transcript/transcript-extension.d.ts.map +1 -1
- package/dist/types/src/components/TranscriptContainer.d.ts.map +1 -1
- package/dist/types/src/components/stories/FileTranscription.stories.d.ts +0 -2
- package/dist/types/src/components/stories/FileTranscription.stories.d.ts.map +1 -1
- package/dist/types/src/components/stories/KeyWordRecognition.stories.d.ts +0 -1
- package/dist/types/src/components/stories/KeyWordRecognition.stories.d.ts.map +1 -1
- package/dist/types/src/components/stories/MicrophoneTranscription.stories.d.ts +0 -2
- package/dist/types/src/components/stories/MicrophoneTranscription.stories.d.ts.map +1 -1
- package/dist/types/src/components/stories/TranscriptionStory.d.ts +3 -3
- package/dist/types/src/components/stories/TranscriptionStory.d.ts.map +1 -1
- package/dist/types/src/functions/open.d.ts +1 -1
- package/dist/types/src/functions/open.d.ts.map +1 -1
- package/dist/types/src/functions/summarize.d.ts +1 -1
- package/dist/types/src/functions/summarize.d.ts.map +1 -1
- package/dist/types/src/hooks/useAudioFile.d.ts.map +1 -1
- package/dist/types/src/hooks/useTranscriber.d.ts.map +1 -1
- package/dist/types/src/meta.d.ts +0 -1
- package/dist/types/src/meta.d.ts.map +1 -1
- package/dist/types/src/segments-normalization/message-normalizer.d.ts +3 -3
- package/dist/types/src/segments-normalization/message-normalizer.d.ts.map +1 -1
- package/dist/types/src/segments-normalization/normalization.d.ts +32 -17
- package/dist/types/src/segments-normalization/normalization.d.ts.map +1 -1
- package/dist/types/src/segments-normalization/utils.d.ts +2 -2
- package/dist/types/src/segments-normalization/utils.d.ts.map +1 -1
- package/dist/types/src/testing/testing.d.ts +6 -6
- package/dist/types/src/testing/testing.d.ts.map +1 -1
- package/dist/types/src/transcriber/transcriber.d.ts +2 -2
- package/dist/types/src/transcriber/transcriber.d.ts.map +1 -1
- package/dist/types/src/transcriber/transcription-manager.d.ts +3 -3
- package/dist/types/src/transcriber/transcription-manager.d.ts.map +1 -1
- package/dist/types/src/types/Transcript.d.ts +3 -3
- package/dist/types/src/types/Transcript.d.ts.map +1 -1
- package/dist/types/src/types/TranscriptAction.d.ts +1 -1
- package/dist/types/src/types/TranscriptAction.d.ts.map +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +64 -63
- package/src/TranscriptionPlugin.tsx +57 -57
- package/src/capabilities/capabilities.ts +3 -3
- package/src/components/Transcript/Transcript.stories.tsx +28 -24
- package/src/components/Transcript/Transcript.tsx +20 -27
- package/src/components/Transcript/deprecated/OldTranscript.tsx +5 -5
- package/src/components/Transcript/transcript-extension.ts +11 -44
- package/src/components/TranscriptContainer.tsx +8 -7
- package/src/components/stories/FileTranscription.stories.tsx +33 -31
- package/src/components/stories/KeyWordRecognition.stories.tsx +4 -6
- package/src/components/stories/MicrophoneTranscription.stories.tsx +40 -37
- package/src/components/stories/TranscriptionStory.tsx +8 -9
- package/src/functions/open.ts +8 -6
- package/src/functions/summarize.ts +10 -4
- package/src/hooks/useAudioFile.ts +50 -45
- package/src/hooks/useTranscriber.ts +7 -2
- package/src/meta.ts +7 -4
- package/src/model/model.test.ts +11 -11
- package/src/segments-normalization/message-normalizer.ts +6 -6
- package/src/segments-normalization/normalization.test.ts +6 -6
- package/src/segments-normalization/normalization.ts +7 -7
- package/src/segments-normalization/utils.ts +2 -2
- package/src/testing/testing.ts +25 -20
- package/src/transcriber/transcriber.browser.test.ts +3 -3
- package/src/transcriber/transcriber.ts +3 -3
- package/src/transcriber/transcription-manager.ts +6 -6
- package/src/types/Transcript.ts +1 -1
- package/src/types/TranscriptAction.ts +1 -1
- package/dist/lib/browser/TranscriptContainer-OX7OOODC.mjs +0 -11
- package/dist/lib/browser/blueprint-definition-QQSGTX5F.mjs.map +0 -7
- package/dist/lib/browser/chunk-3X7HP44M.mjs.map +0 -7
- package/dist/lib/browser/chunk-5I4BMOGE.mjs.map +0 -7
- package/dist/lib/browser/chunk-BA64VLLR.mjs.map +0 -7
- package/dist/lib/browser/chunk-HXVGV7T6.mjs.map +0 -7
- package/dist/lib/browser/chunk-TNEQFHIU.mjs.map +0 -7
- package/dist/lib/node-esm/blueprint-definition-4PLCTH5Q.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-3QAIZGMW.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-DT3AOC3E.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-J5GQ3KGX.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-QARUDLHN.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-W72HVQMD.mjs.map +0 -7
- /package/dist/lib/browser/{TranscriptContainer-OX7OOODC.mjs.map → TranscriptContainer-EKSP5DQ6.mjs.map} +0 -0
- /package/dist/lib/browser/{chunk-26XDG6GQ.mjs.map → chunk-OCI42DWB.mjs.map} +0 -0
- /package/dist/lib/node-esm/{TranscriptContainer-BUWK63SI.mjs.map → TranscriptContainer-WU3EDQR3.mjs.map} +0 -0
- /package/dist/lib/node-esm/{chunk-NWD757KH.mjs.map → chunk-X5BKN32O.mjs.map} +0 -0
|
@@ -8,8 +8,7 @@ import { format } from 'date-fns/format';
|
|
|
8
8
|
import { intervalToDuration } from 'date-fns/intervalToDuration';
|
|
9
9
|
|
|
10
10
|
import { type CleanupFn, addEventListener, combine } from '@dxos/async';
|
|
11
|
-
import { type
|
|
12
|
-
import { type DataType } from '@dxos/schema';
|
|
11
|
+
import { type Message } from '@dxos/types';
|
|
13
12
|
|
|
14
13
|
import { DocumentAdapter, type SerializationModel } from '../../model';
|
|
15
14
|
|
|
@@ -17,15 +16,14 @@ import { DocumentAdapter, type SerializationModel } from '../../model';
|
|
|
17
16
|
* Data structure that maps Chunks queue to lines with transcript state.
|
|
18
17
|
*/
|
|
19
18
|
export type TranscriptOptions = {
|
|
20
|
-
model: SerializationModel<
|
|
19
|
+
model: SerializationModel<Message.Message>;
|
|
21
20
|
started?: Date;
|
|
22
|
-
renderButton?: RenderCallback<{ onClick: () => void }>;
|
|
23
21
|
};
|
|
24
22
|
|
|
25
23
|
/**
|
|
26
24
|
* Scrolling transcript with timestamps.
|
|
27
25
|
*/
|
|
28
|
-
export const transcript = ({ model, started
|
|
26
|
+
export const transcript = ({ model, started }: TranscriptOptions): Extension => {
|
|
29
27
|
return [
|
|
30
28
|
// Show timestamps in the gutter.
|
|
31
29
|
gutter({
|
|
@@ -90,22 +88,6 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
|
|
|
90
88
|
});
|
|
91
89
|
};
|
|
92
90
|
|
|
93
|
-
// Scroll button.
|
|
94
|
-
if (renderButton) {
|
|
95
|
-
this._controls = document.createElement('div');
|
|
96
|
-
this._controls.classList.add('cm-controls', 'transition-opacity', 'duration-300', 'opacity-0');
|
|
97
|
-
view.dom.appendChild(this._controls);
|
|
98
|
-
renderButton(
|
|
99
|
-
this._controls,
|
|
100
|
-
{
|
|
101
|
-
onClick: () => {
|
|
102
|
-
scrollToBottom(false);
|
|
103
|
-
},
|
|
104
|
-
},
|
|
105
|
-
view,
|
|
106
|
-
);
|
|
107
|
-
}
|
|
108
|
-
|
|
109
91
|
// Event listeners.
|
|
110
92
|
this._cleanup = combine(
|
|
111
93
|
addEventListener(view.scrollDOM, 'scroll', () => {
|
|
@@ -131,7 +113,7 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
|
|
|
131
113
|
);
|
|
132
114
|
}
|
|
133
115
|
|
|
134
|
-
update(
|
|
116
|
+
update(_update: ViewUpdate) {
|
|
135
117
|
// Initial sync.
|
|
136
118
|
if (!this._initialized) {
|
|
137
119
|
this._initialized = true;
|
|
@@ -149,27 +131,12 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
|
|
|
149
131
|
),
|
|
150
132
|
|
|
151
133
|
EditorView.theme({
|
|
152
|
-
'.cm-
|
|
153
|
-
|
|
154
|
-
},
|
|
155
|
-
'.cm-hide-scrollbar': {
|
|
156
|
-
scrollbarWidth: 'none',
|
|
157
|
-
'-ms-overflow-style': 'none',
|
|
158
|
-
},
|
|
159
|
-
'.cm-hide-scrollbar::-webkit-scrollbar': {
|
|
160
|
-
display: 'none',
|
|
161
|
-
},
|
|
162
|
-
'.cm-controls': {
|
|
163
|
-
position: 'absolute',
|
|
164
|
-
bottom: '0.5rem',
|
|
165
|
-
right: '0.5rem',
|
|
166
|
-
zIndex: 1000,
|
|
167
|
-
},
|
|
168
|
-
'.cm-line': {
|
|
169
|
-
paddingRight: '1rem',
|
|
134
|
+
'.cm-gutters': {
|
|
135
|
+
backgroundColor: 'var(--dx-baseSurface)',
|
|
170
136
|
},
|
|
171
137
|
'.cm-timestamp-gutter': {
|
|
172
138
|
width: '6rem',
|
|
139
|
+
paddingLeft: '0.5rem',
|
|
173
140
|
paddingRight: '1rem',
|
|
174
141
|
},
|
|
175
142
|
'.cm-timestamp-gutter > .cm-gutterElement > div': {
|
|
@@ -193,11 +160,11 @@ class TimestampMarker extends GutterMarker {
|
|
|
193
160
|
super();
|
|
194
161
|
}
|
|
195
162
|
|
|
196
|
-
override eq(other: this)
|
|
197
|
-
return
|
|
163
|
+
override eq(other: this) {
|
|
164
|
+
return this._timestamp === other._timestamp;
|
|
198
165
|
}
|
|
199
166
|
|
|
200
|
-
override toDOM(view: EditorView)
|
|
167
|
+
override toDOM(view: EditorView) {
|
|
201
168
|
const el = document.createElement('div');
|
|
202
169
|
el.className = 'text-sm text-subdued hover:bg-hoverSurface cursor-pointer';
|
|
203
170
|
el.textContent = formatTimestamp(this._timestamp, this._started);
|
|
@@ -213,7 +180,7 @@ class TimestampMarker extends GutterMarker {
|
|
|
213
180
|
}
|
|
214
181
|
}
|
|
215
182
|
|
|
216
|
-
const getStartTime = (started?: Date, message?:
|
|
183
|
+
const getStartTime = (started?: Date, message?: Message.Message): Date | undefined => {
|
|
217
184
|
if (started) {
|
|
218
185
|
return started;
|
|
219
186
|
}
|
|
@@ -4,15 +4,16 @@
|
|
|
4
4
|
|
|
5
5
|
import React from 'react';
|
|
6
6
|
|
|
7
|
-
import {
|
|
7
|
+
import { getSpace } from '@dxos/client/echo';
|
|
8
|
+
import { Obj } from '@dxos/echo';
|
|
8
9
|
import { useMembers, useQueue } from '@dxos/react-client/echo';
|
|
9
10
|
import { StackItem } from '@dxos/react-ui-stack';
|
|
10
|
-
import { type
|
|
11
|
+
import { type Message } from '@dxos/types';
|
|
11
12
|
|
|
12
13
|
import { useQueueModelAdapter } from '../hooks';
|
|
13
14
|
import { type Transcript } from '../types';
|
|
14
15
|
|
|
15
|
-
import { TranscriptView,
|
|
16
|
+
import { TranscriptView, renderByline } from './Transcript';
|
|
16
17
|
|
|
17
18
|
export type TranscriptionContainerProps = {
|
|
18
19
|
role: string;
|
|
@@ -20,14 +21,14 @@ export type TranscriptionContainerProps = {
|
|
|
20
21
|
};
|
|
21
22
|
|
|
22
23
|
export const TranscriptionContainer = ({ transcript }: TranscriptionContainerProps) => {
|
|
23
|
-
const attendableId =
|
|
24
|
+
const attendableId = Obj.getDXN(transcript).toString();
|
|
24
25
|
const space = getSpace(transcript);
|
|
25
26
|
const members = useMembers(space?.key).map((member) => member.identity);
|
|
26
|
-
const queue = useQueue<
|
|
27
|
-
const model = useQueueModelAdapter(
|
|
27
|
+
const queue = useQueue<Message.Message>(transcript.queue.dxn, { pollInterval: 1_000 });
|
|
28
|
+
const model = useQueueModelAdapter(renderByline(members), queue);
|
|
28
29
|
|
|
29
30
|
return (
|
|
30
|
-
<StackItem.Content
|
|
31
|
+
<StackItem.Content>
|
|
31
32
|
<TranscriptView attendableId={attendableId} space={space} model={model} transcript={transcript} />
|
|
32
33
|
</StackItem.Content>
|
|
33
34
|
);
|
|
@@ -2,8 +2,6 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import '@dxos-theme';
|
|
6
|
-
|
|
7
5
|
import { type Meta, type StoryObj } from '@storybook/react-vite';
|
|
8
6
|
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
|
9
7
|
|
|
@@ -12,26 +10,27 @@ import { withPluginManager } from '@dxos/app-framework/testing';
|
|
|
12
10
|
import { scheduleTask } from '@dxos/async';
|
|
13
11
|
import { Context } from '@dxos/context';
|
|
14
12
|
import { Obj } from '@dxos/echo';
|
|
13
|
+
import { createQueueDXN } from '@dxos/echo/internal';
|
|
15
14
|
import { MemoryQueue } from '@dxos/echo-db';
|
|
16
|
-
import {
|
|
17
|
-
import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
|
|
15
|
+
import { FunctionExecutor, ServiceContainer } from '@dxos/functions-runtime';
|
|
18
16
|
import { log } from '@dxos/log';
|
|
19
17
|
import { ClientPlugin } from '@dxos/plugin-client';
|
|
20
18
|
import { PreviewPlugin } from '@dxos/plugin-preview';
|
|
21
19
|
import { SpacePlugin } from '@dxos/plugin-space';
|
|
22
20
|
import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
|
|
23
21
|
import { ThemePlugin } from '@dxos/plugin-theme';
|
|
22
|
+
import { withLayout, withTheme } from '@dxos/react-ui/testing';
|
|
24
23
|
import { defaultTx } from '@dxos/react-ui-theme';
|
|
25
|
-
import {
|
|
26
|
-
import {
|
|
27
|
-
import {
|
|
24
|
+
import { Testing } from '@dxos/schema/testing';
|
|
25
|
+
import { type Actor, Message, Organization, Person } from '@dxos/types';
|
|
26
|
+
import { seedTestData } from '@dxos/types/testing';
|
|
28
27
|
|
|
29
28
|
import { useAudioFile, useQueueModelAdapter, useTranscriber } from '../../hooks';
|
|
30
29
|
import { MessageNormalizer, getActorId } from '../../segments-normalization';
|
|
31
30
|
import { TestItem } from '../../testing';
|
|
32
31
|
import { type MediaStreamRecorderParams, type TranscriberParams } from '../../transcriber';
|
|
33
32
|
import { TranscriptionPlugin } from '../../TranscriptionPlugin';
|
|
34
|
-
import {
|
|
33
|
+
import { renderByline } from '../Transcript';
|
|
35
34
|
|
|
36
35
|
import { TranscriptionStory } from './TranscriptionStory';
|
|
37
36
|
import { useIsSpeaking } from './useIsSpeaking';
|
|
@@ -52,7 +51,7 @@ const AudioFile = ({
|
|
|
52
51
|
audioConstraints?: MediaTrackConstraints;
|
|
53
52
|
}) => {
|
|
54
53
|
const [running, setRunning] = useState(false);
|
|
55
|
-
const actor:
|
|
54
|
+
const actor: Actor.Actor = useMemo(() => ({ name: 'You' }), []);
|
|
56
55
|
|
|
57
56
|
// Audio.
|
|
58
57
|
const { audio, track, stream } = useAudioFile(audioUrl, audioConstraints);
|
|
@@ -81,13 +80,13 @@ const AudioFile = ({
|
|
|
81
80
|
// Transcriber.
|
|
82
81
|
// TODO(dmaretskyi): Use space.queues.create() instead.
|
|
83
82
|
const queueDxn = useMemo(() => createQueueDXN(), []);
|
|
84
|
-
const queue = useMemo(() => new MemoryQueue<
|
|
83
|
+
const queue = useMemo(() => new MemoryQueue<Message.Message>(queueDxn), [queueDxn]);
|
|
85
84
|
|
|
86
|
-
const model = useQueueModelAdapter(
|
|
85
|
+
const model = useQueueModelAdapter(renderByline([]), queue);
|
|
87
86
|
const handleSegments = useCallback<TranscriberParams['onSegments']>(
|
|
88
87
|
async (blocks) => {
|
|
89
88
|
void queue?.append([
|
|
90
|
-
Obj.make(
|
|
89
|
+
Obj.make(Message.Message, {
|
|
91
90
|
created: new Date().toISOString(),
|
|
92
91
|
sender: actor,
|
|
93
92
|
blocks,
|
|
@@ -189,12 +188,12 @@ const AudioFile = ({
|
|
|
189
188
|
const meta = {
|
|
190
189
|
title: 'plugins/plugin-transcription/FileTranscription',
|
|
191
190
|
decorators: [
|
|
191
|
+
withTheme,
|
|
192
|
+
withLayout({ container: 'column' }),
|
|
192
193
|
withPluginManager({
|
|
193
194
|
plugins: [
|
|
194
|
-
ThemePlugin({ tx: defaultTx }),
|
|
195
|
-
StorybookLayoutPlugin(),
|
|
196
195
|
ClientPlugin({
|
|
197
|
-
types: [TestItem,
|
|
196
|
+
types: [TestItem, Person.Person, Organization.Organization, Testing.DocumentType],
|
|
198
197
|
onClientInitialized: async ({ client }) => {
|
|
199
198
|
await client.halo.createIdentity();
|
|
200
199
|
await client.spaces.waitUntilReady();
|
|
@@ -202,16 +201,18 @@ const meta = {
|
|
|
202
201
|
await seedTestData(client.spaces.default);
|
|
203
202
|
},
|
|
204
203
|
}),
|
|
205
|
-
SpacePlugin(),
|
|
204
|
+
SpacePlugin({}),
|
|
205
|
+
IntentPlugin(),
|
|
206
|
+
|
|
207
|
+
// UI
|
|
208
|
+
ThemePlugin({ tx: defaultTx }),
|
|
206
209
|
SettingsPlugin(),
|
|
207
210
|
PreviewPlugin(),
|
|
208
|
-
IntentPlugin(),
|
|
209
211
|
TranscriptionPlugin(),
|
|
212
|
+
StorybookLayoutPlugin({}),
|
|
210
213
|
],
|
|
211
214
|
fireEvents: [Events.SetupAppGraph],
|
|
212
215
|
}),
|
|
213
|
-
withTheme,
|
|
214
|
-
withLayout({ fullscreen: true, classNames: 'justify-center' }),
|
|
215
216
|
],
|
|
216
217
|
} satisfies Meta;
|
|
217
218
|
|
|
@@ -241,15 +242,16 @@ export const Default: StoryObj<typeof AudioFile> = {
|
|
|
241
242
|
},
|
|
242
243
|
};
|
|
243
244
|
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
}
|
|
245
|
+
// TODO(mykola): Fix sentence normalization.
|
|
246
|
+
// export const WithSentenceNormalization: StoryObj<typeof AudioFile> = {
|
|
247
|
+
// render: AudioFile,
|
|
248
|
+
// args: {
|
|
249
|
+
// detectSpeaking: true,
|
|
250
|
+
// normalizeSentences: true,
|
|
251
|
+
// // https://learnenglish.britishcouncil.org/general-english/audio-zone/living-london
|
|
252
|
+
// audioUrl: 'https://dxos.network/audio-london.m4a',
|
|
253
|
+
// // textUrl: 'https://dxos.network/audio-london.txt',
|
|
254
|
+
// transcriberConfig: TRANSCRIBER_CONFIG,
|
|
255
|
+
// recorderConfig: RECORDER_CONFIG,
|
|
256
|
+
// },
|
|
257
|
+
// };
|
|
@@ -2,15 +2,13 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import '@dxos-theme';
|
|
6
|
-
|
|
7
5
|
import { type Meta, type StoryObj } from '@storybook/react-vite';
|
|
8
6
|
import React, { useEffect, useMemo, useState } from 'react';
|
|
9
7
|
|
|
10
8
|
import { log } from '@dxos/log';
|
|
11
9
|
import { Toolbar } from '@dxos/react-ui';
|
|
10
|
+
import { withLayout, withTheme } from '@dxos/react-ui/testing';
|
|
12
11
|
import { mx } from '@dxos/react-ui-theme';
|
|
13
|
-
import { withLayout, withTheme } from '@dxos/storybook-utils';
|
|
14
12
|
|
|
15
13
|
type Word = {
|
|
16
14
|
text: string;
|
|
@@ -88,7 +86,7 @@ const DefaultStory = ({ keywords }: StoryProps) => {
|
|
|
88
86
|
}, [running, recognition]);
|
|
89
87
|
|
|
90
88
|
return (
|
|
91
|
-
<div className={mx('flex flex-col gap-2
|
|
89
|
+
<div className={mx('flex flex-col gap-2 is-[30rem]')}>
|
|
92
90
|
<Toolbar.Root>
|
|
93
91
|
<Toolbar.IconButton
|
|
94
92
|
iconOnly
|
|
@@ -103,7 +101,7 @@ const DefaultStory = ({ keywords }: StoryProps) => {
|
|
|
103
101
|
<span
|
|
104
102
|
key={index}
|
|
105
103
|
className={mx(
|
|
106
|
-
'
|
|
104
|
+
'pli-3 plb-1 m-2 rounded border',
|
|
107
105
|
word.matched ? 'bg-red-200 text-red-800 border-red-300' : 'bg-gray-200 text-gray-600 border-gray-300',
|
|
108
106
|
)}
|
|
109
107
|
>
|
|
@@ -120,7 +118,7 @@ const DefaultStory = ({ keywords }: StoryProps) => {
|
|
|
120
118
|
const meta = {
|
|
121
119
|
title: 'plugins/plugin-transcription/KeyWordDetection',
|
|
122
120
|
render: DefaultStory,
|
|
123
|
-
decorators: [withTheme, withLayout({
|
|
121
|
+
decorators: [withTheme, withLayout({ container: 'column' })],
|
|
124
122
|
} satisfies Meta<typeof DefaultStory>;
|
|
125
123
|
|
|
126
124
|
export default meta;
|
|
@@ -2,8 +2,6 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import '@dxos-theme';
|
|
6
|
-
|
|
7
5
|
import { type Meta, type StoryObj } from '@storybook/react-vite';
|
|
8
6
|
import React, { useCallback, useEffect, useMemo, useState } from 'react';
|
|
9
7
|
|
|
@@ -11,15 +9,15 @@ import { Events, IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
|
|
|
11
9
|
import { withPluginManager } from '@dxos/app-framework/testing';
|
|
12
10
|
import {
|
|
13
11
|
type ExtractionFunction,
|
|
14
|
-
|
|
15
|
-
|
|
12
|
+
extractionAnthropicFunction,
|
|
13
|
+
extractionNerFunction,
|
|
16
14
|
getNer,
|
|
17
15
|
processTranscriptMessage,
|
|
18
16
|
} from '@dxos/assistant/extraction';
|
|
19
17
|
import { Filter, Obj, type Type } from '@dxos/echo';
|
|
18
|
+
import { createQueueDXN } from '@dxos/echo/internal';
|
|
20
19
|
import { MemoryQueue } from '@dxos/echo-db';
|
|
21
|
-
import {
|
|
22
|
-
import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
|
|
20
|
+
import { FunctionExecutor, ServiceContainer } from '@dxos/functions-runtime';
|
|
23
21
|
import { invariant } from '@dxos/invariant';
|
|
24
22
|
import { log } from '@dxos/log';
|
|
25
23
|
import { ClientPlugin } from '@dxos/plugin-client';
|
|
@@ -28,16 +26,17 @@ import { SpacePlugin } from '@dxos/plugin-space';
|
|
|
28
26
|
import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
|
|
29
27
|
import { ThemePlugin } from '@dxos/plugin-theme';
|
|
30
28
|
import { IndexKind, useSpace } from '@dxos/react-client/echo';
|
|
29
|
+
import { withLayout, withTheme } from '@dxos/react-ui/testing';
|
|
31
30
|
import { defaultTx } from '@dxos/react-ui-theme';
|
|
32
|
-
import {
|
|
33
|
-
import {
|
|
34
|
-
import {
|
|
31
|
+
import { Testing } from '@dxos/schema/testing';
|
|
32
|
+
import { Message, Organization, Person } from '@dxos/types';
|
|
33
|
+
import { seedTestData } from '@dxos/types/testing';
|
|
35
34
|
|
|
36
35
|
import { useAudioTrack, useQueueModelAdapter, useTranscriber } from '../../hooks';
|
|
37
36
|
import { TestItem } from '../../testing';
|
|
38
37
|
import { type MediaStreamRecorderParams, type TranscriberParams } from '../../transcriber';
|
|
39
38
|
import { TranscriptionPlugin } from '../../TranscriptionPlugin';
|
|
40
|
-
import {
|
|
39
|
+
import { renderByline } from '../Transcript';
|
|
41
40
|
|
|
42
41
|
import { TranscriptionStory } from './TranscriptionStory';
|
|
43
42
|
import { useIsSpeaking } from './useIsSpeaking';
|
|
@@ -77,8 +76,8 @@ const DefaultStory = ({
|
|
|
77
76
|
// Queue.
|
|
78
77
|
// TODO(dmaretskyi): Use space.queues.create() instead.
|
|
79
78
|
const queueDxn = useMemo(() => createQueueDXN(), []);
|
|
80
|
-
const queue = useMemo(() => new MemoryQueue<
|
|
81
|
-
const model = useQueueModelAdapter(
|
|
79
|
+
const queue = useMemo(() => new MemoryQueue<Message.Message>(queueDxn), [queueDxn]);
|
|
80
|
+
const model = useQueueModelAdapter(renderByline([]), queue);
|
|
82
81
|
const space = useSpace();
|
|
83
82
|
|
|
84
83
|
useEffect(() => {
|
|
@@ -101,14 +100,14 @@ const DefaultStory = ({
|
|
|
101
100
|
if (entityExtraction === 'ner') {
|
|
102
101
|
// Init model loading. Takes time.
|
|
103
102
|
void getNer();
|
|
104
|
-
extractionFunction =
|
|
103
|
+
extractionFunction = extractionNerFunction;
|
|
105
104
|
} else if (entityExtraction === 'llm') {
|
|
106
|
-
extractionFunction =
|
|
105
|
+
extractionFunction = extractionAnthropicFunction;
|
|
107
106
|
objects = space.db
|
|
108
107
|
.query(
|
|
109
108
|
Filter.or(
|
|
110
|
-
Filter.type(
|
|
111
|
-
Filter.type(
|
|
109
|
+
Filter.type(Person.Person),
|
|
110
|
+
Filter.type(Organization.Organization),
|
|
112
111
|
Filter.type(Testing.DocumentType),
|
|
113
112
|
),
|
|
114
113
|
)
|
|
@@ -134,7 +133,7 @@ const DefaultStory = ({
|
|
|
134
133
|
// Transcriber.
|
|
135
134
|
const handleSegments = useCallback<TranscriberParams['onSegments']>(
|
|
136
135
|
async (blocks) => {
|
|
137
|
-
const message = Obj.make(
|
|
136
|
+
const message = Obj.make(Message.Message, {
|
|
138
137
|
sender: { name: 'You' },
|
|
139
138
|
created: new Date().toISOString(),
|
|
140
139
|
blocks,
|
|
@@ -197,12 +196,12 @@ const meta = {
|
|
|
197
196
|
title: 'plugins/plugin-transcription/MicrophoneTranscription',
|
|
198
197
|
render: DefaultStory,
|
|
199
198
|
decorators: [
|
|
199
|
+
withTheme,
|
|
200
|
+
withLayout({ container: 'column' }),
|
|
200
201
|
withPluginManager({
|
|
201
202
|
plugins: [
|
|
202
|
-
ThemePlugin({ tx: defaultTx }),
|
|
203
|
-
StorybookLayoutPlugin(),
|
|
204
203
|
ClientPlugin({
|
|
205
|
-
types: [TestItem,
|
|
204
|
+
types: [TestItem, Person.Person, Organization.Organization, Testing.DocumentType],
|
|
206
205
|
onClientInitialized: async ({ client }) => {
|
|
207
206
|
await client.halo.createIdentity();
|
|
208
207
|
await client.spaces.waitUntilReady();
|
|
@@ -223,16 +222,18 @@ const meta = {
|
|
|
223
222
|
await seedTestData(client.spaces.default);
|
|
224
223
|
},
|
|
225
224
|
}),
|
|
226
|
-
SpacePlugin(),
|
|
225
|
+
SpacePlugin({}),
|
|
226
|
+
IntentPlugin(),
|
|
227
227
|
SettingsPlugin(),
|
|
228
|
+
|
|
229
|
+
// UI
|
|
230
|
+
ThemePlugin({ tx: defaultTx }),
|
|
228
231
|
PreviewPlugin(),
|
|
229
|
-
IntentPlugin(),
|
|
230
232
|
TranscriptionPlugin(),
|
|
233
|
+
StorybookLayoutPlugin({}),
|
|
231
234
|
],
|
|
232
235
|
fireEvents: [Events.SetupAppGraph],
|
|
233
236
|
}),
|
|
234
|
-
withTheme,
|
|
235
|
-
withLayout({ fullscreen: true, classNames: 'justify-center' }),
|
|
236
237
|
],
|
|
237
238
|
} satisfies Meta<typeof DefaultStory>;
|
|
238
239
|
|
|
@@ -254,19 +255,21 @@ export const Default: Story = {
|
|
|
254
255
|
},
|
|
255
256
|
};
|
|
256
257
|
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
}
|
|
258
|
+
// NOTE: We are running out of free quota on hugging face entity extraction.
|
|
259
|
+
// TODO(mykola): Fix hugging face quota issues.
|
|
260
|
+
// export const EntityExtraction: Story = {
|
|
261
|
+
// args: {
|
|
262
|
+
// detectSpeaking: true,
|
|
263
|
+
// entityExtraction: 'ner',
|
|
264
|
+
// transcriberConfig: TRANSCRIBER_CONFIG,
|
|
265
|
+
// recorderConfig: RECORDER_CONFIG,
|
|
266
|
+
// audioConstraints: {
|
|
267
|
+
// echoCancellation: true,
|
|
268
|
+
// noiseSuppression: true,
|
|
269
|
+
// autoGainControl: true,
|
|
270
|
+
// },
|
|
271
|
+
// },
|
|
272
|
+
// };
|
|
270
273
|
|
|
271
274
|
export const SpeechDetection: Story = {
|
|
272
275
|
args: {
|
|
@@ -7,24 +7,23 @@ import '@dxos-theme';
|
|
|
7
7
|
import React, { type Dispatch, type FC, type RefObject, type SetStateAction } from 'react';
|
|
8
8
|
|
|
9
9
|
import { useSpace } from '@dxos/react-client/echo';
|
|
10
|
-
import { IconButton, Toolbar } from '@dxos/react-ui';
|
|
11
|
-
import {
|
|
12
|
-
import { type DataType } from '@dxos/schema';
|
|
10
|
+
import { IconButton, ScrollContainer, Toolbar } from '@dxos/react-ui';
|
|
11
|
+
import { type Message } from '@dxos/types';
|
|
13
12
|
|
|
14
13
|
import { type SerializationModel } from '../../model';
|
|
15
14
|
import { TranscriptView } from '../Transcript';
|
|
16
15
|
|
|
17
16
|
export const TranscriptionStory: FC<{
|
|
18
|
-
model: SerializationModel<
|
|
17
|
+
model: SerializationModel<Message.Message>;
|
|
19
18
|
disabled?: boolean;
|
|
20
19
|
running: boolean;
|
|
21
20
|
onRunningChange: Dispatch<SetStateAction<boolean>>;
|
|
22
|
-
audioRef?: RefObject<HTMLAudioElement>;
|
|
21
|
+
audioRef?: RefObject<HTMLAudioElement | null>;
|
|
23
22
|
}> = ({ model, running, onRunningChange, audioRef, disabled }) => {
|
|
24
23
|
const space = useSpace();
|
|
25
24
|
|
|
26
25
|
return (
|
|
27
|
-
<div className='flex flex-col
|
|
26
|
+
<div className='flex flex-col is-[30rem]'>
|
|
28
27
|
{audioRef && <audio ref={audioRef} autoPlay />}
|
|
29
28
|
<Toolbar.Root>
|
|
30
29
|
<IconButton
|
|
@@ -36,9 +35,9 @@ export const TranscriptionStory: FC<{
|
|
|
36
35
|
/>
|
|
37
36
|
</Toolbar.Root>
|
|
38
37
|
<ScrollContainer.Root pin>
|
|
39
|
-
<ScrollContainer.
|
|
40
|
-
<TranscriptView space={space} model={model}
|
|
41
|
-
</ScrollContainer.
|
|
38
|
+
<ScrollContainer.Viewport>
|
|
39
|
+
<TranscriptView space={space} model={model} />
|
|
40
|
+
</ScrollContainer.Viewport>
|
|
42
41
|
</ScrollContainer.Root>
|
|
43
42
|
</div>
|
|
44
43
|
);
|
package/src/functions/open.ts
CHANGED
|
@@ -2,18 +2,20 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import
|
|
5
|
+
import * as Effect from 'effect/Effect';
|
|
6
|
+
import * as Schema from 'effect/Schema';
|
|
6
7
|
|
|
7
8
|
import { ArtifactId } from '@dxos/assistant';
|
|
8
9
|
import { Obj } from '@dxos/echo';
|
|
9
10
|
import { DatabaseService, QueueService, defineFunction } from '@dxos/functions';
|
|
10
|
-
import {
|
|
11
|
+
import { Message } from '@dxos/types';
|
|
11
12
|
|
|
12
|
-
import {
|
|
13
|
+
import { renderByline } from '../components';
|
|
13
14
|
import { Transcript } from '../types';
|
|
14
15
|
|
|
15
16
|
export default defineFunction({
|
|
16
|
-
|
|
17
|
+
key: 'dxos.org/function/transcription/open',
|
|
18
|
+
name: 'Open',
|
|
17
19
|
description: 'Opens and reads the contents of a transcription object.',
|
|
18
20
|
inputSchema: Schema.Struct({
|
|
19
21
|
id: ArtifactId.annotations({
|
|
@@ -29,8 +31,8 @@ export default defineFunction({
|
|
|
29
31
|
const queue = yield* QueueService.getQueue(dxn);
|
|
30
32
|
yield* Effect.promise(() => queue?.queryObjects());
|
|
31
33
|
const content = queue?.objects
|
|
32
|
-
.filter((message) => Obj.instanceOf(
|
|
33
|
-
.flatMap((message, index) =>
|
|
34
|
+
.filter((message) => Obj.instanceOf(Message.Message, message))
|
|
35
|
+
.flatMap((message, index) => renderByline([])(message, index))
|
|
34
36
|
.join('\n\n');
|
|
35
37
|
return { content };
|
|
36
38
|
}),
|
|
@@ -2,7 +2,12 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import
|
|
5
|
+
import * as Array from 'effect/Array';
|
|
6
|
+
import * as Effect from 'effect/Effect';
|
|
7
|
+
import * as Function from 'effect/Function';
|
|
8
|
+
import * as Layer from 'effect/Layer';
|
|
9
|
+
import * as Option from 'effect/Option';
|
|
10
|
+
import * as Schema from 'effect/Schema';
|
|
6
11
|
|
|
7
12
|
import { AiService, ConsolePrinter, ToolExecutionService, ToolResolverService } from '@dxos/ai';
|
|
8
13
|
import { AiSession, GenerationObserver } from '@dxos/assistant';
|
|
@@ -13,7 +18,8 @@ import { trim } from '@dxos/util';
|
|
|
13
18
|
* Summarize a transcript of a meeting.
|
|
14
19
|
*/
|
|
15
20
|
export default defineFunction({
|
|
16
|
-
|
|
21
|
+
key: 'dxos.org/function/transcription/summarize',
|
|
22
|
+
name: 'Summarize',
|
|
17
23
|
description: 'Summarize a transcript of a meeting.',
|
|
18
24
|
inputSchema: Schema.Struct({
|
|
19
25
|
transcript: Schema.String.annotations({
|
|
@@ -37,11 +43,11 @@ export default defineFunction({
|
|
|
37
43
|
observer: GenerationObserver.fromPrinter(new ConsolePrinter({ tag: 'summarize' })),
|
|
38
44
|
});
|
|
39
45
|
|
|
40
|
-
const summary = pipe(
|
|
46
|
+
const summary = Function.pipe(
|
|
41
47
|
result,
|
|
42
48
|
Array.findLast((msg) => msg.sender.role === 'assistant' && msg.blocks.some((block) => block._tag === 'text')),
|
|
43
49
|
Option.flatMap((msg) =>
|
|
44
|
-
pipe(
|
|
50
|
+
Function.pipe(
|
|
45
51
|
msg.blocks,
|
|
46
52
|
Array.findLast((block) => block._tag === 'text'),
|
|
47
53
|
Option.map((block) => block.text),
|