@dxos/plugin-transcription 0.8.3 → 0.8.4-main.c1de068
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/TranscriptContainer-6WCKCXQ6.mjs +11 -0
- package/dist/lib/browser/{chunk-UDCCT32F.mjs → chunk-4FJIYH7T.mjs} +3 -3
- package/dist/lib/browser/{chunk-53AXUEIQ.mjs → chunk-5KPYIC3N.mjs} +34 -35
- package/dist/lib/browser/chunk-5KPYIC3N.mjs.map +7 -0
- package/dist/lib/browser/{chunk-GOF7MJNV.mjs → chunk-5WH45KBY.mjs} +2 -2
- package/dist/lib/browser/{chunk-IQ5ZRMHZ.mjs → chunk-FFEPEPCU.mjs} +12 -12
- package/dist/lib/browser/chunk-FFEPEPCU.mjs.map +7 -0
- package/dist/lib/browser/{chunk-OSISJM2F.mjs → chunk-R7INNNUQ.mjs} +4 -4
- package/dist/lib/browser/{chunk-W2KNYJD6.mjs → chunk-U6QBOGTN.mjs} +3 -3
- package/dist/lib/browser/{chunk-W2KNYJD6.mjs.map → chunk-U6QBOGTN.mjs.map} +1 -1
- package/dist/lib/browser/index.mjs +12 -12
- package/dist/lib/browser/index.mjs.map +3 -3
- package/dist/lib/browser/{intent-resolver-BGADCHVM.mjs → intent-resolver-3O33MMP5.mjs} +4 -4
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/{react-surface-HMJS5FSS.mjs → react-surface-E5D5AE3F.mjs} +7 -7
- package/dist/lib/browser/{transcriber-OYPLZFG5.mjs → transcriber-PXT4QBFY.mjs} +6 -6
- package/dist/lib/browser/types/index.mjs +2 -2
- package/dist/lib/node-esm/{TranscriptContainer-G75LDEO5.mjs → TranscriptContainer-VBAOM5DG.mjs} +4 -4
- package/dist/lib/node-esm/{chunk-PRL64SRU.mjs → chunk-HT3W7LDP.mjs} +12 -12
- package/dist/lib/node-esm/chunk-HT3W7LDP.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-MFJPEIVB.mjs → chunk-RHF2ALML.mjs} +3 -3
- package/dist/lib/node-esm/{chunk-MFJPEIVB.mjs.map → chunk-RHF2ALML.mjs.map} +1 -1
- package/dist/lib/node-esm/{chunk-56QHOX4A.mjs → chunk-TJKAO7VR.mjs} +34 -35
- package/dist/lib/node-esm/chunk-TJKAO7VR.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-FH75X4JC.mjs → chunk-U65ZWK3L.mjs} +4 -4
- package/dist/lib/node-esm/{chunk-TXMDLLBR.mjs → chunk-XMUCZBP3.mjs} +2 -2
- package/dist/lib/node-esm/{chunk-MFJO7EKD.mjs → chunk-Y2B4HWSS.mjs} +3 -3
- package/dist/lib/node-esm/index.mjs +12 -12
- package/dist/lib/node-esm/index.mjs.map +3 -3
- package/dist/lib/node-esm/{intent-resolver-5LVCYSJS.mjs → intent-resolver-YMN5SXNC.mjs} +4 -4
- package/dist/lib/node-esm/meta.json +1 -1
- package/dist/lib/node-esm/{react-surface-JGQSYGRR.mjs → react-surface-QRDJOHN5.mjs} +7 -7
- package/dist/lib/node-esm/{transcriber-TVG2MBBN.mjs → transcriber-YUTEVANE.mjs} +6 -6
- package/dist/lib/node-esm/types/index.mjs +2 -2
- package/dist/types/src/capabilities/capabilities.d.ts +2 -2
- package/dist/types/src/capabilities/capabilities.d.ts.map +1 -1
- package/dist/types/src/components/Transcript/Transcript.stories.d.ts +1 -1
- package/dist/types/src/components/Transcript/Transcript.stories.d.ts.map +1 -1
- package/dist/types/src/components/TranscriptContainer.d.ts +4 -3
- package/dist/types/src/components/TranscriptContainer.d.ts.map +1 -1
- package/dist/types/src/components/index.d.ts +1 -4
- package/dist/types/src/components/index.d.ts.map +1 -1
- package/dist/types/src/components/stories/FileTranscription.stories.d.ts +1 -1
- package/dist/types/src/components/stories/FileTranscription.stories.d.ts.map +1 -1
- package/dist/types/src/components/stories/KeyWordRecognition.stories.d.ts +1 -1
- package/dist/types/src/components/stories/KeyWordRecognition.stories.d.ts.map +1 -1
- package/dist/types/src/components/stories/MicrophoneTranscription.stories.d.ts +1 -1
- package/dist/types/src/components/stories/MicrophoneTranscription.stories.d.ts.map +1 -1
- package/dist/types/src/hooks/useTranscriber.d.ts +1 -1
- package/dist/types/src/segments-normalization/normalization.d.ts +150 -63
- package/dist/types/src/segments-normalization/normalization.d.ts.map +1 -1
- package/dist/types/src/testing/testing.d.ts +2 -2
- package/dist/types/src/testing/testing.d.ts.map +1 -1
- package/dist/types/src/transcriber/index.d.ts +1 -1
- package/dist/types/src/transcriber/index.d.ts.map +1 -1
- package/dist/types/src/transcriber/transcriber.d.ts +1 -1
- package/dist/types/src/transcriber/transcriber.d.ts.map +1 -1
- package/dist/types/src/translations.d.ts +11 -11
- package/dist/types/src/translations.d.ts.map +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +57 -57
- package/src/TranscriptionPlugin.tsx +1 -1
- package/src/capabilities/capabilities.ts +2 -2
- package/src/components/Transcript/Transcript.stories.tsx +2 -2
- package/src/components/Transcript/Transcript.tsx +2 -2
- package/src/components/Transcript/transcript-extension.ts +2 -2
- package/src/components/TranscriptContainer.tsx +8 -3
- package/src/components/stories/FileTranscription.stories.tsx +16 -13
- package/src/components/stories/KeyWordRecognition.stories.tsx +1 -1
- package/src/components/stories/MicrophoneTranscription.stories.tsx +11 -9
- package/src/hooks/useTranscriber.ts +5 -5
- package/src/model/model.test.ts +8 -8
- package/src/segments-normalization/normalization.test.ts +18 -20
- package/src/segments-normalization/normalization.ts +19 -10
- package/src/testing/testing.ts +10 -10
- package/src/transcriber/index.ts +1 -1
- package/src/transcriber/transcriber.browser.test.ts +2 -2
- package/src/transcriber/transcriber.ts +3 -6
- package/src/transcriber/transcription-manager.ts +3 -3
- package/src/translations.ts +6 -4
- package/dist/lib/browser/TranscriptContainer-H3AZQ4CQ.mjs +0 -11
- package/dist/lib/browser/chunk-53AXUEIQ.mjs.map +0 -7
- package/dist/lib/browser/chunk-IQ5ZRMHZ.mjs.map +0 -7
- package/dist/lib/node/TranscriptContainer-KIG3GMAB.cjs +0 -32
- package/dist/lib/node/TranscriptContainer-KIG3GMAB.cjs.map +0 -7
- package/dist/lib/node/chunk-3PYT3BDM.cjs +0 -40
- package/dist/lib/node/chunk-3PYT3BDM.cjs.map +0 -7
- package/dist/lib/node/chunk-6LW5RA3K.cjs +0 -75
- package/dist/lib/node/chunk-6LW5RA3K.cjs.map +0 -7
- package/dist/lib/node/chunk-AFUZI63A.cjs +0 -42
- package/dist/lib/node/chunk-AFUZI63A.cjs.map +0 -7
- package/dist/lib/node/chunk-CVUN3UO7.cjs +0 -778
- package/dist/lib/node/chunk-CVUN3UO7.cjs.map +0 -7
- package/dist/lib/node/chunk-FKDJNJ57.cjs +0 -35
- package/dist/lib/node/chunk-FKDJNJ57.cjs.map +0 -7
- package/dist/lib/node/chunk-ZFAUMZXE.cjs +0 -518
- package/dist/lib/node/chunk-ZFAUMZXE.cjs.map +0 -7
- package/dist/lib/node/index.cjs +0 -133
- package/dist/lib/node/index.cjs.map +0 -7
- package/dist/lib/node/intent-resolver-LL3TTI4H.cjs +0 -45
- package/dist/lib/node/intent-resolver-LL3TTI4H.cjs.map +0 -7
- package/dist/lib/node/meta.json +0 -1
- package/dist/lib/node/react-surface-N25TBBG4.cjs +0 -56
- package/dist/lib/node/react-surface-N25TBBG4.cjs.map +0 -7
- package/dist/lib/node/transcriber-6J4FVFSZ.cjs +0 -68
- package/dist/lib/node/transcriber-6J4FVFSZ.cjs.map +0 -7
- package/dist/lib/node/types/index.cjs +0 -36
- package/dist/lib/node/types/index.cjs.map +0 -7
- package/dist/lib/node-esm/chunk-56QHOX4A.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-PRL64SRU.mjs.map +0 -7
- /package/dist/lib/browser/{TranscriptContainer-H3AZQ4CQ.mjs.map → TranscriptContainer-6WCKCXQ6.mjs.map} +0 -0
- /package/dist/lib/browser/{chunk-UDCCT32F.mjs.map → chunk-4FJIYH7T.mjs.map} +0 -0
- /package/dist/lib/browser/{chunk-GOF7MJNV.mjs.map → chunk-5WH45KBY.mjs.map} +0 -0
- /package/dist/lib/browser/{chunk-OSISJM2F.mjs.map → chunk-R7INNNUQ.mjs.map} +0 -0
- /package/dist/lib/browser/{intent-resolver-BGADCHVM.mjs.map → intent-resolver-3O33MMP5.mjs.map} +0 -0
- /package/dist/lib/browser/{react-surface-HMJS5FSS.mjs.map → react-surface-E5D5AE3F.mjs.map} +0 -0
- /package/dist/lib/browser/{transcriber-OYPLZFG5.mjs.map → transcriber-PXT4QBFY.mjs.map} +0 -0
- /package/dist/lib/node-esm/{TranscriptContainer-G75LDEO5.mjs.map → TranscriptContainer-VBAOM5DG.mjs.map} +0 -0
- /package/dist/lib/node-esm/{chunk-FH75X4JC.mjs.map → chunk-U65ZWK3L.mjs.map} +0 -0
- /package/dist/lib/node-esm/{chunk-TXMDLLBR.mjs.map → chunk-XMUCZBP3.mjs.map} +0 -0
- /package/dist/lib/node-esm/{chunk-MFJO7EKD.mjs.map → chunk-Y2B4HWSS.mjs.map} +0 -0
- /package/dist/lib/node-esm/{intent-resolver-5LVCYSJS.mjs.map → intent-resolver-YMN5SXNC.mjs.map} +0 -0
- /package/dist/lib/node-esm/{react-surface-JGQSYGRR.mjs.map → react-surface-QRDJOHN5.mjs.map} +0 -0
- /package/dist/lib/node-esm/{transcriber-TVG2MBBN.mjs.map → transcriber-YUTEVANE.mjs.map} +0 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dxos/plugin-transcription",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.4-main.c1de068",
|
|
4
4
|
"description": "Voice transcription.",
|
|
5
5
|
"homepage": "https://dxos.org",
|
|
6
6
|
"bugs": "https://github.com/dxos/dxos/issues",
|
|
@@ -40,77 +40,77 @@
|
|
|
40
40
|
"extendable-media-recorder": "^9.2.19",
|
|
41
41
|
"extendable-media-recorder-wav-encoder": "^7.0.123",
|
|
42
42
|
"main-thread-scheduling": "^14.1.1",
|
|
43
|
-
"react-resize-detector": "^
|
|
43
|
+
"react-resize-detector": "^11.0.1",
|
|
44
44
|
"react-router-dom": "^6.4.0",
|
|
45
45
|
"react-use": "^17.5.1",
|
|
46
46
|
"wavefile": "^11.0.0",
|
|
47
|
-
"@dxos/ai": "0.8.
|
|
48
|
-
"@dxos/app-framework": "0.8.
|
|
49
|
-
"@dxos/
|
|
50
|
-
"@dxos/
|
|
51
|
-
"@dxos/async": "0.8.
|
|
52
|
-
"@dxos/client": "0.8.
|
|
53
|
-
"@dxos/codec-protobuf": "0.8.
|
|
54
|
-
"@dxos/context": "0.8.
|
|
55
|
-
"@dxos/debug": "0.8.
|
|
56
|
-
"@dxos/display-name": "0.8.
|
|
57
|
-
"@dxos/echo": "0.8.
|
|
58
|
-
"@dxos/echo-db": "0.8.
|
|
59
|
-
"@dxos/echo-schema": "0.8.
|
|
60
|
-
"@dxos/
|
|
61
|
-
"@dxos/
|
|
62
|
-
"@dxos/
|
|
63
|
-
"@dxos/
|
|
64
|
-
"@dxos/local-storage": "0.8.
|
|
65
|
-
"@dxos/log": "0.8.
|
|
66
|
-
"@dxos/plugin-client": "0.8.
|
|
67
|
-
"@dxos/plugin-deck": "0.8.
|
|
68
|
-
"@dxos/plugin-graph": "0.8.
|
|
69
|
-
"@dxos/plugin-markdown": "0.8.
|
|
70
|
-
"@dxos/plugin-
|
|
71
|
-
"@dxos/
|
|
72
|
-
"@dxos/
|
|
73
|
-
"@dxos/react-edge-client": "0.8.
|
|
74
|
-
"@dxos/
|
|
75
|
-
"@dxos/react-ui-
|
|
76
|
-
"@dxos/react-ui-
|
|
77
|
-
"@dxos/react-ui-
|
|
78
|
-
"@dxos/react-ui-
|
|
79
|
-
"@dxos/react-ui-grid": "0.8.
|
|
80
|
-
"@dxos/
|
|
81
|
-
"@dxos/
|
|
82
|
-
"@dxos/
|
|
83
|
-
"@dxos/
|
|
84
|
-
"@dxos/
|
|
47
|
+
"@dxos/ai": "0.8.4-main.c1de068",
|
|
48
|
+
"@dxos/app-framework": "0.8.4-main.c1de068",
|
|
49
|
+
"@dxos/assistant": "0.8.4-main.c1de068",
|
|
50
|
+
"@dxos/blueprints": "0.8.4-main.c1de068",
|
|
51
|
+
"@dxos/async": "0.8.4-main.c1de068",
|
|
52
|
+
"@dxos/client": "0.8.4-main.c1de068",
|
|
53
|
+
"@dxos/codec-protobuf": "0.8.4-main.c1de068",
|
|
54
|
+
"@dxos/context": "0.8.4-main.c1de068",
|
|
55
|
+
"@dxos/debug": "0.8.4-main.c1de068",
|
|
56
|
+
"@dxos/display-name": "0.8.4-main.c1de068",
|
|
57
|
+
"@dxos/echo": "0.8.4-main.c1de068",
|
|
58
|
+
"@dxos/echo-db": "0.8.4-main.c1de068",
|
|
59
|
+
"@dxos/echo-schema": "0.8.4-main.c1de068",
|
|
60
|
+
"@dxos/functions": "0.8.4-main.c1de068",
|
|
61
|
+
"@dxos/invariant": "0.8.4-main.c1de068",
|
|
62
|
+
"@dxos/keys": "0.8.4-main.c1de068",
|
|
63
|
+
"@dxos/live-object": "0.8.4-main.c1de068",
|
|
64
|
+
"@dxos/local-storage": "0.8.4-main.c1de068",
|
|
65
|
+
"@dxos/log": "0.8.4-main.c1de068",
|
|
66
|
+
"@dxos/plugin-client": "0.8.4-main.c1de068",
|
|
67
|
+
"@dxos/plugin-deck": "0.8.4-main.c1de068",
|
|
68
|
+
"@dxos/plugin-graph": "0.8.4-main.c1de068",
|
|
69
|
+
"@dxos/plugin-markdown": "0.8.4-main.c1de068",
|
|
70
|
+
"@dxos/plugin-space": "0.8.4-main.c1de068",
|
|
71
|
+
"@dxos/plugin-thread": "0.8.4-main.c1de068",
|
|
72
|
+
"@dxos/protocols": "0.8.4-main.c1de068",
|
|
73
|
+
"@dxos/react-edge-client": "0.8.4-main.c1de068",
|
|
74
|
+
"@dxos/react-ui-attention": "0.8.4-main.c1de068",
|
|
75
|
+
"@dxos/react-ui-components": "0.8.4-main.c1de068",
|
|
76
|
+
"@dxos/react-ui-form": "0.8.4-main.c1de068",
|
|
77
|
+
"@dxos/react-ui-editor": "0.8.4-main.c1de068",
|
|
78
|
+
"@dxos/react-ui-sfx": "0.8.4-main.c1de068",
|
|
79
|
+
"@dxos/react-ui-grid": "0.8.4-main.c1de068",
|
|
80
|
+
"@dxos/react-client": "0.8.4-main.c1de068",
|
|
81
|
+
"@dxos/react-ui-stack": "0.8.4-main.c1de068",
|
|
82
|
+
"@dxos/schema": "0.8.4-main.c1de068",
|
|
83
|
+
"@dxos/util": "0.8.4-main.c1de068",
|
|
84
|
+
"@dxos/tracing": "0.8.4-main.c1de068"
|
|
85
85
|
},
|
|
86
86
|
"devDependencies": {
|
|
87
|
-
"@effect-rx/rx-react": "
|
|
88
|
-
"@effect/platform": "0.
|
|
87
|
+
"@effect-rx/rx-react": "0.38.0",
|
|
88
|
+
"@effect/platform": "0.89.0",
|
|
89
89
|
"@types/dom-speech-recognition": "^0.0.6",
|
|
90
90
|
"@types/react": "~18.2.0",
|
|
91
|
-
"effect": "3.
|
|
91
|
+
"effect": "3.17.0",
|
|
92
92
|
"react": "~18.2.0",
|
|
93
93
|
"react-dom": "~18.2.0",
|
|
94
94
|
"tailwindcss": "~3.4.1",
|
|
95
|
-
"@dxos/plugin-preview": "0.8.
|
|
96
|
-
"@dxos/
|
|
97
|
-
"@dxos/
|
|
98
|
-
"@dxos/
|
|
99
|
-
"@dxos/react-ui-form": "0.8.
|
|
100
|
-
"@dxos/react-ui
|
|
101
|
-
"@dxos/
|
|
102
|
-
"@dxos/react-ui-theme": "0.8.
|
|
103
|
-
"@dxos/storybook-utils": "0.8.
|
|
104
|
-
"@dxos/test-utils": "0.8.
|
|
95
|
+
"@dxos/plugin-preview": "0.8.4-main.c1de068",
|
|
96
|
+
"@dxos/plugin-theme": "0.8.4-main.c1de068",
|
|
97
|
+
"@dxos/plugin-storybook-layout": "0.8.4-main.c1de068",
|
|
98
|
+
"@dxos/random": "0.8.4-main.c1de068",
|
|
99
|
+
"@dxos/react-ui-form": "0.8.4-main.c1de068",
|
|
100
|
+
"@dxos/react-ui": "0.8.4-main.c1de068",
|
|
101
|
+
"@dxos/react-ui-syntax-highlighter": "0.8.4-main.c1de068",
|
|
102
|
+
"@dxos/react-ui-theme": "0.8.4-main.c1de068",
|
|
103
|
+
"@dxos/storybook-utils": "0.8.4-main.c1de068",
|
|
104
|
+
"@dxos/test-utils": "0.8.4-main.c1de068"
|
|
105
105
|
},
|
|
106
|
-
"
|
|
106
|
+
"peerDependencies": {
|
|
107
107
|
"@effect-rx/rx-react": "^0.34.1",
|
|
108
|
-
"@effect/platform": "0.80.12",
|
|
108
|
+
"@effect/platform": "^0.80.12",
|
|
109
109
|
"effect": "^3.13.3",
|
|
110
110
|
"react": "~18.2.0",
|
|
111
111
|
"react-dom": "~18.2.0",
|
|
112
|
-
"@dxos/react-ui
|
|
113
|
-
"@dxos/react-ui": "0.8.
|
|
112
|
+
"@dxos/react-ui": "0.8.4-main.c1de068",
|
|
113
|
+
"@dxos/react-ui-theme": "0.8.4-main.c1de068"
|
|
114
114
|
},
|
|
115
115
|
"publishConfig": {
|
|
116
116
|
"access": "public"
|
|
@@ -11,7 +11,7 @@ import { DataType } from '@dxos/schema';
|
|
|
11
11
|
import { IntentResolver, ReactSurface, Transcriber } from './capabilities';
|
|
12
12
|
import { renderMarkdown } from './components';
|
|
13
13
|
import { meta } from './meta';
|
|
14
|
-
import translations from './translations';
|
|
14
|
+
import { translations } from './translations';
|
|
15
15
|
import { TranscriptType } from './types';
|
|
16
16
|
|
|
17
17
|
export const TranscriptionPlugin = () =>
|
|
@@ -16,9 +16,9 @@ import {
|
|
|
16
16
|
export namespace TranscriptionCapabilities {
|
|
17
17
|
export type GetTranscriberProps = {
|
|
18
18
|
audioStreamTrack: MediaStreamTrack;
|
|
19
|
-
onSegments: TranscriberParams['onSegments'];
|
|
20
|
-
transcriberConfig?: Partial<TranscriberParams['config']>;
|
|
21
19
|
recorderConfig?: Partial<MediaStreamRecorderParams['config']>;
|
|
20
|
+
transcriberConfig?: Partial<TranscriberParams['config']>;
|
|
21
|
+
onSegments: TranscriberParams['onSegments'];
|
|
22
22
|
};
|
|
23
23
|
export type GetTranscriber = (props: GetTranscriberProps) => Transcriber;
|
|
24
24
|
export const Transcriber = defineCapability<GetTranscriber>(`${TRANSCRIPTION_PLUGIN}/capability/transcriber`);
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
import '@dxos-theme';
|
|
6
6
|
|
|
7
|
-
import { type Meta, type StoryObj } from '@storybook/react';
|
|
7
|
+
import { type Meta, type StoryObj } from '@storybook/react-vite';
|
|
8
8
|
import React, { type FC, useEffect, useMemo, useState } from 'react';
|
|
9
9
|
|
|
10
10
|
import { IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
|
|
@@ -33,7 +33,7 @@ import {
|
|
|
33
33
|
useTestTranscriptionQueue,
|
|
34
34
|
useTestTranscriptionQueueWithEntityExtraction,
|
|
35
35
|
} from '../../testing';
|
|
36
|
-
import translations from '../../translations';
|
|
36
|
+
import { translations } from '../../translations';
|
|
37
37
|
|
|
38
38
|
faker.seed(1);
|
|
39
39
|
|
|
@@ -30,7 +30,7 @@ export const renderMarkdown =
|
|
|
30
30
|
(message: DataType.Message, index: number, debug = false): string[] => {
|
|
31
31
|
if (message.sender.role === 'assistant') {
|
|
32
32
|
// Start/stop block.
|
|
33
|
-
return [message.blocks.find((block) => block.
|
|
33
|
+
return [message.blocks.find((block) => block._tag === 'transcript')?.text ?? '', ''];
|
|
34
34
|
}
|
|
35
35
|
|
|
36
36
|
// TODO(burdon): Use link/reference markup for users (with popover).
|
|
@@ -42,7 +42,7 @@ export const renderMarkdown =
|
|
|
42
42
|
message.sender.name ??
|
|
43
43
|
message.sender.email ??
|
|
44
44
|
message.sender.identityDid;
|
|
45
|
-
const blocks = message.blocks.filter((block) => block.
|
|
45
|
+
const blocks = message.blocks.filter((block) => block._tag === 'transcript');
|
|
46
46
|
return [
|
|
47
47
|
`###### ${name}` + (debug ? ` [${index}]:${message.id}` : ''),
|
|
48
48
|
blocks.map((block) => block.text.trim()).join(' '),
|
|
@@ -38,7 +38,7 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
|
|
|
38
38
|
let line = view.state.doc.lineAt(from);
|
|
39
39
|
while (line.from <= to) {
|
|
40
40
|
const block = model.getChunkAtLine(line.number)?.blocks[0];
|
|
41
|
-
const timestamp = block?.
|
|
41
|
+
const timestamp = block?._tag === 'transcript' && block.started;
|
|
42
42
|
if (timestamp) {
|
|
43
43
|
builder.add(line.from, line.from, new TimestampMarker(line.number, new Date(timestamp), start));
|
|
44
44
|
}
|
|
@@ -218,7 +218,7 @@ const getStartTime = (started?: Date, message?: DataType.Message): Date | undefi
|
|
|
218
218
|
return started;
|
|
219
219
|
}
|
|
220
220
|
|
|
221
|
-
if (message?.blocks[0]?.
|
|
221
|
+
if (message?.blocks[0]?._tag === 'transcript' && message.blocks[0].started) {
|
|
222
222
|
return new Date(message.blocks[0].started);
|
|
223
223
|
}
|
|
224
224
|
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Copyright 2024 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import React
|
|
5
|
+
import React from 'react';
|
|
6
6
|
|
|
7
7
|
import { fullyQualifiedId, getSpace } from '@dxos/client/echo';
|
|
8
8
|
import { useMembers, useQueue } from '@dxos/react-client/echo';
|
|
@@ -13,7 +13,12 @@ import { Transcript, renderMarkdown } from './Transcript';
|
|
|
13
13
|
import { useQueueModelAdapter } from '../hooks';
|
|
14
14
|
import { type TranscriptType } from '../types';
|
|
15
15
|
|
|
16
|
-
export
|
|
16
|
+
export type TranscriptionContainerProps = {
|
|
17
|
+
role: string;
|
|
18
|
+
transcript: TranscriptType;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
export const TranscriptionContainer = ({ transcript }: TranscriptionContainerProps) => {
|
|
17
22
|
const attendableId = fullyQualifiedId(transcript);
|
|
18
23
|
const space = getSpace(transcript);
|
|
19
24
|
const members = useMembers(space?.key).map((member) => member.identity);
|
|
@@ -21,7 +26,7 @@ export const TranscriptionContainer: FC<{ role: string; transcript: TranscriptTy
|
|
|
21
26
|
const model = useQueueModelAdapter(renderMarkdown(members), queue);
|
|
22
27
|
|
|
23
28
|
return (
|
|
24
|
-
<StackItem.Content
|
|
29
|
+
<StackItem.Content classNames='container-max-width pli-2'>
|
|
25
30
|
<Transcript attendableId={attendableId} space={space} model={model} transcript={transcript} />
|
|
26
31
|
</StackItem.Content>
|
|
27
32
|
);
|
|
@@ -4,11 +4,9 @@
|
|
|
4
4
|
|
|
5
5
|
import '@dxos-theme';
|
|
6
6
|
|
|
7
|
-
import { type Meta, type StoryObj } from '@storybook/react';
|
|
7
|
+
import { type Meta, type StoryObj } from '@storybook/react-vite';
|
|
8
8
|
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
|
9
9
|
|
|
10
|
-
import { EdgeAiServiceClient } from '@dxos/ai';
|
|
11
|
-
import { AI_SERVICE_ENDPOINT } from '@dxos/ai/testing';
|
|
12
10
|
import { Events, IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
|
|
13
11
|
import { withPluginManager } from '@dxos/app-framework/testing';
|
|
14
12
|
import { scheduleTask } from '@dxos/async';
|
|
@@ -88,8 +86,13 @@ const AudioFile = ({
|
|
|
88
86
|
const model = useQueueModelAdapter(renderMarkdown([]), queue);
|
|
89
87
|
const handleSegments = useCallback<TranscriberParams['onSegments']>(
|
|
90
88
|
async (blocks) => {
|
|
91
|
-
|
|
92
|
-
|
|
89
|
+
void queue?.append([
|
|
90
|
+
Obj.make(DataType.Message, {
|
|
91
|
+
created: new Date().toISOString(),
|
|
92
|
+
sender: actor,
|
|
93
|
+
blocks,
|
|
94
|
+
}),
|
|
95
|
+
]);
|
|
93
96
|
},
|
|
94
97
|
[queue],
|
|
95
98
|
);
|
|
@@ -108,14 +111,14 @@ const AudioFile = ({
|
|
|
108
111
|
}
|
|
109
112
|
const executor = new FunctionExecutor(
|
|
110
113
|
new ServiceContainer().setServices({
|
|
111
|
-
ai: {
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
},
|
|
114
|
+
// ai: {
|
|
115
|
+
// client: new Edge AiServiceClient({
|
|
116
|
+
// endpoint: AI_SERVICE_ENDPOINT.REMOTE,
|
|
117
|
+
// defaultGenerationOptions: {
|
|
118
|
+
// model: '@anthropic/claude-3-5-sonnet-20241022',
|
|
119
|
+
// },
|
|
120
|
+
// }),
|
|
121
|
+
// },
|
|
119
122
|
}),
|
|
120
123
|
);
|
|
121
124
|
|
|
@@ -4,19 +4,17 @@
|
|
|
4
4
|
|
|
5
5
|
import '@dxos-theme';
|
|
6
6
|
|
|
7
|
-
import { type Meta, type StoryObj } from '@storybook/react';
|
|
7
|
+
import { type Meta, type StoryObj } from '@storybook/react-vite';
|
|
8
8
|
import React, { useCallback, useEffect, useMemo, useState } from 'react';
|
|
9
9
|
|
|
10
|
-
import { EdgeAiServiceClient } from '@dxos/ai';
|
|
11
|
-
import { AI_SERVICE_ENDPOINT } from '@dxos/ai/testing';
|
|
12
10
|
import { Events, IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
|
|
13
11
|
import { withPluginManager } from '@dxos/app-framework/testing';
|
|
14
12
|
import {
|
|
15
|
-
extractionAnthropicFn,
|
|
16
13
|
type ExtractionFunction,
|
|
14
|
+
extractionAnthropicFn,
|
|
17
15
|
extractionNerFn,
|
|
18
|
-
processTranscriptMessage,
|
|
19
16
|
getNer,
|
|
17
|
+
processTranscriptMessage,
|
|
20
18
|
} from '@dxos/assistant';
|
|
21
19
|
import { Filter, Obj, type Type } from '@dxos/echo';
|
|
22
20
|
import { MemoryQueue } from '@dxos/echo-db';
|
|
@@ -117,11 +115,15 @@ const DefaultStory = ({
|
|
|
117
115
|
.then((result) => result.objects);
|
|
118
116
|
}
|
|
119
117
|
if (entityExtraction !== 'none') {
|
|
120
|
-
const AiService = new EdgeAiServiceClient({
|
|
121
|
-
endpoint: AI_SERVICE_ENDPOINT.REMOTE,
|
|
122
|
-
});
|
|
123
118
|
executor = new FunctionExecutor(
|
|
124
|
-
new ServiceContainer().setServices({
|
|
119
|
+
new ServiceContainer().setServices({
|
|
120
|
+
// ai: {
|
|
121
|
+
// client: new Edge AiServiceClient({
|
|
122
|
+
// endpoint: AI_SERVICE_ENDPOINT.REMOTE,
|
|
123
|
+
// }),
|
|
124
|
+
// },
|
|
125
|
+
// database: { db: space!.db },
|
|
126
|
+
}),
|
|
125
127
|
);
|
|
126
128
|
}
|
|
127
129
|
|
|
@@ -14,20 +14,20 @@ import { type Transcriber } from '../transcriber';
|
|
|
14
14
|
*/
|
|
15
15
|
export const useTranscriber = ({
|
|
16
16
|
audioStreamTrack,
|
|
17
|
-
onSegments,
|
|
18
|
-
transcriberConfig,
|
|
19
17
|
recorderConfig,
|
|
18
|
+
transcriberConfig,
|
|
19
|
+
onSegments,
|
|
20
20
|
}: Partial<TranscriptionCapabilities.GetTranscriberProps>) => {
|
|
21
21
|
const [getTranscriber] = useCapabilities(TranscriptionCapabilities.Transcriber);
|
|
22
22
|
|
|
23
23
|
// Initialize audio transcription.
|
|
24
24
|
const transcriber = useMemo<Transcriber | undefined>(() => {
|
|
25
|
-
if (!
|
|
25
|
+
if (!getTranscriber || !audioStreamTrack || !onSegments) {
|
|
26
26
|
return undefined;
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
-
return getTranscriber({ audioStreamTrack,
|
|
30
|
-
}, [
|
|
29
|
+
return getTranscriber({ audioStreamTrack, recorderConfig, transcriberConfig, onSegments });
|
|
30
|
+
}, [getTranscriber, audioStreamTrack, recorderConfig, transcriberConfig, onSegments]);
|
|
31
31
|
|
|
32
32
|
useEffect(() => {
|
|
33
33
|
return () => {
|
package/src/model/model.test.ts
CHANGED
|
@@ -18,7 +18,7 @@ const blockToMarkdown: ChunkRenderer<DataType.Message> = (
|
|
|
18
18
|
): string[] => {
|
|
19
19
|
return [
|
|
20
20
|
`###### ${message.sender.name}`,
|
|
21
|
-
...message.blocks.filter((block) => block.
|
|
21
|
+
...message.blocks.filter((block) => block._tag === 'transcript').map((block) => block.text),
|
|
22
22
|
'',
|
|
23
23
|
];
|
|
24
24
|
};
|
|
@@ -37,7 +37,7 @@ describe('SerializationModel', () => {
|
|
|
37
37
|
sender: { name: 'Alice' },
|
|
38
38
|
blocks: [
|
|
39
39
|
{
|
|
40
|
-
|
|
40
|
+
_tag: 'transcript',
|
|
41
41
|
started: createDate(),
|
|
42
42
|
text: 'Hello world!',
|
|
43
43
|
},
|
|
@@ -51,7 +51,7 @@ describe('SerializationModel', () => {
|
|
|
51
51
|
}
|
|
52
52
|
|
|
53
53
|
// Update message.
|
|
54
|
-
message.blocks.push({
|
|
54
|
+
message.blocks.push({ _tag: 'transcript', started: createDate(), text: 'Hello again!' });
|
|
55
55
|
model.updateChunk(message);
|
|
56
56
|
{
|
|
57
57
|
const text = model.doc.toString();
|
|
@@ -74,7 +74,7 @@ describe('SerializationModel', () => {
|
|
|
74
74
|
sender: { name: 'Alice' },
|
|
75
75
|
blocks: [
|
|
76
76
|
{
|
|
77
|
-
|
|
77
|
+
_tag: 'transcript',
|
|
78
78
|
started: createDate(),
|
|
79
79
|
text: 'Hello world!',
|
|
80
80
|
},
|
|
@@ -92,7 +92,7 @@ describe('SerializationModel', () => {
|
|
|
92
92
|
sender: { name: 'Bob' },
|
|
93
93
|
blocks: [
|
|
94
94
|
{
|
|
95
|
-
|
|
95
|
+
_tag: 'transcript',
|
|
96
96
|
started: createDate(),
|
|
97
97
|
text: 'Hello world!',
|
|
98
98
|
},
|
|
@@ -119,7 +119,7 @@ describe('SerializationModel', () => {
|
|
|
119
119
|
sender: { name: 'Alice' },
|
|
120
120
|
blocks: [
|
|
121
121
|
{
|
|
122
|
-
|
|
122
|
+
_tag: 'transcript',
|
|
123
123
|
started: createDate(),
|
|
124
124
|
text: 'Hello world!',
|
|
125
125
|
},
|
|
@@ -131,7 +131,7 @@ describe('SerializationModel', () => {
|
|
|
131
131
|
expect(view.state.doc.toString()).to.eq('###### Alice\nHello world!\n\n');
|
|
132
132
|
|
|
133
133
|
// Update message (add block).
|
|
134
|
-
message.blocks.push({
|
|
134
|
+
message.blocks.push({ _tag: 'transcript', started: createDate(), text: 'Hello again!' });
|
|
135
135
|
model.updateChunk(message);
|
|
136
136
|
model.sync(adapter);
|
|
137
137
|
expect(view.state.doc.toString()).to.eq('###### Alice\nHello world!\nHello again!\n\n');
|
|
@@ -150,7 +150,7 @@ describe('SerializationModel', () => {
|
|
|
150
150
|
const message = Obj.make(DataType.Message, {
|
|
151
151
|
created: createDate(),
|
|
152
152
|
sender: { name: 'Bob' },
|
|
153
|
-
blocks: [{
|
|
153
|
+
blocks: [{ _tag: 'transcript', started: createDate(), text: 'Hello again!' }],
|
|
154
154
|
});
|
|
155
155
|
model.appendChunk(message);
|
|
156
156
|
model.sync(adapter);
|
|
@@ -5,8 +5,6 @@
|
|
|
5
5
|
import { effect } from '@preact/signals-core';
|
|
6
6
|
import { describe, test } from 'vitest';
|
|
7
7
|
|
|
8
|
-
import { EdgeAiServiceClient, OllamaAiServiceClient } from '@dxos/ai';
|
|
9
|
-
import { AI_SERVICE_ENDPOINT } from '@dxos/ai/testing';
|
|
10
8
|
import { scheduleTaskInterval } from '@dxos/async';
|
|
11
9
|
import { Context } from '@dxos/context';
|
|
12
10
|
import { Obj } from '@dxos/echo';
|
|
@@ -53,31 +51,31 @@ const messages: MessageWithRangeId[] = [
|
|
|
53
51
|
Obj.make(DataType.Message, {
|
|
54
52
|
created: new Date(Date.now() + 1000 * index).toISOString(),
|
|
55
53
|
sender,
|
|
56
|
-
blocks: [{
|
|
57
|
-
|
|
58
|
-
} as any),
|
|
54
|
+
blocks: [{ _tag: 'transcript', started: new Date(Date.now() + 1000 * index).toISOString(), text: string }],
|
|
55
|
+
}),
|
|
59
56
|
);
|
|
60
57
|
|
|
61
|
-
|
|
58
|
+
// eslint-disable-next-line no-unused-vars
|
|
59
|
+
const _REMOTE_AI = true;
|
|
62
60
|
|
|
63
61
|
describe.skip('SentenceNormalization', () => {
|
|
64
62
|
const getExecutor = () => {
|
|
65
63
|
return new FunctionExecutor(
|
|
66
64
|
new ServiceContainer().setServices({
|
|
67
|
-
ai: {
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
},
|
|
65
|
+
// ai: {
|
|
66
|
+
// client: REMOTE_AI
|
|
67
|
+
// ? new Edge AiServiceClient({
|
|
68
|
+
// endpoint: AI_SERVICE_ENDPOINT.REMOTE,
|
|
69
|
+
// defaultGenerationOptions: {
|
|
70
|
+
// model: '@anthropic/claude-3-5-sonnet-20241022',
|
|
71
|
+
// },
|
|
72
|
+
// })
|
|
73
|
+
// : new Ollama AiServiceClient({
|
|
74
|
+
// overrides: {
|
|
75
|
+
// model: 'llama3.1:8b',
|
|
76
|
+
// },
|
|
77
|
+
// }),
|
|
78
|
+
// },
|
|
81
79
|
}),
|
|
82
80
|
);
|
|
83
81
|
};
|
|
@@ -2,12 +2,15 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
+
// TODO(burdon): Needs refactoring.
|
|
6
|
+
// @ts-nocheck
|
|
7
|
+
|
|
5
8
|
import { Schema } from 'effect';
|
|
6
9
|
|
|
7
|
-
import { DEFAULT_EDGE_MODEL,
|
|
8
|
-
import {
|
|
10
|
+
import { DEFAULT_EDGE_MODEL, AiService } from '@dxos/ai';
|
|
11
|
+
import { AiSession } from '@dxos/assistant';
|
|
9
12
|
import { Obj } from '@dxos/echo';
|
|
10
|
-
import {
|
|
13
|
+
import { defineFunction } from '@dxos/functions';
|
|
11
14
|
import { ObjectId } from '@dxos/keys';
|
|
12
15
|
import { log } from '@dxos/log';
|
|
13
16
|
import { DataType } from '@dxos/schema';
|
|
@@ -80,21 +83,27 @@ export const sentenceNormalization = defineFunction<NormalizationInput, Normaliz
|
|
|
80
83
|
handler: async ({ data: { messages }, context }) => {
|
|
81
84
|
log.info('input', { messages });
|
|
82
85
|
const ai = context.getService(AiService);
|
|
83
|
-
const session = new
|
|
86
|
+
const session = new AiSession({ operationModel: 'configured' });
|
|
84
87
|
|
|
85
|
-
|
|
86
|
-
|
|
88
|
+
// TODO(dmaretskyi): This got broken after effect-ai transition.
|
|
89
|
+
const response = session.runStructured(NormalizationOutput, {
|
|
90
|
+
generationOptions: {
|
|
91
|
+
model: DEFAULT_EDGE_MODEL,
|
|
92
|
+
},
|
|
87
93
|
client: ai.client,
|
|
88
94
|
tools: [],
|
|
89
95
|
artifacts: [],
|
|
90
96
|
history: [
|
|
91
|
-
Obj.make(Message, {
|
|
92
|
-
|
|
93
|
-
|
|
97
|
+
Obj.make(DataType.Message, {
|
|
98
|
+
created: new Date().toISOString(),
|
|
99
|
+
sender: {
|
|
100
|
+
role: 'user',
|
|
101
|
+
},
|
|
102
|
+
blocks: messages.map((message) => ({ _tag: 'text', text: JSON.stringify(message) }) as const),
|
|
94
103
|
}),
|
|
95
104
|
],
|
|
96
105
|
prompt,
|
|
97
|
-
});
|
|
106
|
+
}) as any;
|
|
98
107
|
|
|
99
108
|
response.sentences.forEach((sentence) => {
|
|
100
109
|
sentence.id = ObjectId.random();
|
package/src/testing/testing.ts
CHANGED
|
@@ -5,13 +5,11 @@
|
|
|
5
5
|
import { Schema } from 'effect';
|
|
6
6
|
import { useEffect, useMemo, useState } from 'react';
|
|
7
7
|
|
|
8
|
-
import { EdgeAiServiceClient } from '@dxos/ai';
|
|
9
|
-
import { AI_SERVICE_ENDPOINT } from '@dxos/ai/testing';
|
|
10
8
|
import { extractionAnthropicFn, processTranscriptMessage } from '@dxos/assistant';
|
|
11
9
|
import { scheduleTaskInterval } from '@dxos/async';
|
|
12
10
|
import { Filter, type Queue } from '@dxos/client/echo';
|
|
13
11
|
import { Context } from '@dxos/context';
|
|
14
|
-
import {
|
|
12
|
+
import { Obj, Ref, Type, type Key } from '@dxos/echo';
|
|
15
13
|
import { createQueueDXN } from '@dxos/echo-schema';
|
|
16
14
|
import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
|
|
17
15
|
import { IdentityDid } from '@dxos/keys';
|
|
@@ -69,7 +67,7 @@ export class MessageBuilder extends AbstractMessageBuilder {
|
|
|
69
67
|
});
|
|
70
68
|
}
|
|
71
69
|
|
|
72
|
-
createBlock(): DataType.MessageBlock.
|
|
70
|
+
createBlock(): DataType.MessageBlock.Transcript {
|
|
73
71
|
let text = faker.lorem.paragraph();
|
|
74
72
|
if (this._space) {
|
|
75
73
|
const label = faker.commerce.productName();
|
|
@@ -81,7 +79,7 @@ export class MessageBuilder extends AbstractMessageBuilder {
|
|
|
81
79
|
}
|
|
82
80
|
|
|
83
81
|
return {
|
|
84
|
-
|
|
82
|
+
_tag: 'transcript',
|
|
85
83
|
started: this.next().toISOString(),
|
|
86
84
|
text,
|
|
87
85
|
};
|
|
@@ -95,11 +93,13 @@ export class MessageBuilder extends AbstractMessageBuilder {
|
|
|
95
93
|
|
|
96
94
|
// TODO(burdon): Reconcile with BlockBuilder.
|
|
97
95
|
class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
96
|
+
executor = new FunctionExecutor(
|
|
97
|
+
new ServiceContainer().setServices({
|
|
98
|
+
// ai: {
|
|
99
|
+
// client: this.AiService,
|
|
100
|
+
// },
|
|
101
|
+
}),
|
|
102
|
+
);
|
|
103
103
|
|
|
104
104
|
space: Space | undefined;
|
|
105
105
|
currentMessage: number = 0;
|
package/src/transcriber/index.ts
CHANGED