@dxos/plugin-transcription 0.8.3 → 0.8.4-main.28f8d3d
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/TranscriptContainer-FH7VAWYA.mjs +11 -0
- package/dist/lib/browser/{chunk-GOF7MJNV.mjs → chunk-5WH45KBY.mjs} +2 -2
- package/dist/lib/browser/{chunk-53AXUEIQ.mjs → chunk-BJWYA2BU.mjs} +332 -314
- package/dist/lib/browser/chunk-BJWYA2BU.mjs.map +7 -0
- package/dist/lib/browser/{chunk-IQ5ZRMHZ.mjs → chunk-KJIQQHJB.mjs} +45 -34
- package/dist/lib/browser/chunk-KJIQQHJB.mjs.map +7 -0
- package/dist/lib/browser/{chunk-OSISJM2F.mjs → chunk-KWOOVS5G.mjs} +4 -4
- package/dist/lib/browser/{chunk-OSISJM2F.mjs.map → chunk-KWOOVS5G.mjs.map} +2 -2
- package/dist/lib/browser/{chunk-W2KNYJD6.mjs → chunk-SREIAQSP.mjs} +3 -3
- package/dist/lib/browser/{chunk-W2KNYJD6.mjs.map → chunk-SREIAQSP.mjs.map} +1 -1
- package/dist/lib/browser/{chunk-UDCCT32F.mjs → chunk-UKUSJTQW.mjs} +3 -3
- package/dist/lib/browser/index.mjs +12 -12
- package/dist/lib/browser/index.mjs.map +3 -3
- package/dist/lib/browser/{intent-resolver-BGADCHVM.mjs → intent-resolver-AOVQ7JI4.mjs} +4 -4
- package/dist/lib/{node-esm/intent-resolver-5LVCYSJS.mjs.map → browser/intent-resolver-AOVQ7JI4.mjs.map} +2 -2
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/{react-surface-HMJS5FSS.mjs → react-surface-4MEJMKYQ.mjs} +7 -7
- package/dist/lib/browser/{transcriber-OYPLZFG5.mjs → transcriber-V77CN7XX.mjs} +6 -6
- package/dist/lib/browser/transcriber-V77CN7XX.mjs.map +7 -0
- package/dist/lib/browser/types/index.mjs +2 -2
- package/dist/lib/node-esm/{TranscriptContainer-G75LDEO5.mjs → TranscriptContainer-GEUOOYHX.mjs} +4 -4
- package/dist/lib/node-esm/{chunk-PRL64SRU.mjs → chunk-A33GNR44.mjs} +45 -34
- package/dist/lib/node-esm/chunk-A33GNR44.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-FH75X4JC.mjs → chunk-IKQMDRY6.mjs} +4 -4
- package/dist/lib/node-esm/{chunk-FH75X4JC.mjs.map → chunk-IKQMDRY6.mjs.map} +2 -2
- package/dist/lib/node-esm/{chunk-MFJO7EKD.mjs → chunk-PHP7MCAY.mjs} +3 -3
- package/dist/lib/node-esm/{chunk-MFJPEIVB.mjs → chunk-TM2ASOZA.mjs} +3 -3
- package/dist/lib/node-esm/{chunk-MFJPEIVB.mjs.map → chunk-TM2ASOZA.mjs.map} +1 -1
- package/dist/lib/node-esm/{chunk-TXMDLLBR.mjs → chunk-XMUCZBP3.mjs} +2 -2
- package/dist/lib/node-esm/{chunk-56QHOX4A.mjs → chunk-Y7PCJ34K.mjs} +332 -314
- package/dist/lib/node-esm/chunk-Y7PCJ34K.mjs.map +7 -0
- package/dist/lib/node-esm/index.mjs +12 -12
- package/dist/lib/node-esm/index.mjs.map +3 -3
- package/dist/lib/node-esm/{intent-resolver-5LVCYSJS.mjs → intent-resolver-P4XPIKQF.mjs} +4 -4
- package/dist/lib/{browser/intent-resolver-BGADCHVM.mjs.map → node-esm/intent-resolver-P4XPIKQF.mjs.map} +2 -2
- package/dist/lib/node-esm/meta.json +1 -1
- package/dist/lib/node-esm/{react-surface-JGQSYGRR.mjs → react-surface-EZUPNHUC.mjs} +7 -7
- package/dist/lib/node-esm/{transcriber-TVG2MBBN.mjs → transcriber-3SX3DNLV.mjs} +6 -6
- package/dist/lib/node-esm/transcriber-3SX3DNLV.mjs.map +7 -0
- package/dist/lib/node-esm/types/index.mjs +2 -2
- package/dist/types/src/capabilities/capabilities.d.ts +3 -3
- package/dist/types/src/capabilities/capabilities.d.ts.map +1 -1
- package/dist/types/src/capabilities/intent-resolver.d.ts.map +1 -1
- package/dist/types/src/capabilities/transcriber.d.ts.map +1 -1
- package/dist/types/src/components/Transcript/Transcript.d.ts.map +1 -1
- package/dist/types/src/components/Transcript/Transcript.stories.d.ts +1 -1
- package/dist/types/src/components/Transcript/Transcript.stories.d.ts.map +1 -1
- package/dist/types/src/components/TranscriptContainer.d.ts +4 -3
- package/dist/types/src/components/TranscriptContainer.d.ts.map +1 -1
- package/dist/types/src/components/index.d.ts +1 -4
- package/dist/types/src/components/index.d.ts.map +1 -1
- package/dist/types/src/components/stories/FileTranscription.stories.d.ts +1 -1
- package/dist/types/src/components/stories/FileTranscription.stories.d.ts.map +1 -1
- package/dist/types/src/components/stories/KeyWordRecognition.stories.d.ts +1 -1
- package/dist/types/src/components/stories/KeyWordRecognition.stories.d.ts.map +1 -1
- package/dist/types/src/components/stories/MicrophoneTranscription.stories.d.ts +1 -1
- package/dist/types/src/components/stories/MicrophoneTranscription.stories.d.ts.map +1 -1
- package/dist/types/src/hooks/useTranscriber.d.ts +1 -1
- package/dist/types/src/hooks/useVoiceInput.d.ts.map +1 -1
- package/dist/types/src/segments-normalization/normalization.d.ts +159 -66
- package/dist/types/src/segments-normalization/normalization.d.ts.map +1 -1
- package/dist/types/src/testing/testing.d.ts +1 -1
- package/dist/types/src/testing/testing.d.ts.map +1 -1
- package/dist/types/src/transcriber/index.d.ts +1 -1
- package/dist/types/src/transcriber/index.d.ts.map +1 -1
- package/dist/types/src/transcriber/media-stream-recorder.d.ts +1 -1
- package/dist/types/src/transcriber/media-stream-recorder.d.ts.map +1 -1
- package/dist/types/src/transcriber/transcriber.d.ts +1 -1
- package/dist/types/src/transcriber/transcriber.d.ts.map +1 -1
- package/dist/types/src/translations.d.ts +11 -11
- package/dist/types/src/translations.d.ts.map +1 -1
- package/dist/types/src/types/types.d.ts.map +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +59 -57
- package/src/TranscriptionPlugin.tsx +1 -1
- package/src/capabilities/capabilities.ts +5 -5
- package/src/capabilities/intent-resolver.ts +2 -2
- package/src/capabilities/transcriber.ts +3 -2
- package/src/components/Transcript/Transcript.stories.tsx +5 -4
- package/src/components/Transcript/Transcript.tsx +5 -4
- package/src/components/Transcript/deprecated/OldTranscript.tsx +2 -2
- package/src/components/Transcript/transcript-extension.ts +2 -2
- package/src/components/TranscriptContainer.tsx +10 -4
- package/src/components/stories/FileTranscription.stories.tsx +23 -19
- package/src/components/stories/KeyWordRecognition.stories.tsx +1 -1
- package/src/components/stories/MicrophoneTranscription.stories.tsx +18 -15
- package/src/hooks/useTranscriber.ts +5 -5
- package/src/hooks/useVoiceInput.ts +3 -2
- package/src/model/model.test.ts +9 -9
- package/src/segments-normalization/message-normalizer.ts +1 -1
- package/src/segments-normalization/normalization.test.ts +19 -21
- package/src/segments-normalization/normalization.ts +54 -44
- package/src/testing/testing.ts +11 -11
- package/src/transcriber/index.ts +1 -1
- package/src/transcriber/media-stream-recorder.ts +1 -1
- package/src/transcriber/transcriber.browser.test.ts +4 -3
- package/src/transcriber/transcriber.ts +5 -7
- package/src/transcriber/transcription-manager.ts +3 -3
- package/src/translations.ts +6 -4
- package/src/types/types.ts +2 -1
- package/dist/lib/browser/TranscriptContainer-H3AZQ4CQ.mjs +0 -11
- package/dist/lib/browser/chunk-53AXUEIQ.mjs.map +0 -7
- package/dist/lib/browser/chunk-IQ5ZRMHZ.mjs.map +0 -7
- package/dist/lib/browser/transcriber-OYPLZFG5.mjs.map +0 -7
- package/dist/lib/node/TranscriptContainer-KIG3GMAB.cjs +0 -32
- package/dist/lib/node/TranscriptContainer-KIG3GMAB.cjs.map +0 -7
- package/dist/lib/node/chunk-3PYT3BDM.cjs +0 -40
- package/dist/lib/node/chunk-3PYT3BDM.cjs.map +0 -7
- package/dist/lib/node/chunk-6LW5RA3K.cjs +0 -75
- package/dist/lib/node/chunk-6LW5RA3K.cjs.map +0 -7
- package/dist/lib/node/chunk-AFUZI63A.cjs +0 -42
- package/dist/lib/node/chunk-AFUZI63A.cjs.map +0 -7
- package/dist/lib/node/chunk-CVUN3UO7.cjs +0 -778
- package/dist/lib/node/chunk-CVUN3UO7.cjs.map +0 -7
- package/dist/lib/node/chunk-FKDJNJ57.cjs +0 -35
- package/dist/lib/node/chunk-FKDJNJ57.cjs.map +0 -7
- package/dist/lib/node/chunk-ZFAUMZXE.cjs +0 -518
- package/dist/lib/node/chunk-ZFAUMZXE.cjs.map +0 -7
- package/dist/lib/node/index.cjs +0 -133
- package/dist/lib/node/index.cjs.map +0 -7
- package/dist/lib/node/intent-resolver-LL3TTI4H.cjs +0 -45
- package/dist/lib/node/intent-resolver-LL3TTI4H.cjs.map +0 -7
- package/dist/lib/node/meta.json +0 -1
- package/dist/lib/node/react-surface-N25TBBG4.cjs +0 -56
- package/dist/lib/node/react-surface-N25TBBG4.cjs.map +0 -7
- package/dist/lib/node/transcriber-6J4FVFSZ.cjs +0 -68
- package/dist/lib/node/transcriber-6J4FVFSZ.cjs.map +0 -7
- package/dist/lib/node/types/index.cjs +0 -36
- package/dist/lib/node/types/index.cjs.map +0 -7
- package/dist/lib/node-esm/chunk-56QHOX4A.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-PRL64SRU.mjs.map +0 -7
- package/dist/lib/node-esm/transcriber-TVG2MBBN.mjs.map +0 -7
- /package/dist/lib/browser/{TranscriptContainer-H3AZQ4CQ.mjs.map → TranscriptContainer-FH7VAWYA.mjs.map} +0 -0
- /package/dist/lib/browser/{chunk-GOF7MJNV.mjs.map → chunk-5WH45KBY.mjs.map} +0 -0
- /package/dist/lib/browser/{chunk-UDCCT32F.mjs.map → chunk-UKUSJTQW.mjs.map} +0 -0
- /package/dist/lib/browser/{react-surface-HMJS5FSS.mjs.map → react-surface-4MEJMKYQ.mjs.map} +0 -0
- /package/dist/lib/node-esm/{TranscriptContainer-G75LDEO5.mjs.map → TranscriptContainer-GEUOOYHX.mjs.map} +0 -0
- /package/dist/lib/node-esm/{chunk-MFJO7EKD.mjs.map → chunk-PHP7MCAY.mjs.map} +0 -0
- /package/dist/lib/node-esm/{chunk-TXMDLLBR.mjs.map → chunk-XMUCZBP3.mjs.map} +0 -0
- /package/dist/lib/node-esm/{react-surface-JGQSYGRR.mjs.map → react-surface-EZUPNHUC.mjs.map} +0 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dxos/plugin-transcription",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.4-main.28f8d3d",
|
|
4
4
|
"description": "Voice transcription.",
|
|
5
5
|
"homepage": "https://dxos.org",
|
|
6
6
|
"bugs": "https://github.com/dxos/dxos/issues",
|
|
@@ -9,11 +9,13 @@
|
|
|
9
9
|
"type": "module",
|
|
10
10
|
"exports": {
|
|
11
11
|
".": {
|
|
12
|
+
"source": "./src/index.ts",
|
|
12
13
|
"types": "./dist/types/src/index.d.ts",
|
|
13
14
|
"browser": "./dist/lib/browser/index.mjs",
|
|
14
15
|
"node": "./dist/lib/node-esm/index.mjs"
|
|
15
16
|
},
|
|
16
17
|
"./types": {
|
|
18
|
+
"source": "./src/types/index.ts",
|
|
17
19
|
"types": "./dist/types/src/types/index.d.ts",
|
|
18
20
|
"browser": "./dist/lib/browser/types/index.mjs",
|
|
19
21
|
"node": "./dist/lib/node-esm/types/index.mjs"
|
|
@@ -40,77 +42,77 @@
|
|
|
40
42
|
"extendable-media-recorder": "^9.2.19",
|
|
41
43
|
"extendable-media-recorder-wav-encoder": "^7.0.123",
|
|
42
44
|
"main-thread-scheduling": "^14.1.1",
|
|
43
|
-
"react-resize-detector": "^
|
|
45
|
+
"react-resize-detector": "^11.0.1",
|
|
44
46
|
"react-router-dom": "^6.4.0",
|
|
45
47
|
"react-use": "^17.5.1",
|
|
46
48
|
"wavefile": "^11.0.0",
|
|
47
|
-
"@dxos/
|
|
48
|
-
"@dxos/
|
|
49
|
-
"@dxos/
|
|
50
|
-
"@dxos/
|
|
51
|
-
"@dxos/
|
|
52
|
-
"@dxos/
|
|
53
|
-
"@dxos/
|
|
54
|
-
"@dxos/context": "0.8.
|
|
55
|
-
"@dxos/debug": "0.8.
|
|
56
|
-
"@dxos/
|
|
57
|
-
"@dxos/echo": "0.8.
|
|
58
|
-
"@dxos/echo-db": "0.8.
|
|
59
|
-
"@dxos/
|
|
60
|
-
"@dxos/
|
|
61
|
-
"@dxos/
|
|
62
|
-
"@dxos/live-object": "0.8.
|
|
63
|
-
"@dxos/
|
|
64
|
-
"@dxos/
|
|
65
|
-
"@dxos/
|
|
66
|
-
"@dxos/plugin-
|
|
67
|
-
"@dxos/plugin-
|
|
68
|
-
"@dxos/plugin-
|
|
69
|
-
"@dxos/plugin-
|
|
70
|
-
"@dxos/plugin-thread": "0.8.
|
|
71
|
-
"@dxos/
|
|
72
|
-
"@dxos/
|
|
73
|
-
"@dxos/react-edge-client": "0.8.
|
|
74
|
-
"@dxos/
|
|
75
|
-
"@dxos/react-ui-attention": "0.8.
|
|
76
|
-
"@dxos/react-ui-editor": "0.8.
|
|
77
|
-
"@dxos/react-ui-
|
|
78
|
-
"@dxos/react-ui-
|
|
79
|
-
"@dxos/react-ui-
|
|
80
|
-
"@dxos/
|
|
81
|
-
"@dxos/
|
|
82
|
-
"@dxos/
|
|
83
|
-
"@dxos/react-ui-
|
|
84
|
-
"@dxos/util": "0.8.
|
|
49
|
+
"@dxos/app-framework": "0.8.4-main.28f8d3d",
|
|
50
|
+
"@dxos/assistant": "0.8.4-main.28f8d3d",
|
|
51
|
+
"@dxos/async": "0.8.4-main.28f8d3d",
|
|
52
|
+
"@dxos/client": "0.8.4-main.28f8d3d",
|
|
53
|
+
"@dxos/codec-protobuf": "0.8.4-main.28f8d3d",
|
|
54
|
+
"@dxos/blueprints": "0.8.4-main.28f8d3d",
|
|
55
|
+
"@dxos/ai": "0.8.4-main.28f8d3d",
|
|
56
|
+
"@dxos/context": "0.8.4-main.28f8d3d",
|
|
57
|
+
"@dxos/debug": "0.8.4-main.28f8d3d",
|
|
58
|
+
"@dxos/echo": "0.8.4-main.28f8d3d",
|
|
59
|
+
"@dxos/echo-schema": "0.8.4-main.28f8d3d",
|
|
60
|
+
"@dxos/echo-db": "0.8.4-main.28f8d3d",
|
|
61
|
+
"@dxos/invariant": "0.8.4-main.28f8d3d",
|
|
62
|
+
"@dxos/functions": "0.8.4-main.28f8d3d",
|
|
63
|
+
"@dxos/display-name": "0.8.4-main.28f8d3d",
|
|
64
|
+
"@dxos/live-object": "0.8.4-main.28f8d3d",
|
|
65
|
+
"@dxos/local-storage": "0.8.4-main.28f8d3d",
|
|
66
|
+
"@dxos/keys": "0.8.4-main.28f8d3d",
|
|
67
|
+
"@dxos/plugin-client": "0.8.4-main.28f8d3d",
|
|
68
|
+
"@dxos/plugin-graph": "0.8.4-main.28f8d3d",
|
|
69
|
+
"@dxos/plugin-space": "0.8.4-main.28f8d3d",
|
|
70
|
+
"@dxos/plugin-markdown": "0.8.4-main.28f8d3d",
|
|
71
|
+
"@dxos/plugin-deck": "0.8.4-main.28f8d3d",
|
|
72
|
+
"@dxos/plugin-thread": "0.8.4-main.28f8d3d",
|
|
73
|
+
"@dxos/react-client": "0.8.4-main.28f8d3d",
|
|
74
|
+
"@dxos/protocols": "0.8.4-main.28f8d3d",
|
|
75
|
+
"@dxos/react-edge-client": "0.8.4-main.28f8d3d",
|
|
76
|
+
"@dxos/log": "0.8.4-main.28f8d3d",
|
|
77
|
+
"@dxos/react-ui-attention": "0.8.4-main.28f8d3d",
|
|
78
|
+
"@dxos/react-ui-editor": "0.8.4-main.28f8d3d",
|
|
79
|
+
"@dxos/react-ui-form": "0.8.4-main.28f8d3d",
|
|
80
|
+
"@dxos/react-ui-grid": "0.8.4-main.28f8d3d",
|
|
81
|
+
"@dxos/react-ui-sfx": "0.8.4-main.28f8d3d",
|
|
82
|
+
"@dxos/react-ui-stack": "0.8.4-main.28f8d3d",
|
|
83
|
+
"@dxos/schema": "0.8.4-main.28f8d3d",
|
|
84
|
+
"@dxos/tracing": "0.8.4-main.28f8d3d",
|
|
85
|
+
"@dxos/react-ui-components": "0.8.4-main.28f8d3d",
|
|
86
|
+
"@dxos/util": "0.8.4-main.28f8d3d"
|
|
85
87
|
},
|
|
86
88
|
"devDependencies": {
|
|
87
|
-
"@effect-rx/rx-react": "
|
|
88
|
-
"@effect/platform": "0.
|
|
89
|
+
"@effect-rx/rx-react": "0.38.0",
|
|
90
|
+
"@effect/platform": "0.90.2",
|
|
89
91
|
"@types/dom-speech-recognition": "^0.0.6",
|
|
90
92
|
"@types/react": "~18.2.0",
|
|
91
|
-
"effect": "3.
|
|
93
|
+
"effect": "3.17.7",
|
|
92
94
|
"react": "~18.2.0",
|
|
93
95
|
"react-dom": "~18.2.0",
|
|
94
96
|
"tailwindcss": "~3.4.1",
|
|
95
|
-
"@dxos/plugin-preview": "0.8.
|
|
96
|
-
"@dxos/
|
|
97
|
-
"@dxos/
|
|
98
|
-
"@dxos/
|
|
99
|
-
"@dxos/react-ui
|
|
100
|
-
"@dxos/react-ui-
|
|
101
|
-
"@dxos/
|
|
102
|
-
"@dxos/react-ui-theme": "0.8.
|
|
103
|
-
"@dxos/storybook-utils": "0.8.
|
|
104
|
-
"@dxos/test-utils": "0.8.
|
|
97
|
+
"@dxos/plugin-preview": "0.8.4-main.28f8d3d",
|
|
98
|
+
"@dxos/plugin-storybook-layout": "0.8.4-main.28f8d3d",
|
|
99
|
+
"@dxos/plugin-theme": "0.8.4-main.28f8d3d",
|
|
100
|
+
"@dxos/random": "0.8.4-main.28f8d3d",
|
|
101
|
+
"@dxos/react-ui": "0.8.4-main.28f8d3d",
|
|
102
|
+
"@dxos/react-ui-form": "0.8.4-main.28f8d3d",
|
|
103
|
+
"@dxos/react-ui-syntax-highlighter": "0.8.4-main.28f8d3d",
|
|
104
|
+
"@dxos/react-ui-theme": "0.8.4-main.28f8d3d",
|
|
105
|
+
"@dxos/storybook-utils": "0.8.4-main.28f8d3d",
|
|
106
|
+
"@dxos/test-utils": "0.8.4-main.28f8d3d"
|
|
105
107
|
},
|
|
106
|
-
"
|
|
108
|
+
"peerDependencies": {
|
|
107
109
|
"@effect-rx/rx-react": "^0.34.1",
|
|
108
|
-
"@effect/platform": "0.80.12",
|
|
110
|
+
"@effect/platform": "^0.80.12",
|
|
109
111
|
"effect": "^3.13.3",
|
|
110
112
|
"react": "~18.2.0",
|
|
111
113
|
"react-dom": "~18.2.0",
|
|
112
|
-
"@dxos/react-ui
|
|
113
|
-
"@dxos/react-ui": "0.8.
|
|
114
|
+
"@dxos/react-ui": "0.8.4-main.28f8d3d",
|
|
115
|
+
"@dxos/react-ui-theme": "0.8.4-main.28f8d3d"
|
|
114
116
|
},
|
|
115
117
|
"publishConfig": {
|
|
116
118
|
"access": "public"
|
|
@@ -11,7 +11,7 @@ import { DataType } from '@dxos/schema';
|
|
|
11
11
|
import { IntentResolver, ReactSurface, Transcriber } from './capabilities';
|
|
12
12
|
import { renderMarkdown } from './components';
|
|
13
13
|
import { meta } from './meta';
|
|
14
|
-
import translations from './translations';
|
|
14
|
+
import { translations } from './translations';
|
|
15
15
|
import { TranscriptType } from './types';
|
|
16
16
|
|
|
17
17
|
export const TranscriptionPlugin = () =>
|
|
@@ -6,19 +6,19 @@ import { defineCapability } from '@dxos/app-framework';
|
|
|
6
6
|
|
|
7
7
|
import { TRANSCRIPTION_PLUGIN } from '../meta';
|
|
8
8
|
import {
|
|
9
|
-
type
|
|
9
|
+
type MediaStreamRecorderParams,
|
|
10
10
|
type Transcriber,
|
|
11
|
-
type
|
|
11
|
+
type TranscriberParams,
|
|
12
12
|
type TranscriptMessageEnricher,
|
|
13
|
-
type
|
|
13
|
+
type TranscriptionManager,
|
|
14
14
|
} from '../transcriber';
|
|
15
15
|
|
|
16
16
|
export namespace TranscriptionCapabilities {
|
|
17
17
|
export type GetTranscriberProps = {
|
|
18
18
|
audioStreamTrack: MediaStreamTrack;
|
|
19
|
-
onSegments: TranscriberParams['onSegments'];
|
|
20
|
-
transcriberConfig?: Partial<TranscriberParams['config']>;
|
|
21
19
|
recorderConfig?: Partial<MediaStreamRecorderParams['config']>;
|
|
20
|
+
transcriberConfig?: Partial<TranscriberParams['config']>;
|
|
21
|
+
onSegments: TranscriberParams['onSegments'];
|
|
22
22
|
};
|
|
23
23
|
export type GetTranscriber = (props: GetTranscriberProps) => Transcriber;
|
|
24
24
|
export const Transcriber = defineCapability<GetTranscriber>(`${TRANSCRIPTION_PLUGIN}/capability/transcriber`);
|
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import { Capabilities, contributes, createResolver
|
|
5
|
+
import { Capabilities, type PluginContext, contributes, createResolver } from '@dxos/app-framework';
|
|
6
6
|
import { type Key, Obj, Ref } from '@dxos/echo';
|
|
7
7
|
import { createQueueDXN } from '@dxos/echo-schema';
|
|
8
8
|
|
|
9
|
-
import {
|
|
9
|
+
import { TranscriptType, TranscriptionAction } from '../types';
|
|
10
10
|
|
|
11
11
|
export default (context: PluginContext) =>
|
|
12
12
|
contributes(Capabilities.IntentResolver, [
|
|
@@ -2,12 +2,13 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import {
|
|
5
|
+
import { type PluginContext, contributes } from '@dxos/app-framework';
|
|
6
6
|
import { ClientCapabilities } from '@dxos/plugin-client';
|
|
7
7
|
|
|
8
|
-
import { TranscriptionCapabilities } from './capabilities';
|
|
9
8
|
import { MediaStreamRecorder, Transcriber, TranscriptionManager } from '../transcriber';
|
|
10
9
|
|
|
10
|
+
import { TranscriptionCapabilities } from './capabilities';
|
|
11
|
+
|
|
11
12
|
// TODO(burdon): Move to config?
|
|
12
13
|
|
|
13
14
|
/**
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
import '@dxos-theme';
|
|
6
6
|
|
|
7
|
-
import { type Meta, type StoryObj } from '@storybook/react';
|
|
7
|
+
import { type Meta, type StoryObj } from '@storybook/react-vite';
|
|
8
8
|
import React, { type FC, useEffect, useMemo, useState } from 'react';
|
|
9
9
|
|
|
10
10
|
import { IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
|
|
@@ -24,7 +24,6 @@ import { DataType } from '@dxos/schema';
|
|
|
24
24
|
import { Testing } from '@dxos/schema/testing';
|
|
25
25
|
import { withLayout } from '@dxos/storybook-utils';
|
|
26
26
|
|
|
27
|
-
import { renderMarkdown, Transcript, type TranscriptProps } from './Transcript';
|
|
28
27
|
import { useQueueModelAdapter } from '../../hooks';
|
|
29
28
|
import { SerializationModel } from '../../model';
|
|
30
29
|
import {
|
|
@@ -33,7 +32,9 @@ import {
|
|
|
33
32
|
useTestTranscriptionQueue,
|
|
34
33
|
useTestTranscriptionQueueWithEntityExtraction,
|
|
35
34
|
} from '../../testing';
|
|
36
|
-
import translations from '../../translations';
|
|
35
|
+
import { translations } from '../../translations';
|
|
36
|
+
|
|
37
|
+
import { Transcript, type TranscriptProps, renderMarkdown } from './Transcript';
|
|
37
38
|
|
|
38
39
|
faker.seed(1);
|
|
39
40
|
|
|
@@ -187,7 +188,7 @@ const meta: Meta<typeof QueueStory> = {
|
|
|
187
188
|
StorybookLayoutPlugin(),
|
|
188
189
|
ClientPlugin({
|
|
189
190
|
types: [TestItem, Testing.DocumentType, DataType.Person, DataType.Organization],
|
|
190
|
-
onClientInitialized: async (
|
|
191
|
+
onClientInitialized: async ({ client }) => {
|
|
191
192
|
await client.halo.createIdentity();
|
|
192
193
|
},
|
|
193
194
|
}),
|
|
@@ -21,16 +21,17 @@ import { mx } from '@dxos/react-ui-theme';
|
|
|
21
21
|
import { type DataType } from '@dxos/schema';
|
|
22
22
|
import { isNotFalsy } from '@dxos/util';
|
|
23
23
|
|
|
24
|
-
import { transcript } from './transcript-extension';
|
|
25
24
|
import { type SerializationModel } from '../../model';
|
|
26
25
|
import { type TranscriptType } from '../../types';
|
|
27
26
|
|
|
27
|
+
import { transcript } from './transcript-extension';
|
|
28
|
+
|
|
28
29
|
export const renderMarkdown =
|
|
29
30
|
(identities: Identity[]) =>
|
|
30
31
|
(message: DataType.Message, index: number, debug = false): string[] => {
|
|
31
32
|
if (message.sender.role === 'assistant') {
|
|
32
33
|
// Start/stop block.
|
|
33
|
-
return [message.blocks.find((block) => block.
|
|
34
|
+
return [message.blocks.find((block) => block._tag === 'transcript')?.text ?? '', ''];
|
|
34
35
|
}
|
|
35
36
|
|
|
36
37
|
// TODO(burdon): Use link/reference markup for users (with popover).
|
|
@@ -42,7 +43,7 @@ export const renderMarkdown =
|
|
|
42
43
|
message.sender.name ??
|
|
43
44
|
message.sender.email ??
|
|
44
45
|
message.sender.identityDid;
|
|
45
|
-
const blocks = message.blocks.filter((block) => block.
|
|
46
|
+
const blocks = message.blocks.filter((block) => block._tag === 'transcript');
|
|
46
47
|
return [
|
|
47
48
|
`###### ${name}` + (debug ? ` [${index}]:${message.id}` : ''),
|
|
48
49
|
blocks.map((block) => block.text.trim()).join(' '),
|
|
@@ -74,7 +75,7 @@ export const Transcript = ({
|
|
|
74
75
|
const { parentRef } = useTextEditor(() => {
|
|
75
76
|
return {
|
|
76
77
|
extensions: [
|
|
77
|
-
createBasicExtensions({ readOnly: true, lineWrapping: true }),
|
|
78
|
+
createBasicExtensions({ readOnly: true, lineWrapping: true, search: true }),
|
|
78
79
|
createMarkdownExtensions({ themeMode }),
|
|
79
80
|
createThemeExtensions({ themeMode }),
|
|
80
81
|
decorateMarkdown(),
|
|
@@ -5,16 +5,16 @@
|
|
|
5
5
|
import { intervalToDuration } from 'date-fns/intervalToDuration';
|
|
6
6
|
import { Schema } from 'effect';
|
|
7
7
|
import { yieldOrContinue } from 'main-thread-scheduling';
|
|
8
|
-
import React, { type FC, useCallback, useEffect, useMemo, useRef, useState
|
|
8
|
+
import React, { type FC, type WheelEvent, useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
|
9
9
|
import { type OnResizeCallback, useResizeDetector } from 'react-resize-detector';
|
|
10
10
|
|
|
11
11
|
import { Type } from '@dxos/echo';
|
|
12
12
|
import { IconButton, type ThemedClassName, useTranslation } from '@dxos/react-ui';
|
|
13
13
|
import { useAttention } from '@dxos/react-ui-attention';
|
|
14
14
|
import {
|
|
15
|
+
type DxGridAxisMeta,
|
|
15
16
|
type DxGridCellValue,
|
|
16
17
|
type DxGridElement,
|
|
17
|
-
type DxGridAxisMeta,
|
|
18
18
|
type DxGridPlaneCells,
|
|
19
19
|
Grid,
|
|
20
20
|
toPlaneCellIndex,
|
|
@@ -38,7 +38,7 @@ export const transcript = ({ model, started, renderButton }: TranscriptOptions):
|
|
|
38
38
|
let line = view.state.doc.lineAt(from);
|
|
39
39
|
while (line.from <= to) {
|
|
40
40
|
const block = model.getChunkAtLine(line.number)?.blocks[0];
|
|
41
|
-
const timestamp = block?.
|
|
41
|
+
const timestamp = block?._tag === 'transcript' && block.started;
|
|
42
42
|
if (timestamp) {
|
|
43
43
|
builder.add(line.from, line.from, new TimestampMarker(line.number, new Date(timestamp), start));
|
|
44
44
|
}
|
|
@@ -218,7 +218,7 @@ const getStartTime = (started?: Date, message?: DataType.Message): Date | undefi
|
|
|
218
218
|
return started;
|
|
219
219
|
}
|
|
220
220
|
|
|
221
|
-
if (message?.blocks[0]?.
|
|
221
|
+
if (message?.blocks[0]?._tag === 'transcript' && message.blocks[0].started) {
|
|
222
222
|
return new Date(message.blocks[0].started);
|
|
223
223
|
}
|
|
224
224
|
|
|
@@ -2,18 +2,24 @@
|
|
|
2
2
|
// Copyright 2024 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import React
|
|
5
|
+
import React from 'react';
|
|
6
6
|
|
|
7
7
|
import { fullyQualifiedId, getSpace } from '@dxos/client/echo';
|
|
8
8
|
import { useMembers, useQueue } from '@dxos/react-client/echo';
|
|
9
9
|
import { StackItem } from '@dxos/react-ui-stack';
|
|
10
10
|
import { type DataType } from '@dxos/schema';
|
|
11
11
|
|
|
12
|
-
import { Transcript, renderMarkdown } from './Transcript';
|
|
13
12
|
import { useQueueModelAdapter } from '../hooks';
|
|
14
13
|
import { type TranscriptType } from '../types';
|
|
15
14
|
|
|
16
|
-
|
|
15
|
+
import { Transcript, renderMarkdown } from './Transcript';
|
|
16
|
+
|
|
17
|
+
export type TranscriptionContainerProps = {
|
|
18
|
+
role: string;
|
|
19
|
+
transcript: TranscriptType;
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
export const TranscriptionContainer = ({ transcript }: TranscriptionContainerProps) => {
|
|
17
23
|
const attendableId = fullyQualifiedId(transcript);
|
|
18
24
|
const space = getSpace(transcript);
|
|
19
25
|
const members = useMembers(space?.key).map((member) => member.identity);
|
|
@@ -21,7 +27,7 @@ export const TranscriptionContainer: FC<{ role: string; transcript: TranscriptTy
|
|
|
21
27
|
const model = useQueueModelAdapter(renderMarkdown(members), queue);
|
|
22
28
|
|
|
23
29
|
return (
|
|
24
|
-
<StackItem.Content
|
|
30
|
+
<StackItem.Content classNames='container-max-width pli-2'>
|
|
25
31
|
<Transcript attendableId={attendableId} space={space} model={model} transcript={transcript} />
|
|
26
32
|
</StackItem.Content>
|
|
27
33
|
);
|
|
@@ -4,11 +4,9 @@
|
|
|
4
4
|
|
|
5
5
|
import '@dxos-theme';
|
|
6
6
|
|
|
7
|
-
import { type Meta, type StoryObj } from '@storybook/react';
|
|
7
|
+
import { type Meta, type StoryObj } from '@storybook/react-vite';
|
|
8
8
|
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
|
9
9
|
|
|
10
|
-
import { EdgeAiServiceClient } from '@dxos/ai';
|
|
11
|
-
import { AI_SERVICE_ENDPOINT } from '@dxos/ai/testing';
|
|
12
10
|
import { Events, IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
|
|
13
11
|
import { withPluginManager } from '@dxos/app-framework/testing';
|
|
14
12
|
import { scheduleTask } from '@dxos/async';
|
|
@@ -25,18 +23,19 @@ import { StorybookLayoutPlugin } from '@dxos/plugin-storybook-layout';
|
|
|
25
23
|
import { ThemePlugin } from '@dxos/plugin-theme';
|
|
26
24
|
import { defaultTx } from '@dxos/react-ui-theme';
|
|
27
25
|
import { DataType } from '@dxos/schema';
|
|
28
|
-
import {
|
|
26
|
+
import { Testing, seedTestData } from '@dxos/schema/testing';
|
|
29
27
|
import { withLayout, withTheme } from '@dxos/storybook-utils';
|
|
30
28
|
|
|
31
|
-
import { TranscriptionStory } from './TranscriptionStory';
|
|
32
|
-
import { useIsSpeaking } from './useIsSpeaking';
|
|
33
|
-
import { TranscriptionPlugin } from '../../TranscriptionPlugin';
|
|
34
29
|
import { useAudioFile, useQueueModelAdapter, useTranscriber } from '../../hooks';
|
|
35
|
-
import {
|
|
30
|
+
import { MessageNormalizer, getActorId } from '../../segments-normalization';
|
|
36
31
|
import { TestItem } from '../../testing';
|
|
37
32
|
import { type MediaStreamRecorderParams, type TranscriberParams } from '../../transcriber';
|
|
33
|
+
import { TranscriptionPlugin } from '../../TranscriptionPlugin';
|
|
38
34
|
import { renderMarkdown } from '../Transcript';
|
|
39
35
|
|
|
36
|
+
import { TranscriptionStory } from './TranscriptionStory';
|
|
37
|
+
import { useIsSpeaking } from './useIsSpeaking';
|
|
38
|
+
|
|
40
39
|
const AudioFile = ({
|
|
41
40
|
detectSpeaking,
|
|
42
41
|
audioUrl,
|
|
@@ -88,8 +87,13 @@ const AudioFile = ({
|
|
|
88
87
|
const model = useQueueModelAdapter(renderMarkdown([]), queue);
|
|
89
88
|
const handleSegments = useCallback<TranscriberParams['onSegments']>(
|
|
90
89
|
async (blocks) => {
|
|
91
|
-
|
|
92
|
-
|
|
90
|
+
void queue?.append([
|
|
91
|
+
Obj.make(DataType.Message, {
|
|
92
|
+
created: new Date().toISOString(),
|
|
93
|
+
sender: actor,
|
|
94
|
+
blocks,
|
|
95
|
+
}),
|
|
96
|
+
]);
|
|
93
97
|
},
|
|
94
98
|
[queue],
|
|
95
99
|
);
|
|
@@ -108,14 +112,14 @@ const AudioFile = ({
|
|
|
108
112
|
}
|
|
109
113
|
const executor = new FunctionExecutor(
|
|
110
114
|
new ServiceContainer().setServices({
|
|
111
|
-
ai: {
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
},
|
|
115
|
+
// ai: {
|
|
116
|
+
// client: new Edge AiServiceClient({
|
|
117
|
+
// endpoint: AI_SERVICE_ENDPOINT.REMOTE,
|
|
118
|
+
// defaultGenerationOptions: {
|
|
119
|
+
// model: '@anthropic/claude-3-5-sonnet-20241022',
|
|
120
|
+
// },
|
|
121
|
+
// }),
|
|
122
|
+
// },
|
|
119
123
|
}),
|
|
120
124
|
);
|
|
121
125
|
|
|
@@ -184,7 +188,7 @@ const meta: Meta<typeof AudioFile> = {
|
|
|
184
188
|
StorybookLayoutPlugin(),
|
|
185
189
|
ClientPlugin({
|
|
186
190
|
types: [TestItem, DataType.Person, DataType.Organization, Testing.DocumentType],
|
|
187
|
-
onClientInitialized: async (
|
|
191
|
+
onClientInitialized: async ({ client }) => {
|
|
188
192
|
await client.halo.createIdentity();
|
|
189
193
|
await client.spaces.waitUntilReady();
|
|
190
194
|
await client.spaces.default.waitUntilReady();
|
|
@@ -4,20 +4,18 @@
|
|
|
4
4
|
|
|
5
5
|
import '@dxos-theme';
|
|
6
6
|
|
|
7
|
-
import { type Meta, type StoryObj } from '@storybook/react';
|
|
7
|
+
import { type Meta, type StoryObj } from '@storybook/react-vite';
|
|
8
8
|
import React, { useCallback, useEffect, useMemo, useState } from 'react';
|
|
9
9
|
|
|
10
|
-
import { EdgeAiServiceClient } from '@dxos/ai';
|
|
11
|
-
import { AI_SERVICE_ENDPOINT } from '@dxos/ai/testing';
|
|
12
10
|
import { Events, IntentPlugin, SettingsPlugin } from '@dxos/app-framework';
|
|
13
11
|
import { withPluginManager } from '@dxos/app-framework/testing';
|
|
14
12
|
import {
|
|
15
|
-
extractionAnthropicFn,
|
|
16
13
|
type ExtractionFunction,
|
|
14
|
+
extractionAnthropicFn,
|
|
17
15
|
extractionNerFn,
|
|
18
|
-
processTranscriptMessage,
|
|
19
16
|
getNer,
|
|
20
|
-
|
|
17
|
+
processTranscriptMessage,
|
|
18
|
+
} from '@dxos/assistant/extraction';
|
|
21
19
|
import { Filter, Obj, type Type } from '@dxos/echo';
|
|
22
20
|
import { MemoryQueue } from '@dxos/echo-db';
|
|
23
21
|
import { createQueueDXN } from '@dxos/echo-schema';
|
|
@@ -32,17 +30,18 @@ import { ThemePlugin } from '@dxos/plugin-theme';
|
|
|
32
30
|
import { IndexKind, useSpace } from '@dxos/react-client/echo';
|
|
33
31
|
import { defaultTx } from '@dxos/react-ui-theme';
|
|
34
32
|
import { DataType } from '@dxos/schema';
|
|
35
|
-
import {
|
|
33
|
+
import { Testing, seedTestData } from '@dxos/schema/testing';
|
|
36
34
|
import { withLayout, withTheme } from '@dxos/storybook-utils';
|
|
37
35
|
|
|
38
|
-
import { TranscriptionStory } from './TranscriptionStory';
|
|
39
|
-
import { useIsSpeaking } from './useIsSpeaking';
|
|
40
|
-
import { TranscriptionPlugin } from '../../TranscriptionPlugin';
|
|
41
36
|
import { useAudioTrack, useQueueModelAdapter, useTranscriber } from '../../hooks';
|
|
42
37
|
import { TestItem } from '../../testing';
|
|
43
38
|
import { type MediaStreamRecorderParams, type TranscriberParams } from '../../transcriber';
|
|
39
|
+
import { TranscriptionPlugin } from '../../TranscriptionPlugin';
|
|
44
40
|
import { renderMarkdown } from '../Transcript';
|
|
45
41
|
|
|
42
|
+
import { TranscriptionStory } from './TranscriptionStory';
|
|
43
|
+
import { useIsSpeaking } from './useIsSpeaking';
|
|
44
|
+
|
|
46
45
|
const TRANSCRIBER_CONFIG = {
|
|
47
46
|
transcribeAfterChunksAmount: 50,
|
|
48
47
|
prefixBufferChunksAmount: 10,
|
|
@@ -117,11 +116,15 @@ const DefaultStory = ({
|
|
|
117
116
|
.then((result) => result.objects);
|
|
118
117
|
}
|
|
119
118
|
if (entityExtraction !== 'none') {
|
|
120
|
-
const AiService = new EdgeAiServiceClient({
|
|
121
|
-
endpoint: AI_SERVICE_ENDPOINT.REMOTE,
|
|
122
|
-
});
|
|
123
119
|
executor = new FunctionExecutor(
|
|
124
|
-
new ServiceContainer().setServices({
|
|
120
|
+
new ServiceContainer().setServices({
|
|
121
|
+
// ai: {
|
|
122
|
+
// client: new Edge AiServiceClient({
|
|
123
|
+
// endpoint: AI_SERVICE_ENDPOINT.REMOTE,
|
|
124
|
+
// }),
|
|
125
|
+
// },
|
|
126
|
+
// database: { db: space!.db },
|
|
127
|
+
}),
|
|
125
128
|
);
|
|
126
129
|
}
|
|
127
130
|
|
|
@@ -200,7 +203,7 @@ const meta: Meta<typeof DefaultStory> = {
|
|
|
200
203
|
StorybookLayoutPlugin(),
|
|
201
204
|
ClientPlugin({
|
|
202
205
|
types: [TestItem, DataType.Person, DataType.Organization, Testing.DocumentType],
|
|
203
|
-
onClientInitialized: async (
|
|
206
|
+
onClientInitialized: async ({ client }) => {
|
|
204
207
|
await client.halo.createIdentity();
|
|
205
208
|
await client.spaces.waitUntilReady();
|
|
206
209
|
await client.spaces.default.waitUntilReady();
|
|
@@ -14,20 +14,20 @@ import { type Transcriber } from '../transcriber';
|
|
|
14
14
|
*/
|
|
15
15
|
export const useTranscriber = ({
|
|
16
16
|
audioStreamTrack,
|
|
17
|
-
onSegments,
|
|
18
|
-
transcriberConfig,
|
|
19
17
|
recorderConfig,
|
|
18
|
+
transcriberConfig,
|
|
19
|
+
onSegments,
|
|
20
20
|
}: Partial<TranscriptionCapabilities.GetTranscriberProps>) => {
|
|
21
21
|
const [getTranscriber] = useCapabilities(TranscriptionCapabilities.Transcriber);
|
|
22
22
|
|
|
23
23
|
// Initialize audio transcription.
|
|
24
24
|
const transcriber = useMemo<Transcriber | undefined>(() => {
|
|
25
|
-
if (!
|
|
25
|
+
if (!getTranscriber || !audioStreamTrack || !onSegments) {
|
|
26
26
|
return undefined;
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
-
return getTranscriber({ audioStreamTrack,
|
|
30
|
-
}, [
|
|
29
|
+
return getTranscriber({ audioStreamTrack, recorderConfig, transcriberConfig, onSegments });
|
|
30
|
+
}, [getTranscriber, audioStreamTrack, recorderConfig, transcriberConfig, onSegments]);
|
|
31
31
|
|
|
32
32
|
useEffect(() => {
|
|
33
33
|
return () => {
|
|
@@ -2,16 +2,17 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import {
|
|
5
|
+
import { useCallback, useEffect, useState } from 'react';
|
|
6
6
|
|
|
7
7
|
import { scheduleMicroTask } from '@dxos/async';
|
|
8
8
|
import { Context } from '@dxos/context';
|
|
9
9
|
import { log } from '@dxos/log';
|
|
10
10
|
import { useSoundEffect } from '@dxos/react-ui-sfx';
|
|
11
11
|
|
|
12
|
+
import { type TranscriberParams } from '../transcriber';
|
|
13
|
+
|
|
12
14
|
import { useAudioTrack } from './useAudioTrack';
|
|
13
15
|
import { useTranscriber } from './useTranscriber';
|
|
14
|
-
import { type TranscriberParams } from '../transcriber';
|
|
15
16
|
|
|
16
17
|
export type UseVoiceInputProps = {
|
|
17
18
|
active?: boolean;
|