@dxos/plugin-transcription 0.9.0 → 0.9.1-main.c7dcc2e112
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PLUGIN.mdl +20 -0
- package/dist/lib/neutral/{TranscriptionArticle-L6XANBH5.mjs → TranscriptionArticle-S56VVOSF.mjs} +3 -3
- package/dist/lib/neutral/{TranscriptionArticle-L6XANBH5.mjs.map → TranscriptionArticle-S56VVOSF.mjs.map} +3 -3
- package/dist/lib/neutral/TranscriptionPlugin.mjs +6 -4
- package/dist/lib/neutral/TranscriptionPlugin.mjs.map +4 -4
- package/dist/lib/neutral/capabilities/index.mjs +5 -3
- package/dist/lib/neutral/capabilities/index.mjs.map +3 -3
- package/dist/lib/neutral/{chunk-KMT3BMMQ.mjs → chunk-2URVUJT4.mjs} +57 -22
- package/dist/lib/neutral/chunk-2URVUJT4.mjs.map +7 -0
- package/dist/lib/neutral/chunk-J4ONEYPJ.mjs +8 -0
- package/dist/lib/neutral/{chunk-KXJZEYS7.mjs.map → chunk-J4ONEYPJ.mjs.map} +2 -2
- package/dist/lib/neutral/{chunk-TO77KAQK.mjs → chunk-XMHH34NI.mjs} +25 -5
- package/dist/lib/neutral/{chunk-TO77KAQK.mjs.map → chunk-XMHH34NI.mjs.map} +3 -3
- package/dist/lib/neutral/{chunk-KAEXRX3Y.mjs → chunk-XOXHWMXR.mjs} +1 -1
- package/dist/lib/neutral/{chunk-TX3NQISZ.mjs → chunk-XWC5BBOY.mjs} +3 -3
- package/dist/lib/neutral/{chunk-TX3NQISZ.mjs.map → chunk-XWC5BBOY.mjs.map} +3 -3
- package/dist/lib/neutral/chunk-XZVDAYM6.mjs +54 -0
- package/dist/lib/neutral/chunk-XZVDAYM6.mjs.map +7 -0
- package/dist/lib/neutral/containers/index.mjs +1 -1
- package/dist/lib/neutral/{create-RVR7VC33.mjs → create-7NGOP7JP.mjs} +2 -2
- package/dist/lib/neutral/enrich-message-F2AOM53F.mjs +24 -0
- package/dist/lib/neutral/enrich-message-F2AOM53F.mjs.map +7 -0
- package/dist/lib/neutral/hooks/index.mjs +1 -1
- package/dist/lib/neutral/index.mjs +7 -5
- package/dist/lib/neutral/meta.json +1 -1
- package/dist/lib/neutral/meta.mjs +1 -1
- package/dist/lib/neutral/{open-A5KXKRH4.mjs → open-42H72C4G.mjs} +3 -3
- package/dist/lib/neutral/operations/index.mjs +1 -1
- package/dist/lib/neutral/plugin.mjs +2 -2
- package/dist/lib/neutral/settings-DUS32HOM.mjs +30 -0
- package/dist/lib/neutral/settings-DUS32HOM.mjs.map +7 -0
- package/dist/lib/neutral/{summarize-5XPPMVTE.mjs → summarize-45WVUPED.mjs} +3 -3
- package/dist/lib/neutral/{summarize-5XPPMVTE.mjs.map → summarize-45WVUPED.mjs.map} +1 -1
- package/dist/lib/neutral/{text-content-KBB37HSM.mjs → text-content-XZ3HK4OT.mjs} +9 -7
- package/dist/lib/neutral/text-content-XZ3HK4OT.mjs.map +7 -0
- package/dist/lib/neutral/{transcriber-OGSF3XO5.mjs → transcriber-KYV6NWI5.mjs} +3 -3
- package/dist/lib/neutral/translations.mjs +1 -1
- package/dist/lib/neutral/translations.mjs.map +3 -3
- package/dist/lib/neutral/types/index.mjs +3 -1
- package/dist/types/dx.config.d.ts +28 -0
- package/dist/types/dx.config.d.ts.map +1 -0
- package/dist/types/src/TranscriptionPlugin.d.ts.map +1 -1
- package/dist/types/src/capabilities/index.d.ts +5 -8
- package/dist/types/src/capabilities/index.d.ts.map +1 -1
- package/dist/types/src/capabilities/react-surface.d.ts +2 -2
- package/dist/types/src/capabilities/react-surface.d.ts.map +1 -1
- package/dist/types/src/capabilities/settings.d.ts +7 -0
- package/dist/types/src/capabilities/settings.d.ts.map +1 -0
- package/dist/types/src/hooks/useTranscriptionRecording.d.ts.map +1 -1
- package/dist/types/src/meta.d.ts +28 -2
- package/dist/types/src/meta.d.ts.map +1 -1
- package/dist/types/src/operations/enrich-message.d.ts +9 -0
- package/dist/types/src/operations/enrich-message.d.ts.map +1 -0
- package/dist/types/src/operations/index.d.ts.map +1 -1
- package/dist/types/src/stories/LiveTranscription.stories.d.ts +1 -2
- package/dist/types/src/stories/LiveTranscription.stories.d.ts.map +1 -1
- package/dist/types/src/stories/ProperNounExtraction.stories.d.ts +13 -0
- package/dist/types/src/stories/ProperNounExtraction.stories.d.ts.map +1 -0
- package/dist/types/src/stories/common.d.ts.map +1 -1
- package/dist/types/src/testing/testing.d.ts +0 -4
- package/dist/types/src/testing/testing.d.ts.map +1 -1
- package/dist/types/src/types/Settings.d.ts +7 -0
- package/dist/types/src/types/Settings.d.ts.map +1 -0
- package/dist/types/src/types/TranscriptOperation.d.ts +407 -0
- package/dist/types/src/types/TranscriptOperation.d.ts.map +1 -1
- package/dist/types/src/types/TranscriptionCapabilities.d.ts +3 -0
- package/dist/types/src/types/TranscriptionCapabilities.d.ts.map +1 -1
- package/dist/types/src/types/index.d.ts +1 -0
- package/dist/types/src/types/index.d.ts.map +1 -1
- package/dist/types/src/util/index.d.ts +0 -1
- package/dist/types/src/util/index.d.ts.map +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/dx.config.ts +42 -0
- package/package.json +43 -42
- package/src/TranscriptionPlugin.test.ts +1 -1
- package/src/TranscriptionPlugin.tsx +10 -2
- package/src/capabilities/index.ts +2 -7
- package/src/capabilities/settings.ts +33 -0
- package/src/capabilities/text-content.ts +9 -5
- package/src/containers/TranscriptionArticle/TranscriptionArticle.tsx +1 -1
- package/src/enrichment/operations.ts +1 -1
- package/src/hooks/useTranscriptionRecording.ts +26 -4
- package/src/meta.ts +2 -35
- package/src/operations/enrich-message.ts +27 -0
- package/src/operations/index.ts +1 -0
- package/src/operations/summarize.ts +1 -1
- package/src/stories/LiveTranscription.stories.tsx +13 -67
- package/src/stories/ProperNounExtraction.stories.tsx +99 -0
- package/src/stories/common.ts +2 -1
- package/src/testing/testing.ts +4 -89
- package/src/transcriber/transcription-manager.ts +1 -1
- package/src/translations.ts +1 -1
- package/src/types/Settings.ts +21 -0
- package/src/types/TranscriptOperation.ts +17 -1
- package/src/types/TranscriptionCapabilities.ts +7 -2
- package/src/types/index.ts +1 -0
- package/src/util/index.ts +0 -1
- package/dist/lib/neutral/chunk-FHNI26QW.mjs +0 -45
- package/dist/lib/neutral/chunk-FHNI26QW.mjs.map +0 -7
- package/dist/lib/neutral/chunk-KMT3BMMQ.mjs.map +0 -7
- package/dist/lib/neutral/chunk-KXJZEYS7.mjs +0 -8
- package/dist/lib/neutral/text-content-KBB37HSM.mjs.map +0 -7
- package/dist/types/src/util/monitor-audio-level.d.ts +0 -6
- package/dist/types/src/util/monitor-audio-level.d.ts.map +0 -1
- package/src/util/monitor-audio-level.ts +0 -58
- /package/dist/lib/neutral/{chunk-KAEXRX3Y.mjs.map → chunk-XOXHWMXR.mjs.map} +0 -0
- /package/dist/lib/neutral/{create-RVR7VC33.mjs.map → create-7NGOP7JP.mjs.map} +0 -0
- /package/dist/lib/neutral/{open-A5KXKRH4.mjs.map → open-42H72C4G.mjs.map} +0 -0
- /package/dist/lib/neutral/{transcriber-OGSF3XO5.mjs.map → transcriber-KYV6NWI5.mjs.map} +0 -0
package/dx.config.ts
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2025 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import { Config2 } from '@dxos/app-framework/config';
|
|
6
|
+
import { trim } from '@dxos/util';
|
|
7
|
+
|
|
8
|
+
export default Config2.make({
|
|
9
|
+
plugin: {
|
|
10
|
+
key: 'org.dxos.plugin.transcription',
|
|
11
|
+
name: 'Transcription',
|
|
12
|
+
author: 'DXOS',
|
|
13
|
+
description: trim`
|
|
14
|
+
Real-time voice-to-text transcription for DXOS Composer.
|
|
15
|
+
Captures microphone audio in short PCM chunks, batches them into WAV payloads, and streams
|
|
16
|
+
the results to a Whisper-compatible speech-recognition endpoint.
|
|
17
|
+
Transcribed segments are timestamped, deduplicated across chunk boundaries, and persisted
|
|
18
|
+
as Message objects in an ECHO Feed so that every participant in a shared space sees the
|
|
19
|
+
live transcript without any manual sync step.
|
|
20
|
+
|
|
21
|
+
Each recording session is represented as a Transcript object linked to its Feed.
|
|
22
|
+
The read-only editor renders messages with speaker bylines and wall-clock timestamps,
|
|
23
|
+
supporting markdown decoration and document preview.
|
|
24
|
+
An optional message enricher pipeline lets plugins annotate messages before they are
|
|
25
|
+
committed — for example, to attach speaker identity from a calls service.
|
|
26
|
+
|
|
27
|
+
The plugin ships two blueprint-compatible operations: Open (loads and formats a transcript
|
|
28
|
+
for reading by an AI assistant) and Summarize (sends the transcript text to an LLM and
|
|
29
|
+
returns a structured Markdown summary with key points, verbatim quotes, and actionable
|
|
30
|
+
tasks extracted from the conversation).
|
|
31
|
+
|
|
32
|
+
A sentence-normalization pass merges raw word-level tokens from the recognizer into
|
|
33
|
+
grammatically complete sentences, improving readability of the stored transcript without
|
|
34
|
+
altering the underlying audio timeline.
|
|
35
|
+
`,
|
|
36
|
+
source: 'https://github.com/dxos/dxos/tree/main/packages/plugins/plugin-transcription',
|
|
37
|
+
icon: { key: 'ph--microphone--regular', hue: 'sky' },
|
|
38
|
+
spec: 'PLUGIN.mdl',
|
|
39
|
+
tags: ['labs'],
|
|
40
|
+
screenshots: [],
|
|
41
|
+
},
|
|
42
|
+
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dxos/plugin-transcription",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.1-main.c7dcc2e112",
|
|
4
4
|
"description": "Voice transcription.",
|
|
5
5
|
"homepage": "https://dxos.org",
|
|
6
6
|
"bugs": "https://github.com/dxos/dxos/issues",
|
|
@@ -111,6 +111,7 @@
|
|
|
111
111
|
"types": "dist/types/src/index.d.ts",
|
|
112
112
|
"files": [
|
|
113
113
|
"dist",
|
|
114
|
+
"dx.config.ts",
|
|
114
115
|
"src",
|
|
115
116
|
"PLUGIN.mdl"
|
|
116
117
|
],
|
|
@@ -123,35 +124,35 @@
|
|
|
123
124
|
"extendable-media-recorder": "^9.2.19",
|
|
124
125
|
"extendable-media-recorder-wav-encoder": "^7.0.123",
|
|
125
126
|
"wavefile": "^11.0.0",
|
|
126
|
-
"@dxos/ai": "0.9.
|
|
127
|
-
"@dxos/app-
|
|
128
|
-
"@dxos/
|
|
129
|
-
"@dxos/
|
|
130
|
-
"@dxos/
|
|
131
|
-
"@dxos/
|
|
132
|
-
"@dxos/
|
|
133
|
-
"@dxos/
|
|
134
|
-
"@dxos/
|
|
135
|
-
"@dxos/
|
|
136
|
-
"@dxos/
|
|
137
|
-
"@dxos/
|
|
138
|
-
"@dxos/
|
|
139
|
-
"@dxos/
|
|
140
|
-
"@dxos/
|
|
141
|
-
"@dxos/
|
|
142
|
-
"@dxos/react-client": "0.9.
|
|
143
|
-
"@dxos/react-ui-audio": "0.9.
|
|
144
|
-
"@dxos/react-ui-components": "0.9.
|
|
145
|
-
"@dxos/react-ui-graph": "0.9.
|
|
146
|
-
"@dxos/
|
|
147
|
-
"@dxos/
|
|
148
|
-
"@dxos/react-ui-
|
|
149
|
-
"@dxos/
|
|
150
|
-
"@dxos/types": "0.9.
|
|
151
|
-
"@dxos/
|
|
152
|
-
"@dxos/ui-types": "0.9.
|
|
153
|
-
"@dxos/
|
|
154
|
-
"@dxos/
|
|
127
|
+
"@dxos/ai": "0.9.1-main.c7dcc2e112",
|
|
128
|
+
"@dxos/app-framework": "0.9.1-main.c7dcc2e112",
|
|
129
|
+
"@dxos/assistant": "0.9.1-main.c7dcc2e112",
|
|
130
|
+
"@dxos/app-toolkit": "0.9.1-main.c7dcc2e112",
|
|
131
|
+
"@dxos/async": "0.9.1-main.c7dcc2e112",
|
|
132
|
+
"@dxos/av": "0.9.1-main.c7dcc2e112",
|
|
133
|
+
"@dxos/client": "0.9.1-main.c7dcc2e112",
|
|
134
|
+
"@dxos/context": "0.9.1-main.c7dcc2e112",
|
|
135
|
+
"@dxos/compute": "0.9.1-main.c7dcc2e112",
|
|
136
|
+
"@dxos/echo": "0.9.1-main.c7dcc2e112",
|
|
137
|
+
"@dxos/functions-runtime": "0.9.1-main.c7dcc2e112",
|
|
138
|
+
"@dxos/invariant": "0.9.1-main.c7dcc2e112",
|
|
139
|
+
"@dxos/keys": "0.9.1-main.c7dcc2e112",
|
|
140
|
+
"@dxos/log": "0.9.1-main.c7dcc2e112",
|
|
141
|
+
"@dxos/react-client": "0.9.1-main.c7dcc2e112",
|
|
142
|
+
"@dxos/plugin-client": "0.9.1-main.c7dcc2e112",
|
|
143
|
+
"@dxos/react-edge-client": "0.9.1-main.c7dcc2e112",
|
|
144
|
+
"@dxos/react-ui-audio": "0.9.1-main.c7dcc2e112",
|
|
145
|
+
"@dxos/react-ui-components": "0.9.1-main.c7dcc2e112",
|
|
146
|
+
"@dxos/react-ui-graph": "0.9.1-main.c7dcc2e112",
|
|
147
|
+
"@dxos/tracing": "0.9.1-main.c7dcc2e112",
|
|
148
|
+
"@dxos/react-ui-menu": "0.9.1-main.c7dcc2e112",
|
|
149
|
+
"@dxos/react-ui-editor": "0.9.1-main.c7dcc2e112",
|
|
150
|
+
"@dxos/schema": "0.9.1-main.c7dcc2e112",
|
|
151
|
+
"@dxos/types": "0.9.1-main.c7dcc2e112",
|
|
152
|
+
"@dxos/ui-editor": "0.9.1-main.c7dcc2e112",
|
|
153
|
+
"@dxos/ui-types": "0.9.1-main.c7dcc2e112",
|
|
154
|
+
"@dxos/util": "0.9.1-main.c7dcc2e112",
|
|
155
|
+
"@dxos/effect": "0.9.1-main.c7dcc2e112"
|
|
155
156
|
},
|
|
156
157
|
"devDependencies": {
|
|
157
158
|
"@effect-atom/atom-react": "^0.5.0",
|
|
@@ -162,16 +163,16 @@
|
|
|
162
163
|
"react": "~19.2.3",
|
|
163
164
|
"react-dom": "~19.2.3",
|
|
164
165
|
"tailwindcss": "^4.3.0",
|
|
165
|
-
"@dxos/plugin-preview": "0.9.
|
|
166
|
-
"@dxos/plugin-testing": "0.9.
|
|
167
|
-
"@dxos/random": "0.9.
|
|
168
|
-
"@dxos/plugin-theme": "0.9.
|
|
169
|
-
"@dxos/react-ui": "0.9.
|
|
170
|
-
"@dxos/react-ui-form": "0.9.
|
|
171
|
-
"@dxos/
|
|
172
|
-
"@dxos/
|
|
173
|
-
"@dxos/
|
|
174
|
-
"@dxos/
|
|
166
|
+
"@dxos/plugin-preview": "0.9.1-main.c7dcc2e112",
|
|
167
|
+
"@dxos/plugin-testing": "0.9.1-main.c7dcc2e112",
|
|
168
|
+
"@dxos/random": "0.9.1-main.c7dcc2e112",
|
|
169
|
+
"@dxos/plugin-theme": "0.9.1-main.c7dcc2e112",
|
|
170
|
+
"@dxos/react-ui": "0.9.1-main.c7dcc2e112",
|
|
171
|
+
"@dxos/react-ui-form": "0.9.1-main.c7dcc2e112",
|
|
172
|
+
"@dxos/storybook-utils": "0.9.1-main.c7dcc2e112",
|
|
173
|
+
"@dxos/react-ui-syntax-highlighter": "0.9.1-main.c7dcc2e112",
|
|
174
|
+
"@dxos/test-utils": "0.9.1-main.c7dcc2e112",
|
|
175
|
+
"@dxos/ui-theme": "0.9.1-main.c7dcc2e112"
|
|
175
176
|
},
|
|
176
177
|
"peerDependencies": {
|
|
177
178
|
"@effect-atom/atom-react": "^0.5.0",
|
|
@@ -179,8 +180,8 @@
|
|
|
179
180
|
"effect": "3.21.3",
|
|
180
181
|
"react": "~19.2.3",
|
|
181
182
|
"react-dom": "~19.2.3",
|
|
182
|
-
"@dxos/ui
|
|
183
|
-
"@dxos/
|
|
183
|
+
"@dxos/react-ui": "0.9.1-main.c7dcc2e112",
|
|
184
|
+
"@dxos/ui-theme": "0.9.1-main.c7dcc2e112"
|
|
184
185
|
},
|
|
185
186
|
"publishConfig": {
|
|
186
187
|
"access": "public"
|
|
@@ -11,7 +11,7 @@ import { createComposerTestApp } from '@dxos/plugin-testing/harness';
|
|
|
11
11
|
import { meta } from '#meta';
|
|
12
12
|
import { TranscriptionPlugin } from '#plugin';
|
|
13
13
|
|
|
14
|
-
const moduleId = (name: string) => `${meta.
|
|
14
|
+
const moduleId = (name: string) => `${meta.profile.key}.module.${name}`;
|
|
15
15
|
|
|
16
16
|
describe('TranscriptionPlugin', () => {
|
|
17
17
|
test('modules activate on the expected events', async ({ expect }) => {
|
|
@@ -6,7 +6,14 @@ import { Plugin } from '@dxos/app-framework';
|
|
|
6
6
|
import { AppActivationEvents, AppPlugin } from '@dxos/app-toolkit';
|
|
7
7
|
import { Transcript } from '@dxos/types';
|
|
8
8
|
|
|
9
|
-
import {
|
|
9
|
+
import {
|
|
10
|
+
BlueprintDefinition,
|
|
11
|
+
OperationHandler,
|
|
12
|
+
ReactSurface,
|
|
13
|
+
TextContent,
|
|
14
|
+
Transcriber,
|
|
15
|
+
TranscriptionSettings,
|
|
16
|
+
} from '#capabilities';
|
|
10
17
|
import { meta } from '#meta';
|
|
11
18
|
import { translations } from '#translations';
|
|
12
19
|
|
|
@@ -20,13 +27,14 @@ export const TranscriptionPlugin = Plugin.define(meta).pipe(
|
|
|
20
27
|
AppPlugin.addSchemaModule({ schema: [Transcript.Transcript] }),
|
|
21
28
|
AppPlugin.addSurfaceModule({ activate: ReactSurface }),
|
|
22
29
|
AppPlugin.addTranslationsModule({ translations }),
|
|
30
|
+
AppPlugin.addSettingsModule({ activate: TranscriptionSettings }),
|
|
23
31
|
Plugin.addModule({
|
|
24
32
|
id: 'transcription',
|
|
25
33
|
activatesOn: AppActivationEvents.SetupAppGraph,
|
|
26
34
|
activate: Transcriber,
|
|
27
35
|
}),
|
|
28
36
|
AppPlugin.addPluginAssetModule({
|
|
29
|
-
asset: { pluginId: meta.
|
|
37
|
+
asset: { pluginId: meta.profile.key, path: 'PLUGIN.mdl', content: pluginSpec, mimeType: 'application/x-mdl' },
|
|
30
38
|
}),
|
|
31
39
|
Plugin.make,
|
|
32
40
|
);
|
|
@@ -3,15 +3,9 @@
|
|
|
3
3
|
//
|
|
4
4
|
|
|
5
5
|
import { Capability } from '@dxos/app-framework';
|
|
6
|
-
import { type AppCapabilities } from '@dxos/app-toolkit';
|
|
7
6
|
import type { OperationHandlerSet } from '@dxos/compute';
|
|
8
7
|
|
|
9
|
-
|
|
10
|
-
// wrapper needs an explicit annotation to keep the inferred type portable (TS2883).
|
|
11
|
-
export const BlueprintDefinition: Capability.LazyCapability<
|
|
12
|
-
void,
|
|
13
|
-
Capability.Capability<typeof AppCapabilities.BlueprintDefinition>[]
|
|
14
|
-
> = Capability.lazy('BlueprintDefinition', () => import('./blueprint-definition'));
|
|
8
|
+
export const BlueprintDefinition = Capability.lazy('BlueprintDefinition', () => import('./blueprint-definition'));
|
|
15
9
|
export const TextContent = Capability.lazy('TextContent', () => import('./text-content'));
|
|
16
10
|
export const OperationHandler = Capability.lazy<OperationHandlerSet.OperationHandlerSet>(
|
|
17
11
|
'OperationHandler',
|
|
@@ -19,3 +13,4 @@ export const OperationHandler = Capability.lazy<OperationHandlerSet.OperationHan
|
|
|
19
13
|
);
|
|
20
14
|
export const ReactSurface = Capability.lazy('ReactSurface', () => import('./react-surface'));
|
|
21
15
|
export const Transcriber = Capability.lazy('Transcriber', () => import('./transcriber'));
|
|
16
|
+
export const TranscriptionSettings = Capability.lazy('TranscriptionSettings', () => import('./settings'));
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import * as Effect from 'effect/Effect';
|
|
6
|
+
|
|
7
|
+
import { Capability } from '@dxos/app-framework';
|
|
8
|
+
import { AppCapabilities } from '@dxos/app-toolkit';
|
|
9
|
+
import { createKvsStore } from '@dxos/effect';
|
|
10
|
+
|
|
11
|
+
import { meta } from '#meta';
|
|
12
|
+
import { Settings, TranscriptionCapabilities } from '#types';
|
|
13
|
+
|
|
14
|
+
export default Capability.makeModule(() =>
|
|
15
|
+
Effect.sync(() => {
|
|
16
|
+
const settingsAtom = createKvsStore({
|
|
17
|
+
key: meta.profile.key,
|
|
18
|
+
schema: Settings.Settings,
|
|
19
|
+
defaultValue: () => ({
|
|
20
|
+
entityExtraction: true,
|
|
21
|
+
}),
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
return [
|
|
25
|
+
Capability.contributes(TranscriptionCapabilities.Settings, settingsAtom),
|
|
26
|
+
Capability.contributes(AppCapabilities.Settings, {
|
|
27
|
+
prefix: meta.profile.key,
|
|
28
|
+
schema: Settings.Settings,
|
|
29
|
+
atom: settingsAtom,
|
|
30
|
+
}),
|
|
31
|
+
];
|
|
32
|
+
}),
|
|
33
|
+
);
|
|
@@ -6,7 +6,7 @@ import * as Effect from 'effect/Effect';
|
|
|
6
6
|
|
|
7
7
|
import { Capability } from '@dxos/app-framework';
|
|
8
8
|
import { AppCapabilities } from '@dxos/app-toolkit';
|
|
9
|
-
import { Feed, Obj, Type } from '@dxos/echo';
|
|
9
|
+
import { Feed, Filter, Obj, Query, Scope, Type } from '@dxos/echo';
|
|
10
10
|
import { getSpace } from '@dxos/react-client/echo';
|
|
11
11
|
import { Message, Transcript } from '@dxos/types';
|
|
12
12
|
|
|
@@ -20,10 +20,14 @@ export default Capability.makeModule(
|
|
|
20
20
|
const space = getSpace(transcript);
|
|
21
21
|
const members = space?.members.get().map((member) => member.identity) ?? [];
|
|
22
22
|
const feed = await transcript.feed.load();
|
|
23
|
-
const feedDXN = Feed.getQueueUri(feed);
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
23
|
+
const feedDXN = feed ? Feed.getQueueUri(feed) : undefined;
|
|
24
|
+
if (!space || !feedDXN) {
|
|
25
|
+
return undefined;
|
|
26
|
+
}
|
|
27
|
+
const messages = await space.db
|
|
28
|
+
.query(Query.select(Filter.type(Message.Message)).from(Scope.feed(feedDXN)))
|
|
29
|
+
.run();
|
|
30
|
+
return messages
|
|
27
31
|
.filter((message) => Obj.instanceOf(Message.Message, message))
|
|
28
32
|
.flatMap((message, index) => renderByline(members)(message, index))
|
|
29
33
|
.join('\n\n');
|
|
@@ -36,7 +36,7 @@ export const TranscriptionArticle = ({ role, subject: transcript, attendableId }
|
|
|
36
36
|
.action(
|
|
37
37
|
'toggle-recording',
|
|
38
38
|
{
|
|
39
|
-
label: [recording ? 'stop-recording.label' : 'start-recording.label', { ns: meta.
|
|
39
|
+
label: [recording ? 'stop-recording.label' : 'start-recording.label', { ns: meta.profile.key }],
|
|
40
40
|
icon: recording ? 'ph--stop-circle--regular' : 'ph--microphone--regular',
|
|
41
41
|
disposition: 'toolbar',
|
|
42
42
|
testId: 'transcription.toggle-recording',
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
import * as Effect from 'effect/Effect';
|
|
6
6
|
import { useCallback, useEffect, useState } from 'react';
|
|
7
7
|
|
|
8
|
+
import { useAtomCapability, useOperationInvoker } from '@dxos/app-framework/ui';
|
|
8
9
|
import { createFeedServiceLayer } from '@dxos/client/echo';
|
|
9
10
|
import { Feed, Obj } from '@dxos/echo';
|
|
10
11
|
import { EffectEx } from '@dxos/effect';
|
|
@@ -13,6 +14,8 @@ import { getSpace } from '@dxos/react-client/echo';
|
|
|
13
14
|
import { useIdentity } from '@dxos/react-client/halo';
|
|
14
15
|
import { Message, type Transcript } from '@dxos/types';
|
|
15
16
|
|
|
17
|
+
import { TranscriptOperation, TranscriptionCapabilities } from '#types';
|
|
18
|
+
|
|
16
19
|
import { useAudioTrack } from './useAudioTrack';
|
|
17
20
|
import { useTranscriber } from './useTranscriber';
|
|
18
21
|
|
|
@@ -33,23 +36,42 @@ export const useTranscriptionRecording = (transcript: Transcript.Transcript): Tr
|
|
|
33
36
|
|
|
34
37
|
const [recording, setRecording] = useState(false);
|
|
35
38
|
const track = useAudioTrack(recording);
|
|
39
|
+
const { invokePromise } = useOperationInvoker();
|
|
40
|
+
const settings = useAtomCapability(TranscriptionCapabilities.Settings);
|
|
36
41
|
|
|
37
|
-
// Append a transcribed message batch to the transcript's feed
|
|
42
|
+
// Append a transcribed message batch to the transcript's feed, enriching it first (when entity
|
|
43
|
+
// extraction is enabled) with references to known entities mentioned in the transcript.
|
|
38
44
|
const handleSegments = useCallback(
|
|
39
45
|
async (blocks: Message.Message['blocks']) => {
|
|
40
46
|
if (!space || !feed || blocks.length === 0) {
|
|
41
47
|
return;
|
|
42
48
|
}
|
|
43
49
|
const sender = identity ? { identityDid: identity.did, name: identity.profile?.displayName } : {};
|
|
44
|
-
|
|
50
|
+
let message = Obj.make(Message.Message, {
|
|
45
51
|
sender,
|
|
46
52
|
created: new Date().toISOString(),
|
|
47
53
|
blocks,
|
|
48
54
|
});
|
|
49
|
-
|
|
55
|
+
|
|
56
|
+
// `spaceId` provides the space context required by the process-affinity AiService/Database.
|
|
57
|
+
// Fall back to the raw message if extraction fails (e.g. AI service unavailable).
|
|
58
|
+
if (settings?.entityExtraction ?? true) {
|
|
59
|
+
const { data, error } = await invokePromise(
|
|
60
|
+
TranscriptOperation.EnrichMessage,
|
|
61
|
+
{ message },
|
|
62
|
+
{ spaceId: space.id },
|
|
63
|
+
);
|
|
64
|
+
if (error) {
|
|
65
|
+
log.warn('entity extraction failed; using raw transcript', { error });
|
|
66
|
+
} else if (data?.message) {
|
|
67
|
+
message = data.message;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const feedServiceLayer = createFeedServiceLayer(space.db);
|
|
50
72
|
await Feed.append(feed, [message]).pipe(Effect.provide(feedServiceLayer), EffectEx.runAndForwardErrors);
|
|
51
73
|
},
|
|
52
|
-
[space, feed, identity],
|
|
74
|
+
[space, feed, identity, invokePromise, settings],
|
|
53
75
|
);
|
|
54
76
|
|
|
55
77
|
// Drive the transcriber lifecycle off the recording flag + audio track.
|
package/src/meta.ts
CHANGED
|
@@ -3,40 +3,7 @@
|
|
|
3
3
|
//
|
|
4
4
|
|
|
5
5
|
import { Plugin } from '@dxos/app-framework';
|
|
6
|
-
import { DXN } from '@dxos/keys';
|
|
7
|
-
import { trim } from '@dxos/util';
|
|
8
6
|
|
|
9
|
-
|
|
10
|
-
key: DXN.make('org.dxos.plugin.transcription'),
|
|
11
|
-
name: 'Transcription',
|
|
12
|
-
author: 'DXOS',
|
|
13
|
-
description: trim`
|
|
14
|
-
Real-time voice-to-text transcription for DXOS Composer.
|
|
15
|
-
Captures microphone audio in short PCM chunks, batches them into WAV payloads, and streams
|
|
16
|
-
the results to a Whisper-compatible speech-recognition endpoint.
|
|
17
|
-
Transcribed segments are timestamped, deduplicated across chunk boundaries, and persisted
|
|
18
|
-
as Message objects in an ECHO Feed so that every participant in a shared space sees the
|
|
19
|
-
live transcript without any manual sync step.
|
|
7
|
+
import config from '../dx.config';
|
|
20
8
|
|
|
21
|
-
|
|
22
|
-
The read-only editor renders messages with speaker bylines and wall-clock timestamps,
|
|
23
|
-
supporting markdown decoration and document preview.
|
|
24
|
-
An optional message enricher pipeline lets plugins annotate messages before they are
|
|
25
|
-
committed — for example, to attach speaker identity from a calls service.
|
|
26
|
-
|
|
27
|
-
The plugin ships two blueprint-compatible operations: Open (loads and formats a transcript
|
|
28
|
-
for reading by an AI assistant) and Summarize (sends the transcript text to an LLM and
|
|
29
|
-
returns a structured Markdown summary with key points, verbatim quotes, and actionable
|
|
30
|
-
tasks extracted from the conversation).
|
|
31
|
-
|
|
32
|
-
A sentence-normalization pass merges raw word-level tokens from the recognizer into
|
|
33
|
-
grammatically complete sentences, improving readability of the stored transcript without
|
|
34
|
-
altering the underlying audio timeline.
|
|
35
|
-
`,
|
|
36
|
-
icon: 'ph--microphone--regular',
|
|
37
|
-
iconHue: 'sky',
|
|
38
|
-
source: 'https://github.com/dxos/dxos/tree/main/packages/plugins/plugin-transcription',
|
|
39
|
-
spec: 'PLUGIN.mdl',
|
|
40
|
-
tags: ['labs'],
|
|
41
|
-
screenshots: [],
|
|
42
|
-
});
|
|
9
|
+
export const meta = Plugin.getMetaFromConfig(config);
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import * as Effect from 'effect/Effect';
|
|
6
|
+
|
|
7
|
+
import { enrichTranscriptMessage } from '@dxos/assistant/extraction';
|
|
8
|
+
import { Operation } from '@dxos/compute';
|
|
9
|
+
import { Database } from '@dxos/echo';
|
|
10
|
+
|
|
11
|
+
import { TranscriptOperation } from '../types';
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Extracts proper nouns from a transcript message and links them to objects in the space via
|
|
15
|
+
* full-text search, rewriting the text with reference links.
|
|
16
|
+
*/
|
|
17
|
+
const handler: Operation.WithHandler<typeof TranscriptOperation.EnrichMessage> = TranscriptOperation.EnrichMessage.pipe(
|
|
18
|
+
Operation.withHandler(
|
|
19
|
+
Effect.fnUntraced(function* ({ message }) {
|
|
20
|
+
const { db } = yield* Database.Service;
|
|
21
|
+
const enriched = yield* enrichTranscriptMessage(message, { db });
|
|
22
|
+
return { message: enriched };
|
|
23
|
+
}),
|
|
24
|
+
),
|
|
25
|
+
);
|
|
26
|
+
|
|
27
|
+
export default handler;
|
package/src/operations/index.ts
CHANGED
|
@@ -49,7 +49,7 @@ const handler: Operation.WithHandler<typeof TranscriptOperation.Summarize> = Tra
|
|
|
49
49
|
},
|
|
50
50
|
Effect.provide(
|
|
51
51
|
Layer.mergeAll(
|
|
52
|
-
AiService.model('ai.claude.model.claude-sonnet-4-
|
|
52
|
+
AiService.model('ai.claude.model.claude-sonnet-4-6'),
|
|
53
53
|
ToolResolverService.layerEmpty,
|
|
54
54
|
ToolExecutionService.layerEmpty,
|
|
55
55
|
Trace.writerLayerNoop,
|
|
@@ -3,19 +3,9 @@
|
|
|
3
3
|
//
|
|
4
4
|
|
|
5
5
|
import { type Meta, type StoryObj } from '@storybook/react-vite';
|
|
6
|
-
import React, { useCallback,
|
|
7
|
-
|
|
8
|
-
import {
|
|
9
|
-
type ExtractionFunction,
|
|
10
|
-
extractionAnthropicFunction,
|
|
11
|
-
extractionNerFunction,
|
|
12
|
-
getNer,
|
|
13
|
-
processTranscriptMessage,
|
|
14
|
-
} from '@dxos/assistant/extraction';
|
|
15
|
-
import { Filter, type Obj } from '@dxos/echo';
|
|
16
|
-
import { invariant } from '@dxos/invariant';
|
|
17
|
-
import { useSpaces } from '@dxos/react-client/echo';
|
|
18
|
-
import { Message, Organization, Person } from '@dxos/types';
|
|
6
|
+
import React, { useCallback, useState } from 'react';
|
|
7
|
+
|
|
8
|
+
import { Message } from '@dxos/types';
|
|
19
9
|
|
|
20
10
|
import { useAudioTrack } from '#hooks';
|
|
21
11
|
|
|
@@ -37,12 +27,10 @@ type DefaultStoryProps = {
|
|
|
37
27
|
transcriberConfig?: TranscriberProps['config'];
|
|
38
28
|
recorderConfig?: MediaStreamRecorderProps['config'];
|
|
39
29
|
audioConstraints?: MediaTrackConstraints;
|
|
40
|
-
entityExtraction?: 'ner' | 'llm';
|
|
41
30
|
};
|
|
42
31
|
|
|
43
32
|
const DefaultStory = ({
|
|
44
33
|
detectSpeaking,
|
|
45
|
-
entityExtraction,
|
|
46
34
|
transcriberConfig = DEFAULT_TRANSCRIBER_CONFIG,
|
|
47
35
|
recorderConfig = DEFAULT_RECORDER_CONFIG,
|
|
48
36
|
audioConstraints,
|
|
@@ -51,48 +39,19 @@ const DefaultStory = ({
|
|
|
51
39
|
const track = useAudioTrack(running, audioConstraints);
|
|
52
40
|
const isSpeaking = detectSpeaking ? useIsSpeaking(track) : true;
|
|
53
41
|
const { model, appendMessage } = useStoryMessageModel();
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
// Resolve the entity-extraction function + nearby objects once per `entityExtraction` toggle.
|
|
57
|
-
const { extractionFunction, objects } = useMemo(() => {
|
|
58
|
-
if (!space || !entityExtraction) {
|
|
59
|
-
return {};
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
let extractionFunction: ExtractionFunction | undefined;
|
|
63
|
-
let objects: Promise<Obj.Unknown[]> | undefined;
|
|
64
|
-
if (entityExtraction === 'ner') {
|
|
65
|
-
void getNer(); // Init model loading. Takes time.
|
|
66
|
-
extractionFunction = extractionNerFunction;
|
|
67
|
-
} else if (entityExtraction === 'llm') {
|
|
68
|
-
extractionFunction = extractionAnthropicFunction;
|
|
69
|
-
objects = space.db.query(Filter.or(Filter.type(Person.Person), Filter.type(Organization.Organization))).run();
|
|
70
|
-
}
|
|
71
|
-
return { extractionFunction, objects };
|
|
72
|
-
}, [entityExtraction, space]);
|
|
73
|
-
|
|
74
|
-
// Build a Message from each transcribed batch; optionally enrich via entity extraction.
|
|
42
|
+
|
|
43
|
+
// Build a Message from each transcribed batch and append it to the local buffer.
|
|
75
44
|
const handleSegments = useCallback<TranscriberProps['onSegments']>(
|
|
76
45
|
async (blocks) => {
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
invariant(extractionFunction, 'extractionFunction is required');
|
|
85
|
-
const result = await processTranscriptMessage({
|
|
86
|
-
function: extractionFunction,
|
|
87
|
-
input: { message, objects: await objects },
|
|
88
|
-
options: { fallbackToRaw: true, timeout: 30_000 },
|
|
89
|
-
});
|
|
90
|
-
appendMessage(result.message);
|
|
91
|
-
} else {
|
|
92
|
-
appendMessage(message);
|
|
93
|
-
}
|
|
46
|
+
appendMessage(
|
|
47
|
+
Message.make({
|
|
48
|
+
sender: { name: 'You' },
|
|
49
|
+
created: new Date().toISOString(),
|
|
50
|
+
blocks,
|
|
51
|
+
}),
|
|
52
|
+
);
|
|
94
53
|
},
|
|
95
|
-
[appendMessage
|
|
54
|
+
[appendMessage],
|
|
96
55
|
);
|
|
97
56
|
|
|
98
57
|
useStoryTranscriber({
|
|
@@ -138,16 +97,3 @@ export const SpeechDetection: Story = {
|
|
|
138
97
|
},
|
|
139
98
|
},
|
|
140
99
|
};
|
|
141
|
-
|
|
142
|
-
// TODO(mykola): Fix hugging face quota issues.
|
|
143
|
-
// export const EntityExtraction: Story = {
|
|
144
|
-
// args: {
|
|
145
|
-
// detectSpeaking: true,
|
|
146
|
-
// entityExtraction: 'ner',
|
|
147
|
-
// audioConstraints: {
|
|
148
|
-
// echoCancellation: true,
|
|
149
|
-
// noiseSuppression: true,
|
|
150
|
-
// autoGainControl: true,
|
|
151
|
-
// },
|
|
152
|
-
// },
|
|
153
|
-
// };
|