@dxos/plugin-transcription 0.9.0 → 0.9.1-main.c7dcc2e112
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PLUGIN.mdl +20 -0
- package/dist/lib/neutral/{TranscriptionArticle-L6XANBH5.mjs → TranscriptionArticle-S56VVOSF.mjs} +3 -3
- package/dist/lib/neutral/{TranscriptionArticle-L6XANBH5.mjs.map → TranscriptionArticle-S56VVOSF.mjs.map} +3 -3
- package/dist/lib/neutral/TranscriptionPlugin.mjs +6 -4
- package/dist/lib/neutral/TranscriptionPlugin.mjs.map +4 -4
- package/dist/lib/neutral/capabilities/index.mjs +5 -3
- package/dist/lib/neutral/capabilities/index.mjs.map +3 -3
- package/dist/lib/neutral/{chunk-KMT3BMMQ.mjs → chunk-2URVUJT4.mjs} +57 -22
- package/dist/lib/neutral/chunk-2URVUJT4.mjs.map +7 -0
- package/dist/lib/neutral/chunk-J4ONEYPJ.mjs +8 -0
- package/dist/lib/neutral/{chunk-KXJZEYS7.mjs.map → chunk-J4ONEYPJ.mjs.map} +2 -2
- package/dist/lib/neutral/{chunk-TO77KAQK.mjs → chunk-XMHH34NI.mjs} +25 -5
- package/dist/lib/neutral/{chunk-TO77KAQK.mjs.map → chunk-XMHH34NI.mjs.map} +3 -3
- package/dist/lib/neutral/{chunk-KAEXRX3Y.mjs → chunk-XOXHWMXR.mjs} +1 -1
- package/dist/lib/neutral/{chunk-TX3NQISZ.mjs → chunk-XWC5BBOY.mjs} +3 -3
- package/dist/lib/neutral/{chunk-TX3NQISZ.mjs.map → chunk-XWC5BBOY.mjs.map} +3 -3
- package/dist/lib/neutral/chunk-XZVDAYM6.mjs +54 -0
- package/dist/lib/neutral/chunk-XZVDAYM6.mjs.map +7 -0
- package/dist/lib/neutral/containers/index.mjs +1 -1
- package/dist/lib/neutral/{create-RVR7VC33.mjs → create-7NGOP7JP.mjs} +2 -2
- package/dist/lib/neutral/enrich-message-F2AOM53F.mjs +24 -0
- package/dist/lib/neutral/enrich-message-F2AOM53F.mjs.map +7 -0
- package/dist/lib/neutral/hooks/index.mjs +1 -1
- package/dist/lib/neutral/index.mjs +7 -5
- package/dist/lib/neutral/meta.json +1 -1
- package/dist/lib/neutral/meta.mjs +1 -1
- package/dist/lib/neutral/{open-A5KXKRH4.mjs → open-42H72C4G.mjs} +3 -3
- package/dist/lib/neutral/operations/index.mjs +1 -1
- package/dist/lib/neutral/plugin.mjs +2 -2
- package/dist/lib/neutral/settings-DUS32HOM.mjs +30 -0
- package/dist/lib/neutral/settings-DUS32HOM.mjs.map +7 -0
- package/dist/lib/neutral/{summarize-5XPPMVTE.mjs → summarize-45WVUPED.mjs} +3 -3
- package/dist/lib/neutral/{summarize-5XPPMVTE.mjs.map → summarize-45WVUPED.mjs.map} +1 -1
- package/dist/lib/neutral/{text-content-KBB37HSM.mjs → text-content-XZ3HK4OT.mjs} +9 -7
- package/dist/lib/neutral/text-content-XZ3HK4OT.mjs.map +7 -0
- package/dist/lib/neutral/{transcriber-OGSF3XO5.mjs → transcriber-KYV6NWI5.mjs} +3 -3
- package/dist/lib/neutral/translations.mjs +1 -1
- package/dist/lib/neutral/translations.mjs.map +3 -3
- package/dist/lib/neutral/types/index.mjs +3 -1
- package/dist/types/dx.config.d.ts +28 -0
- package/dist/types/dx.config.d.ts.map +1 -0
- package/dist/types/src/TranscriptionPlugin.d.ts.map +1 -1
- package/dist/types/src/capabilities/index.d.ts +5 -8
- package/dist/types/src/capabilities/index.d.ts.map +1 -1
- package/dist/types/src/capabilities/react-surface.d.ts +2 -2
- package/dist/types/src/capabilities/react-surface.d.ts.map +1 -1
- package/dist/types/src/capabilities/settings.d.ts +7 -0
- package/dist/types/src/capabilities/settings.d.ts.map +1 -0
- package/dist/types/src/hooks/useTranscriptionRecording.d.ts.map +1 -1
- package/dist/types/src/meta.d.ts +28 -2
- package/dist/types/src/meta.d.ts.map +1 -1
- package/dist/types/src/operations/enrich-message.d.ts +9 -0
- package/dist/types/src/operations/enrich-message.d.ts.map +1 -0
- package/dist/types/src/operations/index.d.ts.map +1 -1
- package/dist/types/src/stories/LiveTranscription.stories.d.ts +1 -2
- package/dist/types/src/stories/LiveTranscription.stories.d.ts.map +1 -1
- package/dist/types/src/stories/ProperNounExtraction.stories.d.ts +13 -0
- package/dist/types/src/stories/ProperNounExtraction.stories.d.ts.map +1 -0
- package/dist/types/src/stories/common.d.ts.map +1 -1
- package/dist/types/src/testing/testing.d.ts +0 -4
- package/dist/types/src/testing/testing.d.ts.map +1 -1
- package/dist/types/src/types/Settings.d.ts +7 -0
- package/dist/types/src/types/Settings.d.ts.map +1 -0
- package/dist/types/src/types/TranscriptOperation.d.ts +407 -0
- package/dist/types/src/types/TranscriptOperation.d.ts.map +1 -1
- package/dist/types/src/types/TranscriptionCapabilities.d.ts +3 -0
- package/dist/types/src/types/TranscriptionCapabilities.d.ts.map +1 -1
- package/dist/types/src/types/index.d.ts +1 -0
- package/dist/types/src/types/index.d.ts.map +1 -1
- package/dist/types/src/util/index.d.ts +0 -1
- package/dist/types/src/util/index.d.ts.map +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/dx.config.ts +42 -0
- package/package.json +43 -42
- package/src/TranscriptionPlugin.test.ts +1 -1
- package/src/TranscriptionPlugin.tsx +10 -2
- package/src/capabilities/index.ts +2 -7
- package/src/capabilities/settings.ts +33 -0
- package/src/capabilities/text-content.ts +9 -5
- package/src/containers/TranscriptionArticle/TranscriptionArticle.tsx +1 -1
- package/src/enrichment/operations.ts +1 -1
- package/src/hooks/useTranscriptionRecording.ts +26 -4
- package/src/meta.ts +2 -35
- package/src/operations/enrich-message.ts +27 -0
- package/src/operations/index.ts +1 -0
- package/src/operations/summarize.ts +1 -1
- package/src/stories/LiveTranscription.stories.tsx +13 -67
- package/src/stories/ProperNounExtraction.stories.tsx +99 -0
- package/src/stories/common.ts +2 -1
- package/src/testing/testing.ts +4 -89
- package/src/transcriber/transcription-manager.ts +1 -1
- package/src/translations.ts +1 -1
- package/src/types/Settings.ts +21 -0
- package/src/types/TranscriptOperation.ts +17 -1
- package/src/types/TranscriptionCapabilities.ts +7 -2
- package/src/types/index.ts +1 -0
- package/src/util/index.ts +0 -1
- package/dist/lib/neutral/chunk-FHNI26QW.mjs +0 -45
- package/dist/lib/neutral/chunk-FHNI26QW.mjs.map +0 -7
- package/dist/lib/neutral/chunk-KMT3BMMQ.mjs.map +0 -7
- package/dist/lib/neutral/chunk-KXJZEYS7.mjs +0 -8
- package/dist/lib/neutral/text-content-KBB37HSM.mjs.map +0 -7
- package/dist/types/src/util/monitor-audio-level.d.ts +0 -6
- package/dist/types/src/util/monitor-audio-level.d.ts.map +0 -1
- package/src/util/monitor-audio-level.ts +0 -58
- /package/dist/lib/neutral/{chunk-KAEXRX3Y.mjs.map → chunk-XOXHWMXR.mjs.map} +0 -0
- /package/dist/lib/neutral/{create-RVR7VC33.mjs.map → create-7NGOP7JP.mjs.map} +0 -0
- /package/dist/lib/neutral/{open-A5KXKRH4.mjs.map → open-42H72C4G.mjs.map} +0 -0
- /package/dist/lib/neutral/{transcriber-OGSF3XO5.mjs.map → transcriber-KYV6NWI5.mjs.map} +0 -0
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
// Replays a seeded transcript and runs each message through the shared `enrichTranscriptMessage`
|
|
6
|
+
// core (Haiku proper-noun extraction → full-text search → reference links) — the same core the
|
|
7
|
+
// meeting integration invokes via the EnrichMessage Operation. Here it runs with a live edge Haiku
|
|
8
|
+
// preset; the `<Transcription>` editor's `preview()` extension renders the links as chips.
|
|
9
|
+
|
|
10
|
+
import { type Meta, type StoryObj } from '@storybook/react-vite';
|
|
11
|
+
import * as Effect from 'effect/Effect';
|
|
12
|
+
import React, { useEffect, useRef, useState } from 'react';
|
|
13
|
+
|
|
14
|
+
import { AiServiceTestingPreset } from '@dxos/ai/testing';
|
|
15
|
+
import { enrichTranscriptMessage } from '@dxos/assistant/extraction';
|
|
16
|
+
import { EffectEx } from '@dxos/effect';
|
|
17
|
+
import { type Space, useSpaces } from '@dxos/react-client/echo';
|
|
18
|
+
import { IconButton, Panel, Toolbar } from '@dxos/react-ui';
|
|
19
|
+
import { type Message } from '@dxos/types';
|
|
20
|
+
import { seedTestData } from '@dxos/types/testing';
|
|
21
|
+
|
|
22
|
+
import { Transcription } from '../components/Transcription';
|
|
23
|
+
import { createStoryDecorators, useStoryMessageModel } from './common';
|
|
24
|
+
|
|
25
|
+
/** Enrich a message with a live edge Haiku; fall back to the raw message if the edge is unreachable. */
|
|
26
|
+
const enrich = (message: Message.Message, db: Space['db']): Promise<Message.Message> =>
|
|
27
|
+
EffectEx.runPromise(
|
|
28
|
+
enrichTranscriptMessage(message, { db }).pipe(Effect.provide(AiServiceTestingPreset('edge-remote'))),
|
|
29
|
+
).catch(() => message);
|
|
30
|
+
|
|
31
|
+
const DefaultStory = ({ interval = 4_000 }: { interval?: number }) => {
|
|
32
|
+
const [space] = useSpaces();
|
|
33
|
+
const { model, appendMessage } = useStoryMessageModel();
|
|
34
|
+
const [running, setRunning] = useState(true);
|
|
35
|
+
const indexRef = useRef(0);
|
|
36
|
+
|
|
37
|
+
// The decorator seeds the objects; `seedTestData` also returns the transcript script we replay.
|
|
38
|
+
const [messages, setMessages] = useState<Message.Message[]>([]);
|
|
39
|
+
useEffect(() => {
|
|
40
|
+
if (!space) {
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
void seedTestData(space).then(({ transcriptMessages }) => setMessages(transcriptMessages));
|
|
44
|
+
}, [space]);
|
|
45
|
+
|
|
46
|
+
// Replay the transcript one message at a time, enriching each before it is appended.
|
|
47
|
+
useEffect(() => {
|
|
48
|
+
if (!space || !running || messages.length === 0) {
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
let cancelled = false;
|
|
53
|
+
const pump = async () => {
|
|
54
|
+
while (!cancelled && indexRef.current < messages.length) {
|
|
55
|
+
const enriched = await enrich(messages[indexRef.current], space.db);
|
|
56
|
+
if (cancelled) {
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
indexRef.current += 1;
|
|
60
|
+
appendMessage(enriched);
|
|
61
|
+
await new Promise((resolve) => setTimeout(resolve, interval));
|
|
62
|
+
}
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
void pump();
|
|
66
|
+
return () => {
|
|
67
|
+
cancelled = true;
|
|
68
|
+
};
|
|
69
|
+
}, [space, running, messages, interval, appendMessage]);
|
|
70
|
+
|
|
71
|
+
return (
|
|
72
|
+
<Panel.Root>
|
|
73
|
+
<Panel.Toolbar asChild>
|
|
74
|
+
<Toolbar.Root classNames='justify-end'>
|
|
75
|
+
<IconButton
|
|
76
|
+
icon={running ? 'ph--pause--regular' : 'ph--play--regular'}
|
|
77
|
+
label={running ? 'Pause' : 'Resume'}
|
|
78
|
+
onClick={() => setRunning((value) => !value)}
|
|
79
|
+
/>
|
|
80
|
+
</Toolbar.Root>
|
|
81
|
+
</Panel.Toolbar>
|
|
82
|
+
<Panel.Content className='dx-document'>
|
|
83
|
+
<Transcription model={model} />
|
|
84
|
+
</Panel.Content>
|
|
85
|
+
</Panel.Root>
|
|
86
|
+
);
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
const meta = {
|
|
90
|
+
title: 'plugins/plugin-transcription/stories/ProperNounExtraction',
|
|
91
|
+
render: DefaultStory,
|
|
92
|
+
decorators: createStoryDecorators({ enableVectorIndex: true }),
|
|
93
|
+
} satisfies Meta<typeof DefaultStory>;
|
|
94
|
+
|
|
95
|
+
export default meta;
|
|
96
|
+
|
|
97
|
+
type Story = StoryObj<typeof meta>;
|
|
98
|
+
|
|
99
|
+
export const Default: Story = {};
|
package/src/stories/common.ts
CHANGED
|
@@ -14,9 +14,9 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
|
|
14
14
|
import { withPluginManager } from '@dxos/app-framework/testing';
|
|
15
15
|
import { AppActivationEvents } from '@dxos/app-toolkit';
|
|
16
16
|
import { scheduleTask } from '@dxos/async';
|
|
17
|
+
import { SpeakingMonitor } from '@dxos/av';
|
|
17
18
|
import { Context } from '@dxos/context';
|
|
18
19
|
import { Obj } from '@dxos/echo';
|
|
19
|
-
import { SpeakingMonitor } from '@dxos/plugin-calls';
|
|
20
20
|
import { ClientPlugin, initializeIdentity } from '@dxos/plugin-client/testing';
|
|
21
21
|
import { PreviewPlugin } from '@dxos/plugin-preview/testing';
|
|
22
22
|
import { StorybookPlugin, corePlugins } from '@dxos/plugin-testing';
|
|
@@ -226,6 +226,7 @@ const enableQueryIndexes = (services: { QueryService?: any }) =>
|
|
|
226
226
|
{ kind: IndexKind.Kind.SCHEMA_MATCH },
|
|
227
227
|
{ kind: IndexKind.Kind.GRAPH },
|
|
228
228
|
{ kind: IndexKind.Kind.VECTOR },
|
|
229
|
+
{ kind: IndexKind.Kind.FULL_TEXT },
|
|
229
230
|
],
|
|
230
231
|
}),
|
|
231
232
|
);
|
package/src/testing/testing.ts
CHANGED
|
@@ -4,20 +4,14 @@
|
|
|
4
4
|
|
|
5
5
|
import * as Effect from 'effect/Effect';
|
|
6
6
|
import * as Schema from 'effect/Schema';
|
|
7
|
-
import { useEffect, useMemo
|
|
7
|
+
import { useEffect, useMemo } from 'react';
|
|
8
8
|
|
|
9
|
-
import { extractionAnthropicFunction, processTranscriptMessage } from '@dxos/assistant/extraction';
|
|
10
|
-
import { scheduleTaskInterval } from '@dxos/async';
|
|
11
9
|
import { createFeedServiceLayer, type Space } from '@dxos/client/echo';
|
|
12
|
-
import {
|
|
13
|
-
import { DXN, Feed, Filter, Obj, Ref, Type } from '@dxos/echo';
|
|
10
|
+
import { DXN, Feed, Obj, Ref, Type } from '@dxos/echo';
|
|
14
11
|
import { EffectEx } from '@dxos/effect';
|
|
15
12
|
import { IdentityDid } from '@dxos/keys';
|
|
16
|
-
import { log } from '@dxos/log';
|
|
17
13
|
import { random } from '@dxos/random';
|
|
18
|
-
import {
|
|
19
|
-
import { type ContentBlock, Message, Organization, Person } from '@dxos/types';
|
|
20
|
-
import { seedTestData } from '@dxos/types/testing';
|
|
14
|
+
import { type ContentBlock, Message } from '@dxos/types';
|
|
21
15
|
|
|
22
16
|
// TODO(burdon): Reconcile with plugin-markdown. Move to @dxos/schema/testing.
|
|
23
17
|
export const TestItem = Schema.Struct({
|
|
@@ -91,47 +85,6 @@ export class MessageBuilder extends AbstractMessageBuilder {
|
|
|
91
85
|
}
|
|
92
86
|
}
|
|
93
87
|
|
|
94
|
-
// TODO(burdon): Reconcile with BlockBuilder.
|
|
95
|
-
class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
|
|
96
|
-
space: Space | undefined;
|
|
97
|
-
currentMessage: number = 0;
|
|
98
|
-
transcriptMessages: Message.Message[] = [];
|
|
99
|
-
|
|
100
|
-
async connect(space: Space): Promise<void> {
|
|
101
|
-
this.space = space;
|
|
102
|
-
const { transcriptMessages } = await seedTestData(space);
|
|
103
|
-
this.transcriptMessages = transcriptMessages;
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
override async createMessage(): Promise<Message.Message> {
|
|
107
|
-
if (!this.space) {
|
|
108
|
-
throw new Error('Space not connected');
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
const objects = await this.space.db
|
|
112
|
-
.query(
|
|
113
|
-
Filter.or(
|
|
114
|
-
Filter.type(Person.Person),
|
|
115
|
-
Filter.type(Organization.Organization),
|
|
116
|
-
Filter.type(TestSchema.DocumentType),
|
|
117
|
-
),
|
|
118
|
-
)
|
|
119
|
-
.run();
|
|
120
|
-
|
|
121
|
-
log.info('context', { objects });
|
|
122
|
-
const message = this.transcriptMessages[this.currentMessage];
|
|
123
|
-
this.currentMessage++;
|
|
124
|
-
this.currentMessage = this.currentMessage % this.transcriptMessages.length;
|
|
125
|
-
|
|
126
|
-
const { message: enhancedMessage } = await processTranscriptMessage({
|
|
127
|
-
input: { message },
|
|
128
|
-
function: extractionAnthropicFunction,
|
|
129
|
-
});
|
|
130
|
-
|
|
131
|
-
return enhancedMessage;
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
|
|
135
88
|
type UseTestTranscriptionQueue = (
|
|
136
89
|
space: Space | undefined,
|
|
137
90
|
running?: boolean,
|
|
@@ -153,7 +106,7 @@ export const useTestTranscriptionQueue: UseTestTranscriptionQueue = (
|
|
|
153
106
|
if (!space || !feed || !running) {
|
|
154
107
|
return;
|
|
155
108
|
}
|
|
156
|
-
const feedServiceLayer = createFeedServiceLayer(space.
|
|
109
|
+
const feedServiceLayer = createFeedServiceLayer(space.db);
|
|
157
110
|
|
|
158
111
|
const i = setInterval(() => {
|
|
159
112
|
void builder.createMessage(Math.ceil(Math.random() * 3)).then(async (message) => {
|
|
@@ -165,41 +118,3 @@ export const useTestTranscriptionQueue: UseTestTranscriptionQueue = (
|
|
|
165
118
|
|
|
166
119
|
return feed;
|
|
167
120
|
};
|
|
168
|
-
|
|
169
|
-
/**
|
|
170
|
-
* Test transcription feed with entity extraction.
|
|
171
|
-
*/
|
|
172
|
-
// TODO(burdon): Reconcile with useTestTranscriptionQueue.
|
|
173
|
-
export const useTestTranscriptionQueueWithEntityExtraction: UseTestTranscriptionQueue = (
|
|
174
|
-
space: Space | undefined,
|
|
175
|
-
running = true,
|
|
176
|
-
interval = 1_000,
|
|
177
|
-
) => {
|
|
178
|
-
const feed = useMemo(() => (space ? space.db.add(Feed.make({ name: 'transcription' })) : undefined), [space]);
|
|
179
|
-
const [builder] = useState(() => new EntityExtractionMessageBuilder());
|
|
180
|
-
|
|
181
|
-
useEffect(() => {
|
|
182
|
-
if (!space || !feed || !running) {
|
|
183
|
-
return;
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
void builder.connect(space);
|
|
187
|
-
|
|
188
|
-
const feedServiceLayer = createFeedServiceLayer(space.queues);
|
|
189
|
-
const ctx = new Context();
|
|
190
|
-
scheduleTaskInterval(
|
|
191
|
-
ctx,
|
|
192
|
-
async () => {
|
|
193
|
-
const message = await builder.createMessage();
|
|
194
|
-
await Feed.append(feed, [message]).pipe(Effect.provide(feedServiceLayer), EffectEx.runAndForwardErrors);
|
|
195
|
-
},
|
|
196
|
-
interval,
|
|
197
|
-
);
|
|
198
|
-
|
|
199
|
-
return () => {
|
|
200
|
-
void ctx.dispose();
|
|
201
|
-
};
|
|
202
|
-
}, [space, feed, running, interval]);
|
|
203
|
-
|
|
204
|
-
return feed;
|
|
205
|
-
};
|
|
@@ -77,7 +77,7 @@ export class TranscriptionManager extends Resource {
|
|
|
77
77
|
|
|
78
78
|
setFeed(space: Space, feed: Feed.Feed): this {
|
|
79
79
|
this._feed = feed;
|
|
80
|
-
this._feedServiceLayer = createFeedServiceLayer(space.
|
|
80
|
+
this._feedServiceLayer = createFeedServiceLayer(space.db);
|
|
81
81
|
return this;
|
|
82
82
|
}
|
|
83
83
|
|
package/src/translations.ts
CHANGED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
// @import-as-namespace
|
|
6
|
+
|
|
7
|
+
import * as Schema from 'effect/Schema';
|
|
8
|
+
|
|
9
|
+
export const Settings = Schema.mutable(
|
|
10
|
+
Schema.Struct({
|
|
11
|
+
entityExtraction: Schema.optional(
|
|
12
|
+
Schema.Boolean.annotations({
|
|
13
|
+
title: 'Entity extraction',
|
|
14
|
+
description:
|
|
15
|
+
'While transcribing, use the Assistant to detect and annotate mentions of known objects such as people or organizations.',
|
|
16
|
+
}),
|
|
17
|
+
).pipe(Schema.withConstructorDefault(() => true)),
|
|
18
|
+
}),
|
|
19
|
+
);
|
|
20
|
+
|
|
21
|
+
export interface Settings extends Schema.Schema.Type<typeof Settings> {}
|
|
@@ -14,7 +14,7 @@ import { Message, Transcript } from '@dxos/types';
|
|
|
14
14
|
|
|
15
15
|
import { meta } from '#meta';
|
|
16
16
|
|
|
17
|
-
const makeKey = (name: string) => DXN.make(`${meta.
|
|
17
|
+
const makeKey = (name: string) => DXN.make(`${meta.profile.key}.operation.${name}`);
|
|
18
18
|
|
|
19
19
|
export const Create = Operation.make({
|
|
20
20
|
meta: { key: makeKey('create'), name: 'Create Transcript', icon: 'ph--microphone--regular' },
|
|
@@ -79,6 +79,22 @@ export const Summarize = Operation.make({
|
|
|
79
79
|
services: [AiService.AiService],
|
|
80
80
|
});
|
|
81
81
|
|
|
82
|
+
export const EnrichMessage = Operation.make({
|
|
83
|
+
meta: {
|
|
84
|
+
key: DXN.make('org.dxos.function.transcription.enrichMessage'),
|
|
85
|
+
name: 'Enrich Transcript Message',
|
|
86
|
+
description: 'Extract proper nouns from a transcript message and link them to objects in the space.',
|
|
87
|
+
icon: 'ph--text-t--regular',
|
|
88
|
+
},
|
|
89
|
+
input: Schema.Struct({
|
|
90
|
+
message: Type.getSchema(Message.Message),
|
|
91
|
+
}),
|
|
92
|
+
output: Schema.Struct({
|
|
93
|
+
message: Type.getSchema(Message.Message),
|
|
94
|
+
}),
|
|
95
|
+
services: [AiService.AiService, Database.Service],
|
|
96
|
+
});
|
|
97
|
+
|
|
82
98
|
export const SentenceNormalizationInput = Schema.Struct({
|
|
83
99
|
messages: Schema.Array(MessageWithRangeId).annotations({
|
|
84
100
|
description: 'Messages to normalize into sentences.',
|
|
@@ -4,6 +4,8 @@
|
|
|
4
4
|
|
|
5
5
|
// @import-as-namespace
|
|
6
6
|
|
|
7
|
+
import { type Atom } from '@effect-atom/atom-react';
|
|
8
|
+
|
|
7
9
|
import { Capability } from '@dxos/app-framework';
|
|
8
10
|
|
|
9
11
|
import { meta } from '#meta';
|
|
@@ -15,6 +17,7 @@ import {
|
|
|
15
17
|
type TranscriptMessageEnricher,
|
|
16
18
|
type TranscriptionManager as TranscriptionManagerType,
|
|
17
19
|
} from '../transcriber';
|
|
20
|
+
import * as Settings$ from './Settings'; // eslint-disable-line
|
|
18
21
|
|
|
19
22
|
export type TranscriberProviderProps = {
|
|
20
23
|
audioStreamTrack: MediaStreamTrack;
|
|
@@ -32,8 +35,10 @@ export type TranscriptionManagerProviderProps = {
|
|
|
32
35
|
|
|
33
36
|
export type TranscriptionManagerProvider = (props: TranscriptionManagerProviderProps) => TranscriptionManagerType;
|
|
34
37
|
|
|
35
|
-
export const TranscriberProvider = Capability.make<TranscriberProvider>(`${meta.
|
|
38
|
+
export const TranscriberProvider = Capability.make<TranscriberProvider>(`${meta.profile.key}.capability.transcriber`);
|
|
36
39
|
|
|
37
40
|
export const TranscriptionManagerProvider = Capability.make<TranscriptionManagerProvider>(
|
|
38
|
-
`${meta.
|
|
41
|
+
`${meta.profile.key}.capability.transcription-manager`,
|
|
39
42
|
);
|
|
43
|
+
|
|
44
|
+
export const Settings = Capability.make<Atom.Writable<Settings$.Settings>>(`${meta.profile.key}.capability.settings`);
|
package/src/types/index.ts
CHANGED
package/src/util/index.ts
CHANGED
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
// src/meta.ts
|
|
2
|
-
import { Plugin } from "@dxos/app-framework";
|
|
3
|
-
import { DXN } from "@dxos/keys";
|
|
4
|
-
import { trim } from "@dxos/util";
|
|
5
|
-
var meta = Plugin.makeMeta({
|
|
6
|
-
key: DXN.make("org.dxos.plugin.transcription"),
|
|
7
|
-
name: "Transcription",
|
|
8
|
-
author: "DXOS",
|
|
9
|
-
description: trim`
|
|
10
|
-
Real-time voice-to-text transcription for DXOS Composer.
|
|
11
|
-
Captures microphone audio in short PCM chunks, batches them into WAV payloads, and streams
|
|
12
|
-
the results to a Whisper-compatible speech-recognition endpoint.
|
|
13
|
-
Transcribed segments are timestamped, deduplicated across chunk boundaries, and persisted
|
|
14
|
-
as Message objects in an ECHO Feed so that every participant in a shared space sees the
|
|
15
|
-
live transcript without any manual sync step.
|
|
16
|
-
|
|
17
|
-
Each recording session is represented as a Transcript object linked to its Feed.
|
|
18
|
-
The read-only editor renders messages with speaker bylines and wall-clock timestamps,
|
|
19
|
-
supporting markdown decoration and document preview.
|
|
20
|
-
An optional message enricher pipeline lets plugins annotate messages before they are
|
|
21
|
-
committed — for example, to attach speaker identity from a calls service.
|
|
22
|
-
|
|
23
|
-
The plugin ships two blueprint-compatible operations: Open (loads and formats a transcript
|
|
24
|
-
for reading by an AI assistant) and Summarize (sends the transcript text to an LLM and
|
|
25
|
-
returns a structured Markdown summary with key points, verbatim quotes, and actionable
|
|
26
|
-
tasks extracted from the conversation).
|
|
27
|
-
|
|
28
|
-
A sentence-normalization pass merges raw word-level tokens from the recognizer into
|
|
29
|
-
grammatically complete sentences, improving readability of the stored transcript without
|
|
30
|
-
altering the underlying audio timeline.
|
|
31
|
-
`,
|
|
32
|
-
icon: "ph--microphone--regular",
|
|
33
|
-
iconHue: "sky",
|
|
34
|
-
source: "https://github.com/dxos/dxos/tree/main/packages/plugins/plugin-transcription",
|
|
35
|
-
spec: "PLUGIN.mdl",
|
|
36
|
-
tags: [
|
|
37
|
-
"labs"
|
|
38
|
-
],
|
|
39
|
-
screenshots: []
|
|
40
|
-
});
|
|
41
|
-
|
|
42
|
-
export {
|
|
43
|
-
meta
|
|
44
|
-
};
|
|
45
|
-
//# sourceMappingURL=chunk-FHNI26QW.mjs.map
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"version": 3,
|
|
3
|
-
"sources": ["../../../src/meta.ts"],
|
|
4
|
-
"sourcesContent": ["//\n// Copyright 2023 DXOS.org\n//\n\nimport { Plugin } from '@dxos/app-framework';\nimport { DXN } from '@dxos/keys';\nimport { trim } from '@dxos/util';\n\nexport const meta = Plugin.makeMeta({\n key: DXN.make('org.dxos.plugin.transcription'),\n name: 'Transcription',\n author: 'DXOS',\n description: trim`\n Real-time voice-to-text transcription for DXOS Composer.\n Captures microphone audio in short PCM chunks, batches them into WAV payloads, and streams\n the results to a Whisper-compatible speech-recognition endpoint.\n Transcribed segments are timestamped, deduplicated across chunk boundaries, and persisted\n as Message objects in an ECHO Feed so that every participant in a shared space sees the\n live transcript without any manual sync step.\n\n Each recording session is represented as a Transcript object linked to its Feed.\n The read-only editor renders messages with speaker bylines and wall-clock timestamps,\n supporting markdown decoration and document preview.\n An optional message enricher pipeline lets plugins annotate messages before they are\n committed — for example, to attach speaker identity from a calls service.\n\n The plugin ships two blueprint-compatible operations: Open (loads and formats a transcript\n for reading by an AI assistant) and Summarize (sends the transcript text to an LLM and\n returns a structured Markdown summary with key points, verbatim quotes, and actionable\n tasks extracted from the conversation).\n\n A sentence-normalization pass merges raw word-level tokens from the recognizer into\n grammatically complete sentences, improving readability of the stored transcript without\n altering the underlying audio timeline.\n `,\n icon: 'ph--microphone--regular',\n iconHue: 'sky',\n source: 'https://github.com/dxos/dxos/tree/main/packages/plugins/plugin-transcription',\n spec: 'PLUGIN.mdl',\n tags: ['labs'],\n screenshots: [],\n});\n"],
|
|
5
|
-
"mappings": ";AAIA,SAASA,cAAc;AACvB,SAASC,WAAW;AACpB,SAASC,YAAY;AAEd,IAAMC,OAAOH,OAAOI,SAAS;EAClCC,KAAKJ,IAAIK,KAAK,+BAAA;EACdC,MAAM;EACNC,QAAQ;EACRC,aAAaP;;;;;;;;;;;;;;;;;;;;;;;EAuBbQ,MAAM;EACNC,SAAS;EACTC,QAAQ;EACRC,MAAM;EACNC,MAAM;IAAC;;EACPC,aAAa,CAAA;AACf,CAAA;",
|
|
6
|
-
"names": ["Plugin", "DXN", "trim", "meta", "makeMeta", "key", "make", "name", "author", "description", "icon", "iconHue", "source", "spec", "tags", "screenshots"]
|
|
7
|
-
}
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"version": 3,
|
|
3
|
-
"sources": ["../../../src/types/TranscriptionCapabilities.ts", "../../../src/types/TranscriptOperation.ts"],
|
|
4
|
-
"sourcesContent": ["//\n// Copyright 2025 DXOS.org\n//\n\n// @import-as-namespace\n\nimport { Capability } from '@dxos/app-framework';\n\nimport { meta } from '#meta';\n\nimport {\n type MediaStreamRecorderProps,\n type Transcriber as TranscriberType,\n type TranscriberProps,\n type TranscriptMessageEnricher,\n type TranscriptionManager as TranscriptionManagerType,\n} from '../transcriber';\n\nexport type TranscriberProviderProps = {\n audioStreamTrack: MediaStreamTrack;\n recorderConfig?: Partial<MediaStreamRecorderProps['config']>;\n transcriberConfig?: Partial<TranscriberProps['config']>;\n onSegments: TranscriberProps['onSegments'];\n transcribe?: TranscriberProps['transcribe'];\n};\n\nexport type TranscriberProvider = (props: TranscriberProviderProps) => TranscriberType;\n\nexport type TranscriptionManagerProviderProps = {\n messageEnricher?: TranscriptMessageEnricher;\n};\n\nexport type TranscriptionManagerProvider = (props: TranscriptionManagerProviderProps) => TranscriptionManagerType;\n\nexport const TranscriberProvider = Capability.make<TranscriberProvider>(`${meta.id}.capability.transcriber`);\n\nexport const TranscriptionManagerProvider = Capability.make<TranscriptionManagerProvider>(\n `${meta.id}.capability.transcription-manager`,\n);\n", "//\n// Copyright 2025 DXOS.org\n//\n\n// @import-as-namespace\n\nimport * as Schema from 'effect/Schema';\n\nimport { AiService } from '@dxos/ai';\nimport { SpaceSchema } from '@dxos/client/echo';\nimport { Operation } from '@dxos/compute';\nimport { Database, Feed, Ref, Type, DXN } from '@dxos/echo';\nimport { Message, Transcript } from '@dxos/types';\n\nimport { meta } from '#meta';\n\nconst makeKey = (name: string) => DXN.make(`${meta.id}.operation.${name}`);\n\nexport const Create = Operation.make({\n meta: { key: makeKey('create'), name: 'Create Transcript', icon: 'ph--microphone--regular' },\n input: Schema.Struct({\n name: Schema.optional(Schema.String),\n space: SpaceSchema,\n }),\n output: Schema.Struct({\n object: Type.getSchema(Transcript.Transcript),\n }),\n});\n\nexport const MessageWithRangeId = Schema.extend(\n Type.getSchema(Message.Message),\n Schema.Struct({\n rangeId: Schema.optional(Schema.Array(Schema.String)).annotations({\n description: 'The IDs of the messages that contain the sentences.',\n }),\n }),\n);\n\nexport type MessageWithRangeIdType = Schema.Schema.Type<typeof MessageWithRangeId>;\n\nexport const Open = Operation.make({\n meta: {\n key: DXN.make('org.dxos.function.transcription.open'),\n name: 'Open',\n description: 'Opens and reads the contents of a transcription object.',\n icon: 'ph--folder-open--regular',\n },\n input: Schema.Struct({\n transcript: Ref.Ref(Transcript.Transcript).annotations({\n description: 'The ID of the transcription object.',\n }),\n }),\n output: Schema.Struct({\n content: Schema.String,\n }),\n services: [Database.Service, Feed.FeedService],\n});\n\nexport const Summarize = Operation.make({\n meta: {\n key: DXN.make('org.dxos.function.transcription.summarize'),\n name: 'Summarize',\n description: 'Summarize a transcript of a meeting.',\n icon: 'ph--text-align-left--regular',\n },\n input: Schema.Struct({\n transcript: Schema.String.annotations({\n description: 'The transcript of the meeting.',\n }),\n notes: Schema.optional(Schema.String).annotations({\n description: 'Additional notes from the participants.',\n }),\n }),\n output: Schema.Struct({\n summary: Schema.String.annotations({\n description: 'The summary of the transcript.',\n }),\n }),\n services: [AiService.AiService],\n});\n\nexport const SentenceNormalizationInput = Schema.Struct({\n messages: Schema.Array(MessageWithRangeId).annotations({\n description: 'Messages to normalize into sentences.',\n }),\n});\n\nexport type SentenceNormalizationInputType = Schema.Schema.Type<typeof SentenceNormalizationInput>;\n\nexport const SentenceNormalizationOutput = Schema.Struct({\n sentences: Schema.Array(MessageWithRangeId.pipe(Schema.mutable)).pipe(Schema.mutable).annotations({\n description: 'The sentences of the transcript.',\n }),\n});\n\nexport const SentenceNormalization = Operation.make({\n meta: {\n key: DXN.make('org.dxos.function.transcription.sentenceNormalization'),\n name: 'Sentence Normalization',\n description: 'Post process of transcription for sentence normalization',\n icon: 'ph--text-t--regular',\n },\n input: SentenceNormalizationInput,\n output: SentenceNormalizationOutput,\n services: [],\n});\n"],
|
|
5
|
-
"mappings": ";;;;;AAAA;;;;;AAMA,SAASA,kBAAkB;AAE3B,SAASC,YAAY;AA0Bd,IAAMC,sBAAsBF,WAAWG,KAA0B,GAAGF,KAAKG,EAAE,yBAAyB;AAEpG,IAAMC,+BAA+BL,WAAWG,KACrD,GAAGF,KAAKG,EAAE,mCAAmC;;;ACrC/C;;;;;;;;;;AAMA,YAAYE,YAAY;AAExB,SAASC,iBAAiB;AAC1B,SAASC,mBAAmB;AAC5B,SAASC,iBAAiB;AAC1B,SAASC,UAAUC,MAAMC,KAAKC,MAAMC,WAAW;AAC/C,SAASC,SAASC,kBAAkB;AAEpC,SAASC,QAAAA,aAAY;AAErB,IAAMC,UAAU,CAACC,SAAiBL,IAAIM,KAAK,GAAGH,MAAKI,EAAE,cAAcF,IAAAA,EAAM;AAElE,IAAMG,SAASb,UAAUW,KAAK;EACnCH,MAAM;IAAEM,KAAKL,QAAQ,QAAA;IAAWC,MAAM;IAAqBK,MAAM;EAA0B;EAC3FC,OAAcC,cAAO;IACnBP,MAAaQ,gBAAgBC,aAAM;IACnCC,OAAOrB;EACT,CAAA;EACAsB,QAAeJ,cAAO;IACpBK,QAAQlB,KAAKmB,UAAUhB,WAAWA,UAAU;EAC9C,CAAA;AACF,CAAA;AAEO,IAAMiB,qBAA4BC,cACvCrB,KAAKmB,UAAUjB,QAAQA,OAAO,GACvBW,cAAO;EACZS,SAAgBR,gBAAgBS,aAAaR,aAAM,CAAA,EAAGS,YAAY;IAChEC,aAAa;EACf,CAAA;AACF,CAAA,CAAA;AAKK,IAAMC,OAAO9B,UAAUW,KAAK;EACjCH,MAAM;IACJM,KAAKT,IAAIM,KAAK,sCAAA;IACdD,MAAM;IACNmB,aAAa;IACbd,MAAM;EACR;EACAC,OAAcC,cAAO;IACnBc,YAAY5B,IAAIA,IAAII,WAAWA,UAAU,EAAEqB,YAAY;MACrDC,aAAa;IACf,CAAA;EACF,CAAA;EACAR,QAAeJ,cAAO;IACpBe,SAAgBb;EAClB,CAAA;EACAc,UAAU;IAAChC,SAASiC;IAAShC,KAAKiC;;AACpC,CAAA;AAEO,IAAMC,YAAYpC,UAAUW,KAAK;EACtCH,MAAM;IACJM,KAAKT,IAAIM,KAAK,2CAAA;IACdD,MAAM;IACNmB,aAAa;IACbd,MAAM;EACR;EACAC,OAAcC,cAAO;IACnBc,YAAmBZ,cAAOS,YAAY;MACpCC,aAAa;IACf,CAAA;IACAQ,OAAcnB,gBAAgBC,aAAM,EAAES,YAAY;MAChDC,aAAa;IACf,CAAA;EACF,CAAA;EACAR,QAAeJ,cAAO;IACpBqB,SAAgBnB,cAAOS,YAAY;MACjCC,aAAa;IACf,CAAA;EACF,CAAA;EACAI,UAAU;IAACnC,UAAUA;;AACvB,CAAA;AAEO,IAAMyC,6BAAoCtB,cAAO;EACtDuB,UAAiBb,aAAMH,kBAAAA,EAAoBI,YAAY;IACrDC,aAAa;EACf,CAAA;AACF,CAAA;AAIO,IAAMY,8BAAqCxB,cAAO;EACvDyB,WAAkBf,aAAMH,mBAAmBmB,KAAYC,cAAO,CAAA,EAAGD,KAAYC,cAAO,EAAEhB,YAAY;IAChGC,aAAa;EACf,CAAA;AACF,CAAA;AAEO,IAAMgB,wBAAwB7C,UAAUW,KAAK;EAClDH,MAAM;IACJM,KAAKT,IAAIM,KAAK,uDAAA;IACdD,MAAM;IACNmB,aAAa;IACbd,MAAM;EACR;EACAC,OAAOuB;EACPlB,QAAQoB;EACRR,UAAU,CAAA;AACZ,CAAA;",
|
|
6
|
-
"names": ["Capability", "meta", "TranscriberProvider", "make", "id", "TranscriptionManagerProvider", "Schema", "AiService", "SpaceSchema", "Operation", "Database", "Feed", "Ref", "Type", "DXN", "Message", "Transcript", "meta", "makeKey", "name", "make", "id", "Create", "key", "icon", "input", "Struct", "optional", "String", "space", "output", "object", "getSchema", "MessageWithRangeId", "extend", "rangeId", "Array", "annotations", "description", "Open", "transcript", "content", "services", "Service", "FeedService", "Summarize", "notes", "summary", "SentenceNormalizationInput", "messages", "SentenceNormalizationOutput", "sentences", "pipe", "mutable", "SentenceNormalization"]
|
|
7
|
-
}
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
// src/operations/index.ts
|
|
2
|
-
import { OperationHandlerSet } from "@dxos/compute";
|
|
3
|
-
var TranscriptionOperationHandlerSet = OperationHandlerSet.lazy(() => import("./create-RVR7VC33.mjs"), () => import("./open-A5KXKRH4.mjs"), () => import("./summarize-5XPPMVTE.mjs"));
|
|
4
|
-
|
|
5
|
-
export {
|
|
6
|
-
TranscriptionOperationHandlerSet
|
|
7
|
-
};
|
|
8
|
-
//# sourceMappingURL=chunk-KXJZEYS7.mjs.map
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"version": 3,
|
|
3
|
-
"sources": ["../../../src/capabilities/text-content.ts"],
|
|
4
|
-
"sourcesContent": ["//\n// Copyright 2025 DXOS.org\n//\n\nimport * as Effect from 'effect/Effect';\n\nimport { Capability } from '@dxos/app-framework';\nimport { AppCapabilities } from '@dxos/app-toolkit';\nimport { Feed, Obj, Type } from '@dxos/echo';\nimport { getSpace } from '@dxos/react-client/echo';\nimport { Message, Transcript } from '@dxos/types';\n\nimport { renderByline } from '../util';\n\nexport default Capability.makeModule(\n Effect.fnUntraced(function* () {\n return Capability.contributes(AppCapabilities.TextContent, {\n id: Type.getTypename(Transcript.Transcript),\n getTextContent: async (transcript: Transcript.Transcript) => {\n const space = getSpace(transcript);\n const members = space?.members.get().map((member) => member.identity) ?? [];\n const feed = await transcript.feed.load();\n const feedDXN = Feed.getQueueUri(feed);\n const queue = feedDXN ? space?.queues.get<Message.Message>(feedDXN) : undefined;\n await queue?.refresh();\n return queue?.objects\n .filter((message) => Obj.instanceOf(Message.Message, message))\n .flatMap((message, index) => renderByline(members)(message, index))\n .join('\\n\\n');\n },\n });\n }),\n);\n"],
|
|
5
|
-
"mappings": ";;;;;;AAIA,YAAYA,YAAY;AAExB,SAASC,kBAAkB;AAC3B,SAASC,uBAAuB;AAChC,SAASC,MAAMC,KAAKC,YAAY;AAChC,SAASC,gBAAgB;AACzB,SAASC,SAASC,kBAAkB;AAIpC,IAAA,uBAAeC,WAAWC,WACjBC,kBAAW,aAAA;AAChB,SAAOF,WAAWG,YAAYC,gBAAgBC,aAAa;IACzDC,IAAIC,KAAKC,YAAYC,WAAWA,UAAU;IAC1CC,gBAAgB,OAAOC,eAAAA;AACrB,YAAMC,QAAQC,SAASF,UAAAA;AACvB,YAAMG,UAAUF,OAAOE,QAAQC,IAAAA,EAAMC,IAAI,CAACC,WAAWA,OAAOC,QAAQ,KAAK,CAAA;AACzE,YAAMC,OAAO,MAAMR,WAAWQ,KAAKC,KAAI;AACvC,YAAMC,UAAUC,KAAKC,YAAYJ,IAAAA;AACjC,YAAMK,QAAQH,UAAUT,OAAOa,OAAOV,IAAqBM,OAAAA,IAAWK;AACtE,YAAMF,OAAOG,QAAAA;AACb,aAAOH,OAAOI,QACXC,OAAO,CAACC,YAAYC,IAAIC,WAAWC,QAAQA,SAASH,OAAAA,CAAAA,EACpDI,QAAQ,CAACJ,SAASK,UAAUC,aAAatB,OAAAA,EAASgB,SAASK,KAAAA,CAAAA,EAC3DE,KAAK,MAAA;IACV;EACF,CAAA;AACF,CAAA,CAAA;",
|
|
6
|
-
"names": ["Effect", "Capability", "AppCapabilities", "Feed", "Obj", "Type", "getSpace", "Message", "Transcript", "Capability", "makeModule", "fnUntraced", "contributes", "AppCapabilities", "TextContent", "id", "Type", "getTypename", "Transcript", "getTextContent", "transcript", "space", "getSpace", "members", "get", "map", "member", "identity", "feed", "load", "feedDXN", "Feed", "getQueueUri", "queue", "queues", "undefined", "refresh", "objects", "filter", "message", "Obj", "instanceOf", "Message", "flatMap", "index", "renderByline", "join"]
|
|
7
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"monitor-audio-level.d.ts","sourceRoot":"","sources":["../../../../src/util/monitor-audio-level.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,iBAAiB,iDAI3B;IACD,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,SAAS,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IACnC,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,eA6CA,CAAC"}
|
|
@@ -1,58 +0,0 @@
|
|
|
1
|
-
//
|
|
2
|
-
// Copyright 2025 DXOS.org
|
|
3
|
-
//
|
|
4
|
-
|
|
5
|
-
export const monitorAudioLevel = ({
|
|
6
|
-
mediaStreamTrack,
|
|
7
|
-
onMeasure,
|
|
8
|
-
intervalMs = 1000 / 10, // 12 fps.
|
|
9
|
-
}: {
|
|
10
|
-
mediaStreamTrack: MediaStreamTrack;
|
|
11
|
-
onMeasure: (value: number) => void;
|
|
12
|
-
intervalMs?: number;
|
|
13
|
-
}) => {
|
|
14
|
-
let timeout = -1;
|
|
15
|
-
let interval = -1;
|
|
16
|
-
const audioContext = new AudioContext();
|
|
17
|
-
const stream = new MediaStream();
|
|
18
|
-
stream.addTrack(mediaStreamTrack);
|
|
19
|
-
const mediaStreamAudioSourceNode = audioContext.createMediaStreamSource(stream);
|
|
20
|
-
const analyserNode = audioContext.createAnalyser();
|
|
21
|
-
mediaStreamAudioSourceNode.connect(analyserNode);
|
|
22
|
-
// Since we just need a rough approximation and will be measuring
|
|
23
|
-
// frequently, we want to drop this down from the default of 2048.
|
|
24
|
-
analyserNode.fftSize = 32;
|
|
25
|
-
|
|
26
|
-
const pcmData = new Float32Array(analyserNode.fftSize);
|
|
27
|
-
let peak = 0;
|
|
28
|
-
|
|
29
|
-
interval = window.setInterval(() => {
|
|
30
|
-
onMeasure(peak);
|
|
31
|
-
peak = 0;
|
|
32
|
-
}, intervalMs);
|
|
33
|
-
|
|
34
|
-
const tick = () => {
|
|
35
|
-
timeout = window.setTimeout(() => {
|
|
36
|
-
analyserNode.getFloatTimeDomainData(pcmData);
|
|
37
|
-
let sumSquares = 0.0;
|
|
38
|
-
for (const amplitude of pcmData) {
|
|
39
|
-
sumSquares += amplitude * amplitude;
|
|
40
|
-
}
|
|
41
|
-
const current = Math.sqrt(sumSquares / pcmData.length);
|
|
42
|
-
if (current > peak) {
|
|
43
|
-
peak = current;
|
|
44
|
-
}
|
|
45
|
-
tick();
|
|
46
|
-
}, intervalMs);
|
|
47
|
-
};
|
|
48
|
-
tick();
|
|
49
|
-
|
|
50
|
-
return () => {
|
|
51
|
-
mediaStreamAudioSourceNode.disconnect();
|
|
52
|
-
analyserNode.disconnect();
|
|
53
|
-
void audioContext.close();
|
|
54
|
-
clearInterval(interval);
|
|
55
|
-
clearTimeout(timeout);
|
|
56
|
-
stream.removeTrack(mediaStreamTrack);
|
|
57
|
-
};
|
|
58
|
-
};
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|