@dxos/plugin-transcription 0.8.4-main.abd8ff62ef → 0.8.4-main.bc2380dfbc
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +102 -5
- package/README.md +1 -1
- package/dist/lib/neutral/TranscriptionArticle-L6XANBH5.mjs +54 -0
- package/dist/lib/neutral/TranscriptionArticle-L6XANBH5.mjs.map +7 -0
- package/dist/lib/neutral/TranscriptionPlugin.mjs +34 -0
- package/dist/lib/neutral/TranscriptionPlugin.mjs.map +7 -0
- package/dist/lib/neutral/TranscriptionPlugin.node.mjs +25 -0
- package/dist/lib/neutral/TranscriptionPlugin.node.mjs.map +7 -0
- package/dist/lib/neutral/TranscriptionPlugin.workerd.mjs +21 -0
- package/dist/lib/neutral/TranscriptionPlugin.workerd.mjs.map +7 -0
- package/dist/lib/neutral/blueprint-definition-HOXDG5QP.mjs +15 -0
- package/dist/lib/neutral/blueprint-definition-HOXDG5QP.mjs.map +7 -0
- package/dist/lib/neutral/blueprints/index.mjs +8 -0
- package/dist/lib/neutral/capabilities/index.mjs +17 -0
- package/dist/lib/neutral/capabilities/index.mjs.map +7 -0
- package/dist/lib/{browser/chunk-NQQK7UZQ.mjs → neutral/chunk-2O3ONPRZ.mjs} +94 -40
- package/dist/lib/neutral/chunk-2O3ONPRZ.mjs.map +7 -0
- package/dist/lib/{browser/chunk-L4HW35L6.mjs → neutral/chunk-5FXU36PN.mjs} +19 -10
- package/dist/lib/neutral/chunk-5FXU36PN.mjs.map +7 -0
- package/dist/lib/neutral/chunk-GL4RS533.mjs +22 -0
- package/dist/lib/neutral/chunk-GL4RS533.mjs.map +7 -0
- package/dist/lib/{browser/index.mjs → neutral/chunk-ILSA3DU7.mjs} +32 -70
- package/dist/lib/neutral/chunk-ILSA3DU7.mjs.map +7 -0
- package/dist/lib/neutral/chunk-Y5EQAFFR.mjs +8 -0
- package/dist/lib/neutral/chunk-Y5EQAFFR.mjs.map +7 -0
- package/dist/lib/{browser/blueprints/index.mjs → neutral/chunk-ZR4BH4QQ.mjs} +4 -5
- package/dist/lib/neutral/chunk-ZR4BH4QQ.mjs.map +7 -0
- package/dist/lib/{browser → neutral}/components/index.mjs +83 -18
- package/dist/lib/neutral/components/index.mjs.map +7 -0
- package/dist/lib/neutral/containers/index.mjs +9 -0
- package/dist/lib/neutral/containers/index.mjs.map +7 -0
- package/dist/lib/{browser/create-ULR5DPJZ.mjs → neutral/create-RRWRIUSR.mjs} +4 -4
- package/dist/lib/neutral/create-RRWRIUSR.mjs.map +7 -0
- package/dist/lib/{browser → neutral}/hooks/index.mjs +5 -3
- package/dist/lib/neutral/index.mjs +44 -0
- package/dist/lib/neutral/meta.json +1 -0
- package/dist/lib/neutral/meta.mjs +8 -0
- package/dist/lib/{browser/open-SFMPLZVW.mjs → neutral/open-CVMX5KXZ.mjs} +5 -5
- package/dist/lib/neutral/open-CVMX5KXZ.mjs.map +7 -0
- package/dist/lib/neutral/operation-handler-NXMOAXFP.mjs +13 -0
- package/dist/lib/neutral/operation-handler-NXMOAXFP.mjs.map +7 -0
- package/dist/lib/neutral/operations/index.mjs +8 -0
- package/dist/lib/neutral/plugin.mjs +16 -0
- package/dist/lib/neutral/plugin.mjs.map +7 -0
- package/dist/lib/neutral/react-surface-YMZRO242.mjs +26 -0
- package/dist/lib/neutral/react-surface-YMZRO242.mjs.map +7 -0
- package/dist/lib/{browser/summarize-BDFX4APN.mjs → neutral/summarize-LA7TRW7A.mjs} +5 -5
- package/dist/lib/neutral/summarize-LA7TRW7A.mjs.map +7 -0
- package/dist/lib/neutral/text-content-F4ASMITB.mjs +30 -0
- package/dist/lib/neutral/text-content-F4ASMITB.mjs.map +7 -0
- package/dist/lib/neutral/transcriber-ABHATBVH.mjs +59 -0
- package/dist/lib/neutral/transcriber-ABHATBVH.mjs.map +7 -0
- package/dist/lib/neutral/translations.mjs +24 -0
- package/dist/lib/neutral/translations.mjs.map +7 -0
- package/dist/lib/neutral/types/index.mjs +10 -0
- package/dist/lib/neutral/types/index.mjs.map +7 -0
- package/dist/types/src/TranscriptionPlugin.d.ts.map +1 -1
- package/dist/types/src/TranscriptionPlugin.node.d.ts.map +1 -1
- package/dist/types/src/TranscriptionPlugin.workerd.d.ts +4 -0
- package/dist/types/src/TranscriptionPlugin.workerd.d.ts.map +1 -0
- package/dist/types/src/capabilities/index.d.ts +5 -1
- package/dist/types/src/capabilities/index.d.ts.map +1 -1
- package/dist/types/src/capabilities/text-content.d.ts +8 -0
- package/dist/types/src/capabilities/text-content.d.ts.map +1 -0
- package/dist/types/src/components/Oracle/Oracle.d.ts +24 -0
- package/dist/types/src/components/Oracle/Oracle.d.ts.map +1 -0
- package/dist/types/src/components/Oracle/Oracle.stories.d.ts +32 -0
- package/dist/types/src/components/Oracle/Oracle.stories.d.ts.map +1 -0
- package/dist/types/src/components/Oracle/index.d.ts +2 -0
- package/dist/types/src/components/Oracle/index.d.ts.map +1 -0
- package/dist/types/src/components/Transcription/Transcription.d.ts +0 -5
- package/dist/types/src/components/Transcription/Transcription.d.ts.map +1 -1
- package/dist/types/src/components/Transcription/Transcription.stories.d.ts +13 -16
- package/dist/types/src/components/Transcription/Transcription.stories.d.ts.map +1 -1
- package/dist/types/src/components/index.d.ts +1 -0
- package/dist/types/src/components/index.d.ts.map +1 -1
- package/dist/types/src/containers/TranscriptionArticle/TranscriptionArticle.d.ts +6 -0
- package/dist/types/src/containers/TranscriptionArticle/TranscriptionArticle.d.ts.map +1 -0
- package/dist/types/src/containers/TranscriptionArticle/TranscriptionArticle.stories.d.ts +28 -0
- package/dist/types/src/containers/TranscriptionArticle/TranscriptionArticle.stories.d.ts.map +1 -0
- package/dist/types/src/containers/TranscriptionArticle/index.d.ts +2 -0
- package/dist/types/src/containers/TranscriptionArticle/index.d.ts.map +1 -0
- package/dist/types/src/containers/index.d.ts +1 -1
- package/dist/types/src/containers/index.d.ts.map +1 -1
- package/dist/types/src/enrichment/conversation.test.d.ts +2 -0
- package/dist/types/src/enrichment/conversation.test.d.ts.map +1 -0
- package/dist/types/src/enrichment/operations.d.ts +200 -0
- package/dist/types/src/enrichment/operations.d.ts.map +1 -0
- package/dist/types/src/enrichment/parser.d.ts +26 -0
- package/dist/types/src/enrichment/parser.d.ts.map +1 -0
- package/dist/types/src/enrichment/stream-simulator.d.ts +38 -0
- package/dist/types/src/enrichment/stream-simulator.d.ts.map +1 -0
- package/dist/types/src/hooks/index.d.ts +2 -1
- package/dist/types/src/hooks/index.d.ts.map +1 -1
- package/dist/types/src/hooks/useFeedModelAdapter.d.ts +12 -0
- package/dist/types/src/hooks/useFeedModelAdapter.d.ts.map +1 -0
- package/dist/types/src/hooks/useTranscriptionRecording.d.ts +12 -0
- package/dist/types/src/hooks/useTranscriptionRecording.d.ts.map +1 -0
- package/dist/types/src/index.d.ts +3 -4
- package/dist/types/src/index.d.ts.map +1 -1
- package/dist/types/src/normalization/message-normalizer.d.ts +7 -5
- package/dist/types/src/normalization/message-normalizer.d.ts.map +1 -1
- package/dist/types/src/normalization/normalization.d.ts +180 -1
- package/dist/types/src/normalization/normalization.d.ts.map +1 -1
- package/dist/types/src/operations/create.d.ts +2 -2
- package/dist/types/src/operations/create.d.ts.map +1 -1
- package/dist/types/src/operations/index.d.ts +0 -1
- package/dist/types/src/operations/index.d.ts.map +1 -1
- package/dist/types/src/operations/open.d.ts +2 -2
- package/dist/types/src/operations/open.d.ts.map +1 -1
- package/dist/types/src/operations/summarize.d.ts +2 -2
- package/dist/types/src/operations/summarize.d.ts.map +1 -1
- package/dist/types/src/plugin.d.ts +4 -0
- package/dist/types/src/plugin.d.ts.map +1 -0
- package/dist/types/src/stories/FileTranscription.stories.d.ts +5 -3
- package/dist/types/src/stories/FileTranscription.stories.d.ts.map +1 -1
- package/dist/types/src/stories/KeyWordRecognition.stories.d.ts.map +1 -1
- package/dist/types/src/stories/{MicrophoneTranscription.stories.d.ts → LiveTranscription.stories.d.ts} +4 -4
- package/dist/types/src/stories/LiveTranscription.stories.d.ts.map +1 -0
- package/dist/types/src/stories/TranscriptionStory.d.ts +11 -13
- package/dist/types/src/stories/TranscriptionStory.d.ts.map +1 -1
- package/dist/types/src/stories/common.d.ts +46 -0
- package/dist/types/src/stories/common.d.ts.map +1 -0
- package/dist/types/src/testing/testing.d.ts +5 -6
- package/dist/types/src/testing/testing.d.ts.map +1 -1
- package/dist/types/src/transcriber/transcriber.d.ts +4 -4
- package/dist/types/src/transcriber/transcriber.d.ts.map +1 -1
- package/dist/types/src/transcriber/transcription-manager.d.ts +6 -4
- package/dist/types/src/transcriber/transcription-manager.d.ts.map +1 -1
- package/dist/types/src/translations.d.ts +2 -0
- package/dist/types/src/translations.d.ts.map +1 -1
- package/dist/types/src/{operations/definitions.d.ts → types/TranscriptOperation.d.ts} +1 -1
- package/dist/types/src/types/TranscriptOperation.d.ts.map +1 -0
- package/dist/types/src/types/TranscriptionCapabilities.d.ts +17 -0
- package/dist/types/src/types/TranscriptionCapabilities.d.ts.map +1 -0
- package/dist/types/src/types/index.d.ts +2 -2
- package/dist/types/src/types/index.d.ts.map +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +125 -94
- package/src/TranscriptionPlugin.node.ts +3 -31
- package/src/TranscriptionPlugin.test.ts +3 -3
- package/src/TranscriptionPlugin.tsx +3 -31
- package/src/TranscriptionPlugin.workerd.ts +18 -0
- package/src/blueprints/transcription-blueprint.ts +1 -1
- package/src/capabilities/index.ts +1 -0
- package/src/capabilities/react-surface.tsx +2 -2
- package/src/capabilities/text-content.ts +33 -0
- package/src/capabilities/transcriber.ts +1 -1
- package/src/components/Oracle/Oracle.stories.tsx +147 -0
- package/src/components/Oracle/Oracle.tsx +141 -0
- package/src/components/Oracle/index.ts +5 -0
- package/src/components/Transcription/Transcription.stories.tsx +116 -150
- package/src/components/Transcription/Transcription.tsx +7 -13
- package/src/components/index.ts +1 -0
- package/src/containers/TranscriptionArticle/TranscriptionArticle.stories.tsx +61 -0
- package/src/containers/TranscriptionArticle/TranscriptionArticle.tsx +62 -0
- package/src/containers/TranscriptionArticle/index.ts +5 -0
- package/src/containers/index.ts +1 -1
- package/src/enrichment/conversation.test.ts +162 -0
- package/src/enrichment/conversation.txt +23 -0
- package/src/enrichment/operations.ts +268 -0
- package/src/enrichment/parser.ts +101 -0
- package/src/enrichment/stream-simulator.ts +125 -0
- package/src/hooks/index.ts +2 -1
- package/src/hooks/useFeedModelAdapter.ts +46 -0
- package/src/hooks/useTranscriptionRecording.ts +87 -0
- package/src/index.ts +3 -9
- package/src/normalization/message-normalizer.ts +25 -13
- package/src/normalization/normalization.test.ts +3 -37
- package/src/normalization/normalization.ts +6 -7
- package/src/operations/create.ts +2 -2
- package/src/operations/index.ts +0 -2
- package/src/operations/open.ts +2 -2
- package/src/operations/summarize.ts +3 -3
- package/src/plugin.ts +11 -0
- package/src/stories/FileTranscription.stories.tsx +49 -175
- package/src/stories/KeyWordRecognition.stories.tsx +7 -9
- package/src/stories/LiveTranscription.stories.tsx +153 -0
- package/src/stories/TranscriptionStory.tsx +32 -24
- package/src/stories/common.ts +233 -0
- package/src/testing/testing.ts +20 -27
- package/src/transcriber/transcriber.test.ts +9 -9
- package/src/transcriber/transcriber.ts +18 -14
- package/src/transcriber/transcription-manager.ts +14 -9
- package/src/translations.ts +3 -0
- package/src/{operations/definitions.ts → types/TranscriptOperation.ts} +2 -0
- package/src/types/TranscriptionCapabilities.ts +39 -0
- package/src/types/index.ts +2 -2
- package/dist/lib/browser/blueprints/index.mjs.map +0 -7
- package/dist/lib/browser/chunk-3L623EVL.mjs +0 -17
- package/dist/lib/browser/chunk-3L623EVL.mjs.map +0 -7
- package/dist/lib/browser/chunk-L4HW35L6.mjs.map +0 -7
- package/dist/lib/browser/chunk-NQQK7UZQ.mjs.map +0 -7
- package/dist/lib/browser/components/index.mjs.map +0 -7
- package/dist/lib/browser/create-ULR5DPJZ.mjs.map +0 -7
- package/dist/lib/browser/index.mjs.map +0 -7
- package/dist/lib/browser/meta.json +0 -1
- package/dist/lib/browser/open-SFMPLZVW.mjs.map +0 -7
- package/dist/lib/browser/operations/index.mjs +0 -13
- package/dist/lib/browser/operations/index.mjs.map +0 -7
- package/dist/lib/browser/summarize-BDFX4APN.mjs.map +0 -7
- package/dist/lib/browser/types/index.mjs +0 -10
- package/dist/lib/node-esm/blueprints/index.mjs +0 -32
- package/dist/lib/node-esm/blueprints/index.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-7UIZWCD4.mjs +0 -112
- package/dist/lib/node-esm/chunk-7UIZWCD4.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-C3AE2NYJ.mjs +0 -181
- package/dist/lib/node-esm/chunk-C3AE2NYJ.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-HSLMI22Q.mjs +0 -11
- package/dist/lib/node-esm/chunk-KO3IOD2O.mjs +0 -19
- package/dist/lib/node-esm/chunk-KO3IOD2O.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-MQK4R6GT.mjs +0 -245
- package/dist/lib/node-esm/chunk-MQK4R6GT.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-O533NBMK.mjs +0 -37
- package/dist/lib/node-esm/chunk-O533NBMK.mjs.map +0 -7
- package/dist/lib/node-esm/components/index.mjs +0 -208
- package/dist/lib/node-esm/components/index.mjs.map +0 -7
- package/dist/lib/node-esm/create-A5QHNWRF.mjs +0 -22
- package/dist/lib/node-esm/create-A5QHNWRF.mjs.map +0 -7
- package/dist/lib/node-esm/hooks/index.mjs +0 -18
- package/dist/lib/node-esm/index.mjs +0 -474
- package/dist/lib/node-esm/index.mjs.map +0 -7
- package/dist/lib/node-esm/meta.json +0 -1
- package/dist/lib/node-esm/open-OI24QNTO.mjs +0 -28
- package/dist/lib/node-esm/open-OI24QNTO.mjs.map +0 -7
- package/dist/lib/node-esm/operations/index.mjs +0 -14
- package/dist/lib/node-esm/operations/index.mjs.map +0 -7
- package/dist/lib/node-esm/summarize-3UNN24I2.mjs +0 -79
- package/dist/lib/node-esm/summarize-3UNN24I2.mjs.map +0 -7
- package/dist/lib/node-esm/types/index.mjs +0 -11
- package/dist/types/src/containers/TranscriptionContainer/TranscriptionContainer.d.ts +0 -6
- package/dist/types/src/containers/TranscriptionContainer/TranscriptionContainer.d.ts.map +0 -1
- package/dist/types/src/containers/TranscriptionContainer/index.d.ts +0 -2
- package/dist/types/src/containers/TranscriptionContainer/index.d.ts.map +0 -1
- package/dist/types/src/hooks/useQueueModelAdapter.d.ts +0 -8
- package/dist/types/src/hooks/useQueueModelAdapter.d.ts.map +0 -1
- package/dist/types/src/operations/definitions.d.ts.map +0 -1
- package/dist/types/src/stories/MicrophoneTranscription.stories.d.ts.map +0 -1
- package/dist/types/src/stories/useIsSpeaking.d.ts +0 -3
- package/dist/types/src/stories/useIsSpeaking.d.ts.map +0 -1
- package/dist/types/src/types/capabilities.d.ts +0 -19
- package/dist/types/src/types/capabilities.d.ts.map +0 -1
- package/dist/types/src/types/defs.d.ts +0 -5
- package/dist/types/src/types/defs.d.ts.map +0 -1
- package/src/containers/TranscriptionContainer/TranscriptionContainer.tsx +0 -35
- package/src/containers/TranscriptionContainer/index.ts +0 -5
- package/src/hooks/useQueueModelAdapter.ts +0 -66
- package/src/stories/MicrophoneTranscription.stories.tsx +0 -264
- package/src/stories/useIsSpeaking.tsx +0 -40
- package/src/types/capabilities.ts +0 -39
- package/src/types/defs.ts +0 -9
- /package/dist/lib/{browser/hooks → neutral/blueprints}/index.mjs.map +0 -0
- /package/dist/lib/{browser → neutral}/chunk-J5LGTIGS.mjs +0 -0
- /package/dist/lib/{browser → neutral}/chunk-J5LGTIGS.mjs.map +0 -0
- /package/dist/lib/{browser → neutral}/chunk-KAEXRX3Y.mjs +0 -0
- /package/dist/lib/{browser → neutral}/chunk-KAEXRX3Y.mjs.map +0 -0
- /package/dist/lib/{browser → neutral}/chunk-MLRJWAQH.mjs +0 -0
- /package/dist/lib/{browser → neutral}/chunk-MLRJWAQH.mjs.map +0 -0
- /package/dist/lib/{browser/types → neutral/hooks}/index.mjs.map +0 -0
- /package/dist/lib/{node-esm/hooks → neutral}/index.mjs.map +0 -0
- /package/dist/lib/{node-esm/chunk-HSLMI22Q.mjs.map → neutral/meta.mjs.map} +0 -0
- /package/dist/lib/{node-esm/types → neutral/operations}/index.mjs.map +0 -0
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2024 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
// Helpers shared between LiveTranscription / FileTranscription stories.
|
|
6
|
+
// Each "live" story differs only in its *audio source* (mic vs file); everything else —
|
|
7
|
+
// the local in-memory message buffer, the transcriber lifecycle, and the plugin-manager
|
|
8
|
+
// decorator stack — is identical and lives here.
|
|
9
|
+
|
|
10
|
+
import { type Decorator } from '@storybook/react-vite';
|
|
11
|
+
import * as Effect from 'effect/Effect';
|
|
12
|
+
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
|
13
|
+
|
|
14
|
+
import { withPluginManager } from '@dxos/app-framework/testing';
|
|
15
|
+
import { AppActivationEvents } from '@dxos/app-toolkit';
|
|
16
|
+
import { scheduleTask } from '@dxos/async';
|
|
17
|
+
import { Context } from '@dxos/context';
|
|
18
|
+
import { Obj } from '@dxos/echo';
|
|
19
|
+
import { SpeakingMonitor } from '@dxos/plugin-calls';
|
|
20
|
+
import { ClientPlugin, initializeIdentity } from '@dxos/plugin-client/testing';
|
|
21
|
+
import { PreviewPlugin } from '@dxos/plugin-preview/testing';
|
|
22
|
+
import { StorybookPlugin, corePlugins } from '@dxos/plugin-testing';
|
|
23
|
+
import { IndexKind } from '@dxos/react-client/echo';
|
|
24
|
+
import { withLayout } from '@dxos/react-ui/testing';
|
|
25
|
+
import { Message, Organization, Person } from '@dxos/types';
|
|
26
|
+
import { seedTestData } from '@dxos/types/testing';
|
|
27
|
+
|
|
28
|
+
import { useFeedModelAdapter, useTranscriber } from '#hooks';
|
|
29
|
+
import { TestItem } from '#testing';
|
|
30
|
+
import { type TranscriptionCapabilities } from '#types';
|
|
31
|
+
|
|
32
|
+
import { type Transcriber, type TranscriberProps } from '../transcriber';
|
|
33
|
+
import { TranscriptionPlugin } from '../TranscriptionPlugin';
|
|
34
|
+
import { renderByline } from '../util';
|
|
35
|
+
|
|
36
|
+
export const useIsSpeaking = (track?: MediaStreamTrack) => {
|
|
37
|
+
const [isSpeaking, setIsSpeaking] = useState(false);
|
|
38
|
+
const speakingMonitor = useMemo(() => {
|
|
39
|
+
if (!track) {
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
return new SpeakingMonitor(track);
|
|
44
|
+
}, [track]);
|
|
45
|
+
|
|
46
|
+
useEffect(() => {
|
|
47
|
+
if (!speakingMonitor) {
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const ctx = new Context();
|
|
52
|
+
scheduleTask(ctx, async () => {
|
|
53
|
+
speakingMonitor.speakingChanged.on(ctx, () => setIsSpeaking(speakingMonitor.isSpeaking));
|
|
54
|
+
await speakingMonitor.open();
|
|
55
|
+
ctx.onDispose(() => speakingMonitor.close());
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
return () => {
|
|
59
|
+
void ctx.dispose();
|
|
60
|
+
};
|
|
61
|
+
}, [speakingMonitor]);
|
|
62
|
+
|
|
63
|
+
return isSpeaking;
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
//
|
|
67
|
+
// Message model.
|
|
68
|
+
//
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* In-memory message buffer + model adapter for storybook use (production wires up a real
|
|
72
|
+
* space-backed `Feed`). Returns the renderable `model` plus an `appendMessage` callback.
|
|
73
|
+
*/
|
|
74
|
+
export const useStoryMessageModel = () => {
|
|
75
|
+
const [messages, setMessages] = useState<Message.Message[]>([]);
|
|
76
|
+
const appendMessage = useCallback((message: Message.Message) => setMessages((prev) => [...prev, message]), []);
|
|
77
|
+
const model = useFeedModelAdapter(renderByline([]), messages);
|
|
78
|
+
return { model, appendMessage };
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Wraps blocks in a `Message.Message` with the given sender and appends it to the local buffer.
|
|
83
|
+
* Provided as a convenience for stories that don't need entity-extraction or custom enrichment.
|
|
84
|
+
*/
|
|
85
|
+
export const useStoryAppendSegments = (
|
|
86
|
+
appendMessage: (message: Message.Message) => void,
|
|
87
|
+
sender: Message.Message['sender'] = {},
|
|
88
|
+
): TranscriberProps['onSegments'] =>
|
|
89
|
+
useCallback<TranscriberProps['onSegments']>(
|
|
90
|
+
async (blocks) => {
|
|
91
|
+
appendMessage(
|
|
92
|
+
Obj.make(Message.Message, {
|
|
93
|
+
created: new Date().toISOString(),
|
|
94
|
+
sender,
|
|
95
|
+
blocks,
|
|
96
|
+
}),
|
|
97
|
+
);
|
|
98
|
+
},
|
|
99
|
+
[appendMessage, sender],
|
|
100
|
+
);
|
|
101
|
+
|
|
102
|
+
//
|
|
103
|
+
// Transcriber lifecycle.
|
|
104
|
+
//
|
|
105
|
+
|
|
106
|
+
export type UseStoryTranscriberOptions = Pick<
|
|
107
|
+
TranscriptionCapabilities.GetTranscriberProps,
|
|
108
|
+
'transcriberConfig' | 'recorderConfig' | 'onSegments'
|
|
109
|
+
> & {
|
|
110
|
+
audioStreamTrack?: MediaStreamTrack;
|
|
111
|
+
/** Whether the user has the toolbar toggle on. */
|
|
112
|
+
running: boolean;
|
|
113
|
+
/** Voice-activity flag — chunks recording only fires while `true`. */
|
|
114
|
+
isSpeaking: boolean;
|
|
115
|
+
};
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Drives the transcriber's open/close + chunk-recording lifecycle off `running`,
|
|
119
|
+
* the open flag, and an `isSpeaking` voice-activity gate. Returns the transcriber instance
|
|
120
|
+
* (`undefined` until the capability resolves and an audio track is available).
|
|
121
|
+
*/
|
|
122
|
+
export const useStoryTranscriber = ({
|
|
123
|
+
audioStreamTrack,
|
|
124
|
+
running,
|
|
125
|
+
isSpeaking,
|
|
126
|
+
transcriberConfig,
|
|
127
|
+
recorderConfig,
|
|
128
|
+
onSegments,
|
|
129
|
+
}: UseStoryTranscriberOptions): Transcriber | undefined => {
|
|
130
|
+
const transcriber = useTranscriber({
|
|
131
|
+
audioStreamTrack,
|
|
132
|
+
onSegments,
|
|
133
|
+
transcriberConfig,
|
|
134
|
+
recorderConfig,
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
// Open the transcriber when it becomes available; close on teardown.
|
|
138
|
+
// `tokenRef` is a per-effect identity so a late `close()` completion from a stale
|
|
139
|
+
// transcriber instance can't clobber `isOpen` after a newer instance has opened.
|
|
140
|
+
const [isOpen, setIsOpen] = useState(false);
|
|
141
|
+
const tokenRef = useRef<symbol | null>(null);
|
|
142
|
+
useEffect(() => {
|
|
143
|
+
if (!transcriber) {
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
const token = Symbol();
|
|
147
|
+
tokenRef.current = token;
|
|
148
|
+
void transcriber.open().then(
|
|
149
|
+
() => tokenRef.current === token && setIsOpen(true),
|
|
150
|
+
() => tokenRef.current === token && setIsOpen(false),
|
|
151
|
+
);
|
|
152
|
+
return () => {
|
|
153
|
+
void transcriber.close().then(() => {
|
|
154
|
+
if (tokenRef.current === token) {
|
|
155
|
+
setIsOpen(false);
|
|
156
|
+
}
|
|
157
|
+
});
|
|
158
|
+
};
|
|
159
|
+
}, [transcriber]);
|
|
160
|
+
|
|
161
|
+
// Start/stop chunk recording based on running + open + isSpeaking.
|
|
162
|
+
useEffect(() => {
|
|
163
|
+
if (!transcriber || !isOpen) {
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
if (running && isSpeaking) {
|
|
167
|
+
transcriber.startChunksRecording();
|
|
168
|
+
} else {
|
|
169
|
+
transcriber.stopChunksRecording();
|
|
170
|
+
}
|
|
171
|
+
}, [transcriber, isOpen, running, isSpeaking]);
|
|
172
|
+
|
|
173
|
+
return transcriber;
|
|
174
|
+
};
|
|
175
|
+
|
|
176
|
+
//
|
|
177
|
+
// Plugin manager decorator.
|
|
178
|
+
//
|
|
179
|
+
|
|
180
|
+
export type StoryDecoratorsOptions = {
|
|
181
|
+
/**
|
|
182
|
+
* Enable schema/graph/vector indexes after the client boots; needed for entity-extraction
|
|
183
|
+
* queries over seeded `Person` / `Organization` objects.
|
|
184
|
+
* @default false
|
|
185
|
+
*/
|
|
186
|
+
enableVectorIndex?: boolean;
|
|
187
|
+
};
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* Standard plugin-manager decorator for transcription stories: core plugins,
|
|
191
|
+
* `ClientPlugin` with `Person`/`Organization`/`TestItem` schemas + seeded test data, plus
|
|
192
|
+
* the `PreviewPlugin` and `TranscriptionPlugin`. Toggle `enableVectorIndex` for entity-extraction.
|
|
193
|
+
*/
|
|
194
|
+
export const createStoryDecorators = ({ enableVectorIndex = false }: StoryDecoratorsOptions = {}): Decorator[] => [
|
|
195
|
+
withLayout({ layout: 'column' }),
|
|
196
|
+
withPluginManager({
|
|
197
|
+
plugins: [
|
|
198
|
+
...corePlugins(),
|
|
199
|
+
StorybookPlugin({}),
|
|
200
|
+
ClientPlugin({
|
|
201
|
+
types: [TestItem, Person.Person, Organization.Organization],
|
|
202
|
+
onClientInitialized: ({ client }) =>
|
|
203
|
+
Effect.gen(function* () {
|
|
204
|
+
const { personalSpace } = yield* initializeIdentity(client);
|
|
205
|
+
if (enableVectorIndex) {
|
|
206
|
+
yield* enableQueryIndexes(client.services.services);
|
|
207
|
+
}
|
|
208
|
+
yield* Effect.promise(() => seedTestData(personalSpace));
|
|
209
|
+
}),
|
|
210
|
+
}),
|
|
211
|
+
PreviewPlugin(),
|
|
212
|
+
TranscriptionPlugin(),
|
|
213
|
+
],
|
|
214
|
+
fireEvents: [AppActivationEvents.SetupAppGraph],
|
|
215
|
+
}),
|
|
216
|
+
];
|
|
217
|
+
|
|
218
|
+
// TODO(mykola): Make API easier to use.
|
|
219
|
+
// TODO(mykola): Delete after enabling vector indexing by default.
|
|
220
|
+
const enableQueryIndexes = (services: { QueryService?: any }) =>
|
|
221
|
+
Effect.gen(function* () {
|
|
222
|
+
yield* Effect.promise(() =>
|
|
223
|
+
services.QueryService!.setConfig({
|
|
224
|
+
enabled: true,
|
|
225
|
+
indexes: [
|
|
226
|
+
{ kind: IndexKind.Kind.SCHEMA_MATCH },
|
|
227
|
+
{ kind: IndexKind.Kind.GRAPH },
|
|
228
|
+
{ kind: IndexKind.Kind.VECTOR },
|
|
229
|
+
],
|
|
230
|
+
}),
|
|
231
|
+
);
|
|
232
|
+
yield* Effect.promise(() => services.QueryService!.reindex());
|
|
233
|
+
});
|
package/src/testing/testing.ts
CHANGED
|
@@ -2,19 +2,19 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
+
import * as Effect from 'effect/Effect';
|
|
5
6
|
import * as Schema from 'effect/Schema';
|
|
6
7
|
import { useEffect, useMemo, useState } from 'react';
|
|
7
8
|
|
|
8
9
|
import { extractionAnthropicFunction, processTranscriptMessage } from '@dxos/assistant/extraction';
|
|
9
10
|
import { scheduleTaskInterval } from '@dxos/async';
|
|
10
|
-
import {
|
|
11
|
+
import { createFeedServiceLayer, type Space } from '@dxos/client/echo';
|
|
11
12
|
import { Context } from '@dxos/context';
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
13
|
+
import { Feed, Filter, Obj, Ref, Type } from '@dxos/echo';
|
|
14
|
+
import { runAndForwardErrors } from '@dxos/effect';
|
|
14
15
|
import { IdentityDid } from '@dxos/keys';
|
|
15
16
|
import { log } from '@dxos/log';
|
|
16
17
|
import { random } from '@dxos/random';
|
|
17
|
-
import { type Space, useQueue } from '@dxos/react-client/echo';
|
|
18
18
|
import { TestSchema } from '@dxos/schema/testing';
|
|
19
19
|
import { type ContentBlock, Message, Organization, Person } from '@dxos/types';
|
|
20
20
|
import { seedTestData } from '@dxos/types/testing';
|
|
@@ -139,71 +139,64 @@ class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
|
|
|
139
139
|
|
|
140
140
|
type UseTestTranscriptionQueue = (
|
|
141
141
|
space: Space | undefined,
|
|
142
|
-
queueId?: Key.ObjectId,
|
|
143
142
|
running?: boolean,
|
|
144
143
|
interval?: number,
|
|
145
|
-
) =>
|
|
144
|
+
) => Feed.Feed | undefined;
|
|
146
145
|
|
|
147
146
|
/**
|
|
148
|
-
* Test
|
|
147
|
+
* Test transcription feed.
|
|
149
148
|
*/
|
|
150
149
|
export const useTestTranscriptionQueue: UseTestTranscriptionQueue = (
|
|
151
150
|
space: Space | undefined,
|
|
152
|
-
queueId?: Key.ObjectId,
|
|
153
151
|
running = true,
|
|
154
152
|
interval = 1_000,
|
|
155
153
|
) => {
|
|
156
|
-
|
|
157
|
-
const queueDxn = useMemo(() => (space ? createQueueDXN(space.id, queueId) : undefined), [space, queueId]);
|
|
158
|
-
const queue = useQueue<Message.Message>(queueDxn);
|
|
154
|
+
const feed = useMemo(() => (space ? space.db.add(Feed.make({ name: 'transcription' })) : undefined), [space]);
|
|
159
155
|
const builder = useMemo(() => new MessageBuilder(space), [space]);
|
|
160
156
|
|
|
161
157
|
useEffect(() => {
|
|
162
|
-
if (!
|
|
158
|
+
if (!space || !feed || !running) {
|
|
163
159
|
return;
|
|
164
160
|
}
|
|
161
|
+
const feedServiceLayer = createFeedServiceLayer(space.queues);
|
|
165
162
|
|
|
166
163
|
const i = setInterval(() => {
|
|
167
164
|
void builder.createMessage(Math.ceil(Math.random() * 3)).then(async (message) => {
|
|
168
|
-
await
|
|
165
|
+
await Feed.append(feed, [message]).pipe(Effect.provide(feedServiceLayer), runAndForwardErrors);
|
|
169
166
|
});
|
|
170
167
|
}, interval);
|
|
171
168
|
return () => clearInterval(i);
|
|
172
|
-
}, [
|
|
169
|
+
}, [space, feed, running, interval]);
|
|
173
170
|
|
|
174
|
-
return
|
|
171
|
+
return feed;
|
|
175
172
|
};
|
|
176
173
|
|
|
177
174
|
/**
|
|
178
|
-
* Test transcription
|
|
175
|
+
* Test transcription feed with entity extraction.
|
|
179
176
|
*/
|
|
180
177
|
// TODO(burdon): Reconcile with useTestTranscriptionQueue.
|
|
181
178
|
export const useTestTranscriptionQueueWithEntityExtraction: UseTestTranscriptionQueue = (
|
|
182
179
|
space: Space | undefined,
|
|
183
|
-
queueId?: Key.ObjectId,
|
|
184
180
|
running = true,
|
|
185
181
|
interval = 1_000,
|
|
186
182
|
) => {
|
|
187
|
-
|
|
188
|
-
const queueDxn = useMemo(() => (space ? createQueueDXN(space.id, queueId) : undefined), [space, queueId]);
|
|
189
|
-
const queue = useQueue<Message.Message>(queueDxn);
|
|
183
|
+
const feed = useMemo(() => (space ? space.db.add(Feed.make({ name: 'transcription' })) : undefined), [space]);
|
|
190
184
|
const [builder] = useState(() => new EntityExtractionMessageBuilder());
|
|
191
185
|
|
|
192
186
|
useEffect(() => {
|
|
193
|
-
if (!
|
|
187
|
+
if (!space || !feed || !running) {
|
|
194
188
|
return;
|
|
195
189
|
}
|
|
196
190
|
|
|
197
|
-
|
|
198
|
-
void builder.connect(space);
|
|
199
|
-
}
|
|
191
|
+
void builder.connect(space);
|
|
200
192
|
|
|
193
|
+
const feedServiceLayer = createFeedServiceLayer(space.queues);
|
|
201
194
|
const ctx = new Context();
|
|
202
195
|
scheduleTaskInterval(
|
|
203
196
|
ctx,
|
|
204
197
|
async () => {
|
|
205
198
|
const message = await builder.createMessage();
|
|
206
|
-
|
|
199
|
+
await Feed.append(feed, [message]).pipe(Effect.provide(feedServiceLayer), runAndForwardErrors);
|
|
207
200
|
},
|
|
208
201
|
interval,
|
|
209
202
|
);
|
|
@@ -211,7 +204,7 @@ export const useTestTranscriptionQueueWithEntityExtraction: UseTestTranscription
|
|
|
211
204
|
return () => {
|
|
212
205
|
void ctx.dispose();
|
|
213
206
|
};
|
|
214
|
-
}, [space,
|
|
207
|
+
}, [space, feed, running, interval]);
|
|
215
208
|
|
|
216
|
-
return
|
|
209
|
+
return feed;
|
|
217
210
|
};
|
|
@@ -31,10 +31,10 @@ describe('alignWhisperSegments', () => {
|
|
|
31
31
|
0,
|
|
32
32
|
);
|
|
33
33
|
expect(result.transcripts).toEqual([]);
|
|
34
|
-
expect(result.
|
|
34
|
+
expect(result.lastTimestamp).toBeUndefined();
|
|
35
35
|
});
|
|
36
36
|
|
|
37
|
-
test('absolute-timestamps and deduplicates segments against
|
|
37
|
+
test('absolute-timestamps and deduplicates segments against lastTimestamp', ({ expect }) => {
|
|
38
38
|
const segments: WhisperSegment[] = [
|
|
39
39
|
segment({
|
|
40
40
|
text: 'hello world',
|
|
@@ -52,15 +52,15 @@ describe('alignWhisperSegments', () => {
|
|
|
52
52
|
expect(first.transcripts[0]._tag).toBe('transcript');
|
|
53
53
|
expect(first.transcripts[0].text).toBe('hello world');
|
|
54
54
|
expect(new Date(first.transcripts[0].started).getTime()).toBe(T0);
|
|
55
|
-
expect(first.
|
|
55
|
+
expect(first.lastTimestamp).toBe(T0 + 1_000);
|
|
56
56
|
|
|
57
|
-
// Same segments, same chunk start: every word's absolute start is <
|
|
58
|
-
const second = alignWhisperSegments(segments, [chunk(0)], first.
|
|
57
|
+
// Same segments, same chunk start: every word's absolute start is < lastTimestamp -> filtered out.
|
|
58
|
+
const second = alignWhisperSegments(segments, [chunk(0)], first.lastTimestamp!);
|
|
59
59
|
expect(second.transcripts).toEqual([]);
|
|
60
|
-
expect(second.
|
|
60
|
+
expect(second.lastTimestamp).toBeUndefined();
|
|
61
61
|
});
|
|
62
62
|
|
|
63
|
-
test('keeps later words when earlier words fall before
|
|
63
|
+
test('keeps later words when earlier words fall before lastTimestamp', ({ expect }) => {
|
|
64
64
|
const segments: WhisperSegment[] = [
|
|
65
65
|
segment({
|
|
66
66
|
text: 'old new',
|
|
@@ -78,10 +78,10 @@ describe('alignWhisperSegments', () => {
|
|
|
78
78
|
expect(result.transcripts).toHaveLength(1);
|
|
79
79
|
expect(result.transcripts[0].text).toBe('new');
|
|
80
80
|
expect(new Date(result.transcripts[0].started).getTime()).toBe(T0 + 1_500);
|
|
81
|
-
expect(result.
|
|
81
|
+
expect(result.lastTimestamp).toBe(T0 + 2_000);
|
|
82
82
|
});
|
|
83
83
|
|
|
84
|
-
test('drops segments that end before
|
|
84
|
+
test('drops segments that end before lastTimestamp', ({ expect }) => {
|
|
85
85
|
const segments: WhisperSegment[] = [
|
|
86
86
|
segment({
|
|
87
87
|
text: 'stale',
|
|
@@ -10,11 +10,14 @@ import { log } from '@dxos/log';
|
|
|
10
10
|
import { trace } from '@dxos/tracing';
|
|
11
11
|
import { type ContentBlock } from '@dxos/types';
|
|
12
12
|
|
|
13
|
-
import { TRANSCRIPTION_URL } from '#types';
|
|
14
|
-
|
|
15
13
|
import { mergeFloat64Arrays } from '../util';
|
|
16
14
|
import { type AudioChunk, type AudioRecorder } from './audio-recorder';
|
|
17
15
|
|
|
16
|
+
/**
|
|
17
|
+
* Endpoint to the calls service.
|
|
18
|
+
*/
|
|
19
|
+
const DEFAULT_TRANSCRIPTION_URL = 'https://calls-service.dxos.workers.dev';
|
|
20
|
+
|
|
18
21
|
export type WhisperWord = {
|
|
19
22
|
word: string;
|
|
20
23
|
|
|
@@ -96,7 +99,7 @@ export type TranscriberProps = {
|
|
|
96
99
|
*/
|
|
97
100
|
export class Transcriber extends Resource {
|
|
98
101
|
private _audioChunks: AudioChunk[] = [];
|
|
99
|
-
private
|
|
102
|
+
private _lastTimestamp = 0;
|
|
100
103
|
|
|
101
104
|
private readonly _openTrigger = new Trigger({ autoReset: true });
|
|
102
105
|
|
|
@@ -217,7 +220,8 @@ export class Transcriber extends Resource {
|
|
|
217
220
|
if (this._transcribeFn) {
|
|
218
221
|
segments = await this._transcribeFn(audio);
|
|
219
222
|
} else {
|
|
220
|
-
|
|
223
|
+
// TODO(burdon): Create separate endpoint?
|
|
224
|
+
const endpoint = this._config.endpoint ?? DEFAULT_TRANSCRIPTION_URL;
|
|
221
225
|
const response = await fetch(`${endpoint}/transcribe`, {
|
|
222
226
|
method: 'POST',
|
|
223
227
|
body: JSON.stringify({ audio }),
|
|
@@ -238,15 +242,15 @@ export class Transcriber extends Resource {
|
|
|
238
242
|
}
|
|
239
243
|
|
|
240
244
|
log.info('transcription response', { segments: segments.length });
|
|
241
|
-
|
|
242
245
|
return segments as WhisperSegment[];
|
|
243
246
|
}
|
|
244
247
|
|
|
245
248
|
private _alignSegments(segments: WhisperSegment[], originalChunks: AudioChunk[]): ContentBlock.Transcript[] {
|
|
246
|
-
const result = alignWhisperSegments(segments, originalChunks, this.
|
|
247
|
-
if (result.
|
|
248
|
-
this.
|
|
249
|
+
const result = alignWhisperSegments(segments, originalChunks, this._lastTimestamp);
|
|
250
|
+
if (result.lastTimestamp !== undefined) {
|
|
251
|
+
this._lastTimestamp = result.lastTimestamp;
|
|
249
252
|
}
|
|
253
|
+
|
|
250
254
|
return result.transcripts;
|
|
251
255
|
}
|
|
252
256
|
}
|
|
@@ -257,14 +261,14 @@ export class Transcriber extends Resource {
|
|
|
257
261
|
*
|
|
258
262
|
* @param segments - Raw segments returned by the Whisper service for `originalChunks`.
|
|
259
263
|
* @param originalChunks - Chunks fed into the transcription request (used for the zero timestamp).
|
|
260
|
-
* @param
|
|
264
|
+
* @param lastTimestamp - Most recent absolute timestamp already emitted; segments whose end
|
|
261
265
|
* falls strictly before this point are deduped out, as are individual words whose start does.
|
|
262
266
|
*/
|
|
263
267
|
export const alignWhisperSegments = (
|
|
264
268
|
segments: WhisperSegment[],
|
|
265
269
|
originalChunks: AudioChunk[],
|
|
266
|
-
|
|
267
|
-
): { transcripts: ContentBlock.Transcript[];
|
|
270
|
+
lastTimestamp: number,
|
|
271
|
+
): { transcripts: ContentBlock.Transcript[]; lastTimestamp?: number } => {
|
|
268
272
|
if (originalChunks.length === 0) {
|
|
269
273
|
return { transcripts: [] };
|
|
270
274
|
}
|
|
@@ -272,9 +276,9 @@ export const alignWhisperSegments = (
|
|
|
272
276
|
const zeroTimestamp = originalChunks[0].timestamp;
|
|
273
277
|
|
|
274
278
|
const filteredSegments = segments
|
|
275
|
-
.filter((segment) => zeroTimestamp + segment.end * 1_000 >=
|
|
279
|
+
.filter((segment) => zeroTimestamp + segment.end * 1_000 >= lastTimestamp)
|
|
276
280
|
.map((segment) => {
|
|
277
|
-
const words = segment.words.filter((word) => zeroTimestamp + word.start * 1_000 >=
|
|
281
|
+
const words = segment.words.filter((word) => zeroTimestamp + word.start * 1_000 >= lastTimestamp);
|
|
278
282
|
return {
|
|
279
283
|
...segment,
|
|
280
284
|
words,
|
|
@@ -289,11 +293,11 @@ export const alignWhisperSegments = (
|
|
|
289
293
|
}
|
|
290
294
|
|
|
291
295
|
return {
|
|
296
|
+
lastTimestamp: zeroTimestamp + filteredSegments.at(-1)!.end * 1_000,
|
|
292
297
|
transcripts: filteredSegments.map((segment) => ({
|
|
293
298
|
_tag: 'transcript',
|
|
294
299
|
started: new Date(zeroTimestamp + segment.start * 1_000).toISOString(),
|
|
295
300
|
text: segment.text.trim(),
|
|
296
301
|
})),
|
|
297
|
-
lastUsedTimestamp: zeroTimestamp + filteredSegments.at(-1)!.end * 1_000,
|
|
298
302
|
};
|
|
299
303
|
};
|
|
@@ -3,11 +3,14 @@
|
|
|
3
3
|
//
|
|
4
4
|
|
|
5
5
|
import { Atom, type Registry } from '@effect-atom/atom-react';
|
|
6
|
+
import * as Effect from 'effect/Effect';
|
|
7
|
+
import * as Layer from 'effect/Layer';
|
|
6
8
|
|
|
7
9
|
import { synchronized } from '@dxos/async';
|
|
10
|
+
import { createFeedServiceLayer, type Space } from '@dxos/client/echo';
|
|
8
11
|
import { Resource } from '@dxos/context';
|
|
9
|
-
import { Obj } from '@dxos/echo';
|
|
10
|
-
import {
|
|
12
|
+
import { Feed, Obj } from '@dxos/echo';
|
|
13
|
+
import { runAndForwardErrors } from '@dxos/effect';
|
|
11
14
|
import { type EdgeHttpClient } from '@dxos/react-edge-client';
|
|
12
15
|
import { type ContentBlock, Message } from '@dxos/types';
|
|
13
16
|
|
|
@@ -37,7 +40,7 @@ export type TranscriptionManagerOptions = {
|
|
|
37
40
|
registry: Registry.Registry;
|
|
38
41
|
|
|
39
42
|
/**
|
|
40
|
-
* Enrich the message before it is written to the transcription
|
|
43
|
+
* Enrich the message before it is written to the transcription feed.
|
|
41
44
|
*/
|
|
42
45
|
messageEnricher?: TranscriptMessageEnricher;
|
|
43
46
|
};
|
|
@@ -53,7 +56,8 @@ export class TranscriptionManager extends Resource {
|
|
|
53
56
|
private _identityDid?: string = undefined;
|
|
54
57
|
private _mediaRecorder?: MediaStreamRecorder = undefined;
|
|
55
58
|
private _transcriber?: Transcriber = undefined;
|
|
56
|
-
private
|
|
59
|
+
private _feed?: Feed.Feed = undefined;
|
|
60
|
+
private _feedServiceLayer?: Layer.Layer<Feed.FeedService> = undefined;
|
|
57
61
|
private _enabledAtom = Atom.make(false);
|
|
58
62
|
|
|
59
63
|
constructor(options: TranscriptionManagerOptions) {
|
|
@@ -71,8 +75,9 @@ export class TranscriptionManager extends Resource {
|
|
|
71
75
|
return this._registry.get(this._enabledAtom);
|
|
72
76
|
}
|
|
73
77
|
|
|
74
|
-
|
|
75
|
-
this.
|
|
78
|
+
setFeed(space: Space, feed: Feed.Feed): this {
|
|
79
|
+
this._feed = feed;
|
|
80
|
+
this._feedServiceLayer = createFeedServiceLayer(space.queues);
|
|
76
81
|
return this;
|
|
77
82
|
}
|
|
78
83
|
|
|
@@ -132,7 +137,7 @@ export class TranscriptionManager extends Resource {
|
|
|
132
137
|
return;
|
|
133
138
|
}
|
|
134
139
|
|
|
135
|
-
// Reinitialize transcriber if
|
|
140
|
+
// Reinitialize transcriber if feed or media stream track has changed.
|
|
136
141
|
let needReinit = false;
|
|
137
142
|
if (this._audioStreamTrack !== this._mediaRecorder?.mediaStreamTrack) {
|
|
138
143
|
this._mediaRecorder = new MediaStreamRecorder({
|
|
@@ -161,7 +166,7 @@ export class TranscriptionManager extends Resource {
|
|
|
161
166
|
}
|
|
162
167
|
|
|
163
168
|
private async _onSegments(segments: ContentBlock.Transcript[]): Promise<void> {
|
|
164
|
-
if (!this.isOpen || !this.
|
|
169
|
+
if (!this.isOpen || !this._feed || !this._feedServiceLayer) {
|
|
165
170
|
return;
|
|
166
171
|
}
|
|
167
172
|
|
|
@@ -175,6 +180,6 @@ export class TranscriptionManager extends Resource {
|
|
|
175
180
|
block = await this._messageEnricher(block);
|
|
176
181
|
}
|
|
177
182
|
|
|
178
|
-
await
|
|
183
|
+
await Feed.append(this._feed, [block]).pipe(Effect.provide(this._feedServiceLayer), runAndForwardErrors);
|
|
179
184
|
}
|
|
180
185
|
}
|
package/src/translations.ts
CHANGED
|
@@ -17,6 +17,9 @@ export const translations = [
|
|
|
17
17
|
'bookmark.button': 'Bookmark',
|
|
18
18
|
'scroll-to-end.label': 'Scroll to latest',
|
|
19
19
|
|
|
20
|
+
'start-recording.label': 'Start recording',
|
|
21
|
+
'stop-recording.label': 'Stop recording',
|
|
22
|
+
|
|
20
23
|
'settings.entity-extraction.label': 'Entity extraction',
|
|
21
24
|
},
|
|
22
25
|
},
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2025 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
// @import-as-namespace
|
|
6
|
+
|
|
7
|
+
import { Capability } from '@dxos/app-framework';
|
|
8
|
+
|
|
9
|
+
import { meta } from '#meta';
|
|
10
|
+
|
|
11
|
+
import {
|
|
12
|
+
type MediaStreamRecorderProps,
|
|
13
|
+
type Transcriber as TranscriberType,
|
|
14
|
+
type TranscriberProps,
|
|
15
|
+
type TranscriptMessageEnricher,
|
|
16
|
+
type TranscriptionManager as TranscriptionManagerType,
|
|
17
|
+
} from '../transcriber';
|
|
18
|
+
|
|
19
|
+
export type GetTranscriberProps = {
|
|
20
|
+
audioStreamTrack: MediaStreamTrack;
|
|
21
|
+
recorderConfig?: Partial<MediaStreamRecorderProps['config']>;
|
|
22
|
+
transcriberConfig?: Partial<TranscriberProps['config']>;
|
|
23
|
+
onSegments: TranscriberProps['onSegments'];
|
|
24
|
+
transcribe?: TranscriberProps['transcribe'];
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
export type GetTranscriber = (props: GetTranscriberProps) => TranscriberType;
|
|
28
|
+
|
|
29
|
+
export type GetTranscriptionManagerProps = {
|
|
30
|
+
messageEnricher?: TranscriptMessageEnricher;
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
export type GetTranscriptionManager = (props: GetTranscriptionManagerProps) => TranscriptionManagerType;
|
|
34
|
+
|
|
35
|
+
export const Transcriber = Capability.make<GetTranscriber>(`${meta.id}.capability.transcriber`);
|
|
36
|
+
|
|
37
|
+
export const TranscriptionManager = Capability.make<GetTranscriptionManager>(
|
|
38
|
+
`${meta.id}.capability.transcription-manager`,
|
|
39
|
+
);
|
package/src/types/index.ts
CHANGED
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"version": 3,
|
|
3
|
-
"sources": ["../../../../src/blueprints/transcription-blueprint.ts"],
|
|
4
|
-
"sourcesContent": ["//\n// Copyright 2025 DXOS.org\n//\n\nimport { Blueprint, Template } from '@dxos/compute';\nimport { trim } from '@dxos/util';\n\nimport { TranscriptOperation } from '#operations';\n\nconst BLUEPRINT_KEY = 'org.dxos.blueprint.transcription';\n\nconst make = () =>\n Blueprint.make({\n key: BLUEPRINT_KEY,\n name: 'Transcription',\n tools: Blueprint.toolDefinitions({ operations: [TranscriptOperation.Open, TranscriptOperation.Summarize] }),\n instructions: Template.make({\n source: trim`\n You can open and summarize a meeting transcript.\n `,\n }),\n });\n\nconst blueprint: Blueprint.Definition = {\n key: BLUEPRINT_KEY,\n make,\n};\n\nexport default blueprint;\n"],
|
|
5
|
-
"mappings": ";;;AAIA,SAASA,WAAWC,gBAAgB;AACpC,SAASC,YAAY;AAErB,SAASC,2BAA2B;AAEpC,IAAMC,gBAAgB;AAEtB,IAAMC,OAAO,MACXL,UAAUK,KAAK;EACbC,KAAKF;EACLG,MAAM;EACNC,OAAOR,UAAUS,gBAAgB;IAAEC,YAAY;MAACP,oBAAoBQ;MAAMR,oBAAoBS;;EAAW,CAAA;EACzGC,cAAcZ,SAASI,KAAK;IAC1BS,QAAQZ;;;EAGV,CAAA;AACF,CAAA;AAEF,IAAMa,YAAkC;EACtCT,KAAKF;EACLC;AACF;AAEA,IAAA,kCAAeU;",
|
|
6
|
-
"names": ["Blueprint", "Template", "trim", "TranscriptOperation", "BLUEPRINT_KEY", "make", "key", "name", "tools", "toolDefinitions", "operations", "Open", "Summarize", "instructions", "source", "blueprint"]
|
|
7
|
-
}
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
// src/types/capabilities.ts
|
|
2
|
-
import { Capability } from "@dxos/app-framework";
|
|
3
|
-
import { meta } from "#meta";
|
|
4
|
-
(function(TranscriptionCapabilities2) {
|
|
5
|
-
TranscriptionCapabilities2.Transcriber = Capability.make(`${meta.id}.capability.transcriber`);
|
|
6
|
-
TranscriptionCapabilities2.TranscriptionManager = Capability.make(`${meta.id}.capability.transcription-manager`);
|
|
7
|
-
})(TranscriptionCapabilities || (TranscriptionCapabilities = {}));
|
|
8
|
-
var TranscriptionCapabilities;
|
|
9
|
-
|
|
10
|
-
// src/types/defs.ts
|
|
11
|
-
var TRANSCRIPTION_URL = "https://calls-service.dxos.workers.dev";
|
|
12
|
-
|
|
13
|
-
export {
|
|
14
|
-
TranscriptionCapabilities,
|
|
15
|
-
TRANSCRIPTION_URL
|
|
16
|
-
};
|
|
17
|
-
//# sourceMappingURL=chunk-3L623EVL.mjs.map
|