@dxos/plugin-transcription 0.8.2-main.fbd8ed0 → 0.8.2-staging.7ac8446
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/{TranscriptContainer-CXA4ZBHW.mjs → TranscriptContainer-JQ3HFHLQ.mjs} +2 -3
- package/dist/lib/browser/app-graph-builder-VDQOVX7P.mjs +191 -0
- package/dist/lib/browser/app-graph-builder-VDQOVX7P.mjs.map +7 -0
- package/dist/lib/browser/chunk-G4B6CPC2.mjs +253 -0
- package/dist/lib/browser/chunk-G4B6CPC2.mjs.map +7 -0
- package/dist/lib/browser/{chunk-KLD7O5GJ.mjs → chunk-GU2BNYKZ.mjs} +12 -2
- package/dist/lib/browser/chunk-GU2BNYKZ.mjs.map +7 -0
- package/dist/lib/browser/chunk-K7FOYAXO.mjs +8 -0
- package/dist/lib/browser/chunk-K7FOYAXO.mjs.map +7 -0
- package/dist/lib/browser/chunk-OGNXNGJB.mjs +73 -0
- package/dist/lib/browser/chunk-OGNXNGJB.mjs.map +7 -0
- package/dist/lib/browser/{chunk-3TKXI6TQ.mjs → chunk-YCZISX63.mjs} +81 -108
- package/dist/lib/browser/chunk-YCZISX63.mjs.map +7 -0
- package/dist/lib/browser/{chunk-LTIQFX4W.mjs → chunk-YEPT334A.mjs} +1 -1
- package/dist/lib/browser/chunk-YEPT334A.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +210 -47
- package/dist/lib/browser/index.mjs.map +4 -4
- package/dist/lib/browser/{intent-resolver-DZ62CJPC.mjs → intent-resolver-4R66ZCKT.mjs} +8 -8
- package/dist/lib/browser/intent-resolver-4R66ZCKT.mjs.map +7 -0
- package/dist/lib/browser/{meeting-transcription-state-MNA2XROE.mjs → meeting-transcription-state-DLFHQK72.mjs} +4 -4
- package/dist/lib/browser/meeting-transcription-state-DLFHQK72.mjs.map +7 -0
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/{react-surface-PUXZNAZL.mjs → react-surface-FQUHU5G7.mjs} +6 -18
- package/dist/lib/browser/react-surface-FQUHU5G7.mjs.map +7 -0
- package/dist/lib/browser/{transcriber-YWWIV6KP.mjs → transcriber-LGMIWDYI.mjs} +7 -8
- package/dist/lib/browser/{transcriber-YWWIV6KP.mjs.map → transcriber-LGMIWDYI.mjs.map} +3 -3
- package/dist/lib/browser/types/index.mjs +3 -3
- package/dist/lib/node/{TranscriptContainer-SSJVLRM6.cjs → TranscriptContainer-QEYTTT2V.cjs} +7 -8
- package/dist/lib/node/TranscriptContainer-QEYTTT2V.cjs.map +7 -0
- package/dist/lib/node/app-graph-builder-IFSIVNUR.cjs +200 -0
- package/dist/lib/node/app-graph-builder-IFSIVNUR.cjs.map +7 -0
- package/dist/lib/node/{chunk-J72PUQ3C.cjs → chunk-3KHUSQNW.cjs} +15 -6
- package/dist/lib/node/chunk-3KHUSQNW.cjs.map +7 -0
- package/dist/lib/node/{chunk-JUGOXWES.cjs → chunk-CZMLKKWT.cjs} +4 -4
- package/dist/lib/node/chunk-CZMLKKWT.cjs.map +7 -0
- package/dist/lib/node/{chunk-WKZLGVEJ.cjs → chunk-FCCJALOT.cjs} +83 -111
- package/dist/lib/node/chunk-FCCJALOT.cjs.map +7 -0
- package/dist/lib/node/{chunk-L27AK6FQ.cjs → chunk-LP4FG423.cjs} +8 -23
- package/dist/lib/node/chunk-LP4FG423.cjs.map +7 -0
- package/dist/lib/node/{chunk-EJWB53WK.cjs → chunk-P4VVD6ZO.cjs} +34 -24
- package/dist/lib/node/chunk-P4VVD6ZO.cjs.map +7 -0
- package/dist/lib/node/chunk-UOFHT4GZ.cjs +280 -0
- package/dist/lib/node/chunk-UOFHT4GZ.cjs.map +7 -0
- package/dist/lib/node/index.cjs +238 -66
- package/dist/lib/node/index.cjs.map +4 -4
- package/dist/lib/node/{intent-resolver-TRBKI24D.cjs → intent-resolver-WMMCOUEH.cjs} +10 -11
- package/dist/lib/node/intent-resolver-WMMCOUEH.cjs.map +7 -0
- package/dist/lib/node/{meeting-transcription-state-5XLTMWPA.cjs → meeting-transcription-state-URIYILEQ.cjs} +7 -7
- package/dist/lib/node/meeting-transcription-state-URIYILEQ.cjs.map +7 -0
- package/dist/lib/node/meta.json +1 -1
- package/dist/lib/node/{react-surface-SZAMHCX7.cjs → react-surface-WI2MGWCC.cjs} +10 -20
- package/dist/lib/node/react-surface-WI2MGWCC.cjs.map +7 -0
- package/dist/lib/node/{transcriber-AFUXFHCR.cjs → transcriber-QG5WPKX4.cjs} +13 -14
- package/dist/lib/node/{transcriber-AFUXFHCR.cjs.map → transcriber-QG5WPKX4.cjs.map} +3 -3
- package/dist/lib/node/types/index.cjs +8 -8
- package/dist/lib/node/types/index.cjs.map +1 -1
- package/dist/lib/node-esm/{TranscriptContainer-M53LI2TX.mjs → TranscriptContainer-ZXUCOWGY.mjs} +2 -3
- package/dist/lib/node-esm/app-graph-builder-UL6XLIZY.mjs +192 -0
- package/dist/lib/node-esm/app-graph-builder-UL6XLIZY.mjs.map +7 -0
- package/dist/lib/node-esm/chunk-6X7X2UZF.mjs +254 -0
- package/dist/lib/node-esm/chunk-6X7X2UZF.mjs.map +7 -0
- package/dist/lib/node-esm/chunk-GIO6J6FE.mjs +74 -0
- package/dist/lib/node-esm/chunk-GIO6J6FE.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-QMHMEKB3.mjs → chunk-HKATMD34.mjs} +12 -2
- package/dist/lib/node-esm/chunk-HKATMD34.mjs.map +7 -0
- package/dist/lib/node-esm/chunk-J4D2RBBH.mjs +10 -0
- package/dist/lib/node-esm/chunk-J4D2RBBH.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-5E2MR5XK.mjs → chunk-PTZHDZ36.mjs} +1 -1
- package/dist/lib/node-esm/chunk-PTZHDZ36.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-V5IWBPYO.mjs → chunk-YQYBLFNI.mjs} +81 -108
- package/dist/lib/node-esm/chunk-YQYBLFNI.mjs.map +7 -0
- package/dist/lib/node-esm/index.mjs +210 -47
- package/dist/lib/node-esm/index.mjs.map +4 -4
- package/dist/lib/node-esm/{intent-resolver-HFVAL274.mjs → intent-resolver-QN2LO5TX.mjs} +8 -8
- package/dist/lib/node-esm/intent-resolver-QN2LO5TX.mjs.map +7 -0
- package/dist/lib/node-esm/{meeting-transcription-state-3WOZ52BQ.mjs → meeting-transcription-state-ZYVRAW6Y.mjs} +4 -4
- package/dist/lib/node-esm/meeting-transcription-state-ZYVRAW6Y.mjs.map +7 -0
- package/dist/lib/node-esm/meta.json +1 -1
- package/dist/lib/node-esm/{react-surface-UEJELL7I.mjs → react-surface-PCH6NTVB.mjs} +6 -18
- package/dist/lib/node-esm/react-surface-PCH6NTVB.mjs.map +7 -0
- package/dist/lib/node-esm/{transcriber-7RJEFPDI.mjs → transcriber-TMPYZ5L4.mjs} +7 -8
- package/dist/lib/node-esm/{transcriber-7RJEFPDI.mjs.map → transcriber-TMPYZ5L4.mjs.map} +3 -3
- package/dist/lib/node-esm/types/index.mjs +3 -3
- package/dist/types/src/TranscriptionPlugin.d.ts.map +1 -1
- package/dist/types/src/capabilities/app-graph-builder.d.ts +179 -2
- package/dist/types/src/capabilities/app-graph-builder.d.ts.map +1 -1
- package/dist/types/src/capabilities/capabilities.d.ts +7 -5
- package/dist/types/src/capabilities/capabilities.d.ts.map +1 -1
- package/dist/types/src/capabilities/index.d.ts +182 -4
- package/dist/types/src/capabilities/index.d.ts.map +1 -1
- package/dist/types/src/capabilities/intent-resolver.d.ts +2 -2
- package/dist/types/src/capabilities/intent-resolver.d.ts.map +1 -1
- package/dist/types/src/capabilities/meeting-transcription-state.d.ts +4 -2
- package/dist/types/src/capabilities/meeting-transcription-state.d.ts.map +1 -1
- package/dist/types/src/capabilities/react-surface.d.ts.map +1 -1
- package/dist/types/src/capabilities/transcriber.d.ts.map +1 -1
- package/dist/types/src/components/Transcript/Transcript.d.ts +4 -14
- package/dist/types/src/components/Transcript/Transcript.d.ts.map +1 -1
- package/dist/types/src/components/Transcript/Transcript.stories.d.ts +4 -19
- package/dist/types/src/components/Transcript/Transcript.stories.d.ts.map +1 -1
- package/dist/types/src/components/TranscriptContainer.d.ts.map +1 -1
- package/dist/types/src/components/index.d.ts +0 -1
- package/dist/types/src/components/index.d.ts.map +1 -1
- package/dist/types/src/components/transcription.stories.d.ts +3 -6
- package/dist/types/src/components/transcription.stories.d.ts.map +1 -1
- package/dist/types/src/hooks/index.d.ts +0 -1
- package/dist/types/src/hooks/index.d.ts.map +1 -1
- package/dist/types/src/hooks/useAudioFile.d.ts.map +1 -1
- package/dist/types/src/hooks/useAudioTrack.d.ts.map +1 -1
- package/dist/types/src/hooks/useTranscriber.d.ts +1 -1
- package/dist/types/src/hooks/useTranscriber.d.ts.map +1 -1
- package/dist/types/src/hooks/useVoiceInput.d.ts.map +1 -1
- package/dist/types/src/transcriber/transcriber.d.ts +2 -2
- package/dist/types/src/transcriber/transcriber.d.ts.map +1 -1
- package/dist/types/src/transcriber/transcriber.test.d.ts +2 -0
- package/dist/types/src/transcriber/transcriber.test.d.ts.map +1 -0
- package/dist/types/src/transcriber/transcription-manager.d.ts +15 -17
- package/dist/types/src/transcriber/transcription-manager.d.ts.map +1 -1
- package/dist/types/src/translations.d.ts +18 -3
- package/dist/types/src/translations.d.ts.map +1 -1
- package/dist/types/src/types/schema.d.ts +40 -26
- package/dist/types/src/types/schema.d.ts.map +1 -1
- package/dist/types/src/types/types.d.ts +7 -18
- package/dist/types/src/types/types.d.ts.map +1 -1
- package/dist/types/src/util/array.d.ts.map +1 -1
- package/dist/types/src/util/get-time-str.d.ts.map +1 -1
- package/dist/types/src/util/index.d.ts +1 -0
- package/dist/types/src/util/index.d.ts.map +1 -1
- package/dist/types/src/util/monitor-audio-level.d.ts.map +1 -1
- package/dist/types/src/util/random-queue-dxn.d.ts +3 -0
- package/dist/types/src/util/random-queue-dxn.d.ts.map +1 -0
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +40 -57
- package/src/TranscriptionPlugin.tsx +13 -30
- package/src/capabilities/app-graph-builder.ts +111 -192
- package/src/capabilities/capabilities.ts +3 -4
- package/src/capabilities/index.ts +0 -1
- package/src/capabilities/intent-resolver.ts +7 -8
- package/src/capabilities/meeting-transcription-state.ts +2 -2
- package/src/capabilities/react-surface.tsx +4 -12
- package/src/capabilities/transcriber.ts +1 -2
- package/src/components/Transcript/Transcript.stories.tsx +43 -201
- package/src/components/Transcript/Transcript.tsx +208 -83
- package/src/components/TranscriptContainer.tsx +6 -11
- package/src/components/index.ts +0 -1
- package/src/components/transcription.stories.tsx +65 -164
- package/src/hooks/index.ts +0 -1
- package/src/hooks/useTranscriber.ts +2 -3
- package/src/transcriber/{transcriber.browser.test.ts → transcriber.test.ts} +3 -3
- package/src/transcriber/transcriber.ts +11 -20
- package/src/transcriber/transcription-manager.ts +58 -67
- package/src/translations.ts +0 -2
- package/src/types/schema.ts +32 -11
- package/src/types/types.ts +9 -19
- package/src/util/index.ts +1 -0
- package/src/util/random-queue-dxn.ts +9 -0
- package/dist/lib/browser/app-graph-builder-REQZ5SC5.mjs +0 -351
- package/dist/lib/browser/app-graph-builder-REQZ5SC5.mjs.map +0 -7
- package/dist/lib/browser/chunk-3TKXI6TQ.mjs.map +0 -7
- package/dist/lib/browser/chunk-JHWEZRWC.mjs +0 -63
- package/dist/lib/browser/chunk-JHWEZRWC.mjs.map +0 -7
- package/dist/lib/browser/chunk-K2LS2QOM.mjs +0 -720
- package/dist/lib/browser/chunk-K2LS2QOM.mjs.map +0 -7
- package/dist/lib/browser/chunk-KLD7O5GJ.mjs.map +0 -7
- package/dist/lib/browser/chunk-LCTMCTYY.mjs +0 -27
- package/dist/lib/browser/chunk-LCTMCTYY.mjs.map +0 -7
- package/dist/lib/browser/chunk-LTIQFX4W.mjs.map +0 -7
- package/dist/lib/browser/intent-resolver-DZ62CJPC.mjs.map +0 -7
- package/dist/lib/browser/meeting-transcription-state-MNA2XROE.mjs.map +0 -7
- package/dist/lib/browser/react-surface-PUXZNAZL.mjs.map +0 -7
- package/dist/lib/browser/settings-JWDS4J3F.mjs +0 -24
- package/dist/lib/browser/settings-JWDS4J3F.mjs.map +0 -7
- package/dist/lib/node/TranscriptContainer-SSJVLRM6.cjs.map +0 -7
- package/dist/lib/node/app-graph-builder-OAPU4GDH.cjs +0 -352
- package/dist/lib/node/app-graph-builder-OAPU4GDH.cjs.map +0 -7
- package/dist/lib/node/chunk-EJWB53WK.cjs.map +0 -7
- package/dist/lib/node/chunk-EVE3H46F.cjs +0 -737
- package/dist/lib/node/chunk-EVE3H46F.cjs.map +0 -7
- package/dist/lib/node/chunk-J72PUQ3C.cjs.map +0 -7
- package/dist/lib/node/chunk-JUGOXWES.cjs.map +0 -7
- package/dist/lib/node/chunk-L27AK6FQ.cjs.map +0 -7
- package/dist/lib/node/chunk-WKZLGVEJ.cjs.map +0 -7
- package/dist/lib/node/intent-resolver-TRBKI24D.cjs.map +0 -7
- package/dist/lib/node/meeting-transcription-state-5XLTMWPA.cjs.map +0 -7
- package/dist/lib/node/react-surface-SZAMHCX7.cjs.map +0 -7
- package/dist/lib/node/settings-AWWMOSCV.cjs +0 -38
- package/dist/lib/node/settings-AWWMOSCV.cjs.map +0 -7
- package/dist/lib/node-esm/app-graph-builder-UBP6EOBR.mjs +0 -352
- package/dist/lib/node-esm/app-graph-builder-UBP6EOBR.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-4KC3E5T4.mjs +0 -721
- package/dist/lib/node-esm/chunk-4KC3E5T4.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-5E2MR5XK.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-MHQP2PAF.mjs +0 -28
- package/dist/lib/node-esm/chunk-MHQP2PAF.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-QMHMEKB3.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-QYG2FOXA.mjs +0 -64
- package/dist/lib/node-esm/chunk-QYG2FOXA.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-V5IWBPYO.mjs.map +0 -7
- package/dist/lib/node-esm/intent-resolver-HFVAL274.mjs.map +0 -7
- package/dist/lib/node-esm/meeting-transcription-state-3WOZ52BQ.mjs.map +0 -7
- package/dist/lib/node-esm/react-surface-UEJELL7I.mjs.map +0 -7
- package/dist/lib/node-esm/settings-QFNFGFJ5.mjs +0 -25
- package/dist/lib/node-esm/settings-QFNFGFJ5.mjs.map +0 -7
- package/dist/types/src/capabilities/settings.d.ts +0 -4
- package/dist/types/src/capabilities/settings.d.ts.map +0 -1
- package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts +0 -27
- package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts.map +0 -1
- package/dist/types/src/components/Transcript/transcript-extension.d.ts +0 -19
- package/dist/types/src/components/Transcript/transcript-extension.d.ts.map +0 -1
- package/dist/types/src/components/TranscriptionSettings/TranscriptionSettings.d.ts +0 -6
- package/dist/types/src/components/TranscriptionSettings/TranscriptionSettings.d.ts.map +0 -1
- package/dist/types/src/components/TranscriptionSettings/index.d.ts +0 -2
- package/dist/types/src/components/TranscriptionSettings/index.d.ts.map +0 -1
- package/dist/types/src/entity-extraction/entity-extraction.d.ts +0 -42
- package/dist/types/src/entity-extraction/entity-extraction.d.ts.map +0 -1
- package/dist/types/src/entity-extraction/entity-extraction.test.d.ts +0 -2
- package/dist/types/src/entity-extraction/entity-extraction.test.d.ts.map +0 -1
- package/dist/types/src/entity-extraction/index.d.ts +0 -2
- package/dist/types/src/entity-extraction/index.d.ts.map +0 -1
- package/dist/types/src/hooks/useQueueModelAdapter.d.ts +0 -7
- package/dist/types/src/hooks/useQueueModelAdapter.d.ts.map +0 -1
- package/dist/types/src/model/index.d.ts +0 -2
- package/dist/types/src/model/index.d.ts.map +0 -1
- package/dist/types/src/model/model.d.ts +0 -68
- package/dist/types/src/model/model.d.ts.map +0 -1
- package/dist/types/src/model/model.test.d.ts +0 -2
- package/dist/types/src/model/model.test.d.ts.map +0 -1
- package/dist/types/src/testing/index.d.ts +0 -2
- package/dist/types/src/testing/index.d.ts.map +0 -1
- package/dist/types/src/testing/testing.d.ts +0 -40
- package/dist/types/src/testing/testing.d.ts.map +0 -1
- package/dist/types/src/transcriber/transcriber.browser.test.d.ts +0 -2
- package/dist/types/src/transcriber/transcriber.browser.test.d.ts.map +0 -1
- package/src/capabilities/settings.ts +0 -20
- package/src/components/Transcript/deprecated/OldTranscript.tsx +0 -260
- package/src/components/Transcript/transcript-extension.ts +0 -236
- package/src/components/TranscriptionSettings/TranscriptionSettings.tsx +0 -26
- package/src/components/TranscriptionSettings/index.ts +0 -5
- package/src/entity-extraction/entity-extraction.test.ts +0 -97
- package/src/entity-extraction/entity-extraction.ts +0 -149
- package/src/entity-extraction/index.ts +0 -5
- package/src/entity-extraction/system-prompt.tpl +0 -38
- package/src/entity-extraction/types.d.ts +0 -8
- package/src/hooks/useQueueModelAdapter.ts +0 -63
- package/src/model/index.ts +0 -5
- package/src/model/model.test.ts +0 -169
- package/src/model/model.ts +0 -229
- package/src/testing/index.ts +0 -5
- package/src/testing/testing.ts +0 -216
- /package/dist/lib/browser/{TranscriptContainer-CXA4ZBHW.mjs.map → TranscriptContainer-JQ3HFHLQ.mjs.map} +0 -0
- /package/dist/lib/node-esm/{TranscriptContainer-M53LI2TX.mjs.map → TranscriptContainer-ZXUCOWGY.mjs.map} +0 -0
|
@@ -5,14 +5,15 @@
|
|
|
5
5
|
import { synchronized } from '@dxos/async';
|
|
6
6
|
import { Resource } from '@dxos/context';
|
|
7
7
|
import { QueueImpl, type Queue } from '@dxos/echo-db';
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
8
|
+
import { createStatic } from '@dxos/echo-schema';
|
|
9
|
+
import { invariant } from '@dxos/invariant';
|
|
10
|
+
import { DXN } from '@dxos/keys';
|
|
11
11
|
import { type EdgeHttpClient } from '@dxos/react-edge-client';
|
|
12
|
-
import {
|
|
12
|
+
import { type HuePalette } from '@dxos/react-ui-theme';
|
|
13
13
|
|
|
14
14
|
import { MediaStreamRecorder } from './media-stream-recorder';
|
|
15
15
|
import { Transcriber } from './transcriber';
|
|
16
|
+
import { TranscriptBlock, type TranscriptSegment } from '../types';
|
|
16
17
|
|
|
17
18
|
/**
|
|
18
19
|
* Length of the chunk in ms.
|
|
@@ -30,34 +31,23 @@ const PREFIXED_CHUNKS_AMOUNT = 10;
|
|
|
30
31
|
*/
|
|
31
32
|
const TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
|
|
32
33
|
|
|
33
|
-
export type TranscriptMessageEnricher = (message: DataType.Message) => Promise<DataType.Message>;
|
|
34
|
-
|
|
35
|
-
export type TranscriptionManagerOptions = {
|
|
36
|
-
edgeClient: EdgeHttpClient;
|
|
37
|
-
|
|
38
|
-
/**
|
|
39
|
-
* Enrich the message before it is written to the transcription queue.
|
|
40
|
-
*/
|
|
41
|
-
messageEnricher?: TranscriptMessageEnricher;
|
|
42
|
-
};
|
|
43
|
-
|
|
44
34
|
/**
|
|
45
35
|
* Manages transcription state for a meeting.
|
|
46
36
|
*/
|
|
37
|
+
// TODO(mykola): Reconcile with transcriber capability.
|
|
47
38
|
export class TranscriptionManager extends Resource {
|
|
48
39
|
private readonly _edgeClient: EdgeHttpClient;
|
|
49
|
-
private readonly _messageEnricher?: TranscriptMessageEnricher;
|
|
50
40
|
private _audioStreamTrack?: MediaStreamTrack = undefined;
|
|
51
|
-
private
|
|
41
|
+
private _name?: string = undefined;
|
|
42
|
+
private _hue?: HuePalette = undefined;
|
|
52
43
|
private _mediaRecorder?: MediaStreamRecorder = undefined;
|
|
53
44
|
private _transcriber?: Transcriber = undefined;
|
|
54
|
-
private _queue?: Queue<DataType.Message> = undefined;
|
|
55
45
|
private _enabled = false;
|
|
46
|
+
private _queue?: Queue<TranscriptBlock> = undefined;
|
|
56
47
|
|
|
57
|
-
constructor(
|
|
48
|
+
constructor(edgeClient: EdgeHttpClient) {
|
|
58
49
|
super();
|
|
59
|
-
this._edgeClient =
|
|
60
|
-
this._messageEnricher = options.messageEnricher;
|
|
50
|
+
this._edgeClient = edgeClient;
|
|
61
51
|
}
|
|
62
52
|
|
|
63
53
|
protected override async _open() {
|
|
@@ -68,24 +58,10 @@ export class TranscriptionManager extends Resource {
|
|
|
68
58
|
void this._transcriber?.close();
|
|
69
59
|
}
|
|
70
60
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
log.info('setQueue', { queueDxn: queueDxn.toString() });
|
|
74
|
-
this._queue = new QueueImpl<DataType.Message>(this._edgeClient, queueDxn);
|
|
75
|
-
}
|
|
76
|
-
return this;
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
setIdentityDid(did: string): TranscriptionManager {
|
|
80
|
-
if (this._identityDid !== did) {
|
|
81
|
-
this._identityDid = did;
|
|
82
|
-
}
|
|
83
|
-
return this;
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
setRecording(recording?: boolean): TranscriptionManager {
|
|
61
|
+
@synchronized
|
|
62
|
+
setRecording(recording?: boolean) {
|
|
87
63
|
if (!this.isOpen || !this._enabled) {
|
|
88
|
-
return
|
|
64
|
+
return;
|
|
89
65
|
}
|
|
90
66
|
|
|
91
67
|
if (recording) {
|
|
@@ -93,7 +69,6 @@ export class TranscriptionManager extends Resource {
|
|
|
93
69
|
} else {
|
|
94
70
|
this._transcriber?.stopChunksRecording();
|
|
95
71
|
}
|
|
96
|
-
return this;
|
|
97
72
|
}
|
|
98
73
|
|
|
99
74
|
/**
|
|
@@ -105,9 +80,7 @@ export class TranscriptionManager extends Resource {
|
|
|
105
80
|
if (this._enabled === enabled) {
|
|
106
81
|
return;
|
|
107
82
|
}
|
|
108
|
-
|
|
109
83
|
this._enabled = enabled ?? false;
|
|
110
|
-
// TODO(burdon): Why is toggle called here?
|
|
111
84
|
this.isOpen && (await this._toggleTranscriber());
|
|
112
85
|
}
|
|
113
86
|
|
|
@@ -116,33 +89,60 @@ export class TranscriptionManager extends Resource {
|
|
|
116
89
|
if (this._audioStreamTrack === track) {
|
|
117
90
|
return;
|
|
118
91
|
}
|
|
119
|
-
|
|
120
92
|
this._audioStreamTrack = track;
|
|
121
93
|
this.isOpen && (await this._toggleTranscriber());
|
|
122
94
|
}
|
|
123
95
|
|
|
124
|
-
|
|
96
|
+
/**
|
|
97
|
+
* Set the queue to save the transcription to.
|
|
98
|
+
* @param queue - The queue to save the transcription to or the DXN of the queue.
|
|
99
|
+
*/
|
|
100
|
+
@synchronized
|
|
101
|
+
setQueue(queue?: Queue<TranscriptBlock> | string) {
|
|
102
|
+
switch (typeof queue) {
|
|
103
|
+
case 'string':
|
|
104
|
+
if (this._queue?.dxn.toString() === queue) {
|
|
105
|
+
return;
|
|
106
|
+
}
|
|
107
|
+
this._queue = new QueueImpl<TranscriptBlock>(this._edgeClient, DXN.parse(queue));
|
|
108
|
+
break;
|
|
109
|
+
case 'object':
|
|
110
|
+
if (this._queue === queue) {
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
invariant(queue instanceof QueueImpl);
|
|
114
|
+
this._queue = queue;
|
|
115
|
+
break;
|
|
116
|
+
case 'undefined':
|
|
117
|
+
this._queue = undefined;
|
|
118
|
+
break;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
@synchronized
|
|
123
|
+
setName(name: string) {
|
|
124
|
+
if (this._name === name) {
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
127
|
+
this._name = name;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
@synchronized
|
|
131
|
+
setHue(hue: HuePalette) {
|
|
132
|
+
if (this._hue === hue) {
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
this._hue = hue;
|
|
136
|
+
}
|
|
137
|
+
|
|
125
138
|
private async _toggleTranscriber() {
|
|
126
139
|
await this._maybeReinitTranscriber();
|
|
127
140
|
|
|
128
141
|
// Open or close transcriber if transcription is enabled or disabled.
|
|
129
142
|
if (this._enabled) {
|
|
130
143
|
await this._transcriber?.open();
|
|
131
|
-
// TODO(burdon): Started and stopped blocks appear twice.
|
|
132
|
-
const block = create(DataType.Message, {
|
|
133
|
-
created: new Date().toISOString(),
|
|
134
|
-
blocks: [{ type: 'transcription', text: 'Started', started: new Date().toISOString() }],
|
|
135
|
-
sender: { role: 'assistant' },
|
|
136
|
-
});
|
|
137
|
-
this._queue?.append([block]);
|
|
138
144
|
} else {
|
|
139
145
|
await this._transcriber?.close();
|
|
140
|
-
const block = create(DataType.Message, {
|
|
141
|
-
created: new Date().toISOString(),
|
|
142
|
-
blocks: [{ type: 'transcription', text: 'Stopped', started: new Date().toISOString() }],
|
|
143
|
-
sender: { role: 'assistant' },
|
|
144
|
-
});
|
|
145
|
-
this._queue?.append([block]);
|
|
146
146
|
}
|
|
147
147
|
}
|
|
148
148
|
|
|
@@ -173,21 +173,12 @@ export class TranscriptionManager extends Resource {
|
|
|
173
173
|
}
|
|
174
174
|
}
|
|
175
175
|
|
|
176
|
-
private async _onSegments(segments:
|
|
176
|
+
private async _onSegments(segments: TranscriptSegment[]) {
|
|
177
177
|
if (!this.isOpen || !this._queue) {
|
|
178
178
|
return;
|
|
179
179
|
}
|
|
180
180
|
|
|
181
|
-
|
|
182
|
-
created: new Date().toISOString(),
|
|
183
|
-
blocks: segments,
|
|
184
|
-
sender: { identityDid: this._identityDid },
|
|
185
|
-
});
|
|
186
|
-
|
|
187
|
-
if (this._messageEnricher) {
|
|
188
|
-
block = await this._messageEnricher(block);
|
|
189
|
-
}
|
|
190
|
-
|
|
181
|
+
const block = createStatic(TranscriptBlock, { authorName: this._name, authorHue: this._hue, segments });
|
|
191
182
|
this._queue.append([block]);
|
|
192
183
|
}
|
|
193
184
|
}
|
package/src/translations.ts
CHANGED
package/src/types/schema.ts
CHANGED
|
@@ -2,19 +2,17 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import {
|
|
6
|
-
|
|
7
|
-
import { Expando, Ref, TypedObject } from '@dxos/echo-schema';
|
|
5
|
+
import { EchoObject, Expando, Ref, S, TypedObject } from '@dxos/echo-schema';
|
|
8
6
|
|
|
9
7
|
/**
|
|
10
8
|
* Root transcript object created when the user starts a transcription.
|
|
11
9
|
*/
|
|
12
|
-
export const TranscriptSchema =
|
|
13
|
-
name:
|
|
10
|
+
export const TranscriptSchema = S.Struct({
|
|
11
|
+
name: S.String,
|
|
14
12
|
|
|
15
|
-
// TODO(burdon): Use
|
|
16
|
-
started:
|
|
17
|
-
ended:
|
|
13
|
+
// TODO(burdon): Use string for dates?
|
|
14
|
+
started: S.optional(S.Date),
|
|
15
|
+
ended: S.optional(S.Date),
|
|
18
16
|
|
|
19
17
|
/**
|
|
20
18
|
* Queue containing TranscriptBlock objects.
|
|
@@ -34,8 +32,31 @@ export class TranscriptType extends TypedObject({
|
|
|
34
32
|
* First message in queue.
|
|
35
33
|
* Contains metadata for the recording and transcript.
|
|
36
34
|
*/
|
|
37
|
-
const TranscriptHeader =
|
|
38
|
-
started:
|
|
35
|
+
const TranscriptHeader = S.Struct({
|
|
36
|
+
started: S.optional(S.Date),
|
|
39
37
|
});
|
|
40
38
|
|
|
41
|
-
export type TranscriptHeader =
|
|
39
|
+
export type TranscriptHeader = S.Schema.Type<typeof TranscriptHeader>;
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Transcription fragment.
|
|
43
|
+
*/
|
|
44
|
+
const TranscriptSegment = S.Struct({
|
|
45
|
+
// TODO(burdon): TS from service is not Unix TS (x1000).
|
|
46
|
+
started: S.Date,
|
|
47
|
+
text: S.String,
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
export type TranscriptSegment = S.Schema.Type<typeof TranscriptSegment>;
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Transcription block (from single speaker).
|
|
54
|
+
*/
|
|
55
|
+
export const TranscriptBlock = S.Struct({
|
|
56
|
+
id: S.String,
|
|
57
|
+
authorName: S.optional(S.String), // TODO(burdon): IdentityDid.
|
|
58
|
+
authorHue: S.optional(S.String),
|
|
59
|
+
segments: S.Array(TranscriptSegment),
|
|
60
|
+
}).pipe(EchoObject({ typename: 'dxos.org/type/TranscriptBlock', version: '0.1.0' }));
|
|
61
|
+
|
|
62
|
+
export type TranscriptBlock = S.Schema.Type<typeof TranscriptBlock>;
|
package/src/types/types.ts
CHANGED
|
@@ -2,10 +2,8 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import {
|
|
6
|
-
|
|
7
|
-
import { isInstanceOf } from '@dxos/echo-schema';
|
|
8
|
-
import { isLiveObject, SpaceId } from '@dxos/react-client/echo';
|
|
5
|
+
import { S, isInstanceOf } from '@dxos/echo-schema';
|
|
6
|
+
import { isReactiveObject } from '@dxos/react-client/echo';
|
|
9
7
|
|
|
10
8
|
import { TranscriptType } from './schema';
|
|
11
9
|
import { TRANSCRIPTION_PLUGIN } from '../meta';
|
|
@@ -21,26 +19,18 @@ export const TRANSCRIPTION_URL = 'https://calls-service.dxos.workers.dev';
|
|
|
21
19
|
export namespace TranscriptionAction {
|
|
22
20
|
const TRANSCRIPTION_ACTION = `${TRANSCRIPTION_PLUGIN}/action`;
|
|
23
21
|
|
|
24
|
-
export class Create extends
|
|
25
|
-
input:
|
|
26
|
-
name:
|
|
27
|
-
|
|
22
|
+
export class Create extends S.TaggedClass<Create>()(`${TRANSCRIPTION_ACTION}/create`, {
|
|
23
|
+
input: S.Struct({
|
|
24
|
+
name: S.optional(S.String),
|
|
25
|
+
// TODO(wittjosiah): SpaceId.
|
|
26
|
+
spaceId: S.String,
|
|
28
27
|
}),
|
|
29
|
-
output:
|
|
28
|
+
output: S.Struct({
|
|
30
29
|
object: TranscriptType,
|
|
31
30
|
}),
|
|
32
31
|
}) {}
|
|
33
32
|
}
|
|
34
33
|
|
|
35
34
|
export const isTranscript = (object: unknown): object is typeof TranscriptType => {
|
|
36
|
-
return
|
|
35
|
+
return isReactiveObject(object) && isInstanceOf(TranscriptType, object);
|
|
37
36
|
};
|
|
38
|
-
|
|
39
|
-
// TODO(burdon): Create with decode consistently: Schema.decodeSync(TranscriptionSettingsSchema)({}))
|
|
40
|
-
export const TranscriptionSettingsSchema = Schema.mutable(
|
|
41
|
-
Schema.Struct({
|
|
42
|
-
entityExtraction: Schema.optional(Schema.Boolean).pipe(Schema.withConstructorDefault(() => true)),
|
|
43
|
-
}),
|
|
44
|
-
);
|
|
45
|
-
|
|
46
|
-
export type TranscriptionSettingsProps = Schema.Schema.Type<typeof TranscriptionSettingsSchema>;
|
package/src/util/index.ts
CHANGED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2025 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import { ObjectId } from '@dxos/echo-schema';
|
|
6
|
+
import { DXN, QueueSubspaceTags, SpaceId } from '@dxos/keys';
|
|
7
|
+
|
|
8
|
+
export const randomQueueDxn = (spaceId?: SpaceId) =>
|
|
9
|
+
new DXN(DXN.kind.QUEUE, [QueueSubspaceTags.DATA, spaceId ?? SpaceId.random(), ObjectId.random()]);
|
|
@@ -1,351 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
TranscriptionManager
|
|
3
|
-
} from "./chunk-3TKXI6TQ.mjs";
|
|
4
|
-
import "./chunk-KLD7O5GJ.mjs";
|
|
5
|
-
import {
|
|
6
|
-
TranscriptType,
|
|
7
|
-
TranscriptionAction
|
|
8
|
-
} from "./chunk-JHWEZRWC.mjs";
|
|
9
|
-
import {
|
|
10
|
-
TranscriptionCapabilities
|
|
11
|
-
} from "./chunk-LTIQFX4W.mjs";
|
|
12
|
-
import {
|
|
13
|
-
TRANSCRIPTION_PLUGIN
|
|
14
|
-
} from "./chunk-6ZR5MDK3.mjs";
|
|
15
|
-
|
|
16
|
-
// packages/plugins/plugin-transcription/src/capabilities/app-graph-builder.ts
|
|
17
|
-
import { Rx } from "@effect-rx/rx-react";
|
|
18
|
-
import { Option, pipe } from "effect";
|
|
19
|
-
import { Capabilities, contributes, createIntent } from "@dxos/app-framework";
|
|
20
|
-
import { AIServiceEdgeClient } from "@dxos/assistant";
|
|
21
|
-
import { AI_SERVICE_ENDPOINT } from "@dxos/assistant/testing";
|
|
22
|
-
import { Filter, fullyQualifiedId, getSpace, makeRef, Query } from "@dxos/client/echo";
|
|
23
|
-
import { getSchemaTypename, isInstanceOf } from "@dxos/echo-schema";
|
|
24
|
-
import { invariant } from "@dxos/invariant";
|
|
25
|
-
import { DXN as DXN2 } from "@dxos/keys";
|
|
26
|
-
import { log as log2 } from "@dxos/log";
|
|
27
|
-
import { ClientCapabilities } from "@dxos/plugin-client";
|
|
28
|
-
import { PLANK_COMPANION_TYPE, ATTENDABLE_PATH_SEPARATOR } from "@dxos/plugin-deck/types";
|
|
29
|
-
import { createExtension, rxFromSignal } from "@dxos/plugin-graph";
|
|
30
|
-
import { DocumentType } from "@dxos/plugin-markdown/types";
|
|
31
|
-
import { MeetingCapabilities } from "@dxos/plugin-meeting";
|
|
32
|
-
import { MeetingType } from "@dxos/plugin-meeting/types";
|
|
33
|
-
import { DataType as DataType2 } from "@dxos/schema";
|
|
34
|
-
|
|
35
|
-
// packages/plugins/plugin-transcription/src/entity-extraction/entity-extraction.ts
|
|
36
|
-
import { Schema } from "effect";
|
|
37
|
-
import { createTemplate, Message, structuredOutputParser } from "@dxos/artifact";
|
|
38
|
-
import { MixedStreamParser } from "@dxos/assistant";
|
|
39
|
-
import { asyncTimeout } from "@dxos/async";
|
|
40
|
-
import { create, ObjectId } from "@dxos/echo-schema";
|
|
41
|
-
import { DXN } from "@dxos/keys";
|
|
42
|
-
import { log } from "@dxos/log";
|
|
43
|
-
import { DataType } from "@dxos/schema";
|
|
44
|
-
|
|
45
|
-
// raw-loader:./system-prompt.tpl?raw
|
|
46
|
-
var system_prompt_default = 'You are an AI assistant specialized in processing and enhancing transcripts with contextual information.\nYour task is to analyze a given transcript and extract references to the context objects.\n\nThe context and transcript are provided in the user message in `context` and `transcript` XML tags.\n\nHere are your instructions:\n\n1. Read the transcript and context information provided above.\n\n2. Process each segment of the transcript separately.\n\n3. For each segment:\n a. Analyze the content and identify relevant information from the context.\n b. Extract a number of references to the context objects that are mentioned in the segment.\n c. Output the references in the format `{ "quote": "earnings report", "id": "01JT0JP9AX0XKGZX4MV4B69VT6" }`.\n d. If there are no references, just output an empty array.\n\n4. Maintain the exact order and structure of the original transcript segments.\n\nOutput Format:\nThe final output should be a JSON array of references.\n\nExample output structure:\n```json\n{\n "references": [\n { "quote": "earnings report", "id": "01JT0JP9AX0XKGZX4MV4B69VT6" },\n { "quote": "John", "id": "01JT0VM0VN3ZKN7GDBM8CV9BA7" }\n ]\n}\n```\n\nRemember:\n- Only quote the text that is in the transcript.\n- Analyze each segment carefully before processing it.\n- Only reference objects that appear in the context and exactly by their id, If the object is not in the context, do not insert a reference. \n\nNow, process the transcript and context provided below according to these instructions.';
|
|
47
|
-
|
|
48
|
-
// packages/plugins/plugin-transcription/src/entity-extraction/entity-extraction.ts
|
|
49
|
-
var __dxlog_file = "/home/runner/work/dxos/dxos/packages/plugins/plugin-transcription/src/entity-extraction/entity-extraction.ts";
|
|
50
|
-
var createSystemPrompt = () => {
|
|
51
|
-
const template = createTemplate(system_prompt_default);
|
|
52
|
-
return template({});
|
|
53
|
-
};
|
|
54
|
-
var ReferencedQuotes = Schema.Struct({
|
|
55
|
-
references: Schema.Array(Schema.Struct({
|
|
56
|
-
quote: Schema.String,
|
|
57
|
-
id: Schema.String
|
|
58
|
-
})).annotations({
|
|
59
|
-
description: "The references to the context objects that are mentioned in the transcript. quote should match the original transcript text exactly, while id is the id of the context object."
|
|
60
|
-
})
|
|
61
|
-
});
|
|
62
|
-
var processTranscriptMessage = async (params) => {
|
|
63
|
-
try {
|
|
64
|
-
const outputParser = structuredOutputParser(ReferencedQuotes);
|
|
65
|
-
const runParser = async () => {
|
|
66
|
-
const startTime = performance.now();
|
|
67
|
-
const result = outputParser.getResult(await new MixedStreamParser().parse(await params.aiService.execStream({
|
|
68
|
-
model: "@anthropic/claude-3-5-haiku-20241022",
|
|
69
|
-
systemPrompt: createSystemPrompt(),
|
|
70
|
-
history: [
|
|
71
|
-
create(Message, {
|
|
72
|
-
role: "user",
|
|
73
|
-
content: [
|
|
74
|
-
{
|
|
75
|
-
type: "text",
|
|
76
|
-
text: `<context>${JSON.stringify(params.context.objects)}</context>`
|
|
77
|
-
},
|
|
78
|
-
{
|
|
79
|
-
type: "text",
|
|
80
|
-
text: `<transcript>${JSON.stringify(params.message.blocks)}</transcript>`
|
|
81
|
-
}
|
|
82
|
-
]
|
|
83
|
-
})
|
|
84
|
-
],
|
|
85
|
-
tools: [
|
|
86
|
-
outputParser.tool
|
|
87
|
-
]
|
|
88
|
-
})));
|
|
89
|
-
log.info("entity extraction result", {
|
|
90
|
-
refs: result.references
|
|
91
|
-
}, {
|
|
92
|
-
F: __dxlog_file,
|
|
93
|
-
L: 98,
|
|
94
|
-
S: void 0,
|
|
95
|
-
C: (f, a) => f(...a)
|
|
96
|
-
});
|
|
97
|
-
return {
|
|
98
|
-
message: create(DataType.Message, {
|
|
99
|
-
...params.message,
|
|
100
|
-
blocks: params.message.blocks.map((block, i) => block.type !== "transcription" ? block : {
|
|
101
|
-
...block,
|
|
102
|
-
text: postprocessText(block.text, result)
|
|
103
|
-
})
|
|
104
|
-
}),
|
|
105
|
-
timeElapsed: performance.now() - startTime
|
|
106
|
-
};
|
|
107
|
-
};
|
|
108
|
-
if (params.options?.timeout && params.options.timeout > 0) {
|
|
109
|
-
return await asyncTimeout(runParser(), params.options.timeout);
|
|
110
|
-
} else {
|
|
111
|
-
return await runParser();
|
|
112
|
-
}
|
|
113
|
-
} catch (error) {
|
|
114
|
-
if (params.options?.fallbackToRaw) {
|
|
115
|
-
log.warn("failed to process transcript message, falling back to raw text", {
|
|
116
|
-
error
|
|
117
|
-
}, {
|
|
118
|
-
F: __dxlog_file,
|
|
119
|
-
L: 123,
|
|
120
|
-
S: void 0,
|
|
121
|
-
C: (f, a) => f(...a)
|
|
122
|
-
});
|
|
123
|
-
return {
|
|
124
|
-
message: params.message,
|
|
125
|
-
timeElapsed: 0
|
|
126
|
-
};
|
|
127
|
-
} else {
|
|
128
|
-
throw error;
|
|
129
|
-
}
|
|
130
|
-
}
|
|
131
|
-
};
|
|
132
|
-
var postprocessText = (text, quotes) => {
|
|
133
|
-
for (const quote of quotes.references) {
|
|
134
|
-
if (!ObjectId.isValid(quote.id)) {
|
|
135
|
-
continue;
|
|
136
|
-
}
|
|
137
|
-
const regex = new RegExp(quote.quote.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "gi");
|
|
138
|
-
text = text.replace(regex, `[${quote.quote}][${DXN.fromLocalObjectId(quote.id).toString()}]`);
|
|
139
|
-
}
|
|
140
|
-
return text;
|
|
141
|
-
};
|
|
142
|
-
|
|
143
|
-
// packages/plugins/plugin-transcription/src/capabilities/app-graph-builder.ts
|
|
144
|
-
var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/plugins/plugin-transcription/src/capabilities/app-graph-builder.ts";
|
|
145
|
-
var getMeetingTranscript = async (context, meeting) => {
|
|
146
|
-
const { dispatchPromise: dispatch } = context.getCapability(Capabilities.IntentDispatcher);
|
|
147
|
-
const typename = getSchemaTypename(TranscriptType);
|
|
148
|
-
const existing = await meeting.artifacts[typename]?.load();
|
|
149
|
-
if (existing instanceof TranscriptType) {
|
|
150
|
-
return existing;
|
|
151
|
-
}
|
|
152
|
-
const space = getSpace(meeting);
|
|
153
|
-
invariant(space, void 0, {
|
|
154
|
-
F: __dxlog_file2,
|
|
155
|
-
L: 49,
|
|
156
|
-
S: void 0,
|
|
157
|
-
A: [
|
|
158
|
-
"space",
|
|
159
|
-
""
|
|
160
|
-
]
|
|
161
|
-
});
|
|
162
|
-
const { data } = await dispatch(createIntent(TranscriptionAction.Create, {
|
|
163
|
-
spaceId: space.id
|
|
164
|
-
}));
|
|
165
|
-
meeting.artifacts[typename] = makeRef(data.object);
|
|
166
|
-
return data?.object;
|
|
167
|
-
};
|
|
168
|
-
var app_graph_builder_default = (context) => contributes(Capabilities.AppGraphBuilder, [
|
|
169
|
-
createExtension({
|
|
170
|
-
id: `${TRANSCRIPTION_PLUGIN}/meeting-transcript`,
|
|
171
|
-
actions: (node) => Rx.make((get) => pipe(get(node), Option.flatMap((node2) => isInstanceOf(MeetingType, node2.data) ? Option.some(node2.data) : Option.none()), Option.map((meeting) => {
|
|
172
|
-
const state = context.getCapability(TranscriptionCapabilities.MeetingTranscriptionState);
|
|
173
|
-
const enabled = get(rxFromSignal(() => state.enabled));
|
|
174
|
-
return [
|
|
175
|
-
{
|
|
176
|
-
id: `${fullyQualifiedId(meeting)}/action/start-stop-transcription`,
|
|
177
|
-
data: async () => {
|
|
178
|
-
const transcript = await getMeetingTranscript(context, meeting);
|
|
179
|
-
invariant(transcript, "Failed to create transcript", {
|
|
180
|
-
F: __dxlog_file2,
|
|
181
|
-
L: 76,
|
|
182
|
-
S: void 0,
|
|
183
|
-
A: [
|
|
184
|
-
"transcript",
|
|
185
|
-
"'Failed to create transcript'"
|
|
186
|
-
]
|
|
187
|
-
});
|
|
188
|
-
const callManager = context.getCapability(MeetingCapabilities.CallManager);
|
|
189
|
-
callManager.setActivity(getSchemaTypename(TranscriptType), {
|
|
190
|
-
queueDxn: transcript.queue.dxn.toString(),
|
|
191
|
-
enabled: !enabled
|
|
192
|
-
});
|
|
193
|
-
},
|
|
194
|
-
properties: {
|
|
195
|
-
label: enabled ? [
|
|
196
|
-
"stop transcription label",
|
|
197
|
-
{
|
|
198
|
-
ns: TRANSCRIPTION_PLUGIN
|
|
199
|
-
}
|
|
200
|
-
] : [
|
|
201
|
-
"start transcription label",
|
|
202
|
-
{
|
|
203
|
-
ns: TRANSCRIPTION_PLUGIN
|
|
204
|
-
}
|
|
205
|
-
],
|
|
206
|
-
icon: "ph--subtitles--regular",
|
|
207
|
-
disposition: "toolbar",
|
|
208
|
-
classNames: enabled ? "bg-callAlert" : ""
|
|
209
|
-
}
|
|
210
|
-
}
|
|
211
|
-
];
|
|
212
|
-
}), Option.getOrElse(() => []))),
|
|
213
|
-
connector: (node) => Rx.make((get) => pipe(get(node), Option.flatMap((node2) => isInstanceOf(MeetingType, node2.data) ? Option.some(node2.data) : Option.none()), Option.map((meeting) => {
|
|
214
|
-
const client = context.getCapability(ClientCapabilities.Client);
|
|
215
|
-
const state = context.getCapability(TranscriptionCapabilities.MeetingTranscriptionState);
|
|
216
|
-
const settingsStore = get(context.capabilities(Capabilities.SettingsStore))[0];
|
|
217
|
-
const settings = get(rxFromSignal(() => settingsStore?.getStore(TRANSCRIPTION_PLUGIN).value));
|
|
218
|
-
const aiClient = new AIServiceEdgeClient({
|
|
219
|
-
// TODO(burdon): Get from config.
|
|
220
|
-
endpoint: AI_SERVICE_ENDPOINT.REMOTE
|
|
221
|
-
});
|
|
222
|
-
return [
|
|
223
|
-
{
|
|
224
|
-
id: `${fullyQualifiedId(meeting)}${ATTENDABLE_PATH_SEPARATOR}${getSchemaTypename(TranscriptType)}`,
|
|
225
|
-
type: PLANK_COMPANION_TYPE,
|
|
226
|
-
data: get(rxFromSignal(() => meeting.artifacts[getSchemaTypename(TranscriptType)]?.target)),
|
|
227
|
-
properties: {
|
|
228
|
-
label: [
|
|
229
|
-
"transcript companion label",
|
|
230
|
-
{
|
|
231
|
-
ns: TRANSCRIPTION_PLUGIN
|
|
232
|
-
}
|
|
233
|
-
],
|
|
234
|
-
icon: "ph--subtitles--regular",
|
|
235
|
-
position: "hoist",
|
|
236
|
-
disposition: "hidden",
|
|
237
|
-
schema: TranscriptType,
|
|
238
|
-
getIntent: ({ space }) => createIntent(TranscriptionAction.Create, {
|
|
239
|
-
spaceId: space.id
|
|
240
|
-
}),
|
|
241
|
-
// TODO(burdon): Is this the right place to inisitalize the state? Change to a capability?
|
|
242
|
-
onJoin: async ({ meeting: meeting2 }) => {
|
|
243
|
-
const identity = client.halo.identity.get();
|
|
244
|
-
invariant(identity, void 0, {
|
|
245
|
-
F: __dxlog_file2,
|
|
246
|
-
L: 134,
|
|
247
|
-
S: void 0,
|
|
248
|
-
A: [
|
|
249
|
-
"identity",
|
|
250
|
-
""
|
|
251
|
-
]
|
|
252
|
-
});
|
|
253
|
-
const space = getSpace(meeting2);
|
|
254
|
-
invariant(space, void 0, {
|
|
255
|
-
F: __dxlog_file2,
|
|
256
|
-
L: 136,
|
|
257
|
-
S: void 0,
|
|
258
|
-
A: [
|
|
259
|
-
"space",
|
|
260
|
-
""
|
|
261
|
-
]
|
|
262
|
-
});
|
|
263
|
-
let messageEnricher;
|
|
264
|
-
if (aiClient && settings.entityExtraction) {
|
|
265
|
-
messageEnricher = createEntityExtractionEnricher({
|
|
266
|
-
aiClient,
|
|
267
|
-
// TODO(dmaretskyi): Have those be discovered from the schema graph or contributed by capabilities?
|
|
268
|
-
// This forced me to add a dependency on markdown plugin.
|
|
269
|
-
// This will be replaced with a vector search index anyway, so its not a big deal.
|
|
270
|
-
contextTypes: [
|
|
271
|
-
DocumentType,
|
|
272
|
-
DataType2.Person,
|
|
273
|
-
DataType2.Organization
|
|
274
|
-
],
|
|
275
|
-
space
|
|
276
|
-
});
|
|
277
|
-
}
|
|
278
|
-
state.transcriptionManager = await new TranscriptionManager({
|
|
279
|
-
edgeClient: client.edge,
|
|
280
|
-
messageEnricher
|
|
281
|
-
}).setIdentityDid(identity.did).open();
|
|
282
|
-
},
|
|
283
|
-
onLeave: async () => {
|
|
284
|
-
await state.transcriptionManager?.close();
|
|
285
|
-
state.transcriptionManager = void 0;
|
|
286
|
-
state.enabled = false;
|
|
287
|
-
},
|
|
288
|
-
onCallStateUpdated: async (callState) => {
|
|
289
|
-
const typename = getSchemaTypename(TranscriptType);
|
|
290
|
-
const transcription = typename ? callState.activities?.[typename] : void 0;
|
|
291
|
-
if (!transcription?.payload) {
|
|
292
|
-
return;
|
|
293
|
-
}
|
|
294
|
-
const payload = transcription.payload;
|
|
295
|
-
state.enabled = !!payload.enabled;
|
|
296
|
-
if (payload.queueDxn) {
|
|
297
|
-
state.transcriptionManager?.setQueue(DXN2.parse(payload.queueDxn));
|
|
298
|
-
}
|
|
299
|
-
await state.transcriptionManager?.setEnabled(payload.enabled);
|
|
300
|
-
},
|
|
301
|
-
onMediaStateUpdated: async ([mediaState, isSpeaking]) => {
|
|
302
|
-
void state.transcriptionManager?.setAudioTrack(mediaState.audioTrack);
|
|
303
|
-
void state.transcriptionManager?.setRecording(isSpeaking);
|
|
304
|
-
}
|
|
305
|
-
}
|
|
306
|
-
}
|
|
307
|
-
];
|
|
308
|
-
}), Option.getOrElse(() => [])))
|
|
309
|
-
})
|
|
310
|
-
]);
|
|
311
|
-
var createEntityExtractionEnricher = ({ aiClient, contextTypes, space }) => {
|
|
312
|
-
return async (message) => {
|
|
313
|
-
const { objects } = await space.db.query(Query.select(Filter.or(...contextTypes.map((s) => Filter.type(s))))).run();
|
|
314
|
-
log2.info("context", {
|
|
315
|
-
objects
|
|
316
|
-
}, {
|
|
317
|
-
F: __dxlog_file2,
|
|
318
|
-
L: 203,
|
|
319
|
-
S: void 0,
|
|
320
|
-
C: (f, a) => f(...a)
|
|
321
|
-
});
|
|
322
|
-
const { message: enhancedMessage, timeElapsed } = await processTranscriptMessage({
|
|
323
|
-
aiService: aiClient,
|
|
324
|
-
message,
|
|
325
|
-
context: {
|
|
326
|
-
objects: await Promise.all(objects.map((o) => processContextObject(o)))
|
|
327
|
-
},
|
|
328
|
-
options: {
|
|
329
|
-
timeout: ENTITY_EXTRACTOR_TIMEOUT,
|
|
330
|
-
fallbackToRaw: true
|
|
331
|
-
}
|
|
332
|
-
});
|
|
333
|
-
log2.info("entity extraction time", {
|
|
334
|
-
timeElapsed
|
|
335
|
-
}, {
|
|
336
|
-
F: __dxlog_file2,
|
|
337
|
-
L: 214,
|
|
338
|
-
S: void 0,
|
|
339
|
-
C: (f, a) => f(...a)
|
|
340
|
-
});
|
|
341
|
-
return enhancedMessage;
|
|
342
|
-
};
|
|
343
|
-
};
|
|
344
|
-
var processContextObject = async (object) => {
|
|
345
|
-
return object;
|
|
346
|
-
};
|
|
347
|
-
var ENTITY_EXTRACTOR_TIMEOUT = 25e3;
|
|
348
|
-
export {
|
|
349
|
-
app_graph_builder_default as default
|
|
350
|
-
};
|
|
351
|
-
//# sourceMappingURL=app-graph-builder-REQZ5SC5.mjs.map
|