@dxos/plugin-transcription 0.8.3 → 0.8.4-main.1068cf700f
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/TranscriptContainer-MV3JBKLS.mjs +12 -0
- package/dist/lib/browser/blueprint-definition-K3E72ZBC.mjs +19 -0
- package/dist/lib/browser/blueprint-definition-K3E72ZBC.mjs.map +7 -0
- package/dist/lib/browser/blueprints/index.mjs +9 -0
- package/dist/lib/browser/chunk-AV44VISC.mjs +148 -0
- package/dist/lib/browser/chunk-AV44VISC.mjs.map +7 -0
- package/dist/lib/browser/chunk-HOOPZ7EM.mjs +35 -0
- package/dist/lib/browser/chunk-HOOPZ7EM.mjs.map +7 -0
- package/dist/lib/browser/chunk-J5LGTIGS.mjs +10 -0
- package/dist/lib/browser/chunk-J5LGTIGS.mjs.map +7 -0
- package/dist/lib/browser/{chunk-53AXUEIQ.mjs → chunk-K3JJ46VV.mjs} +367 -433
- package/dist/lib/browser/chunk-K3JJ46VV.mjs.map +7 -0
- package/dist/lib/browser/chunk-Q5OC3WVG.mjs +72 -0
- package/dist/lib/browser/chunk-Q5OC3WVG.mjs.map +7 -0
- package/dist/lib/browser/{chunk-UDCCT32F.mjs → chunk-RUKY36XW.mjs} +3 -3
- package/dist/lib/browser/{chunk-IQ5ZRMHZ.mjs → chunk-TKFU53OZ.mjs} +70 -105
- package/dist/lib/browser/chunk-TKFU53OZ.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +68 -75
- package/dist/lib/browser/index.mjs.map +4 -4
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/operation-resolver-GLEBGPKH.mjs +22 -0
- package/dist/lib/browser/operation-resolver-GLEBGPKH.mjs.map +7 -0
- package/dist/lib/browser/react-surface-AWZ2A5WZ.mjs +35 -0
- package/dist/lib/browser/react-surface-AWZ2A5WZ.mjs.map +7 -0
- package/dist/lib/browser/{transcriber-OYPLZFG5.mjs → transcriber-SLVQP76E.mjs} +17 -13
- package/dist/lib/browser/transcriber-SLVQP76E.mjs.map +7 -0
- package/dist/lib/browser/types/index.mjs +8 -8
- package/dist/lib/node-esm/{TranscriptContainer-G75LDEO5.mjs → TranscriptContainer-7M43UAZ2.mjs} +5 -4
- package/dist/lib/node-esm/TranscriptContainer-7M43UAZ2.mjs.map +7 -0
- package/dist/lib/node-esm/blueprint-definition-2WIDDA3Y.mjs +20 -0
- package/dist/lib/node-esm/blueprint-definition-2WIDDA3Y.mjs.map +7 -0
- package/dist/lib/node-esm/blueprints/index.mjs +10 -0
- package/dist/lib/node-esm/blueprints/index.mjs.map +7 -0
- package/dist/lib/node-esm/chunk-4BFY6P2D.mjs +73 -0
- package/dist/lib/node-esm/chunk-4BFY6P2D.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-PRL64SRU.mjs → chunk-EPC27WWV.mjs} +70 -105
- package/dist/lib/node-esm/chunk-EPC27WWV.mjs.map +7 -0
- package/dist/lib/node-esm/chunk-HSLMI22Q.mjs +11 -0
- package/dist/lib/node-esm/chunk-HSLMI22Q.mjs.map +7 -0
- package/dist/lib/node-esm/chunk-NVM2QUMX.mjs +37 -0
- package/dist/lib/node-esm/chunk-NVM2QUMX.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-MFJO7EKD.mjs → chunk-SNZAN34I.mjs} +3 -3
- package/dist/lib/node-esm/{chunk-56QHOX4A.mjs → chunk-TIUYXGQN.mjs} +367 -433
- package/dist/lib/node-esm/chunk-TIUYXGQN.mjs.map +7 -0
- package/dist/lib/node-esm/chunk-TTJBW7CY.mjs +149 -0
- package/dist/lib/node-esm/chunk-TTJBW7CY.mjs.map +7 -0
- package/dist/lib/node-esm/index.mjs +68 -75
- package/dist/lib/node-esm/index.mjs.map +4 -4
- package/dist/lib/node-esm/meta.json +1 -1
- package/dist/lib/node-esm/operation-resolver-SMCFTUFM.mjs +23 -0
- package/dist/lib/node-esm/operation-resolver-SMCFTUFM.mjs.map +7 -0
- package/dist/lib/node-esm/react-surface-YBBXC4W5.mjs +36 -0
- package/dist/lib/node-esm/react-surface-YBBXC4W5.mjs.map +7 -0
- package/dist/lib/node-esm/{transcriber-TVG2MBBN.mjs → transcriber-BO2N3UQ4.mjs} +17 -13
- package/dist/lib/node-esm/transcriber-BO2N3UQ4.mjs.map +7 -0
- package/dist/lib/node-esm/types/index.mjs +8 -8
- package/dist/types/src/TranscriptionPlugin.d.ts +2 -1
- package/dist/types/src/TranscriptionPlugin.d.ts.map +1 -1
- package/dist/types/src/blueprints/index.d.ts +2 -0
- package/dist/types/src/blueprints/index.d.ts.map +1 -0
- package/dist/types/src/blueprints/transcription-blueprint.d.ts +21 -0
- package/dist/types/src/blueprints/transcription-blueprint.d.ts.map +1 -0
- package/dist/types/src/capabilities/blueprint-definition/blueprint-definition.d.ts +10 -0
- package/dist/types/src/capabilities/blueprint-definition/blueprint-definition.d.ts.map +1 -0
- package/dist/types/src/capabilities/blueprint-definition/index.d.ts +3 -0
- package/dist/types/src/capabilities/blueprint-definition/index.d.ts.map +1 -0
- package/dist/types/src/capabilities/index.d.ts +4 -4
- package/dist/types/src/capabilities/index.d.ts.map +1 -1
- package/dist/types/src/capabilities/operation-handler/index.d.ts +3 -0
- package/dist/types/src/capabilities/operation-handler/index.d.ts.map +1 -0
- package/dist/types/src/capabilities/operation-handler/operation-handler.d.ts +5 -0
- package/dist/types/src/capabilities/operation-handler/operation-handler.d.ts.map +1 -0
- package/dist/types/src/capabilities/operation-resolver/index.d.ts +3 -0
- package/dist/types/src/capabilities/operation-resolver/index.d.ts.map +1 -0
- package/dist/types/src/capabilities/operation-resolver/operation-resolver.d.ts +5 -0
- package/dist/types/src/capabilities/operation-resolver/operation-resolver.d.ts.map +1 -0
- package/dist/types/src/capabilities/react-surface/index.d.ts +3 -0
- package/dist/types/src/capabilities/react-surface/index.d.ts.map +1 -0
- package/dist/types/src/capabilities/react-surface/react-surface.d.ts +5 -0
- package/dist/types/src/capabilities/react-surface/react-surface.d.ts.map +1 -0
- package/dist/types/src/capabilities/transcriber/index.d.ts +3 -0
- package/dist/types/src/capabilities/transcriber/index.d.ts.map +1 -0
- package/dist/types/src/capabilities/transcriber/transcriber.d.ts +9 -0
- package/dist/types/src/capabilities/transcriber/transcriber.d.ts.map +1 -0
- package/dist/types/src/components/Transcript/Transcript.d.ts +7 -10
- package/dist/types/src/components/Transcript/Transcript.d.ts.map +1 -1
- package/dist/types/src/components/Transcript/Transcript.stories.d.ts +34 -18
- package/dist/types/src/components/Transcript/Transcript.stories.d.ts.map +1 -1
- package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts +12 -12
- package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts.map +1 -1
- package/dist/types/src/components/Transcript/transcript-extension.d.ts +3 -7
- package/dist/types/src/components/Transcript/transcript-extension.d.ts.map +1 -1
- package/dist/types/src/components/TranscriptContainer.d.ts +5 -6
- package/dist/types/src/components/TranscriptContainer.d.ts.map +1 -1
- package/dist/types/src/components/index.d.ts +1 -4
- package/dist/types/src/components/index.d.ts.map +1 -1
- package/dist/types/src/components/stories/FileTranscription.stories.d.ts +8 -7
- package/dist/types/src/components/stories/FileTranscription.stories.d.ts.map +1 -1
- package/dist/types/src/components/stories/KeyWordRecognition.stories.d.ts +10 -6
- package/dist/types/src/components/stories/KeyWordRecognition.stories.d.ts.map +1 -1
- package/dist/types/src/components/stories/MicrophoneTranscription.stories.d.ts +10 -9
- package/dist/types/src/components/stories/MicrophoneTranscription.stories.d.ts.map +1 -1
- package/dist/types/src/components/stories/TranscriptionStory.d.ts +4 -3
- package/dist/types/src/components/stories/TranscriptionStory.d.ts.map +1 -1
- package/dist/types/src/functions/index.d.ts +3 -0
- package/dist/types/src/functions/index.d.ts.map +1 -0
- package/dist/types/src/functions/open.d.ts +11 -0
- package/dist/types/src/functions/open.d.ts.map +1 -0
- package/dist/types/src/functions/summarize.d.ts +11 -0
- package/dist/types/src/functions/summarize.d.ts.map +1 -0
- package/dist/types/src/hooks/useAudioFile.d.ts.map +1 -1
- package/dist/types/src/hooks/useQueueModelAdapter.d.ts +1 -1
- package/dist/types/src/hooks/useQueueModelAdapter.d.ts.map +1 -1
- package/dist/types/src/hooks/useTranscriber.d.ts +2 -2
- package/dist/types/src/hooks/useTranscriber.d.ts.map +1 -1
- package/dist/types/src/hooks/useVoiceInput.d.ts.map +1 -1
- package/dist/types/src/index.d.ts +1 -1
- package/dist/types/src/index.d.ts.map +1 -1
- package/dist/types/src/meta.d.ts +2 -3
- package/dist/types/src/meta.d.ts.map +1 -1
- package/dist/types/src/segments-normalization/message-normalizer.d.ts +5 -5
- package/dist/types/src/segments-normalization/message-normalizer.d.ts.map +1 -1
- package/dist/types/src/segments-normalization/normalization.d.ts +503 -311
- package/dist/types/src/segments-normalization/normalization.d.ts.map +1 -1
- package/dist/types/src/segments-normalization/utils.d.ts +2 -2
- package/dist/types/src/segments-normalization/utils.d.ts.map +1 -1
- package/dist/types/src/testing/testing.d.ts +10 -10
- package/dist/types/src/testing/testing.d.ts.map +1 -1
- package/dist/types/src/transcriber/index.d.ts +1 -1
- package/dist/types/src/transcriber/index.d.ts.map +1 -1
- package/dist/types/src/transcriber/media-stream-recorder.d.ts +3 -3
- package/dist/types/src/transcriber/media-stream-recorder.d.ts.map +1 -1
- package/dist/types/src/transcriber/transcriber.d.ts +4 -4
- package/dist/types/src/transcriber/transcriber.d.ts.map +1 -1
- package/dist/types/src/transcriber/transcription-manager.d.ts +12 -13
- package/dist/types/src/transcriber/transcription-manager.d.ts.map +1 -1
- package/dist/types/src/translations.d.ts +11 -11
- package/dist/types/src/translations.d.ts.map +1 -1
- package/dist/types/src/types/TranscriptAction.d.ts +25 -0
- package/dist/types/src/types/TranscriptAction.d.ts.map +1 -0
- package/dist/types/src/types/capabilities.d.ts +18 -0
- package/dist/types/src/types/capabilities.d.ts.map +1 -0
- package/dist/types/src/types/index.d.ts +3 -1
- package/dist/types/src/types/index.d.ts.map +1 -1
- package/dist/types/src/types/types.d.ts +0 -26
- package/dist/types/src/types/types.d.ts.map +1 -1
- package/dist/types/src/util/index.d.ts +1 -0
- package/dist/types/src/util/index.d.ts.map +1 -1
- package/dist/types/src/util/render-byline.d.ts +4 -0
- package/dist/types/src/util/render-byline.d.ts.map +1 -0
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +88 -72
- package/src/TranscriptionPlugin.tsx +40 -58
- package/src/blueprints/index.ts +5 -0
- package/src/blueprints/transcription-blueprint.ts +25 -0
- package/src/capabilities/blueprint-definition/blueprint-definition.ts +24 -0
- package/src/capabilities/blueprint-definition/index.ts +7 -0
- package/src/capabilities/index.ts +4 -7
- package/src/capabilities/operation-handler/index.ts +7 -0
- package/src/capabilities/operation-handler/operation-handler.ts +25 -0
- package/src/capabilities/operation-resolver/index.ts +7 -0
- package/src/capabilities/operation-resolver/operation-resolver.ts +25 -0
- package/src/capabilities/react-surface/index.ts +7 -0
- package/src/capabilities/react-surface/react-surface.tsx +28 -0
- package/src/capabilities/transcriber/index.ts +7 -0
- package/src/capabilities/transcriber/transcriber.ts +85 -0
- package/src/components/Transcript/Transcript.stories.tsx +61 -57
- package/src/components/Transcript/Transcript.tsx +25 -59
- package/src/components/Transcript/deprecated/OldTranscript.tsx +11 -11
- package/src/components/Transcript/transcript-extension.ts +13 -46
- package/src/components/TranscriptContainer.tsx +19 -15
- package/src/components/stories/FileTranscription.stories.tsx +80 -70
- package/src/components/stories/KeyWordRecognition.stories.tsx +17 -13
- package/src/components/stories/MicrophoneTranscription.stories.tsx +89 -87
- package/src/components/stories/TranscriptionStory.tsx +14 -14
- package/src/functions/index.ts +6 -0
- package/src/functions/open.ts +37 -0
- package/src/functions/summarize.ts +103 -0
- package/src/hooks/useAudioFile.ts +49 -53
- package/src/hooks/useQueueModelAdapter.ts +1 -1
- package/src/hooks/useTranscriber.ts +12 -7
- package/src/hooks/useVoiceInput.ts +4 -3
- package/src/index.ts +1 -1
- package/src/meta.ts +9 -6
- package/src/model/model.test.ts +27 -21
- package/src/segments-normalization/message-normalizer.ts +19 -14
- package/src/segments-normalization/normalization.test.ts +25 -32
- package/src/segments-normalization/normalization.ts +59 -49
- package/src/segments-normalization/utils.ts +2 -2
- package/src/testing/testing.ts +42 -32
- package/src/transcriber/index.ts +1 -1
- package/src/transcriber/media-stream-recorder.ts +4 -4
- package/src/transcriber/transcriber.browser.test.ts +6 -5
- package/src/transcriber/transcriber.ts +9 -11
- package/src/transcriber/transcription-manager.ts +37 -52
- package/src/translations.ts +6 -4
- package/src/types/TranscriptAction.ts +32 -0
- package/src/types/capabilities.ts +33 -0
- package/src/types/index.ts +3 -1
- package/src/types/types.ts +0 -23
- package/src/util/index.ts +1 -0
- package/src/util/render-byline.ts +33 -0
- package/dist/lib/browser/TranscriptContainer-H3AZQ4CQ.mjs +0 -11
- package/dist/lib/browser/chunk-53AXUEIQ.mjs.map +0 -7
- package/dist/lib/browser/chunk-GOF7MJNV.mjs +0 -19
- package/dist/lib/browser/chunk-GOF7MJNV.mjs.map +0 -7
- package/dist/lib/browser/chunk-IQ5ZRMHZ.mjs.map +0 -7
- package/dist/lib/browser/chunk-OSISJM2F.mjs +0 -55
- package/dist/lib/browser/chunk-OSISJM2F.mjs.map +0 -7
- package/dist/lib/browser/chunk-W2KNYJD6.mjs +0 -16
- package/dist/lib/browser/chunk-W2KNYJD6.mjs.map +0 -7
- package/dist/lib/browser/intent-resolver-BGADCHVM.mjs +0 -30
- package/dist/lib/browser/intent-resolver-BGADCHVM.mjs.map +0 -7
- package/dist/lib/browser/react-surface-HMJS5FSS.mjs +0 -34
- package/dist/lib/browser/react-surface-HMJS5FSS.mjs.map +0 -7
- package/dist/lib/browser/transcriber-OYPLZFG5.mjs.map +0 -7
- package/dist/lib/node/TranscriptContainer-KIG3GMAB.cjs +0 -32
- package/dist/lib/node/TranscriptContainer-KIG3GMAB.cjs.map +0 -7
- package/dist/lib/node/chunk-3PYT3BDM.cjs +0 -40
- package/dist/lib/node/chunk-3PYT3BDM.cjs.map +0 -7
- package/dist/lib/node/chunk-6LW5RA3K.cjs +0 -75
- package/dist/lib/node/chunk-6LW5RA3K.cjs.map +0 -7
- package/dist/lib/node/chunk-AFUZI63A.cjs +0 -42
- package/dist/lib/node/chunk-AFUZI63A.cjs.map +0 -7
- package/dist/lib/node/chunk-CVUN3UO7.cjs +0 -778
- package/dist/lib/node/chunk-CVUN3UO7.cjs.map +0 -7
- package/dist/lib/node/chunk-FKDJNJ57.cjs +0 -35
- package/dist/lib/node/chunk-FKDJNJ57.cjs.map +0 -7
- package/dist/lib/node/chunk-ZFAUMZXE.cjs +0 -518
- package/dist/lib/node/chunk-ZFAUMZXE.cjs.map +0 -7
- package/dist/lib/node/index.cjs +0 -133
- package/dist/lib/node/index.cjs.map +0 -7
- package/dist/lib/node/intent-resolver-LL3TTI4H.cjs +0 -45
- package/dist/lib/node/intent-resolver-LL3TTI4H.cjs.map +0 -7
- package/dist/lib/node/meta.json +0 -1
- package/dist/lib/node/react-surface-N25TBBG4.cjs +0 -56
- package/dist/lib/node/react-surface-N25TBBG4.cjs.map +0 -7
- package/dist/lib/node/transcriber-6J4FVFSZ.cjs +0 -68
- package/dist/lib/node/transcriber-6J4FVFSZ.cjs.map +0 -7
- package/dist/lib/node/types/index.cjs +0 -36
- package/dist/lib/node/types/index.cjs.map +0 -7
- package/dist/lib/node-esm/chunk-56QHOX4A.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-FH75X4JC.mjs +0 -56
- package/dist/lib/node-esm/chunk-FH75X4JC.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-MFJPEIVB.mjs +0 -17
- package/dist/lib/node-esm/chunk-MFJPEIVB.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-PRL64SRU.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-TXMDLLBR.mjs +0 -21
- package/dist/lib/node-esm/chunk-TXMDLLBR.mjs.map +0 -7
- package/dist/lib/node-esm/intent-resolver-5LVCYSJS.mjs +0 -31
- package/dist/lib/node-esm/intent-resolver-5LVCYSJS.mjs.map +0 -7
- package/dist/lib/node-esm/react-surface-JGQSYGRR.mjs +0 -35
- package/dist/lib/node-esm/react-surface-JGQSYGRR.mjs.map +0 -7
- package/dist/lib/node-esm/transcriber-TVG2MBBN.mjs.map +0 -7
- package/dist/types/src/capabilities/capabilities.d.ts +0 -17
- package/dist/types/src/capabilities/capabilities.d.ts.map +0 -1
- package/dist/types/src/capabilities/intent-resolver.d.ts +0 -4
- package/dist/types/src/capabilities/intent-resolver.d.ts.map +0 -1
- package/dist/types/src/capabilities/react-surface.d.ts +0 -4
- package/dist/types/src/capabilities/react-surface.d.ts.map +0 -1
- package/dist/types/src/capabilities/transcriber.d.ts +0 -8
- package/dist/types/src/capabilities/transcriber.d.ts.map +0 -1
- package/dist/types/src/types/schema.d.ts +0 -41
- package/dist/types/src/types/schema.d.ts.map +0 -1
- package/src/capabilities/capabilities.ts +0 -33
- package/src/capabilities/intent-resolver.ts +0 -24
- package/src/capabilities/react-surface.tsx +0 -22
- package/src/capabilities/transcriber.ts +0 -76
- package/src/types/schema.ts +0 -41
- /package/dist/lib/browser/{TranscriptContainer-H3AZQ4CQ.mjs.map → TranscriptContainer-MV3JBKLS.mjs.map} +0 -0
- /package/dist/lib/{node-esm/TranscriptContainer-G75LDEO5.mjs.map → browser/blueprints/index.mjs.map} +0 -0
- /package/dist/lib/browser/{chunk-UDCCT32F.mjs.map → chunk-RUKY36XW.mjs.map} +0 -0
- /package/dist/lib/node-esm/{chunk-MFJO7EKD.mjs.map → chunk-SNZAN34I.mjs.map} +0 -0
|
@@ -2,25 +2,22 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import { effect } from '@preact/signals-core';
|
|
6
5
|
import { describe, test } from 'vitest';
|
|
7
6
|
|
|
8
|
-
import { EdgeAiServiceClient, OllamaAiServiceClient } from '@dxos/ai';
|
|
9
|
-
import { AI_SERVICE_ENDPOINT } from '@dxos/ai/testing';
|
|
10
7
|
import { scheduleTaskInterval } from '@dxos/async';
|
|
11
8
|
import { Context } from '@dxos/context';
|
|
12
9
|
import { Obj } from '@dxos/echo';
|
|
10
|
+
import { createQueueDXN } from '@dxos/echo/internal';
|
|
13
11
|
import { MemoryQueue } from '@dxos/echo-db';
|
|
14
|
-
import {
|
|
15
|
-
import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
|
|
12
|
+
import { FunctionExecutor, ServiceContainer } from '@dxos/functions-runtime';
|
|
16
13
|
import { log } from '@dxos/log';
|
|
17
|
-
import {
|
|
14
|
+
import { type Actor, Message } from '@dxos/types';
|
|
18
15
|
|
|
19
16
|
import { MessageNormalizer } from './message-normalizer';
|
|
20
17
|
import { type MessageWithRangeId, sentenceNormalization } from './normalization';
|
|
21
18
|
import { getActorId } from './utils';
|
|
22
19
|
|
|
23
|
-
const sender:
|
|
20
|
+
const sender: Actor.Actor = {
|
|
24
21
|
identityDid: 'did:key:123',
|
|
25
22
|
};
|
|
26
23
|
|
|
@@ -50,34 +47,33 @@ const messages: MessageWithRangeId[] = [
|
|
|
50
47
|
// No punctuation.
|
|
51
48
|
'in classical physics objects have well-defined properties such as position speed and momentum',
|
|
52
49
|
].map((string, index) =>
|
|
53
|
-
Obj.make(
|
|
54
|
-
created: new Date(Date.now() +
|
|
50
|
+
Obj.make(Message.Message, {
|
|
51
|
+
created: new Date(Date.now() + 1_000 * index).toISOString(),
|
|
55
52
|
sender,
|
|
56
|
-
blocks: [{
|
|
57
|
-
|
|
58
|
-
} as any),
|
|
53
|
+
blocks: [{ _tag: 'transcript', started: new Date(Date.now() + 1_000 * index).toISOString(), text: string }],
|
|
54
|
+
}),
|
|
59
55
|
);
|
|
60
56
|
|
|
61
|
-
const REMOTE_AI = true;
|
|
57
|
+
// const REMOTE_AI = true;
|
|
62
58
|
|
|
63
59
|
describe.skip('SentenceNormalization', () => {
|
|
64
60
|
const getExecutor = () => {
|
|
65
61
|
return new FunctionExecutor(
|
|
66
62
|
new ServiceContainer().setServices({
|
|
67
|
-
ai: {
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
},
|
|
63
|
+
// ai: {
|
|
64
|
+
// client: REMOTE_AI
|
|
65
|
+
// ? new Edge AiServiceClient({
|
|
66
|
+
// endpoint: AI_SERVICE_ENDPOINT.REMOTE,
|
|
67
|
+
// defaultGenerationOptions: {
|
|
68
|
+
// model: '@anthropic/claude-3-5-sonnet-20241022',
|
|
69
|
+
// },
|
|
70
|
+
// })
|
|
71
|
+
// : new Ollama AiServiceClient({
|
|
72
|
+
// overrides: {
|
|
73
|
+
// model: 'llama3.1:8b',
|
|
74
|
+
// },
|
|
75
|
+
// }),
|
|
76
|
+
// },
|
|
81
77
|
}),
|
|
82
78
|
);
|
|
83
79
|
};
|
|
@@ -100,6 +96,7 @@ describe.skip('SentenceNormalization', () => {
|
|
|
100
96
|
buffer = sentences.slice(activeSentenceIndex);
|
|
101
97
|
}
|
|
102
98
|
}
|
|
99
|
+
|
|
103
100
|
log.info('sentences', {
|
|
104
101
|
originalMessages: JSON.stringify(messages, null, 2),
|
|
105
102
|
sentences: JSON.stringify(sentences, null, 2),
|
|
@@ -110,7 +107,7 @@ describe.skip('SentenceNormalization', () => {
|
|
|
110
107
|
test.only('queue', { timeout: 120_000 }, async () => {
|
|
111
108
|
// Create queue.
|
|
112
109
|
// TODO(dmaretskyi): Use space.queues.create() instead.
|
|
113
|
-
const queue = new MemoryQueue<
|
|
110
|
+
const queue = new MemoryQueue<Message.Message>(createQueueDXN());
|
|
114
111
|
const ctx = new Context();
|
|
115
112
|
let idx = 0;
|
|
116
113
|
scheduleTaskInterval(
|
|
@@ -135,10 +132,6 @@ describe.skip('SentenceNormalization', () => {
|
|
|
135
132
|
|
|
136
133
|
// Start normalizer.
|
|
137
134
|
await normalizer.open();
|
|
138
|
-
effect(() => {
|
|
139
|
-
log.info('normalizer');
|
|
140
|
-
log.info(JSON.stringify(queue.objects, null, 2));
|
|
141
|
-
});
|
|
142
135
|
|
|
143
136
|
await new Promise(() => {});
|
|
144
137
|
});
|
|
@@ -2,20 +2,24 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
// ISSUE(burdon): defineFunction
|
|
6
|
+
// @ts-nocheck
|
|
6
7
|
|
|
7
|
-
import
|
|
8
|
-
|
|
8
|
+
import * as Schema from 'effect/Schema';
|
|
9
|
+
|
|
10
|
+
import { AiService, DEFAULT_EDGE_MODEL } from '@dxos/ai';
|
|
11
|
+
import { AiSession } from '@dxos/assistant';
|
|
9
12
|
import { Obj } from '@dxos/echo';
|
|
10
|
-
import {
|
|
13
|
+
import { defineFunction } from '@dxos/functions';
|
|
11
14
|
import { ObjectId } from '@dxos/keys';
|
|
12
15
|
import { log } from '@dxos/log';
|
|
13
|
-
import {
|
|
16
|
+
import { Message } from '@dxos/types';
|
|
17
|
+
import { trim } from '@dxos/util';
|
|
14
18
|
|
|
15
19
|
const MessageWithRangeId = Schema.extend(
|
|
16
|
-
|
|
20
|
+
Message.Message,
|
|
17
21
|
Schema.Struct({
|
|
18
|
-
// TODO(mykola): Move to meta in
|
|
22
|
+
// TODO(mykola): Move to meta in Message. Use `Message.Message` instead.
|
|
19
23
|
rangeId: Schema.optional(Schema.Array(Schema.String)).annotations({
|
|
20
24
|
description: 'The IDs of the messages that contain the sentences.',
|
|
21
25
|
}),
|
|
@@ -25,7 +29,7 @@ const MessageWithRangeId = Schema.extend(
|
|
|
25
29
|
export interface MessageWithRangeId extends Schema.Schema.Type<typeof MessageWithRangeId> {}
|
|
26
30
|
|
|
27
31
|
export const NormalizationInput = Schema.Struct({
|
|
28
|
-
// TODO(mykola): Move to meta in
|
|
32
|
+
// TODO(mykola): Move to meta in Message. Use `Message.Message` instead.
|
|
29
33
|
messages: Schema.Array(MessageWithRangeId).annotations({
|
|
30
34
|
description: 'Messages to normalize into sentences.',
|
|
31
35
|
}),
|
|
@@ -39,40 +43,6 @@ export const NormalizationOutput = Schema.Struct({
|
|
|
39
43
|
});
|
|
40
44
|
export interface NormalizationOutput extends Schema.Schema.Type<typeof NormalizationOutput> {}
|
|
41
45
|
|
|
42
|
-
const prompt = `
|
|
43
|
-
You are observing a real-time transcript of a single person speaking.
|
|
44
|
-
The transcription is delivered in chunks of 10 seconds or less. As a result, individual sentences may be split across multiple messages, or multiple sentences may appear within a single message. Additionally, because this is real-time transcription, punctuation and capitalization may be incorrect or missing.
|
|
45
|
-
|
|
46
|
-
# Task Description:
|
|
47
|
-
- Your task is to detect and reconstruct broken or incomplete sentences by merging segments into coherent, grammatically correct sentences where appropriate.
|
|
48
|
-
|
|
49
|
-
# Input Format:
|
|
50
|
-
- The input is an array of messages that contain transcription blocks.
|
|
51
|
-
- Each message is a DataType.Message, as described in the output format.
|
|
52
|
-
|
|
53
|
-
# Output Format:
|
|
54
|
-
- You have been provided with the tool that defines the output format; make sure to query it.
|
|
55
|
-
- Do not output anything other than the expected format.
|
|
56
|
-
|
|
57
|
-
# Segment Handling Rules:
|
|
58
|
-
- Sort messages by timestamp.
|
|
59
|
-
- Leave ID of the first message in a sentence.
|
|
60
|
-
- Each divided sentence should be added to the output as a separate message or block within existing messages, preserving the original order.
|
|
61
|
-
- If one message contains multiple sentences, you should not split them.
|
|
62
|
-
- The last sentence may be incomplete; it should still be output as a separate block or message.
|
|
63
|
-
- Keep track of the original timestamps of the messages and blocks and, use the timestamp of the first message/block of a sentence as the 'started' field of the merged message.
|
|
64
|
-
- The 'rangeId' field of the merged message should include 'messageId'-s and 'rangeId'-s of all messages that have been used to construct this message.
|
|
65
|
-
|
|
66
|
-
# Punctuation and Capitalization:
|
|
67
|
-
- Do not rely on the original punctuation and capitalization—they may be incorrect.
|
|
68
|
-
- Use logical reasoning to apply appropriate punctuation (e.g., period, comma, question mark, exclamation mark) and capitalization.
|
|
69
|
-
|
|
70
|
-
# Restrictions:
|
|
71
|
-
- Do not alter the order of the original messages; maintain the natural flow of speech.
|
|
72
|
-
- Do not interpret or infer meaning beyond reconstructing sentences.
|
|
73
|
-
- Do not add or remove any words or phrases.
|
|
74
|
-
`;
|
|
75
|
-
|
|
76
46
|
export const sentenceNormalization = defineFunction<NormalizationInput, NormalizationOutput>({
|
|
77
47
|
description: 'Post process of transcription for sentence normalization',
|
|
78
48
|
inputSchema: NormalizationInput,
|
|
@@ -80,21 +50,27 @@ export const sentenceNormalization = defineFunction<NormalizationInput, Normaliz
|
|
|
80
50
|
handler: async ({ data: { messages }, context }) => {
|
|
81
51
|
log.info('input', { messages });
|
|
82
52
|
const ai = context.getService(AiService);
|
|
83
|
-
const session = new
|
|
53
|
+
const session = new AiSession({ operationModel: 'configured' });
|
|
84
54
|
|
|
85
|
-
|
|
86
|
-
|
|
55
|
+
// TODO(dmaretskyi): This got broken after effect-ai transition.
|
|
56
|
+
const response = session.runStructured(NormalizationOutput, {
|
|
57
|
+
generationOptions: {
|
|
58
|
+
model: DEFAULT_EDGE_MODEL,
|
|
59
|
+
},
|
|
87
60
|
client: ai.client,
|
|
88
61
|
tools: [],
|
|
89
62
|
artifacts: [],
|
|
90
63
|
history: [
|
|
91
|
-
Obj.make(Message, {
|
|
92
|
-
|
|
93
|
-
|
|
64
|
+
Obj.make(Message.Message, {
|
|
65
|
+
created: new Date().toISOString(),
|
|
66
|
+
sender: {
|
|
67
|
+
role: 'user',
|
|
68
|
+
},
|
|
69
|
+
blocks: messages.map((message) => ({ _tag: 'text', text: JSON.stringify(message) }) as const),
|
|
94
70
|
}),
|
|
95
71
|
],
|
|
96
72
|
prompt,
|
|
97
|
-
});
|
|
73
|
+
}) as any;
|
|
98
74
|
|
|
99
75
|
response.sentences.forEach((sentence) => {
|
|
100
76
|
sentence.id = ObjectId.random();
|
|
@@ -103,3 +79,37 @@ export const sentenceNormalization = defineFunction<NormalizationInput, Normaliz
|
|
|
103
79
|
return response;
|
|
104
80
|
},
|
|
105
81
|
});
|
|
82
|
+
|
|
83
|
+
const prompt = trim`
|
|
84
|
+
You are observing a real-time transcript of a single person speaking.
|
|
85
|
+
The transcription is delivered in chunks of 10 seconds or less. As a result, individual sentences may be split across multiple messages, or multiple sentences may appear within a single message. Additionally, because this is real-time transcription, punctuation and capitalization may be incorrect or missing.
|
|
86
|
+
|
|
87
|
+
# Task Description:
|
|
88
|
+
- Your task is to detect and reconstruct broken or incomplete sentences by merging segments into coherent, grammatically correct sentences where appropriate.
|
|
89
|
+
|
|
90
|
+
# Input Format:
|
|
91
|
+
- The input is an array of messages that contain transcription blocks.
|
|
92
|
+
- Each message is a Message.Message, as described in the output format.
|
|
93
|
+
|
|
94
|
+
# Output Format:
|
|
95
|
+
- You have been provided with the tool that defines the output format; make sure to query it.
|
|
96
|
+
- Do not output anything other than the expected format.
|
|
97
|
+
|
|
98
|
+
# Segment Handling Rules:
|
|
99
|
+
- Sort messages by timestamp.
|
|
100
|
+
- Leave ID of the first message in a sentence.
|
|
101
|
+
- Each divided sentence should be added to the output as a separate message or block within existing messages, preserving the original order.
|
|
102
|
+
- If one message contains multiple sentences, you should not split them.
|
|
103
|
+
- The last sentence may be incomplete; it should still be output as a separate block or message.
|
|
104
|
+
- Keep track of the original timestamps of the messages and blocks and, use the timestamp of the first message/block of a sentence as the 'started' field of the merged message.
|
|
105
|
+
- The 'rangeId' field of the merged message should include 'messageId'-s and 'rangeId'-s of all messages that have been used to construct this message.
|
|
106
|
+
|
|
107
|
+
# Punctuation and Capitalization:
|
|
108
|
+
- Do not rely on the original punctuation and capitalization—they may be incorrect.
|
|
109
|
+
- Use logical reasoning to apply appropriate punctuation (e.g., period, comma, question mark, exclamation mark) and capitalization.
|
|
110
|
+
|
|
111
|
+
# Restrictions:
|
|
112
|
+
- Do not alter the order of the original messages; maintain the natural flow of speech.
|
|
113
|
+
- Do not interpret or infer meaning beyond reconstructing sentences.
|
|
114
|
+
- Do not add or remove any words or phrases.
|
|
115
|
+
`;
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import { type
|
|
5
|
+
import { type Actor } from '@dxos/types';
|
|
6
6
|
|
|
7
|
-
export const getActorId = (actor:
|
|
7
|
+
export const getActorId = (actor: Actor.Actor) => {
|
|
8
8
|
return actor.identityDid || actor.identityKey || actor.email || actor.name || '';
|
|
9
9
|
};
|
package/src/testing/testing.ts
CHANGED
|
@@ -2,24 +2,23 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import
|
|
5
|
+
import * as Schema from 'effect/Schema';
|
|
6
6
|
import { useEffect, useMemo, useState } from 'react';
|
|
7
7
|
|
|
8
|
-
import {
|
|
9
|
-
import { AI_SERVICE_ENDPOINT } from '@dxos/ai/testing';
|
|
10
|
-
import { extractionAnthropicFn, processTranscriptMessage } from '@dxos/assistant';
|
|
8
|
+
import { extractionAnthropicFunction, processTranscriptMessage } from '@dxos/assistant/extraction';
|
|
11
9
|
import { scheduleTaskInterval } from '@dxos/async';
|
|
12
10
|
import { Filter, type Queue } from '@dxos/client/echo';
|
|
13
11
|
import { Context } from '@dxos/context';
|
|
14
12
|
import { type Key, Obj, Ref, Type } from '@dxos/echo';
|
|
15
|
-
import { createQueueDXN } from '@dxos/echo
|
|
16
|
-
import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
|
|
13
|
+
import { createQueueDXN } from '@dxos/echo/internal';
|
|
14
|
+
import { FunctionExecutor, ServiceContainer } from '@dxos/functions-runtime';
|
|
17
15
|
import { IdentityDid } from '@dxos/keys';
|
|
18
16
|
import { log } from '@dxos/log';
|
|
19
17
|
import { faker } from '@dxos/random';
|
|
20
|
-
import {
|
|
21
|
-
import {
|
|
22
|
-
import {
|
|
18
|
+
import { type Space, useQueue } from '@dxos/react-client/echo';
|
|
19
|
+
import { TestSchema } from '@dxos/schema/testing';
|
|
20
|
+
import { type ContentBlock, Message, Organization, Person } from '@dxos/types';
|
|
21
|
+
import { seedTestData } from '@dxos/types/testing';
|
|
23
22
|
|
|
24
23
|
// TODO(burdon): Reconcile with plugin-markdown. Move to @dxos/schema/testing.
|
|
25
24
|
export const TestItem = Schema.Struct({
|
|
@@ -32,7 +31,7 @@ export const TestItem = Schema.Struct({
|
|
|
32
31
|
description: 'Product description',
|
|
33
32
|
}),
|
|
34
33
|
}).pipe(
|
|
35
|
-
Type.
|
|
34
|
+
Type.object({
|
|
36
35
|
typename: 'dxos.org/type/Test',
|
|
37
36
|
version: '0.1.0',
|
|
38
37
|
}),
|
|
@@ -40,7 +39,7 @@ export const TestItem = Schema.Struct({
|
|
|
40
39
|
|
|
41
40
|
// TODO(wittjosiah): Make builder generic and reuse for all message types.
|
|
42
41
|
abstract class AbstractMessageBuilder {
|
|
43
|
-
abstract createMessage(numSegments?: number): Promise<
|
|
42
|
+
abstract createMessage(numSegments?: number): Promise<Message.Message>;
|
|
44
43
|
}
|
|
45
44
|
|
|
46
45
|
/**
|
|
@@ -61,27 +60,32 @@ export class MessageBuilder extends AbstractMessageBuilder {
|
|
|
61
60
|
super();
|
|
62
61
|
}
|
|
63
62
|
|
|
64
|
-
override async createMessage(numSegments = 1): Promise<
|
|
65
|
-
return Obj.make(
|
|
63
|
+
override async createMessage(numSegments = 1): Promise<Message.Message> {
|
|
64
|
+
return Obj.make(Message.Message, {
|
|
66
65
|
created: this.next().toISOString(),
|
|
67
66
|
sender: faker.helpers.arrayElement(this.users),
|
|
68
67
|
blocks: Array.from({ length: numSegments }).map(() => this.createBlock()),
|
|
69
68
|
});
|
|
70
69
|
}
|
|
71
70
|
|
|
72
|
-
createBlock():
|
|
71
|
+
createBlock(): ContentBlock.Transcript {
|
|
73
72
|
let text = faker.lorem.paragraph();
|
|
74
73
|
if (this._space) {
|
|
75
74
|
const label = faker.commerce.productName();
|
|
76
|
-
const obj = this._space.db.add(
|
|
75
|
+
const obj = this._space.db.add(
|
|
76
|
+
Obj.make(TestItem, {
|
|
77
|
+
title: label,
|
|
78
|
+
description: faker.lorem.paragraph(),
|
|
79
|
+
}),
|
|
80
|
+
);
|
|
77
81
|
const dxn = Ref.make(obj).dxn.toString();
|
|
78
82
|
const words = text.split(' ');
|
|
79
|
-
words.splice(Math.floor(Math.random() * words.length), 0, `[${label}]
|
|
83
|
+
words.splice(Math.floor(Math.random() * words.length), 0, `[${label}](${dxn})`);
|
|
80
84
|
text = words.join(' ');
|
|
81
85
|
}
|
|
82
86
|
|
|
83
87
|
return {
|
|
84
|
-
|
|
88
|
+
_tag: 'transcript',
|
|
85
89
|
started: this.next().toISOString(),
|
|
86
90
|
text,
|
|
87
91
|
};
|
|
@@ -95,15 +99,17 @@ export class MessageBuilder extends AbstractMessageBuilder {
|
|
|
95
99
|
|
|
96
100
|
// TODO(burdon): Reconcile with BlockBuilder.
|
|
97
101
|
class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
102
|
+
executor = new FunctionExecutor(
|
|
103
|
+
new ServiceContainer().setServices({
|
|
104
|
+
// ai: {
|
|
105
|
+
// client: this.AiService,
|
|
106
|
+
// },
|
|
107
|
+
}),
|
|
108
|
+
);
|
|
103
109
|
|
|
104
110
|
space: Space | undefined;
|
|
105
111
|
currentMessage: number = 0;
|
|
106
|
-
transcriptMessages:
|
|
112
|
+
transcriptMessages: Message.Message[] = [];
|
|
107
113
|
|
|
108
114
|
async connect(space: Space): Promise<void> {
|
|
109
115
|
this.space = space;
|
|
@@ -111,14 +117,18 @@ class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
|
|
|
111
117
|
this.transcriptMessages = transcriptMessages;
|
|
112
118
|
}
|
|
113
119
|
|
|
114
|
-
override async createMessage(): Promise<
|
|
120
|
+
override async createMessage(): Promise<Message.Message> {
|
|
115
121
|
if (!this.space) {
|
|
116
122
|
throw new Error('Space not connected');
|
|
117
123
|
}
|
|
118
124
|
|
|
119
|
-
const
|
|
125
|
+
const objects = await this.space.db
|
|
120
126
|
.query(
|
|
121
|
-
Filter.or(
|
|
127
|
+
Filter.or(
|
|
128
|
+
Filter.type(Person.Person),
|
|
129
|
+
Filter.type(Organization.Organization),
|
|
130
|
+
Filter.type(TestSchema.DocumentType),
|
|
131
|
+
),
|
|
122
132
|
)
|
|
123
133
|
.run();
|
|
124
134
|
|
|
@@ -130,7 +140,7 @@ class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
|
|
|
130
140
|
const { message: enhancedMessage } = await processTranscriptMessage({
|
|
131
141
|
input: { message },
|
|
132
142
|
executor: this.executor,
|
|
133
|
-
function:
|
|
143
|
+
function: extractionAnthropicFunction,
|
|
134
144
|
});
|
|
135
145
|
|
|
136
146
|
return enhancedMessage;
|
|
@@ -142,7 +152,7 @@ type UseTestTranscriptionQueue = (
|
|
|
142
152
|
queueId?: Key.ObjectId,
|
|
143
153
|
running?: boolean,
|
|
144
154
|
interval?: number,
|
|
145
|
-
) => Queue<
|
|
155
|
+
) => Queue<Message.Message> | undefined;
|
|
146
156
|
|
|
147
157
|
/**
|
|
148
158
|
* Test transcriptionqueue.
|
|
@@ -155,7 +165,7 @@ export const useTestTranscriptionQueue: UseTestTranscriptionQueue = (
|
|
|
155
165
|
) => {
|
|
156
166
|
// TODO(dmaretskyi): Use space.queues.create() instead.
|
|
157
167
|
const queueDxn = useMemo(() => (space ? createQueueDXN(space.id, queueId) : undefined), [space, queueId]);
|
|
158
|
-
const queue = useQueue<
|
|
168
|
+
const queue = useQueue<Message.Message>(queueDxn);
|
|
159
169
|
const builder = useMemo(() => new MessageBuilder(space), [space]);
|
|
160
170
|
|
|
161
171
|
useEffect(() => {
|
|
@@ -165,7 +175,7 @@ export const useTestTranscriptionQueue: UseTestTranscriptionQueue = (
|
|
|
165
175
|
|
|
166
176
|
const i = setInterval(() => {
|
|
167
177
|
void builder.createMessage(Math.ceil(Math.random() * 3)).then(async (message) => {
|
|
168
|
-
await queue.append([Obj.make(
|
|
178
|
+
await queue.append([Obj.make(Message.Message, message)]);
|
|
169
179
|
});
|
|
170
180
|
}, interval);
|
|
171
181
|
return () => clearInterval(i);
|
|
@@ -186,7 +196,7 @@ export const useTestTranscriptionQueueWithEntityExtraction: UseTestTranscription
|
|
|
186
196
|
) => {
|
|
187
197
|
// TODO(dmaretskyi): Use space.queues.create() instead.
|
|
188
198
|
const queueDxn = useMemo(() => (space ? createQueueDXN(space.id, queueId) : undefined), [space, queueId]);
|
|
189
|
-
const queue = useQueue<
|
|
199
|
+
const queue = useQueue<Message.Message>(queueDxn);
|
|
190
200
|
const [builder] = useState(() => new EntityExtractionMessageBuilder());
|
|
191
201
|
|
|
192
202
|
useEffect(() => {
|
|
@@ -203,7 +213,7 @@ export const useTestTranscriptionQueueWithEntityExtraction: UseTestTranscription
|
|
|
203
213
|
ctx,
|
|
204
214
|
async () => {
|
|
205
215
|
const message = await builder.createMessage();
|
|
206
|
-
void queue.append([Obj.make(
|
|
216
|
+
void queue.append([Obj.make(Message.Message, message)]);
|
|
207
217
|
},
|
|
208
218
|
interval,
|
|
209
219
|
);
|
package/src/transcriber/index.ts
CHANGED
|
@@ -10,7 +10,7 @@ import { invariant } from '@dxos/invariant';
|
|
|
10
10
|
import { log } from '@dxos/log';
|
|
11
11
|
import { trace } from '@dxos/tracing';
|
|
12
12
|
|
|
13
|
-
import { type
|
|
13
|
+
import { type AudioChunk, type AudioRecorder, type WavConfig } from './audio-recorder';
|
|
14
14
|
|
|
15
15
|
let initializingPromise: Promise<void> | undefined;
|
|
16
16
|
|
|
@@ -19,7 +19,7 @@ const initializeExtendableMediaRecorder = async () => {
|
|
|
19
19
|
await register(await connect());
|
|
20
20
|
};
|
|
21
21
|
|
|
22
|
-
export type
|
|
22
|
+
export type MediaStreamRecorderProps = {
|
|
23
23
|
mediaStreamTrack: MediaStreamTrack;
|
|
24
24
|
config: {
|
|
25
25
|
interval: number;
|
|
@@ -37,12 +37,12 @@ export class MediaStreamRecorder implements AudioRecorder {
|
|
|
37
37
|
/**
|
|
38
38
|
* Default MediaRecorder implementation do not support wav encoding.
|
|
39
39
|
*/
|
|
40
|
-
private readonly _config:
|
|
40
|
+
private readonly _config: MediaStreamRecorderProps['config'];
|
|
41
41
|
private _mediaRecorder?: IMediaRecorder = undefined;
|
|
42
42
|
private _header?: Uint8Array = undefined;
|
|
43
43
|
private _onChunk?: (chunk: AudioChunk) => void;
|
|
44
44
|
|
|
45
|
-
constructor({ mediaStreamTrack, config }:
|
|
45
|
+
constructor({ mediaStreamTrack, config }: MediaStreamRecorderProps) {
|
|
46
46
|
this._mediaStreamTrack = mediaStreamTrack;
|
|
47
47
|
this._config = config;
|
|
48
48
|
}
|
|
@@ -4,14 +4,15 @@
|
|
|
4
4
|
|
|
5
5
|
import fs from 'node:fs';
|
|
6
6
|
import path from 'node:path';
|
|
7
|
+
|
|
7
8
|
import { describe, expect, test } from 'vitest';
|
|
8
9
|
import { WaveFile } from 'wavefile';
|
|
9
10
|
|
|
10
11
|
import { Trigger } from '@dxos/async';
|
|
11
12
|
import { log } from '@dxos/log';
|
|
12
|
-
import { type DataType } from '@dxos/schema';
|
|
13
13
|
import { openAndClose } from '@dxos/test-utils';
|
|
14
|
-
import {
|
|
14
|
+
import { TRACE_PROCESSOR, trace } from '@dxos/tracing';
|
|
15
|
+
import { type ContentBlock } from '@dxos/types';
|
|
15
16
|
|
|
16
17
|
import { type AudioChunk, type AudioRecorder, Transcriber } from '../transcriber';
|
|
17
18
|
import { mergeFloat64Arrays } from '../util';
|
|
@@ -93,7 +94,7 @@ describe.skip('Transcriber', () => {
|
|
|
93
94
|
});
|
|
94
95
|
|
|
95
96
|
test.skip('transcription of audio recording', { timeout: 10_000 }, async () => {
|
|
96
|
-
const trigger = new Trigger<
|
|
97
|
+
const trigger = new Trigger<ContentBlock.Transcript[]>({ autoReset: true });
|
|
97
98
|
const recorder = new MockAudioRecorder({
|
|
98
99
|
buffer: await readFile('test.wav'),
|
|
99
100
|
chunkDuration: 3_000,
|
|
@@ -127,7 +128,7 @@ describe.skip('Transcriber', () => {
|
|
|
127
128
|
});
|
|
128
129
|
|
|
129
130
|
test.skip('transcription of audio recording with overlapping chunks', { timeout: 20_000 }, async () => {
|
|
130
|
-
const trigger = new Trigger<
|
|
131
|
+
const trigger = new Trigger<ContentBlock.Transcript[]>({ autoReset: true });
|
|
131
132
|
const recorder = new MockAudioRecorder({
|
|
132
133
|
buffer: await readFile('test.wav'),
|
|
133
134
|
chunkDuration: 3_000,
|
|
@@ -212,7 +213,7 @@ class MockAudioRecorder implements AudioRecorder {
|
|
|
212
213
|
|
|
213
214
|
@trace.span()
|
|
214
215
|
async start(): Promise<void> {
|
|
215
|
-
log.info('start mock audio recorder', {
|
|
216
|
+
log.info('start mock audio recorder', { wavProps: this.wav.fmt });
|
|
216
217
|
const now = Date.now();
|
|
217
218
|
const samples = this.wav.getSamples();
|
|
218
219
|
const sampleRate = this.wavConfig.sampleRate;
|
|
@@ -7,13 +7,14 @@ import { WaveFile } from 'wavefile';
|
|
|
7
7
|
import { DeferredTask, Trigger } from '@dxos/async';
|
|
8
8
|
import { type Context, LifecycleState, Resource } from '@dxos/context';
|
|
9
9
|
import { log } from '@dxos/log';
|
|
10
|
-
import { type DataType } from '@dxos/schema';
|
|
11
10
|
import { trace } from '@dxos/tracing';
|
|
11
|
+
import { type ContentBlock } from '@dxos/types';
|
|
12
12
|
|
|
13
|
-
import { type AudioRecorder, type AudioChunk } from './audio-recorder';
|
|
14
13
|
import { TRANSCRIPTION_URL } from '../types';
|
|
15
14
|
import { mergeFloat64Arrays } from '../util';
|
|
16
15
|
|
|
16
|
+
import { type AudioChunk, type AudioRecorder } from './audio-recorder';
|
|
17
|
+
|
|
17
18
|
type WhisperWord = {
|
|
18
19
|
word: string;
|
|
19
20
|
|
|
@@ -62,14 +63,14 @@ export type TranscribeConfig = {
|
|
|
62
63
|
prefixBufferChunksAmount: number;
|
|
63
64
|
};
|
|
64
65
|
|
|
65
|
-
export type
|
|
66
|
+
export type TranscriberProps = {
|
|
66
67
|
config: TranscribeConfig;
|
|
67
68
|
recorder: AudioRecorder;
|
|
68
69
|
/**
|
|
69
70
|
* Callback to handle the transcribed segments, after all segment transformers are applied.
|
|
70
71
|
* @param segments - The transcribed segments.
|
|
71
72
|
*/
|
|
72
|
-
onSegments: (segments:
|
|
73
|
+
onSegments: (segments: ContentBlock.Transcript[]) => Promise<void>;
|
|
73
74
|
};
|
|
74
75
|
|
|
75
76
|
/**
|
|
@@ -85,12 +86,12 @@ export class Transcriber extends Resource {
|
|
|
85
86
|
|
|
86
87
|
private readonly _config: TranscribeConfig;
|
|
87
88
|
private readonly _recorder: AudioRecorder;
|
|
88
|
-
private readonly _onSegments:
|
|
89
|
+
private readonly _onSegments: TranscriberProps['onSegments'];
|
|
89
90
|
|
|
90
91
|
private _recording = false;
|
|
91
92
|
private _transcribeTask?: DeferredTask = undefined;
|
|
92
93
|
|
|
93
|
-
constructor({ config, recorder, onSegments }:
|
|
94
|
+
constructor({ config, recorder, onSegments }: TranscriberProps) {
|
|
94
95
|
super();
|
|
95
96
|
this._config = config;
|
|
96
97
|
this._recorder = recorder;
|
|
@@ -220,10 +221,7 @@ export class Transcriber extends Resource {
|
|
|
220
221
|
return segments;
|
|
221
222
|
}
|
|
222
223
|
|
|
223
|
-
private _alignSegments(
|
|
224
|
-
segments: WhisperSegment[],
|
|
225
|
-
originalChunks: AudioChunk[],
|
|
226
|
-
): DataType.MessageBlock.Transcription[] {
|
|
224
|
+
private _alignSegments(segments: WhisperSegment[], originalChunks: AudioChunk[]): ContentBlock.Transcript[] {
|
|
227
225
|
// Absolute zero for all relative timestamps in the segments.
|
|
228
226
|
const zeroTimestamp = originalChunks.at(0)!.timestamp;
|
|
229
227
|
|
|
@@ -252,7 +250,7 @@ export class Transcriber extends Resource {
|
|
|
252
250
|
|
|
253
251
|
// Add absolute timestamp to each segment.
|
|
254
252
|
return filteredSegments.map((segment) => ({
|
|
255
|
-
|
|
253
|
+
_tag: 'transcript',
|
|
256
254
|
started: new Date(zeroTimestamp + segment.start * 1_000).toISOString(),
|
|
257
255
|
text: segment.text.trim(),
|
|
258
256
|
}));
|