@dxos/plugin-transcription 0.8.4-main.ae835ea → 0.8.4-main.bc2380dfbc
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +102 -5
- package/README.md +1 -1
- package/dist/lib/neutral/TranscriptionArticle-L6XANBH5.mjs +54 -0
- package/dist/lib/neutral/TranscriptionArticle-L6XANBH5.mjs.map +7 -0
- package/dist/lib/neutral/TranscriptionPlugin.mjs +34 -0
- package/dist/lib/neutral/TranscriptionPlugin.mjs.map +7 -0
- package/dist/lib/neutral/TranscriptionPlugin.node.mjs +25 -0
- package/dist/lib/neutral/TranscriptionPlugin.node.mjs.map +7 -0
- package/dist/lib/neutral/TranscriptionPlugin.workerd.mjs +21 -0
- package/dist/lib/neutral/TranscriptionPlugin.workerd.mjs.map +7 -0
- package/dist/lib/neutral/blueprint-definition-HOXDG5QP.mjs +15 -0
- package/dist/lib/neutral/blueprint-definition-HOXDG5QP.mjs.map +7 -0
- package/dist/lib/neutral/blueprints/index.mjs +8 -0
- package/dist/lib/neutral/capabilities/index.mjs +17 -0
- package/dist/lib/neutral/capabilities/index.mjs.map +7 -0
- package/dist/lib/neutral/chunk-2O3ONPRZ.mjs +298 -0
- package/dist/lib/neutral/chunk-2O3ONPRZ.mjs.map +7 -0
- package/dist/lib/neutral/chunk-5FXU36PN.mjs +120 -0
- package/dist/lib/neutral/chunk-5FXU36PN.mjs.map +7 -0
- package/dist/lib/{browser/chunk-VBUIO677.mjs → neutral/chunk-GL4RS533.mjs} +2 -9
- package/dist/lib/neutral/chunk-GL4RS533.mjs.map +7 -0
- package/dist/lib/{browser/chunk-WH36ENNE.mjs → neutral/chunk-ILSA3DU7.mjs} +102 -133
- package/dist/lib/neutral/chunk-ILSA3DU7.mjs.map +7 -0
- package/dist/lib/neutral/chunk-J5LGTIGS.mjs +10 -0
- package/dist/lib/neutral/chunk-KAEXRX3Y.mjs +35 -0
- package/dist/lib/neutral/chunk-KAEXRX3Y.mjs.map +7 -0
- package/dist/lib/neutral/chunk-MLRJWAQH.mjs +179 -0
- package/dist/lib/neutral/chunk-MLRJWAQH.mjs.map +7 -0
- package/dist/lib/neutral/chunk-Y5EQAFFR.mjs +8 -0
- package/dist/lib/neutral/chunk-Y5EQAFFR.mjs.map +7 -0
- package/dist/lib/neutral/chunk-ZR4BH4QQ.mjs +30 -0
- package/dist/lib/neutral/chunk-ZR4BH4QQ.mjs.map +7 -0
- package/dist/lib/neutral/components/index.mjs +272 -0
- package/dist/lib/neutral/components/index.mjs.map +7 -0
- package/dist/lib/neutral/containers/index.mjs +9 -0
- package/dist/lib/neutral/containers/index.mjs.map +7 -0
- package/dist/lib/neutral/create-RRWRIUSR.mjs +21 -0
- package/dist/lib/neutral/create-RRWRIUSR.mjs.map +7 -0
- package/dist/lib/neutral/hooks/index.mjs +19 -0
- package/dist/lib/neutral/index.mjs +44 -0
- package/dist/lib/neutral/meta.json +1 -0
- package/dist/lib/neutral/meta.mjs +8 -0
- package/dist/lib/neutral/meta.mjs.map +7 -0
- package/dist/lib/neutral/open-CVMX5KXZ.mjs +27 -0
- package/dist/lib/neutral/open-CVMX5KXZ.mjs.map +7 -0
- package/dist/lib/neutral/operation-handler-NXMOAXFP.mjs +13 -0
- package/dist/lib/neutral/operation-handler-NXMOAXFP.mjs.map +7 -0
- package/dist/lib/neutral/operations/index.mjs +8 -0
- package/dist/lib/neutral/operations/index.mjs.map +7 -0
- package/dist/lib/neutral/plugin.mjs +16 -0
- package/dist/lib/neutral/plugin.mjs.map +7 -0
- package/dist/lib/neutral/react-surface-YMZRO242.mjs +26 -0
- package/dist/lib/neutral/react-surface-YMZRO242.mjs.map +7 -0
- package/dist/lib/neutral/summarize-LA7TRW7A.mjs +78 -0
- package/dist/lib/neutral/summarize-LA7TRW7A.mjs.map +7 -0
- package/dist/lib/neutral/text-content-F4ASMITB.mjs +30 -0
- package/dist/lib/neutral/text-content-F4ASMITB.mjs.map +7 -0
- package/dist/lib/{node-esm/transcriber-UGB7ECX6.mjs → neutral/transcriber-ABHATBVH.mjs} +20 -18
- package/dist/lib/neutral/transcriber-ABHATBVH.mjs.map +7 -0
- package/dist/lib/neutral/translations.mjs +24 -0
- package/dist/lib/neutral/translations.mjs.map +7 -0
- package/dist/lib/neutral/types/index.mjs +10 -0
- package/dist/lib/neutral/types/index.mjs.map +7 -0
- package/dist/types/src/TranscriptionPlugin.d.ts +3 -1
- package/dist/types/src/TranscriptionPlugin.d.ts.map +1 -1
- package/dist/types/src/TranscriptionPlugin.node.d.ts +4 -0
- package/dist/types/src/TranscriptionPlugin.node.d.ts.map +1 -0
- package/dist/types/src/TranscriptionPlugin.test.d.ts +2 -0
- package/dist/types/src/TranscriptionPlugin.test.d.ts.map +1 -0
- package/dist/types/src/TranscriptionPlugin.workerd.d.ts +4 -0
- package/dist/types/src/TranscriptionPlugin.workerd.d.ts.map +1 -0
- package/dist/types/src/blueprints/index.d.ts +2 -0
- package/dist/types/src/blueprints/index.d.ts.map +1 -0
- package/dist/types/src/blueprints/transcription-blueprint.d.ts +4 -0
- package/dist/types/src/blueprints/transcription-blueprint.d.ts.map +1 -0
- package/dist/types/src/capabilities/blueprint-definition.d.ts +5 -4
- package/dist/types/src/capabilities/blueprint-definition.d.ts.map +1 -1
- package/dist/types/src/capabilities/index.d.ts +10 -5
- package/dist/types/src/capabilities/index.d.ts.map +1 -1
- package/dist/types/src/capabilities/operation-handler.d.ts +6 -0
- package/dist/types/src/capabilities/operation-handler.d.ts.map +1 -0
- package/dist/types/src/capabilities/react-surface.d.ts +3 -2
- package/dist/types/src/capabilities/react-surface.d.ts.map +1 -1
- package/dist/types/src/capabilities/text-content.d.ts +8 -0
- package/dist/types/src/capabilities/text-content.d.ts.map +1 -0
- package/dist/types/src/capabilities/transcriber.d.ts +4 -3
- package/dist/types/src/capabilities/transcriber.d.ts.map +1 -1
- package/dist/types/src/components/Oracle/Oracle.d.ts +24 -0
- package/dist/types/src/components/Oracle/Oracle.d.ts.map +1 -0
- package/dist/types/src/components/Oracle/Oracle.stories.d.ts +32 -0
- package/dist/types/src/components/Oracle/Oracle.stories.d.ts.map +1 -0
- package/dist/types/src/components/Oracle/index.d.ts +2 -0
- package/dist/types/src/components/Oracle/index.d.ts.map +1 -0
- package/dist/types/src/components/Transcription/Transcription.d.ts +13 -0
- package/dist/types/src/components/Transcription/Transcription.d.ts.map +1 -0
- package/dist/types/src/components/Transcription/Transcription.stories.d.ts +37 -0
- package/dist/types/src/components/Transcription/Transcription.stories.d.ts.map +1 -0
- package/dist/types/src/components/Transcription/index.d.ts +2 -0
- package/dist/types/src/components/Transcription/index.d.ts.map +1 -0
- package/dist/types/src/components/Transcription/transcription-extension.d.ts +15 -0
- package/dist/types/src/components/Transcription/transcription-extension.d.ts.map +1 -0
- package/dist/types/src/components/index.d.ts +2 -3
- package/dist/types/src/components/index.d.ts.map +1 -1
- package/dist/types/src/containers/TranscriptionArticle/TranscriptionArticle.d.ts +6 -0
- package/dist/types/src/containers/TranscriptionArticle/TranscriptionArticle.d.ts.map +1 -0
- package/dist/types/src/containers/TranscriptionArticle/TranscriptionArticle.stories.d.ts +28 -0
- package/dist/types/src/containers/TranscriptionArticle/TranscriptionArticle.stories.d.ts.map +1 -0
- package/dist/types/src/containers/TranscriptionArticle/index.d.ts +2 -0
- package/dist/types/src/containers/TranscriptionArticle/index.d.ts.map +1 -0
- package/dist/types/src/containers/index.d.ts +3 -0
- package/dist/types/src/containers/index.d.ts.map +1 -0
- package/dist/types/src/enrichment/conversation.test.d.ts +2 -0
- package/dist/types/src/enrichment/conversation.test.d.ts.map +1 -0
- package/dist/types/src/enrichment/operations.d.ts +200 -0
- package/dist/types/src/enrichment/operations.d.ts.map +1 -0
- package/dist/types/src/enrichment/parser.d.ts +26 -0
- package/dist/types/src/enrichment/parser.d.ts.map +1 -0
- package/dist/types/src/enrichment/stream-simulator.d.ts +38 -0
- package/dist/types/src/enrichment/stream-simulator.d.ts.map +1 -0
- package/dist/types/src/hooks/index.d.ts +2 -1
- package/dist/types/src/hooks/index.d.ts.map +1 -1
- package/dist/types/src/hooks/useAudioFile.d.ts.map +1 -1
- package/dist/types/src/hooks/useAudioTrack.d.ts.map +1 -1
- package/dist/types/src/hooks/useFeedModelAdapter.d.ts +12 -0
- package/dist/types/src/hooks/useFeedModelAdapter.d.ts.map +1 -0
- package/dist/types/src/hooks/useTranscriber.d.ts +2 -2
- package/dist/types/src/hooks/useTranscriber.d.ts.map +1 -1
- package/dist/types/src/hooks/useTranscriptionRecording.d.ts +12 -0
- package/dist/types/src/hooks/useTranscriptionRecording.d.ts.map +1 -0
- package/dist/types/src/hooks/useVoiceInput.d.ts.map +1 -1
- package/dist/types/src/index.d.ts +2 -3
- package/dist/types/src/index.d.ts.map +1 -1
- package/dist/types/src/meta.d.ts +2 -2
- package/dist/types/src/meta.d.ts.map +1 -1
- package/dist/types/src/model/model.d.ts +2 -2
- package/dist/types/src/model/model.d.ts.map +1 -1
- package/dist/types/src/{segments-normalization → normalization}/index.d.ts +1 -1
- package/dist/types/src/normalization/index.d.ts.map +1 -0
- package/dist/types/src/{segments-normalization → normalization}/message-normalizer.d.ts +9 -7
- package/dist/types/src/normalization/message-normalizer.d.ts.map +1 -0
- package/dist/types/src/normalization/normalization.d.ts +543 -0
- package/dist/types/src/normalization/normalization.d.ts.map +1 -0
- package/dist/types/src/normalization/normalization.test.d.ts.map +1 -0
- package/dist/types/src/normalization/utils.d.ts +3 -0
- package/dist/types/src/normalization/utils.d.ts.map +1 -0
- package/dist/types/src/operations/create.d.ts +5 -0
- package/dist/types/src/operations/create.d.ts.map +1 -0
- package/dist/types/src/operations/index.d.ts +3 -0
- package/dist/types/src/operations/index.d.ts.map +1 -0
- package/dist/types/src/operations/open.d.ts +5 -0
- package/dist/types/src/operations/open.d.ts.map +1 -0
- package/dist/types/src/operations/summarize.d.ts +8 -0
- package/dist/types/src/operations/summarize.d.ts.map +1 -0
- package/dist/types/src/plugin.d.ts +4 -0
- package/dist/types/src/plugin.d.ts.map +1 -0
- package/dist/types/src/stories/FileTranscription.stories.d.ts +20 -0
- package/dist/types/src/stories/FileTranscription.stories.d.ts.map +1 -0
- package/dist/types/src/{components/stories → stories}/KeyWordRecognition.stories.d.ts +2 -2
- package/dist/types/src/stories/KeyWordRecognition.stories.d.ts.map +1 -0
- package/dist/types/src/{components/stories/MicrophoneTranscription.stories.d.ts → stories/LiveTranscription.stories.d.ts} +7 -7
- package/dist/types/src/stories/LiveTranscription.stories.d.ts.map +1 -0
- package/dist/types/src/stories/TranscriptionStory.d.ts +21 -0
- package/dist/types/src/stories/TranscriptionStory.d.ts.map +1 -0
- package/dist/types/src/stories/common.d.ts +46 -0
- package/dist/types/src/stories/common.d.ts.map +1 -0
- package/dist/types/src/testing/testing.d.ts +13 -14
- package/dist/types/src/testing/testing.d.ts.map +1 -1
- package/dist/types/src/transcriber/index.d.ts +1 -1
- package/dist/types/src/transcriber/index.d.ts.map +1 -1
- package/dist/types/src/transcriber/media-stream-recorder.d.ts +2 -2
- package/dist/types/src/transcriber/media-stream-recorder.d.ts.map +1 -1
- package/dist/types/src/transcriber/transcriber.d.ts +61 -6
- package/dist/types/src/transcriber/transcriber.d.ts.map +1 -1
- package/dist/types/src/transcriber/transcriber.test.d.ts +2 -0
- package/dist/types/src/transcriber/transcriber.test.d.ts.map +1 -0
- package/dist/types/src/transcriber/transcription-manager.d.ts +14 -9
- package/dist/types/src/transcriber/transcription-manager.d.ts.map +1 -1
- package/dist/types/src/translations.d.ts +9 -8
- package/dist/types/src/translations.d.ts.map +1 -1
- package/dist/types/src/types/TranscriptOperation.d.ts +917 -0
- package/dist/types/src/types/TranscriptOperation.d.ts.map +1 -0
- package/dist/types/src/types/TranscriptionCapabilities.d.ts +17 -0
- package/dist/types/src/types/TranscriptionCapabilities.d.ts.map +1 -0
- package/dist/types/src/types/index.d.ts +2 -3
- package/dist/types/src/types/index.d.ts.map +1 -1
- package/dist/types/src/util/array.d.ts.map +1 -1
- package/dist/types/src/util/get-time-str.d.ts.map +1 -1
- package/dist/types/src/util/index.d.ts +1 -0
- package/dist/types/src/util/index.d.ts.map +1 -1
- package/dist/types/src/util/monitor-audio-level.d.ts.map +1 -1
- package/dist/types/src/util/render-byline.d.ts +4 -0
- package/dist/types/src/util/render-byline.d.ts.map +1 -0
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +157 -82
- package/src/TranscriptionPlugin.node.ts +20 -0
- package/src/TranscriptionPlugin.test.ts +31 -0
- package/src/TranscriptionPlugin.tsx +20 -64
- package/src/TranscriptionPlugin.workerd.ts +18 -0
- package/src/blueprints/index.ts +5 -0
- package/src/blueprints/transcription-blueprint.ts +29 -0
- package/src/capabilities/blueprint-definition.ts +12 -24
- package/src/capabilities/index.ts +11 -7
- package/src/capabilities/operation-handler.ts +16 -0
- package/src/capabilities/react-surface.tsx +23 -14
- package/src/capabilities/text-content.ts +33 -0
- package/src/capabilities/transcriber.ts +53 -42
- package/src/components/Oracle/Oracle.stories.tsx +147 -0
- package/src/components/Oracle/Oracle.tsx +141 -0
- package/src/components/Oracle/index.ts +5 -0
- package/src/components/Transcription/Transcription.stories.tsx +213 -0
- package/src/components/Transcription/Transcription.tsx +58 -0
- package/src/components/{Transcript → Transcription}/index.ts +1 -1
- package/src/components/{Transcript/transcript-extension.ts → Transcription/transcription-extension.ts} +13 -9
- package/src/components/index.ts +2 -6
- package/src/containers/TranscriptionArticle/TranscriptionArticle.stories.tsx +61 -0
- package/src/containers/TranscriptionArticle/TranscriptionArticle.tsx +62 -0
- package/src/containers/TranscriptionArticle/index.ts +5 -0
- package/src/containers/index.ts +7 -0
- package/src/enrichment/conversation.test.ts +162 -0
- package/src/enrichment/conversation.txt +23 -0
- package/src/enrichment/operations.ts +268 -0
- package/src/enrichment/parser.ts +101 -0
- package/src/enrichment/stream-simulator.ts +125 -0
- package/src/hooks/index.ts +2 -1
- package/src/hooks/useFeedModelAdapter.ts +46 -0
- package/src/hooks/useTranscriber.ts +12 -4
- package/src/hooks/useTranscriptionRecording.ts +87 -0
- package/src/hooks/useVoiceInput.ts +2 -3
- package/src/index.ts +4 -5
- package/src/meta.ts +3 -3
- package/src/model/model.test.ts +20 -14
- package/src/model/model.ts +3 -3
- package/src/{segments-normalization → normalization}/message-normalizer.ts +37 -24
- package/src/{segments-normalization → normalization}/normalization.test.ts +8 -47
- package/src/normalization/normalization.ts +76 -0
- package/src/{segments-normalization → normalization}/utils.ts +2 -2
- package/src/operations/create.ts +24 -0
- package/src/operations/index.ts +11 -0
- package/src/operations/open.ts +28 -0
- package/src/operations/summarize.ts +101 -0
- package/src/plugin.ts +11 -0
- package/src/stories/FileTranscription.stories.tsx +143 -0
- package/src/{components/stories → stories}/KeyWordRecognition.stories.tsx +40 -40
- package/src/stories/LiveTranscription.stories.tsx +153 -0
- package/src/stories/TranscriptionStory.tsx +101 -0
- package/src/stories/common.ts +233 -0
- package/src/testing/testing.ts +49 -56
- package/src/transcriber/index.ts +1 -1
- package/src/transcriber/media-stream-recorder.ts +3 -3
- package/src/transcriber/transcriber.browser.test.ts +4 -5
- package/src/transcriber/transcriber.test.ts +104 -0
- package/src/transcriber/transcriber.ts +105 -60
- package/src/transcriber/transcription-manager.ts +33 -22
- package/src/translations.ts +10 -7
- package/src/types/TranscriptOperation.ts +103 -0
- package/src/types/TranscriptionCapabilities.ts +39 -0
- package/src/types/index.ts +2 -3
- package/src/util/index.ts +1 -0
- package/src/util/render-byline.ts +33 -0
- package/dist/lib/browser/TranscriptContainer-2GSL2XB5.mjs +0 -11
- package/dist/lib/browser/blueprint-definition-UAAJYVLZ.mjs +0 -150
- package/dist/lib/browser/blueprint-definition-UAAJYVLZ.mjs.map +0 -7
- package/dist/lib/browser/chunk-3MKZ2MNQ.mjs +0 -16
- package/dist/lib/browser/chunk-3MKZ2MNQ.mjs.map +0 -7
- package/dist/lib/browser/chunk-535OJKFO.mjs +0 -8
- package/dist/lib/browser/chunk-535OJKFO.mjs.map +0 -7
- package/dist/lib/browser/chunk-7DR6HZEF.mjs +0 -732
- package/dist/lib/browser/chunk-7DR6HZEF.mjs.map +0 -7
- package/dist/lib/browser/chunk-VBUIO677.mjs.map +0 -7
- package/dist/lib/browser/chunk-W7WKZTM4.mjs +0 -63
- package/dist/lib/browser/chunk-W7WKZTM4.mjs.map +0 -7
- package/dist/lib/browser/chunk-WH36ENNE.mjs.map +0 -7
- package/dist/lib/browser/index.mjs +0 -130
- package/dist/lib/browser/index.mjs.map +0 -7
- package/dist/lib/browser/intent-resolver-ZTKTEAQX.mjs +0 -25
- package/dist/lib/browser/intent-resolver-ZTKTEAQX.mjs.map +0 -7
- package/dist/lib/browser/meta.json +0 -1
- package/dist/lib/browser/react-surface-KA32A72I.mjs +0 -34
- package/dist/lib/browser/react-surface-KA32A72I.mjs.map +0 -7
- package/dist/lib/browser/transcriber-LWOLYQHC.mjs +0 -56
- package/dist/lib/browser/transcriber-LWOLYQHC.mjs.map +0 -7
- package/dist/lib/browser/types/index.mjs +0 -12
- package/dist/lib/node-esm/TranscriptContainer-GRLDCESB.mjs +0 -12
- package/dist/lib/node-esm/blueprint-definition-Z3LTZ3ID.mjs +0 -151
- package/dist/lib/node-esm/blueprint-definition-Z3LTZ3ID.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-K4AZOB5Y.mjs +0 -733
- package/dist/lib/node-esm/chunk-K4AZOB5Y.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-OLMSTF3M.mjs +0 -64
- package/dist/lib/node-esm/chunk-OLMSTF3M.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-Q6WD7L2J.mjs +0 -10
- package/dist/lib/node-esm/chunk-Q6WD7L2J.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-SWUVPFKR.mjs +0 -17
- package/dist/lib/node-esm/chunk-SWUVPFKR.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-YFN4FJFK.mjs +0 -467
- package/dist/lib/node-esm/chunk-YFN4FJFK.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-YVNQN2J6.mjs +0 -30
- package/dist/lib/node-esm/chunk-YVNQN2J6.mjs.map +0 -7
- package/dist/lib/node-esm/index.mjs +0 -131
- package/dist/lib/node-esm/index.mjs.map +0 -7
- package/dist/lib/node-esm/intent-resolver-55WH2GY6.mjs +0 -26
- package/dist/lib/node-esm/intent-resolver-55WH2GY6.mjs.map +0 -7
- package/dist/lib/node-esm/meta.json +0 -1
- package/dist/lib/node-esm/react-surface-GU7LW2JO.mjs +0 -35
- package/dist/lib/node-esm/react-surface-GU7LW2JO.mjs.map +0 -7
- package/dist/lib/node-esm/transcriber-UGB7ECX6.mjs.map +0 -7
- package/dist/lib/node-esm/types/index.mjs +0 -13
- package/dist/types/src/capabilities/capabilities.d.ts +0 -17
- package/dist/types/src/capabilities/capabilities.d.ts.map +0 -1
- package/dist/types/src/capabilities/intent-resolver.d.ts +0 -4
- package/dist/types/src/capabilities/intent-resolver.d.ts.map +0 -1
- package/dist/types/src/components/Transcript/Transcript.d.ts +0 -20
- package/dist/types/src/components/Transcript/Transcript.d.ts.map +0 -1
- package/dist/types/src/components/Transcript/Transcript.stories.d.ts +0 -39
- package/dist/types/src/components/Transcript/Transcript.stories.d.ts.map +0 -1
- package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts +0 -28
- package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts.map +0 -1
- package/dist/types/src/components/Transcript/index.d.ts +0 -2
- package/dist/types/src/components/Transcript/index.d.ts.map +0 -1
- package/dist/types/src/components/Transcript/transcript-extension.d.ts +0 -15
- package/dist/types/src/components/Transcript/transcript-extension.d.ts.map +0 -1
- package/dist/types/src/components/TranscriptContainer.d.ts +0 -9
- package/dist/types/src/components/TranscriptContainer.d.ts.map +0 -1
- package/dist/types/src/components/stories/FileTranscription.stories.d.ts +0 -18
- package/dist/types/src/components/stories/FileTranscription.stories.d.ts.map +0 -1
- package/dist/types/src/components/stories/KeyWordRecognition.stories.d.ts.map +0 -1
- package/dist/types/src/components/stories/MicrophoneTranscription.stories.d.ts.map +0 -1
- package/dist/types/src/components/stories/TranscriptionStory.d.ts +0 -12
- package/dist/types/src/components/stories/TranscriptionStory.d.ts.map +0 -1
- package/dist/types/src/components/stories/useIsSpeaking.d.ts +0 -3
- package/dist/types/src/components/stories/useIsSpeaking.d.ts.map +0 -1
- package/dist/types/src/functions/index.d.ts +0 -3
- package/dist/types/src/functions/index.d.ts.map +0 -1
- package/dist/types/src/functions/open.d.ts +0 -7
- package/dist/types/src/functions/open.d.ts.map +0 -1
- package/dist/types/src/functions/summarize.d.ts +0 -11
- package/dist/types/src/functions/summarize.d.ts.map +0 -1
- package/dist/types/src/hooks/useQueueModelAdapter.d.ts +0 -8
- package/dist/types/src/hooks/useQueueModelAdapter.d.ts.map +0 -1
- package/dist/types/src/segments-normalization/index.d.ts.map +0 -1
- package/dist/types/src/segments-normalization/message-normalizer.d.ts.map +0 -1
- package/dist/types/src/segments-normalization/normalization.d.ts +0 -476
- package/dist/types/src/segments-normalization/normalization.d.ts.map +0 -1
- package/dist/types/src/segments-normalization/normalization.test.d.ts.map +0 -1
- package/dist/types/src/segments-normalization/utils.d.ts +0 -3
- package/dist/types/src/segments-normalization/utils.d.ts.map +0 -1
- package/dist/types/src/types/Transcript.d.ts +0 -30
- package/dist/types/src/types/Transcript.d.ts.map +0 -1
- package/dist/types/src/types/TranscriptAction.d.ts +0 -23
- package/dist/types/src/types/TranscriptAction.d.ts.map +0 -1
- package/dist/types/src/types/types.d.ts +0 -5
- package/dist/types/src/types/types.d.ts.map +0 -1
- package/src/capabilities/capabilities.ts +0 -33
- package/src/capabilities/intent-resolver.ts +0 -18
- package/src/components/Transcript/Transcript.stories.tsx +0 -245
- package/src/components/Transcript/Transcript.tsx +0 -93
- package/src/components/Transcript/deprecated/OldTranscript.tsx +0 -260
- package/src/components/TranscriptContainer.tsx +0 -36
- package/src/components/stories/FileTranscription.stories.tsx +0 -256
- package/src/components/stories/MicrophoneTranscription.stories.tsx +0 -285
- package/src/components/stories/TranscriptionStory.tsx +0 -45
- package/src/components/stories/useIsSpeaking.tsx +0 -40
- package/src/functions/index.ts +0 -6
- package/src/functions/open.ts +0 -39
- package/src/functions/summarize.ts +0 -103
- package/src/hooks/useQueueModelAdapter.ts +0 -66
- package/src/segments-normalization/normalization.ts +0 -115
- package/src/types/Transcript.ts +0 -44
- package/src/types/TranscriptAction.ts +0 -24
- package/src/types/types.ts +0 -9
- package/dist/lib/{browser/types → neutral/blueprints}/index.mjs.map +0 -0
- package/dist/lib/{browser/TranscriptContainer-2GSL2XB5.mjs.map → neutral/chunk-J5LGTIGS.mjs.map} +0 -0
- package/dist/lib/{node-esm/types → neutral/hooks}/index.mjs.map +0 -0
- package/dist/lib/{node-esm/TranscriptContainer-GRLDCESB.mjs.map → neutral/index.mjs.map} +0 -0
- package/dist/types/src/{segments-normalization → normalization}/normalization.test.d.ts +0 -0
- package/src/{segments-normalization → normalization}/index.ts +1 -1
package/src/testing/testing.ts
CHANGED
|
@@ -2,22 +2,22 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
+
import * as Effect from 'effect/Effect';
|
|
5
6
|
import * as Schema from 'effect/Schema';
|
|
6
7
|
import { useEffect, useMemo, useState } from 'react';
|
|
7
8
|
|
|
8
9
|
import { extractionAnthropicFunction, processTranscriptMessage } from '@dxos/assistant/extraction';
|
|
9
10
|
import { scheduleTaskInterval } from '@dxos/async';
|
|
10
|
-
import {
|
|
11
|
+
import { createFeedServiceLayer, type Space } from '@dxos/client/echo';
|
|
11
12
|
import { Context } from '@dxos/context';
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
|
|
13
|
+
import { Feed, Filter, Obj, Ref, Type } from '@dxos/echo';
|
|
14
|
+
import { runAndForwardErrors } from '@dxos/effect';
|
|
15
15
|
import { IdentityDid } from '@dxos/keys';
|
|
16
16
|
import { log } from '@dxos/log';
|
|
17
|
-
import {
|
|
18
|
-
import {
|
|
19
|
-
import {
|
|
20
|
-
import {
|
|
17
|
+
import { random } from '@dxos/random';
|
|
18
|
+
import { TestSchema } from '@dxos/schema/testing';
|
|
19
|
+
import { type ContentBlock, Message, Organization, Person } from '@dxos/types';
|
|
20
|
+
import { seedTestData } from '@dxos/types/testing';
|
|
21
21
|
|
|
22
22
|
// TODO(burdon): Reconcile with plugin-markdown. Move to @dxos/schema/testing.
|
|
23
23
|
export const TestItem = Schema.Struct({
|
|
@@ -30,15 +30,15 @@ export const TestItem = Schema.Struct({
|
|
|
30
30
|
description: 'Product description',
|
|
31
31
|
}),
|
|
32
32
|
}).pipe(
|
|
33
|
-
Type.
|
|
34
|
-
typename: 'dxos.
|
|
33
|
+
Type.object({
|
|
34
|
+
typename: 'org.dxos.type.test',
|
|
35
35
|
version: '0.1.0',
|
|
36
36
|
}),
|
|
37
37
|
);
|
|
38
38
|
|
|
39
39
|
// TODO(wittjosiah): Make builder generic and reuse for all message types.
|
|
40
40
|
abstract class AbstractMessageBuilder {
|
|
41
|
-
abstract createMessage(numSegments?: number): Promise<
|
|
41
|
+
abstract createMessage(numSegments?: number): Promise<Message.Message>;
|
|
42
42
|
}
|
|
43
43
|
|
|
44
44
|
/**
|
|
@@ -49,8 +49,8 @@ export class MessageBuilder extends AbstractMessageBuilder {
|
|
|
49
49
|
|
|
50
50
|
users = Array.from({ length: 5 }, () => ({
|
|
51
51
|
identityDid: IdentityDid.random().toString(),
|
|
52
|
-
name:
|
|
53
|
-
email:
|
|
52
|
+
name: random.person.fullName(),
|
|
53
|
+
email: random.internet.email(),
|
|
54
54
|
}));
|
|
55
55
|
|
|
56
56
|
start = new Date(Date.now() - 24 * 60 * 60 * 10_000);
|
|
@@ -59,19 +59,24 @@ export class MessageBuilder extends AbstractMessageBuilder {
|
|
|
59
59
|
super();
|
|
60
60
|
}
|
|
61
61
|
|
|
62
|
-
override async createMessage(numSegments = 1): Promise<
|
|
63
|
-
return Obj.make(
|
|
62
|
+
override async createMessage(numSegments = 1): Promise<Message.Message> {
|
|
63
|
+
return Obj.make(Message.Message, {
|
|
64
64
|
created: this.next().toISOString(),
|
|
65
|
-
sender:
|
|
65
|
+
sender: random.helpers.arrayElement(this.users),
|
|
66
66
|
blocks: Array.from({ length: numSegments }).map(() => this.createBlock()),
|
|
67
67
|
});
|
|
68
68
|
}
|
|
69
69
|
|
|
70
|
-
createBlock():
|
|
71
|
-
let text =
|
|
70
|
+
createBlock(): ContentBlock.Transcript {
|
|
71
|
+
let text = random.lorem.paragraph();
|
|
72
72
|
if (this._space) {
|
|
73
|
-
const label =
|
|
74
|
-
const obj = this._space.db.add(
|
|
73
|
+
const label = random.commerce.productName();
|
|
74
|
+
const obj = this._space.db.add(
|
|
75
|
+
Obj.make(TestItem, {
|
|
76
|
+
title: label,
|
|
77
|
+
description: random.lorem.paragraph(),
|
|
78
|
+
}),
|
|
79
|
+
);
|
|
75
80
|
const dxn = Ref.make(obj).dxn.toString();
|
|
76
81
|
const words = text.split(' ');
|
|
77
82
|
words.splice(Math.floor(Math.random() * words.length), 0, `[${label}](${dxn})`);
|
|
@@ -93,17 +98,9 @@ export class MessageBuilder extends AbstractMessageBuilder {
|
|
|
93
98
|
|
|
94
99
|
// TODO(burdon): Reconcile with BlockBuilder.
|
|
95
100
|
class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
|
|
96
|
-
executor = new FunctionExecutor(
|
|
97
|
-
new ServiceContainer().setServices({
|
|
98
|
-
// ai: {
|
|
99
|
-
// client: this.AiService,
|
|
100
|
-
// },
|
|
101
|
-
}),
|
|
102
|
-
);
|
|
103
|
-
|
|
104
101
|
space: Space | undefined;
|
|
105
102
|
currentMessage: number = 0;
|
|
106
|
-
transcriptMessages:
|
|
103
|
+
transcriptMessages: Message.Message[] = [];
|
|
107
104
|
|
|
108
105
|
async connect(space: Space): Promise<void> {
|
|
109
106
|
this.space = space;
|
|
@@ -111,14 +108,18 @@ class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
|
|
|
111
108
|
this.transcriptMessages = transcriptMessages;
|
|
112
109
|
}
|
|
113
110
|
|
|
114
|
-
override async createMessage(): Promise<
|
|
111
|
+
override async createMessage(): Promise<Message.Message> {
|
|
115
112
|
if (!this.space) {
|
|
116
113
|
throw new Error('Space not connected');
|
|
117
114
|
}
|
|
118
115
|
|
|
119
|
-
const
|
|
116
|
+
const objects = await this.space.db
|
|
120
117
|
.query(
|
|
121
|
-
Filter.or(
|
|
118
|
+
Filter.or(
|
|
119
|
+
Filter.type(Person.Person),
|
|
120
|
+
Filter.type(Organization.Organization),
|
|
121
|
+
Filter.type(TestSchema.DocumentType),
|
|
122
|
+
),
|
|
122
123
|
)
|
|
123
124
|
.run();
|
|
124
125
|
|
|
@@ -129,7 +130,6 @@ class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
|
|
|
129
130
|
|
|
130
131
|
const { message: enhancedMessage } = await processTranscriptMessage({
|
|
131
132
|
input: { message },
|
|
132
|
-
executor: this.executor,
|
|
133
133
|
function: extractionAnthropicFunction,
|
|
134
134
|
});
|
|
135
135
|
|
|
@@ -139,71 +139,64 @@ class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
|
|
|
139
139
|
|
|
140
140
|
type UseTestTranscriptionQueue = (
|
|
141
141
|
space: Space | undefined,
|
|
142
|
-
queueId?: Key.ObjectId,
|
|
143
142
|
running?: boolean,
|
|
144
143
|
interval?: number,
|
|
145
|
-
) =>
|
|
144
|
+
) => Feed.Feed | undefined;
|
|
146
145
|
|
|
147
146
|
/**
|
|
148
|
-
* Test
|
|
147
|
+
* Test transcription feed.
|
|
149
148
|
*/
|
|
150
149
|
export const useTestTranscriptionQueue: UseTestTranscriptionQueue = (
|
|
151
150
|
space: Space | undefined,
|
|
152
|
-
queueId?: Key.ObjectId,
|
|
153
151
|
running = true,
|
|
154
152
|
interval = 1_000,
|
|
155
153
|
) => {
|
|
156
|
-
|
|
157
|
-
const queueDxn = useMemo(() => (space ? createQueueDXN(space.id, queueId) : undefined), [space, queueId]);
|
|
158
|
-
const queue = useQueue<DataType.Message>(queueDxn);
|
|
154
|
+
const feed = useMemo(() => (space ? space.db.add(Feed.make({ name: 'transcription' })) : undefined), [space]);
|
|
159
155
|
const builder = useMemo(() => new MessageBuilder(space), [space]);
|
|
160
156
|
|
|
161
157
|
useEffect(() => {
|
|
162
|
-
if (!
|
|
158
|
+
if (!space || !feed || !running) {
|
|
163
159
|
return;
|
|
164
160
|
}
|
|
161
|
+
const feedServiceLayer = createFeedServiceLayer(space.queues);
|
|
165
162
|
|
|
166
163
|
const i = setInterval(() => {
|
|
167
164
|
void builder.createMessage(Math.ceil(Math.random() * 3)).then(async (message) => {
|
|
168
|
-
await
|
|
165
|
+
await Feed.append(feed, [message]).pipe(Effect.provide(feedServiceLayer), runAndForwardErrors);
|
|
169
166
|
});
|
|
170
167
|
}, interval);
|
|
171
168
|
return () => clearInterval(i);
|
|
172
|
-
}, [
|
|
169
|
+
}, [space, feed, running, interval]);
|
|
173
170
|
|
|
174
|
-
return
|
|
171
|
+
return feed;
|
|
175
172
|
};
|
|
176
173
|
|
|
177
174
|
/**
|
|
178
|
-
* Test transcription
|
|
175
|
+
* Test transcription feed with entity extraction.
|
|
179
176
|
*/
|
|
180
177
|
// TODO(burdon): Reconcile with useTestTranscriptionQueue.
|
|
181
178
|
export const useTestTranscriptionQueueWithEntityExtraction: UseTestTranscriptionQueue = (
|
|
182
179
|
space: Space | undefined,
|
|
183
|
-
queueId?: Key.ObjectId,
|
|
184
180
|
running = true,
|
|
185
181
|
interval = 1_000,
|
|
186
182
|
) => {
|
|
187
|
-
|
|
188
|
-
const queueDxn = useMemo(() => (space ? createQueueDXN(space.id, queueId) : undefined), [space, queueId]);
|
|
189
|
-
const queue = useQueue<DataType.Message>(queueDxn);
|
|
183
|
+
const feed = useMemo(() => (space ? space.db.add(Feed.make({ name: 'transcription' })) : undefined), [space]);
|
|
190
184
|
const [builder] = useState(() => new EntityExtractionMessageBuilder());
|
|
191
185
|
|
|
192
186
|
useEffect(() => {
|
|
193
|
-
if (!
|
|
187
|
+
if (!space || !feed || !running) {
|
|
194
188
|
return;
|
|
195
189
|
}
|
|
196
190
|
|
|
197
|
-
|
|
198
|
-
void builder.connect(space);
|
|
199
|
-
}
|
|
191
|
+
void builder.connect(space);
|
|
200
192
|
|
|
193
|
+
const feedServiceLayer = createFeedServiceLayer(space.queues);
|
|
201
194
|
const ctx = new Context();
|
|
202
195
|
scheduleTaskInterval(
|
|
203
196
|
ctx,
|
|
204
197
|
async () => {
|
|
205
198
|
const message = await builder.createMessage();
|
|
206
|
-
|
|
199
|
+
await Feed.append(feed, [message]).pipe(Effect.provide(feedServiceLayer), runAndForwardErrors);
|
|
207
200
|
},
|
|
208
201
|
interval,
|
|
209
202
|
);
|
|
@@ -211,7 +204,7 @@ export const useTestTranscriptionQueueWithEntityExtraction: UseTestTranscription
|
|
|
211
204
|
return () => {
|
|
212
205
|
void ctx.dispose();
|
|
213
206
|
};
|
|
214
|
-
}, [space,
|
|
207
|
+
}, [space, feed, running, interval]);
|
|
215
208
|
|
|
216
|
-
return
|
|
209
|
+
return feed;
|
|
217
210
|
};
|
package/src/transcriber/index.ts
CHANGED
|
@@ -19,7 +19,7 @@ const initializeExtendableMediaRecorder = async () => {
|
|
|
19
19
|
await register(await connect());
|
|
20
20
|
};
|
|
21
21
|
|
|
22
|
-
export type
|
|
22
|
+
export type MediaStreamRecorderProps = {
|
|
23
23
|
mediaStreamTrack: MediaStreamTrack;
|
|
24
24
|
config: {
|
|
25
25
|
interval: number;
|
|
@@ -37,12 +37,12 @@ export class MediaStreamRecorder implements AudioRecorder {
|
|
|
37
37
|
/**
|
|
38
38
|
* Default MediaRecorder implementation do not support wav encoding.
|
|
39
39
|
*/
|
|
40
|
-
private readonly _config:
|
|
40
|
+
private readonly _config: MediaStreamRecorderProps['config'];
|
|
41
41
|
private _mediaRecorder?: IMediaRecorder = undefined;
|
|
42
42
|
private _header?: Uint8Array = undefined;
|
|
43
43
|
private _onChunk?: (chunk: AudioChunk) => void;
|
|
44
44
|
|
|
45
|
-
constructor({ mediaStreamTrack, config }:
|
|
45
|
+
constructor({ mediaStreamTrack, config }: MediaStreamRecorderProps) {
|
|
46
46
|
this._mediaStreamTrack = mediaStreamTrack;
|
|
47
47
|
this._config = config;
|
|
48
48
|
}
|
|
@@ -4,15 +4,14 @@
|
|
|
4
4
|
|
|
5
5
|
import fs from 'node:fs';
|
|
6
6
|
import path from 'node:path';
|
|
7
|
-
|
|
8
7
|
import { describe, expect, test } from 'vitest';
|
|
9
8
|
import { WaveFile } from 'wavefile';
|
|
10
9
|
|
|
11
10
|
import { Trigger } from '@dxos/async';
|
|
12
11
|
import { log } from '@dxos/log';
|
|
13
|
-
import { type DataType } from '@dxos/schema';
|
|
14
12
|
import { openAndClose } from '@dxos/test-utils';
|
|
15
13
|
import { TRACE_PROCESSOR, trace } from '@dxos/tracing';
|
|
14
|
+
import { type ContentBlock } from '@dxos/types';
|
|
16
15
|
|
|
17
16
|
import { type AudioChunk, type AudioRecorder, Transcriber } from '../transcriber';
|
|
18
17
|
import { mergeFloat64Arrays } from '../util';
|
|
@@ -94,7 +93,7 @@ describe.skip('Transcriber', () => {
|
|
|
94
93
|
});
|
|
95
94
|
|
|
96
95
|
test.skip('transcription of audio recording', { timeout: 10_000 }, async () => {
|
|
97
|
-
const trigger = new Trigger<
|
|
96
|
+
const trigger = new Trigger<ContentBlock.Transcript[]>({ autoReset: true });
|
|
98
97
|
const recorder = new MockAudioRecorder({
|
|
99
98
|
buffer: await readFile('test.wav'),
|
|
100
99
|
chunkDuration: 3_000,
|
|
@@ -128,7 +127,7 @@ describe.skip('Transcriber', () => {
|
|
|
128
127
|
});
|
|
129
128
|
|
|
130
129
|
test.skip('transcription of audio recording with overlapping chunks', { timeout: 20_000 }, async () => {
|
|
131
|
-
const trigger = new Trigger<
|
|
130
|
+
const trigger = new Trigger<ContentBlock.Transcript[]>({ autoReset: true });
|
|
132
131
|
const recorder = new MockAudioRecorder({
|
|
133
132
|
buffer: await readFile('test.wav'),
|
|
134
133
|
chunkDuration: 3_000,
|
|
@@ -213,7 +212,7 @@ class MockAudioRecorder implements AudioRecorder {
|
|
|
213
212
|
|
|
214
213
|
@trace.span()
|
|
215
214
|
async start(): Promise<void> {
|
|
216
|
-
log.info('start mock audio recorder', {
|
|
215
|
+
log.info('start mock audio recorder', { wavProps: this.wav.fmt });
|
|
217
216
|
const now = Date.now();
|
|
218
217
|
const samples = this.wav.getSamples();
|
|
219
218
|
const sampleRate = this.wavConfig.sampleRate;
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import { describe, test } from 'vitest';
|
|
6
|
+
|
|
7
|
+
import { type AudioChunk } from './audio-recorder';
|
|
8
|
+
import { alignWhisperSegments, type WhisperSegment } from './transcriber';
|
|
9
|
+
|
|
10
|
+
const T0 = Date.parse('2026-01-01T00:00:00.000Z');
|
|
11
|
+
|
|
12
|
+
const chunk = (offsetMs = 0, samples = 16): AudioChunk => ({
|
|
13
|
+
timestamp: T0 + offsetMs,
|
|
14
|
+
data: new Float64Array(samples),
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
const segment = (overrides: Partial<WhisperSegment>): WhisperSegment => ({
|
|
18
|
+
text: '',
|
|
19
|
+
start: 0,
|
|
20
|
+
end: 0,
|
|
21
|
+
no_speech_prob: 0,
|
|
22
|
+
words: [],
|
|
23
|
+
...overrides,
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
describe('alignWhisperSegments', () => {
|
|
27
|
+
test('returns empty when no chunks were transcribed', ({ expect }) => {
|
|
28
|
+
const result = alignWhisperSegments(
|
|
29
|
+
[segment({ text: 'x', end: 1, words: [{ word: 'x', start: 0, end: 1 }] })],
|
|
30
|
+
[],
|
|
31
|
+
0,
|
|
32
|
+
);
|
|
33
|
+
expect(result.transcripts).toEqual([]);
|
|
34
|
+
expect(result.lastTimestamp).toBeUndefined();
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
test('absolute-timestamps and deduplicates segments against lastTimestamp', ({ expect }) => {
|
|
38
|
+
const segments: WhisperSegment[] = [
|
|
39
|
+
segment({
|
|
40
|
+
text: 'hello world',
|
|
41
|
+
start: 0,
|
|
42
|
+
end: 1,
|
|
43
|
+
words: [
|
|
44
|
+
{ word: 'hello ', start: 0, end: 0.5 },
|
|
45
|
+
{ word: 'world', start: 0.5, end: 1 },
|
|
46
|
+
],
|
|
47
|
+
}),
|
|
48
|
+
];
|
|
49
|
+
|
|
50
|
+
const first = alignWhisperSegments(segments, [chunk(0)], 0);
|
|
51
|
+
expect(first.transcripts).toHaveLength(1);
|
|
52
|
+
expect(first.transcripts[0]._tag).toBe('transcript');
|
|
53
|
+
expect(first.transcripts[0].text).toBe('hello world');
|
|
54
|
+
expect(new Date(first.transcripts[0].started).getTime()).toBe(T0);
|
|
55
|
+
expect(first.lastTimestamp).toBe(T0 + 1_000);
|
|
56
|
+
|
|
57
|
+
// Same segments, same chunk start: every word's absolute start is < lastTimestamp -> filtered out.
|
|
58
|
+
const second = alignWhisperSegments(segments, [chunk(0)], first.lastTimestamp!);
|
|
59
|
+
expect(second.transcripts).toEqual([]);
|
|
60
|
+
expect(second.lastTimestamp).toBeUndefined();
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
test('keeps later words when earlier words fall before lastTimestamp', ({ expect }) => {
|
|
64
|
+
const segments: WhisperSegment[] = [
|
|
65
|
+
segment({
|
|
66
|
+
text: 'old new',
|
|
67
|
+
start: 0,
|
|
68
|
+
end: 2,
|
|
69
|
+
words: [
|
|
70
|
+
{ word: 'old ', start: 0, end: 1 },
|
|
71
|
+
{ word: 'new', start: 1.5, end: 2 },
|
|
72
|
+
],
|
|
73
|
+
}),
|
|
74
|
+
];
|
|
75
|
+
|
|
76
|
+
// First word starts at T0, second at T0+1500 — only the second survives a cursor at T0+1000.
|
|
77
|
+
const result = alignWhisperSegments(segments, [chunk(0)], T0 + 1_000);
|
|
78
|
+
expect(result.transcripts).toHaveLength(1);
|
|
79
|
+
expect(result.transcripts[0].text).toBe('new');
|
|
80
|
+
expect(new Date(result.transcripts[0].started).getTime()).toBe(T0 + 1_500);
|
|
81
|
+
expect(result.lastTimestamp).toBe(T0 + 2_000);
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
test('drops segments that end before lastTimestamp', ({ expect }) => {
|
|
85
|
+
const segments: WhisperSegment[] = [
|
|
86
|
+
segment({
|
|
87
|
+
text: 'stale',
|
|
88
|
+
start: 0,
|
|
89
|
+
end: 0.5,
|
|
90
|
+
words: [{ word: 'stale', start: 0, end: 0.5 }],
|
|
91
|
+
}),
|
|
92
|
+
segment({
|
|
93
|
+
text: 'fresh',
|
|
94
|
+
start: 1,
|
|
95
|
+
end: 2,
|
|
96
|
+
words: [{ word: 'fresh', start: 1.5, end: 2 }],
|
|
97
|
+
}),
|
|
98
|
+
];
|
|
99
|
+
|
|
100
|
+
const result = alignWhisperSegments(segments, [chunk(0)], T0 + 1_000);
|
|
101
|
+
expect(result.transcripts).toHaveLength(1);
|
|
102
|
+
expect(result.transcripts[0].text).toBe('fresh');
|
|
103
|
+
});
|
|
104
|
+
});
|
|
@@ -7,15 +7,18 @@ import { WaveFile } from 'wavefile';
|
|
|
7
7
|
import { DeferredTask, Trigger } from '@dxos/async';
|
|
8
8
|
import { type Context, LifecycleState, Resource } from '@dxos/context';
|
|
9
9
|
import { log } from '@dxos/log';
|
|
10
|
-
import { type DataType } from '@dxos/schema';
|
|
11
10
|
import { trace } from '@dxos/tracing';
|
|
11
|
+
import { type ContentBlock } from '@dxos/types';
|
|
12
12
|
|
|
13
|
-
import { TRANSCRIPTION_URL } from '../types';
|
|
14
13
|
import { mergeFloat64Arrays } from '../util';
|
|
15
|
-
|
|
16
14
|
import { type AudioChunk, type AudioRecorder } from './audio-recorder';
|
|
17
15
|
|
|
18
|
-
|
|
16
|
+
/**
|
|
17
|
+
* Endpoint to the calls service.
|
|
18
|
+
*/
|
|
19
|
+
const DEFAULT_TRANSCRIPTION_URL = 'https://calls-service.dxos.workers.dev';
|
|
20
|
+
|
|
21
|
+
export type WhisperWord = {
|
|
19
22
|
word: string;
|
|
20
23
|
|
|
21
24
|
/**
|
|
@@ -29,7 +32,7 @@ type WhisperWord = {
|
|
|
29
32
|
end: number;
|
|
30
33
|
};
|
|
31
34
|
|
|
32
|
-
type WhisperSegment = {
|
|
35
|
+
export type WhisperSegment = {
|
|
33
36
|
text: string;
|
|
34
37
|
|
|
35
38
|
/**
|
|
@@ -61,16 +64,32 @@ export type TranscribeConfig = {
|
|
|
61
64
|
* This is needed to catch the beginning of the user's speech.
|
|
62
65
|
*/
|
|
63
66
|
prefixBufferChunksAmount: number;
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Override the transcription endpoint base URL.
|
|
70
|
+
* Defaults to the DXOS calls service.
|
|
71
|
+
*/
|
|
72
|
+
endpoint?: string;
|
|
64
73
|
};
|
|
65
74
|
|
|
66
|
-
|
|
75
|
+
/**
|
|
76
|
+
* Function that converts a base64-encoded WAV payload into Whisper segments.
|
|
77
|
+
* Allows callers to swap in alternative providers or mock the transport.
|
|
78
|
+
*/
|
|
79
|
+
export type TranscribeFn = (audio: string) => Promise<WhisperSegment[]>;
|
|
80
|
+
|
|
81
|
+
export type TranscriberProps = {
|
|
67
82
|
config: TranscribeConfig;
|
|
68
83
|
recorder: AudioRecorder;
|
|
69
84
|
/**
|
|
70
85
|
* Callback to handle the transcribed segments, after all segment transformers are applied.
|
|
71
86
|
* @param segments - The transcribed segments.
|
|
72
87
|
*/
|
|
73
|
-
onSegments: (segments:
|
|
88
|
+
onSegments: (segments: ContentBlock.Transcript[]) => Promise<void>;
|
|
89
|
+
/**
|
|
90
|
+
* Optional override of the transcription transport. When provided, supersedes `config.endpoint`.
|
|
91
|
+
*/
|
|
92
|
+
transcribe?: TranscribeFn;
|
|
74
93
|
};
|
|
75
94
|
|
|
76
95
|
/**
|
|
@@ -80,22 +99,24 @@ export type TranscriberParams = {
|
|
|
80
99
|
*/
|
|
81
100
|
export class Transcriber extends Resource {
|
|
82
101
|
private _audioChunks: AudioChunk[] = [];
|
|
83
|
-
private
|
|
102
|
+
private _lastTimestamp = 0;
|
|
84
103
|
|
|
85
104
|
private readonly _openTrigger = new Trigger({ autoReset: true });
|
|
86
105
|
|
|
87
106
|
private readonly _config: TranscribeConfig;
|
|
88
107
|
private readonly _recorder: AudioRecorder;
|
|
89
|
-
private readonly _onSegments:
|
|
108
|
+
private readonly _onSegments: TranscriberProps['onSegments'];
|
|
109
|
+
private readonly _transcribeFn?: TranscribeFn;
|
|
90
110
|
|
|
91
111
|
private _recording = false;
|
|
92
112
|
private _transcribeTask?: DeferredTask = undefined;
|
|
93
113
|
|
|
94
|
-
constructor({ config, recorder, onSegments }:
|
|
114
|
+
constructor({ config, recorder, onSegments, transcribe }: TranscriberProps) {
|
|
95
115
|
super();
|
|
96
116
|
this._config = config;
|
|
97
117
|
this._recorder = recorder;
|
|
98
118
|
this._onSegments = onSegments;
|
|
119
|
+
this._transcribeFn = transcribe;
|
|
99
120
|
}
|
|
100
121
|
|
|
101
122
|
protected override async _open(ctx: Context): Promise<void> {
|
|
@@ -192,67 +213,91 @@ export class Transcriber extends Resource {
|
|
|
192
213
|
@trace.span({ showInBrowserTimeline: true })
|
|
193
214
|
private async _fetchTranscription(audio: string): Promise<WhisperSegment[]> {
|
|
194
215
|
if (audio.length === 0) {
|
|
195
|
-
this._audioChunks = [];
|
|
196
216
|
throw new Error('No audio to send for transcribing');
|
|
197
217
|
}
|
|
198
218
|
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
219
|
+
let segments: unknown;
|
|
220
|
+
if (this._transcribeFn) {
|
|
221
|
+
segments = await this._transcribeFn(audio);
|
|
222
|
+
} else {
|
|
223
|
+
// TODO(burdon): Create separate endpoint?
|
|
224
|
+
const endpoint = this._config.endpoint ?? DEFAULT_TRANSCRIPTION_URL;
|
|
225
|
+
const response = await fetch(`${endpoint}/transcribe`, {
|
|
226
|
+
method: 'POST',
|
|
227
|
+
body: JSON.stringify({ audio }),
|
|
228
|
+
headers: {
|
|
229
|
+
'Content-Type': 'application/json',
|
|
230
|
+
},
|
|
231
|
+
});
|
|
232
|
+
|
|
233
|
+
if (!response.ok) {
|
|
234
|
+
throw new Error(`Transcription failed: ${response.statusText}`);
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
({ segments } = (await response.json()) as { segments?: unknown });
|
|
210
238
|
}
|
|
211
239
|
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
log.info('transcription response', {
|
|
217
|
-
segments: segments.length,
|
|
218
|
-
string: segments.map((segments) => segments.text).join(' '),
|
|
219
|
-
});
|
|
240
|
+
if (!Array.isArray(segments)) {
|
|
241
|
+
throw new Error('Transcription response payload is invalid');
|
|
242
|
+
}
|
|
220
243
|
|
|
221
|
-
|
|
244
|
+
log.info('transcription response', { segments: segments.length });
|
|
245
|
+
return segments as WhisperSegment[];
|
|
222
246
|
}
|
|
223
247
|
|
|
224
|
-
private _alignSegments(segments: WhisperSegment[], originalChunks: AudioChunk[]):
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
const filteredSegments = segments
|
|
229
|
-
// Use segments that end after the last used timestamp.
|
|
230
|
-
// Segments could overlap, so we need to filter words that starts after the last used timestamp.
|
|
231
|
-
.filter((segment) => zeroTimestamp + segment.end * 1_000 >= this._lastUsedTimestamp)
|
|
232
|
-
// Use words that starts after the last used timestamp.
|
|
233
|
-
.map((segment) => {
|
|
234
|
-
const words = segment.words.filter((word) => zeroTimestamp + word.start * 1_000 >= this._lastUsedTimestamp);
|
|
235
|
-
return {
|
|
236
|
-
...segment,
|
|
237
|
-
words,
|
|
238
|
-
text: words.map((word) => word.word).join(''),
|
|
239
|
-
start: words.at(0)?.start ?? segment.end,
|
|
240
|
-
};
|
|
241
|
-
})
|
|
242
|
-
.filter((segment) => segment.words.length > 0);
|
|
243
|
-
|
|
244
|
-
if (filteredSegments.length === 0) {
|
|
245
|
-
return [];
|
|
248
|
+
private _alignSegments(segments: WhisperSegment[], originalChunks: AudioChunk[]): ContentBlock.Transcript[] {
|
|
249
|
+
const result = alignWhisperSegments(segments, originalChunks, this._lastTimestamp);
|
|
250
|
+
if (result.lastTimestamp !== undefined) {
|
|
251
|
+
this._lastTimestamp = result.lastTimestamp;
|
|
246
252
|
}
|
|
247
253
|
|
|
248
|
-
|
|
249
|
-
|
|
254
|
+
return result.transcripts;
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* Pure helper that filters and absolute-timestamps Whisper segments against a chunk timeline.
|
|
260
|
+
* Exported for testing.
|
|
261
|
+
*
|
|
262
|
+
* @param segments - Raw segments returned by the Whisper service for `originalChunks`.
|
|
263
|
+
* @param originalChunks - Chunks fed into the transcription request (used for the zero timestamp).
|
|
264
|
+
* @param lastTimestamp - Most recent absolute timestamp already emitted; segments whose end
|
|
265
|
+
* falls strictly before this point are deduped out, as are individual words whose start does.
|
|
266
|
+
*/
|
|
267
|
+
export const alignWhisperSegments = (
|
|
268
|
+
segments: WhisperSegment[],
|
|
269
|
+
originalChunks: AudioChunk[],
|
|
270
|
+
lastTimestamp: number,
|
|
271
|
+
): { transcripts: ContentBlock.Transcript[]; lastTimestamp?: number } => {
|
|
272
|
+
if (originalChunks.length === 0) {
|
|
273
|
+
return { transcripts: [] };
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
const zeroTimestamp = originalChunks[0].timestamp;
|
|
277
|
+
|
|
278
|
+
const filteredSegments = segments
|
|
279
|
+
.filter((segment) => zeroTimestamp + segment.end * 1_000 >= lastTimestamp)
|
|
280
|
+
.map((segment) => {
|
|
281
|
+
const words = segment.words.filter((word) => zeroTimestamp + word.start * 1_000 >= lastTimestamp);
|
|
282
|
+
return {
|
|
283
|
+
...segment,
|
|
284
|
+
words,
|
|
285
|
+
text: words.map((word) => word.word).join(''),
|
|
286
|
+
start: words.at(0)?.start ?? segment.end,
|
|
287
|
+
};
|
|
288
|
+
})
|
|
289
|
+
.filter((segment) => segment.words.length > 0);
|
|
290
|
+
|
|
291
|
+
if (filteredSegments.length === 0) {
|
|
292
|
+
return { transcripts: [] };
|
|
293
|
+
}
|
|
250
294
|
|
|
251
|
-
|
|
252
|
-
|
|
295
|
+
return {
|
|
296
|
+
lastTimestamp: zeroTimestamp + filteredSegments.at(-1)!.end * 1_000,
|
|
297
|
+
transcripts: filteredSegments.map((segment) => ({
|
|
253
298
|
_tag: 'transcript',
|
|
254
299
|
started: new Date(zeroTimestamp + segment.start * 1_000).toISOString(),
|
|
255
300
|
text: segment.text.trim(),
|
|
256
|
-
}))
|
|
257
|
-
}
|
|
258
|
-
}
|
|
301
|
+
})),
|
|
302
|
+
};
|
|
303
|
+
};
|