@dxos/plugin-transcription 0.8.4-main.ae835ea → 0.8.4-main.bc2380dfbc
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +102 -5
- package/README.md +1 -1
- package/dist/lib/neutral/TranscriptionArticle-L6XANBH5.mjs +54 -0
- package/dist/lib/neutral/TranscriptionArticle-L6XANBH5.mjs.map +7 -0
- package/dist/lib/neutral/TranscriptionPlugin.mjs +34 -0
- package/dist/lib/neutral/TranscriptionPlugin.mjs.map +7 -0
- package/dist/lib/neutral/TranscriptionPlugin.node.mjs +25 -0
- package/dist/lib/neutral/TranscriptionPlugin.node.mjs.map +7 -0
- package/dist/lib/neutral/TranscriptionPlugin.workerd.mjs +21 -0
- package/dist/lib/neutral/TranscriptionPlugin.workerd.mjs.map +7 -0
- package/dist/lib/neutral/blueprint-definition-HOXDG5QP.mjs +15 -0
- package/dist/lib/neutral/blueprint-definition-HOXDG5QP.mjs.map +7 -0
- package/dist/lib/neutral/blueprints/index.mjs +8 -0
- package/dist/lib/neutral/capabilities/index.mjs +17 -0
- package/dist/lib/neutral/capabilities/index.mjs.map +7 -0
- package/dist/lib/neutral/chunk-2O3ONPRZ.mjs +298 -0
- package/dist/lib/neutral/chunk-2O3ONPRZ.mjs.map +7 -0
- package/dist/lib/neutral/chunk-5FXU36PN.mjs +120 -0
- package/dist/lib/neutral/chunk-5FXU36PN.mjs.map +7 -0
- package/dist/lib/{browser/chunk-VBUIO677.mjs → neutral/chunk-GL4RS533.mjs} +2 -9
- package/dist/lib/neutral/chunk-GL4RS533.mjs.map +7 -0
- package/dist/lib/{browser/chunk-WH36ENNE.mjs → neutral/chunk-ILSA3DU7.mjs} +102 -133
- package/dist/lib/neutral/chunk-ILSA3DU7.mjs.map +7 -0
- package/dist/lib/neutral/chunk-J5LGTIGS.mjs +10 -0
- package/dist/lib/neutral/chunk-KAEXRX3Y.mjs +35 -0
- package/dist/lib/neutral/chunk-KAEXRX3Y.mjs.map +7 -0
- package/dist/lib/neutral/chunk-MLRJWAQH.mjs +179 -0
- package/dist/lib/neutral/chunk-MLRJWAQH.mjs.map +7 -0
- package/dist/lib/neutral/chunk-Y5EQAFFR.mjs +8 -0
- package/dist/lib/neutral/chunk-Y5EQAFFR.mjs.map +7 -0
- package/dist/lib/neutral/chunk-ZR4BH4QQ.mjs +30 -0
- package/dist/lib/neutral/chunk-ZR4BH4QQ.mjs.map +7 -0
- package/dist/lib/neutral/components/index.mjs +272 -0
- package/dist/lib/neutral/components/index.mjs.map +7 -0
- package/dist/lib/neutral/containers/index.mjs +9 -0
- package/dist/lib/neutral/containers/index.mjs.map +7 -0
- package/dist/lib/neutral/create-RRWRIUSR.mjs +21 -0
- package/dist/lib/neutral/create-RRWRIUSR.mjs.map +7 -0
- package/dist/lib/neutral/hooks/index.mjs +19 -0
- package/dist/lib/neutral/index.mjs +44 -0
- package/dist/lib/neutral/meta.json +1 -0
- package/dist/lib/neutral/meta.mjs +8 -0
- package/dist/lib/neutral/meta.mjs.map +7 -0
- package/dist/lib/neutral/open-CVMX5KXZ.mjs +27 -0
- package/dist/lib/neutral/open-CVMX5KXZ.mjs.map +7 -0
- package/dist/lib/neutral/operation-handler-NXMOAXFP.mjs +13 -0
- package/dist/lib/neutral/operation-handler-NXMOAXFP.mjs.map +7 -0
- package/dist/lib/neutral/operations/index.mjs +8 -0
- package/dist/lib/neutral/operations/index.mjs.map +7 -0
- package/dist/lib/neutral/plugin.mjs +16 -0
- package/dist/lib/neutral/plugin.mjs.map +7 -0
- package/dist/lib/neutral/react-surface-YMZRO242.mjs +26 -0
- package/dist/lib/neutral/react-surface-YMZRO242.mjs.map +7 -0
- package/dist/lib/neutral/summarize-LA7TRW7A.mjs +78 -0
- package/dist/lib/neutral/summarize-LA7TRW7A.mjs.map +7 -0
- package/dist/lib/neutral/text-content-F4ASMITB.mjs +30 -0
- package/dist/lib/neutral/text-content-F4ASMITB.mjs.map +7 -0
- package/dist/lib/{node-esm/transcriber-UGB7ECX6.mjs → neutral/transcriber-ABHATBVH.mjs} +20 -18
- package/dist/lib/neutral/transcriber-ABHATBVH.mjs.map +7 -0
- package/dist/lib/neutral/translations.mjs +24 -0
- package/dist/lib/neutral/translations.mjs.map +7 -0
- package/dist/lib/neutral/types/index.mjs +10 -0
- package/dist/lib/neutral/types/index.mjs.map +7 -0
- package/dist/types/src/TranscriptionPlugin.d.ts +3 -1
- package/dist/types/src/TranscriptionPlugin.d.ts.map +1 -1
- package/dist/types/src/TranscriptionPlugin.node.d.ts +4 -0
- package/dist/types/src/TranscriptionPlugin.node.d.ts.map +1 -0
- package/dist/types/src/TranscriptionPlugin.test.d.ts +2 -0
- package/dist/types/src/TranscriptionPlugin.test.d.ts.map +1 -0
- package/dist/types/src/TranscriptionPlugin.workerd.d.ts +4 -0
- package/dist/types/src/TranscriptionPlugin.workerd.d.ts.map +1 -0
- package/dist/types/src/blueprints/index.d.ts +2 -0
- package/dist/types/src/blueprints/index.d.ts.map +1 -0
- package/dist/types/src/blueprints/transcription-blueprint.d.ts +4 -0
- package/dist/types/src/blueprints/transcription-blueprint.d.ts.map +1 -0
- package/dist/types/src/capabilities/blueprint-definition.d.ts +5 -4
- package/dist/types/src/capabilities/blueprint-definition.d.ts.map +1 -1
- package/dist/types/src/capabilities/index.d.ts +10 -5
- package/dist/types/src/capabilities/index.d.ts.map +1 -1
- package/dist/types/src/capabilities/operation-handler.d.ts +6 -0
- package/dist/types/src/capabilities/operation-handler.d.ts.map +1 -0
- package/dist/types/src/capabilities/react-surface.d.ts +3 -2
- package/dist/types/src/capabilities/react-surface.d.ts.map +1 -1
- package/dist/types/src/capabilities/text-content.d.ts +8 -0
- package/dist/types/src/capabilities/text-content.d.ts.map +1 -0
- package/dist/types/src/capabilities/transcriber.d.ts +4 -3
- package/dist/types/src/capabilities/transcriber.d.ts.map +1 -1
- package/dist/types/src/components/Oracle/Oracle.d.ts +24 -0
- package/dist/types/src/components/Oracle/Oracle.d.ts.map +1 -0
- package/dist/types/src/components/Oracle/Oracle.stories.d.ts +32 -0
- package/dist/types/src/components/Oracle/Oracle.stories.d.ts.map +1 -0
- package/dist/types/src/components/Oracle/index.d.ts +2 -0
- package/dist/types/src/components/Oracle/index.d.ts.map +1 -0
- package/dist/types/src/components/Transcription/Transcription.d.ts +13 -0
- package/dist/types/src/components/Transcription/Transcription.d.ts.map +1 -0
- package/dist/types/src/components/Transcription/Transcription.stories.d.ts +37 -0
- package/dist/types/src/components/Transcription/Transcription.stories.d.ts.map +1 -0
- package/dist/types/src/components/Transcription/index.d.ts +2 -0
- package/dist/types/src/components/Transcription/index.d.ts.map +1 -0
- package/dist/types/src/components/Transcription/transcription-extension.d.ts +15 -0
- package/dist/types/src/components/Transcription/transcription-extension.d.ts.map +1 -0
- package/dist/types/src/components/index.d.ts +2 -3
- package/dist/types/src/components/index.d.ts.map +1 -1
- package/dist/types/src/containers/TranscriptionArticle/TranscriptionArticle.d.ts +6 -0
- package/dist/types/src/containers/TranscriptionArticle/TranscriptionArticle.d.ts.map +1 -0
- package/dist/types/src/containers/TranscriptionArticle/TranscriptionArticle.stories.d.ts +28 -0
- package/dist/types/src/containers/TranscriptionArticle/TranscriptionArticle.stories.d.ts.map +1 -0
- package/dist/types/src/containers/TranscriptionArticle/index.d.ts +2 -0
- package/dist/types/src/containers/TranscriptionArticle/index.d.ts.map +1 -0
- package/dist/types/src/containers/index.d.ts +3 -0
- package/dist/types/src/containers/index.d.ts.map +1 -0
- package/dist/types/src/enrichment/conversation.test.d.ts +2 -0
- package/dist/types/src/enrichment/conversation.test.d.ts.map +1 -0
- package/dist/types/src/enrichment/operations.d.ts +200 -0
- package/dist/types/src/enrichment/operations.d.ts.map +1 -0
- package/dist/types/src/enrichment/parser.d.ts +26 -0
- package/dist/types/src/enrichment/parser.d.ts.map +1 -0
- package/dist/types/src/enrichment/stream-simulator.d.ts +38 -0
- package/dist/types/src/enrichment/stream-simulator.d.ts.map +1 -0
- package/dist/types/src/hooks/index.d.ts +2 -1
- package/dist/types/src/hooks/index.d.ts.map +1 -1
- package/dist/types/src/hooks/useAudioFile.d.ts.map +1 -1
- package/dist/types/src/hooks/useAudioTrack.d.ts.map +1 -1
- package/dist/types/src/hooks/useFeedModelAdapter.d.ts +12 -0
- package/dist/types/src/hooks/useFeedModelAdapter.d.ts.map +1 -0
- package/dist/types/src/hooks/useTranscriber.d.ts +2 -2
- package/dist/types/src/hooks/useTranscriber.d.ts.map +1 -1
- package/dist/types/src/hooks/useTranscriptionRecording.d.ts +12 -0
- package/dist/types/src/hooks/useTranscriptionRecording.d.ts.map +1 -0
- package/dist/types/src/hooks/useVoiceInput.d.ts.map +1 -1
- package/dist/types/src/index.d.ts +2 -3
- package/dist/types/src/index.d.ts.map +1 -1
- package/dist/types/src/meta.d.ts +2 -2
- package/dist/types/src/meta.d.ts.map +1 -1
- package/dist/types/src/model/model.d.ts +2 -2
- package/dist/types/src/model/model.d.ts.map +1 -1
- package/dist/types/src/{segments-normalization → normalization}/index.d.ts +1 -1
- package/dist/types/src/normalization/index.d.ts.map +1 -0
- package/dist/types/src/{segments-normalization → normalization}/message-normalizer.d.ts +9 -7
- package/dist/types/src/normalization/message-normalizer.d.ts.map +1 -0
- package/dist/types/src/normalization/normalization.d.ts +543 -0
- package/dist/types/src/normalization/normalization.d.ts.map +1 -0
- package/dist/types/src/normalization/normalization.test.d.ts.map +1 -0
- package/dist/types/src/normalization/utils.d.ts +3 -0
- package/dist/types/src/normalization/utils.d.ts.map +1 -0
- package/dist/types/src/operations/create.d.ts +5 -0
- package/dist/types/src/operations/create.d.ts.map +1 -0
- package/dist/types/src/operations/index.d.ts +3 -0
- package/dist/types/src/operations/index.d.ts.map +1 -0
- package/dist/types/src/operations/open.d.ts +5 -0
- package/dist/types/src/operations/open.d.ts.map +1 -0
- package/dist/types/src/operations/summarize.d.ts +8 -0
- package/dist/types/src/operations/summarize.d.ts.map +1 -0
- package/dist/types/src/plugin.d.ts +4 -0
- package/dist/types/src/plugin.d.ts.map +1 -0
- package/dist/types/src/stories/FileTranscription.stories.d.ts +20 -0
- package/dist/types/src/stories/FileTranscription.stories.d.ts.map +1 -0
- package/dist/types/src/{components/stories → stories}/KeyWordRecognition.stories.d.ts +2 -2
- package/dist/types/src/stories/KeyWordRecognition.stories.d.ts.map +1 -0
- package/dist/types/src/{components/stories/MicrophoneTranscription.stories.d.ts → stories/LiveTranscription.stories.d.ts} +7 -7
- package/dist/types/src/stories/LiveTranscription.stories.d.ts.map +1 -0
- package/dist/types/src/stories/TranscriptionStory.d.ts +21 -0
- package/dist/types/src/stories/TranscriptionStory.d.ts.map +1 -0
- package/dist/types/src/stories/common.d.ts +46 -0
- package/dist/types/src/stories/common.d.ts.map +1 -0
- package/dist/types/src/testing/testing.d.ts +13 -14
- package/dist/types/src/testing/testing.d.ts.map +1 -1
- package/dist/types/src/transcriber/index.d.ts +1 -1
- package/dist/types/src/transcriber/index.d.ts.map +1 -1
- package/dist/types/src/transcriber/media-stream-recorder.d.ts +2 -2
- package/dist/types/src/transcriber/media-stream-recorder.d.ts.map +1 -1
- package/dist/types/src/transcriber/transcriber.d.ts +61 -6
- package/dist/types/src/transcriber/transcriber.d.ts.map +1 -1
- package/dist/types/src/transcriber/transcriber.test.d.ts +2 -0
- package/dist/types/src/transcriber/transcriber.test.d.ts.map +1 -0
- package/dist/types/src/transcriber/transcription-manager.d.ts +14 -9
- package/dist/types/src/transcriber/transcription-manager.d.ts.map +1 -1
- package/dist/types/src/translations.d.ts +9 -8
- package/dist/types/src/translations.d.ts.map +1 -1
- package/dist/types/src/types/TranscriptOperation.d.ts +917 -0
- package/dist/types/src/types/TranscriptOperation.d.ts.map +1 -0
- package/dist/types/src/types/TranscriptionCapabilities.d.ts +17 -0
- package/dist/types/src/types/TranscriptionCapabilities.d.ts.map +1 -0
- package/dist/types/src/types/index.d.ts +2 -3
- package/dist/types/src/types/index.d.ts.map +1 -1
- package/dist/types/src/util/array.d.ts.map +1 -1
- package/dist/types/src/util/get-time-str.d.ts.map +1 -1
- package/dist/types/src/util/index.d.ts +1 -0
- package/dist/types/src/util/index.d.ts.map +1 -1
- package/dist/types/src/util/monitor-audio-level.d.ts.map +1 -1
- package/dist/types/src/util/render-byline.d.ts +4 -0
- package/dist/types/src/util/render-byline.d.ts.map +1 -0
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +157 -82
- package/src/TranscriptionPlugin.node.ts +20 -0
- package/src/TranscriptionPlugin.test.ts +31 -0
- package/src/TranscriptionPlugin.tsx +20 -64
- package/src/TranscriptionPlugin.workerd.ts +18 -0
- package/src/blueprints/index.ts +5 -0
- package/src/blueprints/transcription-blueprint.ts +29 -0
- package/src/capabilities/blueprint-definition.ts +12 -24
- package/src/capabilities/index.ts +11 -7
- package/src/capabilities/operation-handler.ts +16 -0
- package/src/capabilities/react-surface.tsx +23 -14
- package/src/capabilities/text-content.ts +33 -0
- package/src/capabilities/transcriber.ts +53 -42
- package/src/components/Oracle/Oracle.stories.tsx +147 -0
- package/src/components/Oracle/Oracle.tsx +141 -0
- package/src/components/Oracle/index.ts +5 -0
- package/src/components/Transcription/Transcription.stories.tsx +213 -0
- package/src/components/Transcription/Transcription.tsx +58 -0
- package/src/components/{Transcript → Transcription}/index.ts +1 -1
- package/src/components/{Transcript/transcript-extension.ts → Transcription/transcription-extension.ts} +13 -9
- package/src/components/index.ts +2 -6
- package/src/containers/TranscriptionArticle/TranscriptionArticle.stories.tsx +61 -0
- package/src/containers/TranscriptionArticle/TranscriptionArticle.tsx +62 -0
- package/src/containers/TranscriptionArticle/index.ts +5 -0
- package/src/containers/index.ts +7 -0
- package/src/enrichment/conversation.test.ts +162 -0
- package/src/enrichment/conversation.txt +23 -0
- package/src/enrichment/operations.ts +268 -0
- package/src/enrichment/parser.ts +101 -0
- package/src/enrichment/stream-simulator.ts +125 -0
- package/src/hooks/index.ts +2 -1
- package/src/hooks/useFeedModelAdapter.ts +46 -0
- package/src/hooks/useTranscriber.ts +12 -4
- package/src/hooks/useTranscriptionRecording.ts +87 -0
- package/src/hooks/useVoiceInput.ts +2 -3
- package/src/index.ts +4 -5
- package/src/meta.ts +3 -3
- package/src/model/model.test.ts +20 -14
- package/src/model/model.ts +3 -3
- package/src/{segments-normalization → normalization}/message-normalizer.ts +37 -24
- package/src/{segments-normalization → normalization}/normalization.test.ts +8 -47
- package/src/normalization/normalization.ts +76 -0
- package/src/{segments-normalization → normalization}/utils.ts +2 -2
- package/src/operations/create.ts +24 -0
- package/src/operations/index.ts +11 -0
- package/src/operations/open.ts +28 -0
- package/src/operations/summarize.ts +101 -0
- package/src/plugin.ts +11 -0
- package/src/stories/FileTranscription.stories.tsx +143 -0
- package/src/{components/stories → stories}/KeyWordRecognition.stories.tsx +40 -40
- package/src/stories/LiveTranscription.stories.tsx +153 -0
- package/src/stories/TranscriptionStory.tsx +101 -0
- package/src/stories/common.ts +233 -0
- package/src/testing/testing.ts +49 -56
- package/src/transcriber/index.ts +1 -1
- package/src/transcriber/media-stream-recorder.ts +3 -3
- package/src/transcriber/transcriber.browser.test.ts +4 -5
- package/src/transcriber/transcriber.test.ts +104 -0
- package/src/transcriber/transcriber.ts +105 -60
- package/src/transcriber/transcription-manager.ts +33 -22
- package/src/translations.ts +10 -7
- package/src/types/TranscriptOperation.ts +103 -0
- package/src/types/TranscriptionCapabilities.ts +39 -0
- package/src/types/index.ts +2 -3
- package/src/util/index.ts +1 -0
- package/src/util/render-byline.ts +33 -0
- package/dist/lib/browser/TranscriptContainer-2GSL2XB5.mjs +0 -11
- package/dist/lib/browser/blueprint-definition-UAAJYVLZ.mjs +0 -150
- package/dist/lib/browser/blueprint-definition-UAAJYVLZ.mjs.map +0 -7
- package/dist/lib/browser/chunk-3MKZ2MNQ.mjs +0 -16
- package/dist/lib/browser/chunk-3MKZ2MNQ.mjs.map +0 -7
- package/dist/lib/browser/chunk-535OJKFO.mjs +0 -8
- package/dist/lib/browser/chunk-535OJKFO.mjs.map +0 -7
- package/dist/lib/browser/chunk-7DR6HZEF.mjs +0 -732
- package/dist/lib/browser/chunk-7DR6HZEF.mjs.map +0 -7
- package/dist/lib/browser/chunk-VBUIO677.mjs.map +0 -7
- package/dist/lib/browser/chunk-W7WKZTM4.mjs +0 -63
- package/dist/lib/browser/chunk-W7WKZTM4.mjs.map +0 -7
- package/dist/lib/browser/chunk-WH36ENNE.mjs.map +0 -7
- package/dist/lib/browser/index.mjs +0 -130
- package/dist/lib/browser/index.mjs.map +0 -7
- package/dist/lib/browser/intent-resolver-ZTKTEAQX.mjs +0 -25
- package/dist/lib/browser/intent-resolver-ZTKTEAQX.mjs.map +0 -7
- package/dist/lib/browser/meta.json +0 -1
- package/dist/lib/browser/react-surface-KA32A72I.mjs +0 -34
- package/dist/lib/browser/react-surface-KA32A72I.mjs.map +0 -7
- package/dist/lib/browser/transcriber-LWOLYQHC.mjs +0 -56
- package/dist/lib/browser/transcriber-LWOLYQHC.mjs.map +0 -7
- package/dist/lib/browser/types/index.mjs +0 -12
- package/dist/lib/node-esm/TranscriptContainer-GRLDCESB.mjs +0 -12
- package/dist/lib/node-esm/blueprint-definition-Z3LTZ3ID.mjs +0 -151
- package/dist/lib/node-esm/blueprint-definition-Z3LTZ3ID.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-K4AZOB5Y.mjs +0 -733
- package/dist/lib/node-esm/chunk-K4AZOB5Y.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-OLMSTF3M.mjs +0 -64
- package/dist/lib/node-esm/chunk-OLMSTF3M.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-Q6WD7L2J.mjs +0 -10
- package/dist/lib/node-esm/chunk-Q6WD7L2J.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-SWUVPFKR.mjs +0 -17
- package/dist/lib/node-esm/chunk-SWUVPFKR.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-YFN4FJFK.mjs +0 -467
- package/dist/lib/node-esm/chunk-YFN4FJFK.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-YVNQN2J6.mjs +0 -30
- package/dist/lib/node-esm/chunk-YVNQN2J6.mjs.map +0 -7
- package/dist/lib/node-esm/index.mjs +0 -131
- package/dist/lib/node-esm/index.mjs.map +0 -7
- package/dist/lib/node-esm/intent-resolver-55WH2GY6.mjs +0 -26
- package/dist/lib/node-esm/intent-resolver-55WH2GY6.mjs.map +0 -7
- package/dist/lib/node-esm/meta.json +0 -1
- package/dist/lib/node-esm/react-surface-GU7LW2JO.mjs +0 -35
- package/dist/lib/node-esm/react-surface-GU7LW2JO.mjs.map +0 -7
- package/dist/lib/node-esm/transcriber-UGB7ECX6.mjs.map +0 -7
- package/dist/lib/node-esm/types/index.mjs +0 -13
- package/dist/types/src/capabilities/capabilities.d.ts +0 -17
- package/dist/types/src/capabilities/capabilities.d.ts.map +0 -1
- package/dist/types/src/capabilities/intent-resolver.d.ts +0 -4
- package/dist/types/src/capabilities/intent-resolver.d.ts.map +0 -1
- package/dist/types/src/components/Transcript/Transcript.d.ts +0 -20
- package/dist/types/src/components/Transcript/Transcript.d.ts.map +0 -1
- package/dist/types/src/components/Transcript/Transcript.stories.d.ts +0 -39
- package/dist/types/src/components/Transcript/Transcript.stories.d.ts.map +0 -1
- package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts +0 -28
- package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts.map +0 -1
- package/dist/types/src/components/Transcript/index.d.ts +0 -2
- package/dist/types/src/components/Transcript/index.d.ts.map +0 -1
- package/dist/types/src/components/Transcript/transcript-extension.d.ts +0 -15
- package/dist/types/src/components/Transcript/transcript-extension.d.ts.map +0 -1
- package/dist/types/src/components/TranscriptContainer.d.ts +0 -9
- package/dist/types/src/components/TranscriptContainer.d.ts.map +0 -1
- package/dist/types/src/components/stories/FileTranscription.stories.d.ts +0 -18
- package/dist/types/src/components/stories/FileTranscription.stories.d.ts.map +0 -1
- package/dist/types/src/components/stories/KeyWordRecognition.stories.d.ts.map +0 -1
- package/dist/types/src/components/stories/MicrophoneTranscription.stories.d.ts.map +0 -1
- package/dist/types/src/components/stories/TranscriptionStory.d.ts +0 -12
- package/dist/types/src/components/stories/TranscriptionStory.d.ts.map +0 -1
- package/dist/types/src/components/stories/useIsSpeaking.d.ts +0 -3
- package/dist/types/src/components/stories/useIsSpeaking.d.ts.map +0 -1
- package/dist/types/src/functions/index.d.ts +0 -3
- package/dist/types/src/functions/index.d.ts.map +0 -1
- package/dist/types/src/functions/open.d.ts +0 -7
- package/dist/types/src/functions/open.d.ts.map +0 -1
- package/dist/types/src/functions/summarize.d.ts +0 -11
- package/dist/types/src/functions/summarize.d.ts.map +0 -1
- package/dist/types/src/hooks/useQueueModelAdapter.d.ts +0 -8
- package/dist/types/src/hooks/useQueueModelAdapter.d.ts.map +0 -1
- package/dist/types/src/segments-normalization/index.d.ts.map +0 -1
- package/dist/types/src/segments-normalization/message-normalizer.d.ts.map +0 -1
- package/dist/types/src/segments-normalization/normalization.d.ts +0 -476
- package/dist/types/src/segments-normalization/normalization.d.ts.map +0 -1
- package/dist/types/src/segments-normalization/normalization.test.d.ts.map +0 -1
- package/dist/types/src/segments-normalization/utils.d.ts +0 -3
- package/dist/types/src/segments-normalization/utils.d.ts.map +0 -1
- package/dist/types/src/types/Transcript.d.ts +0 -30
- package/dist/types/src/types/Transcript.d.ts.map +0 -1
- package/dist/types/src/types/TranscriptAction.d.ts +0 -23
- package/dist/types/src/types/TranscriptAction.d.ts.map +0 -1
- package/dist/types/src/types/types.d.ts +0 -5
- package/dist/types/src/types/types.d.ts.map +0 -1
- package/src/capabilities/capabilities.ts +0 -33
- package/src/capabilities/intent-resolver.ts +0 -18
- package/src/components/Transcript/Transcript.stories.tsx +0 -245
- package/src/components/Transcript/Transcript.tsx +0 -93
- package/src/components/Transcript/deprecated/OldTranscript.tsx +0 -260
- package/src/components/TranscriptContainer.tsx +0 -36
- package/src/components/stories/FileTranscription.stories.tsx +0 -256
- package/src/components/stories/MicrophoneTranscription.stories.tsx +0 -285
- package/src/components/stories/TranscriptionStory.tsx +0 -45
- package/src/components/stories/useIsSpeaking.tsx +0 -40
- package/src/functions/index.ts +0 -6
- package/src/functions/open.ts +0 -39
- package/src/functions/summarize.ts +0 -103
- package/src/hooks/useQueueModelAdapter.ts +0 -66
- package/src/segments-normalization/normalization.ts +0 -115
- package/src/types/Transcript.ts +0 -44
- package/src/types/TranscriptAction.ts +0 -24
- package/src/types/types.ts +0 -9
- package/dist/lib/{browser/types → neutral/blueprints}/index.mjs.map +0 -0
- package/dist/lib/{browser/TranscriptContainer-2GSL2XB5.mjs.map → neutral/chunk-J5LGTIGS.mjs.map} +0 -0
- package/dist/lib/{node-esm/types → neutral/hooks}/index.mjs.map +0 -0
- package/dist/lib/{node-esm/TranscriptContainer-GRLDCESB.mjs.map → neutral/index.mjs.map} +0 -0
- package/dist/types/src/{segments-normalization → normalization}/normalization.test.d.ts +0 -0
- package/src/{segments-normalization → normalization}/index.ts +1 -1
|
@@ -8,22 +8,22 @@ import { format } from 'date-fns/format';
|
|
|
8
8
|
import { intervalToDuration } from 'date-fns/intervalToDuration';
|
|
9
9
|
|
|
10
10
|
import { type CleanupFn, addEventListener, combine } from '@dxos/async';
|
|
11
|
-
import { type
|
|
11
|
+
import { type Message } from '@dxos/types';
|
|
12
12
|
|
|
13
13
|
import { DocumentAdapter, type SerializationModel } from '../../model';
|
|
14
14
|
|
|
15
15
|
/**
|
|
16
16
|
* Data structure that maps Chunks queue to lines with transcript state.
|
|
17
17
|
*/
|
|
18
|
-
export type
|
|
19
|
-
model: SerializationModel<
|
|
18
|
+
export type TranscriptionOptions = {
|
|
19
|
+
model: SerializationModel<Message.Message>;
|
|
20
20
|
started?: Date;
|
|
21
21
|
};
|
|
22
22
|
|
|
23
23
|
/**
|
|
24
24
|
* Scrolling transcript with timestamps.
|
|
25
25
|
*/
|
|
26
|
-
export const
|
|
26
|
+
export const transcription = ({ model, started }: TranscriptionOptions): Extension => {
|
|
27
27
|
return [
|
|
28
28
|
// Show timestamps in the gutter.
|
|
29
29
|
gutter({
|
|
@@ -131,8 +131,12 @@ export const transcript = ({ model, started }: TranscriptOptions): Extension =>
|
|
|
131
131
|
),
|
|
132
132
|
|
|
133
133
|
EditorView.theme({
|
|
134
|
+
'.cm-gutters': {
|
|
135
|
+
backgroundColor: 'var(--color-base-surface)',
|
|
136
|
+
},
|
|
134
137
|
'.cm-timestamp-gutter': {
|
|
135
138
|
width: '6rem',
|
|
139
|
+
paddingLeft: '0.5rem',
|
|
136
140
|
paddingRight: '1rem',
|
|
137
141
|
},
|
|
138
142
|
'.cm-timestamp-gutter > .cm-gutterElement > div': {
|
|
@@ -156,13 +160,13 @@ class TimestampMarker extends GutterMarker {
|
|
|
156
160
|
super();
|
|
157
161
|
}
|
|
158
162
|
|
|
159
|
-
override eq(other: this)
|
|
160
|
-
return
|
|
163
|
+
override eq(other: this) {
|
|
164
|
+
return this._timestamp === other._timestamp;
|
|
161
165
|
}
|
|
162
166
|
|
|
163
|
-
override toDOM(view: EditorView)
|
|
167
|
+
override toDOM(view: EditorView) {
|
|
164
168
|
const el = document.createElement('div');
|
|
165
|
-
el.className = 'text-sm text-subdued hover:bg-
|
|
169
|
+
el.className = 'text-sm text-subdued hover:bg-hover-surface cursor-pointer';
|
|
166
170
|
el.textContent = formatTimestamp(this._timestamp, this._started);
|
|
167
171
|
// TODO(burdon): Click to bookmark or copy hyperlink.
|
|
168
172
|
el.onclick = () => {
|
|
@@ -176,7 +180,7 @@ class TimestampMarker extends GutterMarker {
|
|
|
176
180
|
}
|
|
177
181
|
}
|
|
178
182
|
|
|
179
|
-
const getStartTime = (started?: Date, message?:
|
|
183
|
+
const getStartTime = (started?: Date, message?: Message.Message): Date | undefined => {
|
|
180
184
|
if (started) {
|
|
181
185
|
return started;
|
|
182
186
|
}
|
package/src/components/index.ts
CHANGED
|
@@ -2,9 +2,5 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
export * from './Transcript';
|
|
8
|
-
export * from './TranscriptContainer';
|
|
9
|
-
|
|
10
|
-
export const TranscriptContainer = lazy(() => import('./TranscriptContainer'));
|
|
5
|
+
export * from './Oracle';
|
|
6
|
+
export * from './Transcription';
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2026 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import { type Meta, type StoryObj } from '@storybook/react-vite';
|
|
6
|
+
import * as Effect from 'effect/Effect';
|
|
7
|
+
import React from 'react';
|
|
8
|
+
|
|
9
|
+
import { withPluginManager } from '@dxos/app-framework/testing';
|
|
10
|
+
import { Feed, Query, Ref } from '@dxos/echo';
|
|
11
|
+
import { ClientPlugin, initializeIdentity } from '@dxos/plugin-client/testing';
|
|
12
|
+
import { corePlugins } from '@dxos/plugin-testing';
|
|
13
|
+
import { useQuery, useSpaces } from '@dxos/react-client/echo';
|
|
14
|
+
import { Loading, withLayout } from '@dxos/react-ui/testing';
|
|
15
|
+
import { Message, Transcript } from '@dxos/types';
|
|
16
|
+
|
|
17
|
+
import { translations } from '#translations';
|
|
18
|
+
|
|
19
|
+
import { TestItem } from '../../testing';
|
|
20
|
+
import { TranscriptionPlugin } from '../../TranscriptionPlugin';
|
|
21
|
+
import { TranscriptionArticle } from './TranscriptionArticle';
|
|
22
|
+
|
|
23
|
+
const DefaultStory = () => {
|
|
24
|
+
const [space] = useSpaces();
|
|
25
|
+
const [transcript] = useQuery(space?.db, Query.type(Transcript.Transcript));
|
|
26
|
+
if (!space || !transcript) {
|
|
27
|
+
return <Loading />;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
return <TranscriptionArticle role='article' attendableId='story' subject={transcript} />;
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
const meta = {
|
|
34
|
+
title: 'plugins/plugin-transcription/containers/TranscriptionArticle',
|
|
35
|
+
render: DefaultStory,
|
|
36
|
+
decorators: [
|
|
37
|
+
withLayout({ layout: 'column' }),
|
|
38
|
+
withPluginManager({
|
|
39
|
+
plugins: [
|
|
40
|
+
...corePlugins(),
|
|
41
|
+
ClientPlugin({
|
|
42
|
+
types: [Transcript.Transcript, Feed.Feed, Message.Message, TestItem],
|
|
43
|
+
onClientInitialized: ({ client }) =>
|
|
44
|
+
Effect.gen(function* () {
|
|
45
|
+
const { personalSpace } = yield* initializeIdentity(client);
|
|
46
|
+
const feed = personalSpace.db.add(Feed.make({ name: 'transcription' }));
|
|
47
|
+
personalSpace.db.add(Transcript.make(Ref.make(feed)));
|
|
48
|
+
}),
|
|
49
|
+
}),
|
|
50
|
+
TranscriptionPlugin(),
|
|
51
|
+
],
|
|
52
|
+
}),
|
|
53
|
+
],
|
|
54
|
+
parameters: { layout: 'fullscreen', translations },
|
|
55
|
+
} satisfies Meta<typeof DefaultStory>;
|
|
56
|
+
|
|
57
|
+
export default meta;
|
|
58
|
+
|
|
59
|
+
type Story = StoryObj<typeof meta>;
|
|
60
|
+
|
|
61
|
+
export const Default: Story = {};
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2024 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import React from 'react';
|
|
6
|
+
|
|
7
|
+
import { type AppSurface } from '@dxos/app-toolkit/ui';
|
|
8
|
+
import { Filter, Obj, Query } from '@dxos/echo';
|
|
9
|
+
import { useMembers, useQuery } from '@dxos/react-client/echo';
|
|
10
|
+
import { Panel } from '@dxos/react-ui';
|
|
11
|
+
import { Menu, MenuBuilder, useMenuBuilder } from '@dxos/react-ui-menu';
|
|
12
|
+
import { Message, type Transcript } from '@dxos/types';
|
|
13
|
+
|
|
14
|
+
import { Transcription } from '#components';
|
|
15
|
+
import { useFeedModelAdapter, useTranscriptionRecording } from '#hooks';
|
|
16
|
+
import { meta } from '#meta';
|
|
17
|
+
|
|
18
|
+
import { renderByline } from '../../util';
|
|
19
|
+
|
|
20
|
+
export type TranscriptionArticleProps = AppSurface.ObjectArticleProps<Transcript.Transcript>;
|
|
21
|
+
|
|
22
|
+
export const TranscriptionArticle = ({ role, subject: transcript, attendableId }: TranscriptionArticleProps) => {
|
|
23
|
+
const db = Obj.getDatabase(transcript);
|
|
24
|
+
const members = useMembers(db?.spaceId).map((member) => member.identity);
|
|
25
|
+
const feed = transcript.feed.target;
|
|
26
|
+
const messages = useQuery(
|
|
27
|
+
db,
|
|
28
|
+
feed ? Query.select(Filter.type(Message.Message)).from(feed) : Query.select(Filter.nothing()),
|
|
29
|
+
);
|
|
30
|
+
const model = useFeedModelAdapter(renderByline(members), messages);
|
|
31
|
+
const { recording, toggleRecording } = useTranscriptionRecording(transcript);
|
|
32
|
+
|
|
33
|
+
const menuActions = useMenuBuilder(
|
|
34
|
+
() =>
|
|
35
|
+
MenuBuilder.make()
|
|
36
|
+
.action(
|
|
37
|
+
'toggle-recording',
|
|
38
|
+
{
|
|
39
|
+
label: [recording ? 'stop-recording.label' : 'start-recording.label', { ns: meta.id }],
|
|
40
|
+
icon: recording ? 'ph--stop-circle--regular' : 'ph--microphone--regular',
|
|
41
|
+
disposition: 'toolbar',
|
|
42
|
+
testId: 'transcription.toggle-recording',
|
|
43
|
+
},
|
|
44
|
+
toggleRecording,
|
|
45
|
+
)
|
|
46
|
+
.build(),
|
|
47
|
+
[recording, toggleRecording],
|
|
48
|
+
);
|
|
49
|
+
|
|
50
|
+
return (
|
|
51
|
+
<Panel.Root role={role}>
|
|
52
|
+
<Menu.Root {...menuActions} attendableId={attendableId}>
|
|
53
|
+
<Panel.Toolbar asChild>
|
|
54
|
+
<Menu.Toolbar />
|
|
55
|
+
</Panel.Toolbar>
|
|
56
|
+
</Menu.Root>
|
|
57
|
+
<Panel.Content asChild>
|
|
58
|
+
<Transcription model={model} transcript={transcript} />
|
|
59
|
+
</Panel.Content>
|
|
60
|
+
</Panel.Root>
|
|
61
|
+
);
|
|
62
|
+
};
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2024 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
// Headless test that simulates a two-speaker conversation as a pair of Whisper streams and
|
|
6
|
+
// drives them through the (currently stubbed) Pass A enrichment and Pass B summarization
|
|
7
|
+
// operations. The handlers are deterministic placeholders — they don't call an LLM — so the
|
|
8
|
+
// test runs offline; switching to the real LLM-backed handlers later doesn't change the
|
|
9
|
+
// harness, just the assertions.
|
|
10
|
+
//
|
|
11
|
+
// What this verifies today:
|
|
12
|
+
// - Conversation fixture parses to entities + speaker-tagged utterances.
|
|
13
|
+
// - Stream simulator emits per-batch transcript blocks in chronological order.
|
|
14
|
+
// - Pass A's sliding-window enrichment links seeded ECHO entities by name and surfaces
|
|
15
|
+
// remaining capitalised tokens as candidates.
|
|
16
|
+
// - Pass B accumulates a per-speaker summary and resolves "I" → speaker display name.
|
|
17
|
+
|
|
18
|
+
import { describe, it } from '@effect/vitest';
|
|
19
|
+
import * as Effect from 'effect/Effect';
|
|
20
|
+
import * as fs from 'node:fs';
|
|
21
|
+
import * as path from 'node:path';
|
|
22
|
+
import * as url from 'node:url';
|
|
23
|
+
|
|
24
|
+
import { Blueprint, Operation, OperationHandlerSet } from '@dxos/compute';
|
|
25
|
+
import { Database, Feed, Obj } from '@dxos/echo';
|
|
26
|
+
import { TestHelpers } from '@dxos/effect/testing';
|
|
27
|
+
import { AssistantTestLayer } from '@dxos/functions-runtime/testing';
|
|
28
|
+
import { ObjectId } from '@dxos/keys';
|
|
29
|
+
import { Organization, Person } from '@dxos/types';
|
|
30
|
+
|
|
31
|
+
import { EnrichTranscript, EnrichmentHandlers, SummarizeConversation } from './operations';
|
|
32
|
+
import { parseConversation } from './parser';
|
|
33
|
+
import { simulateStream } from './stream-simulator';
|
|
34
|
+
|
|
35
|
+
ObjectId.dangerouslyDisableRandomness();
|
|
36
|
+
|
|
37
|
+
const TestLayer = AssistantTestLayer({
|
|
38
|
+
operationHandlers: OperationHandlerSet.make(...EnrichmentHandlers),
|
|
39
|
+
types: [Person.Person, Organization.Organization, Blueprint.Blueprint, Feed.Feed],
|
|
40
|
+
disableLlmMemoization: true,
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
describe('Transcript enrichment — conversation simulation', () => {
|
|
44
|
+
it.scoped(
|
|
45
|
+
'streams a two-speaker conversation through Pass A and Pass B (stub handlers)',
|
|
46
|
+
Effect.fnUntraced(
|
|
47
|
+
function* ({ expect }) {
|
|
48
|
+
// 1. Load and parse the fixture.
|
|
49
|
+
const fixturePath = path.join(path.dirname(url.fileURLToPath(import.meta.url)), 'conversation.txt');
|
|
50
|
+
const conversation = parseConversation(fs.readFileSync(fixturePath, 'utf8'));
|
|
51
|
+
expect(conversation.speakers).toEqual(['A', 'B']);
|
|
52
|
+
expect(conversation.utterances.length).toBeGreaterThan(0);
|
|
53
|
+
expect(conversation.entities.length).toBeGreaterThan(0);
|
|
54
|
+
|
|
55
|
+
// 2. Seed the ECHO space with the fixture's entities. Map fixture types → schema types.
|
|
56
|
+
const seeded: { id: string; typename: string; name: string }[] = [];
|
|
57
|
+
for (const entity of conversation.entities) {
|
|
58
|
+
if (entity.kind === 'Person') {
|
|
59
|
+
const fullName = entity.fields.fullName ?? entity.fields.name;
|
|
60
|
+
if (!fullName) {
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
const person = yield* Database.add(Obj.make(Person.Person, { fullName }));
|
|
64
|
+
seeded.push({ id: person.id, typename: Person.Person.typename, name: fullName });
|
|
65
|
+
} else if (entity.kind === 'Organization') {
|
|
66
|
+
const name = entity.fields.name;
|
|
67
|
+
if (!name) {
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
const org = yield* Database.add(Obj.make(Organization.Organization, { name }));
|
|
71
|
+
seeded.push({ id: org.id, typename: Organization.Organization.typename, name });
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
yield* Database.flush();
|
|
75
|
+
expect(seeded.map((s) => s.name)).toContain('Chad');
|
|
76
|
+
expect(seeded.map((s) => s.name)).toContain('Blue Yard');
|
|
77
|
+
|
|
78
|
+
// 3. Generate the per-batch Whisper schedule. Batch by speaker so each batch maps to
|
|
79
|
+
// a single Pass-A invocation, mimicking what the live `useTranscriptEnrichment`
|
|
80
|
+
// hook will receive from the transcriber.
|
|
81
|
+
const batches = simulateStream(conversation, { wordsPerSecond: 2.5, gapSeconds: 0.5 });
|
|
82
|
+
expect(batches.length).toBe(conversation.utterances.length);
|
|
83
|
+
expect(batches[0].offsetMs).toBe(0);
|
|
84
|
+
expect(batches.every((b) => b.blocks.length > 0)).toBe(true);
|
|
85
|
+
|
|
86
|
+
// 4. Run Pass A over each batch — sliding window of the last 4 blocks across speakers.
|
|
87
|
+
const slidingWindow: { speaker: string; text: string; started: string }[] = [];
|
|
88
|
+
const enrichments: {
|
|
89
|
+
speaker: string;
|
|
90
|
+
corrected: string;
|
|
91
|
+
referenceIds: string[];
|
|
92
|
+
candidates: { text: string }[];
|
|
93
|
+
}[] = [];
|
|
94
|
+
for (const batch of batches) {
|
|
95
|
+
for (const block of batch.blocks) {
|
|
96
|
+
slidingWindow.push({ speaker: batch.speaker, text: block.text, started: block.started });
|
|
97
|
+
if (slidingWindow.length > 4) {
|
|
98
|
+
slidingWindow.shift();
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
const result = yield* Operation.invoke(EnrichTranscript, {
|
|
102
|
+
window: slidingWindow,
|
|
103
|
+
knownEntities: seeded,
|
|
104
|
+
});
|
|
105
|
+
// The new batch's entries are the *last* `batch.blocks.length` items in the window.
|
|
106
|
+
const newEntries = result.blocks.slice(-batch.blocks.length);
|
|
107
|
+
for (const block of newEntries) {
|
|
108
|
+
const windowEntry = slidingWindow[block.index];
|
|
109
|
+
enrichments.push({
|
|
110
|
+
speaker: windowEntry.speaker,
|
|
111
|
+
corrected: block.corrected,
|
|
112
|
+
referenceIds: [...block.referenceIds],
|
|
113
|
+
candidates: block.candidates.map((c) => ({ text: c.text })),
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// 4a. Every block got a terminal-punctuated `corrected` value.
|
|
119
|
+
for (const e of enrichments) {
|
|
120
|
+
expect(e.corrected).toMatch(/[.!?]$/);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// 4b. The block containing "Chad" links to the seeded Person.
|
|
124
|
+
const chadId = seeded.find((s) => s.name === 'Chad')!.id;
|
|
125
|
+
const chadMention = enrichments.find((e) => /\bChad\b/.test(e.corrected));
|
|
126
|
+
expect(chadMention).toBeDefined();
|
|
127
|
+
expect(chadMention!.referenceIds).toContain(chadId);
|
|
128
|
+
|
|
129
|
+
// 4c. Place names surface as candidates (not in our seed).
|
|
130
|
+
const allCandidates = enrichments.flatMap((e) => e.candidates.map((c) => c.text));
|
|
131
|
+
expect(allCandidates).toContain('Munich');
|
|
132
|
+
expect(allCandidates).toContain('Los Angeles');
|
|
133
|
+
|
|
134
|
+
// 5. Run Pass B once at the end of the conversation (the real pipeline would fire it
|
|
135
|
+
// after silence + interval; here we trigger it manually after the last batch).
|
|
136
|
+
const speakerNames = { A: 'Speaker A', B: 'Speaker B' };
|
|
137
|
+
const summary = yield* Operation.invoke(SummarizeConversation, {
|
|
138
|
+
previousSummary: '',
|
|
139
|
+
newUtterances: enrichments.map((e) => ({ speaker: e.speaker, text: e.corrected })),
|
|
140
|
+
speakerNames,
|
|
141
|
+
recentReferences: seeded.map((s) => ({ id: s.id, name: s.name })),
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
// 5a. Summary is non-empty and mentions both speakers.
|
|
145
|
+
expect(summary.summary.length).toBeGreaterThan(0);
|
|
146
|
+
expect(summary.summary).toContain('Speaker A');
|
|
147
|
+
expect(summary.summary).toContain('Speaker B');
|
|
148
|
+
|
|
149
|
+
// 5b. Resolved referents: every utterance that contains "I" picks up the speaker's name.
|
|
150
|
+
const iUtterances = enrichments.filter((e) => /\bI\b/.test(e.corrected));
|
|
151
|
+
expect(summary.resolvedReferents.length).toBe(iUtterances.length);
|
|
152
|
+
for (const referent of summary.resolvedReferents) {
|
|
153
|
+
expect(referent.surface).toBe('I');
|
|
154
|
+
expect([speakerNames.A, speakerNames.B]).toContain(referent.referent);
|
|
155
|
+
}
|
|
156
|
+
},
|
|
157
|
+
Effect.provide(TestLayer),
|
|
158
|
+
TestHelpers.provideTestContext,
|
|
159
|
+
),
|
|
160
|
+
{ timeout: 30_000 },
|
|
161
|
+
);
|
|
162
|
+
});
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Conversation fixture for transcription enrichment tests.
|
|
2
|
+
# Format:
|
|
3
|
+
# `Entities:` block — declares ECHO objects to seed into the test space.
|
|
4
|
+
# Each entry is `<Type>:` followed by indented `<field>: <value>` lines.
|
|
5
|
+
# `Conversation:` block — one utterance per line, `<speaker>: <text>`.
|
|
6
|
+
# Speakers map to distinct virtual Whisper streams in the simulator.
|
|
7
|
+
# Lines starting with `#` are comments and are ignored by the parser.
|
|
8
|
+
|
|
9
|
+
Entities:
|
|
10
|
+
Person:
|
|
11
|
+
fullName: Chad
|
|
12
|
+
employer: Blue Yard
|
|
13
|
+
Organization:
|
|
14
|
+
name: Blue Yard
|
|
15
|
+
|
|
16
|
+
Conversation:
|
|
17
|
+
A: I think I'm going to go.
|
|
18
|
+
B: Are you meeting with them?
|
|
19
|
+
A: Yes I think so.
|
|
20
|
+
B: But I thought they were in Munich?
|
|
21
|
+
A: They are, but the offsite is in Los Angeles.
|
|
22
|
+
B: Is Chad going to be there?
|
|
23
|
+
A: Yes, the whole team. It's their annual offsite.
|
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2024 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
// Operation definitions for the transcript enrichment pipeline (see ENRICHMENT.md).
|
|
6
|
+
//
|
|
7
|
+
// **Stub status.** The handlers below return deterministic dummy values rather than calling
|
|
8
|
+
// an LLM; they exist so the end-to-end test harness (`conversation.test.ts`) can exercise
|
|
9
|
+
// the schemas, the operation wiring, and the streaming/orchestration code paths without
|
|
10
|
+
// requiring an Anthropic key on first run. The real handlers will replace these stubs in
|
|
11
|
+
// a follow-up PR (calling `LanguageModel.generateObject(...)`) while keeping the same
|
|
12
|
+
// inputs / outputs.
|
|
13
|
+
|
|
14
|
+
import * as Effect from 'effect/Effect';
|
|
15
|
+
import * as Schema from 'effect/Schema';
|
|
16
|
+
|
|
17
|
+
import { Operation } from '@dxos/compute';
|
|
18
|
+
|
|
19
|
+
import { meta } from '#meta';
|
|
20
|
+
|
|
21
|
+
const NAMESPACE = `${meta.id}.enrichment`;
|
|
22
|
+
|
|
23
|
+
//
|
|
24
|
+
// Shared sub-schemas.
|
|
25
|
+
//
|
|
26
|
+
|
|
27
|
+
/** A noun / proper noun surfaced for follow-up actions (create object, research, link…). */
|
|
28
|
+
export const CandidateMention = Schema.Struct({
|
|
29
|
+
text: Schema.String.annotations({ description: 'Surface form as it appears in the transcript.' }),
|
|
30
|
+
kind: Schema.Literal('noun', 'proper-noun').annotations({
|
|
31
|
+
description: 'Whether this is a common noun (e.g. "team") or a proper noun (e.g. "Munich").',
|
|
32
|
+
}),
|
|
33
|
+
start: Schema.Number.annotations({ description: 'Character offset within the corrected text.' }),
|
|
34
|
+
end: Schema.Number.annotations({ description: 'Character offset within the corrected text.' }),
|
|
35
|
+
suggested: Schema.optional(
|
|
36
|
+
Schema.Struct({
|
|
37
|
+
typename: Schema.String.annotations({
|
|
38
|
+
description: 'Optional hint for the ECHO type the user might create (e.g. "org.dxos.type.organization").',
|
|
39
|
+
}),
|
|
40
|
+
}),
|
|
41
|
+
),
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
/** Per-block enrichment output produced by Pass A. */
|
|
45
|
+
export const EnrichmentBlock = Schema.Struct({
|
|
46
|
+
/** Index into the input window; lets the caller match enrichment to the originating block. */
|
|
47
|
+
index: Schema.Number,
|
|
48
|
+
/** Corrected text — punctuation, capitalization, words split across batch boundaries reassembled. */
|
|
49
|
+
corrected: Schema.String,
|
|
50
|
+
/**
|
|
51
|
+
* IDs of ECHO objects the LLM is confident the block mentions. Strings rather than `Ref<T>`
|
|
52
|
+
* to keep the schema serialisable; the consumer resolves them via `Database.resolve(...)`.
|
|
53
|
+
*/
|
|
54
|
+
referenceIds: Schema.Array(Schema.String),
|
|
55
|
+
/** Candidate nouns / proper nouns not already covered by `referenceIds`. */
|
|
56
|
+
candidates: Schema.Array(CandidateMention),
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
//
|
|
60
|
+
// Pass A — enrichment (per Whisper batch / sliding window).
|
|
61
|
+
//
|
|
62
|
+
|
|
63
|
+
export const EnrichmentInput = Schema.Struct({
|
|
64
|
+
/** Sliding-window blocks. Each carries the speaker that produced it. */
|
|
65
|
+
window: Schema.Array(
|
|
66
|
+
Schema.Struct({
|
|
67
|
+
speaker: Schema.String,
|
|
68
|
+
text: Schema.String,
|
|
69
|
+
started: Schema.String,
|
|
70
|
+
}),
|
|
71
|
+
),
|
|
72
|
+
/** ECHO objects available in the space that the LLM may pick from when linking references. */
|
|
73
|
+
knownEntities: Schema.optional(
|
|
74
|
+
Schema.Array(
|
|
75
|
+
Schema.Struct({
|
|
76
|
+
id: Schema.String,
|
|
77
|
+
typename: Schema.String,
|
|
78
|
+
name: Schema.String,
|
|
79
|
+
}),
|
|
80
|
+
),
|
|
81
|
+
),
|
|
82
|
+
});
|
|
83
|
+
export type EnrichmentInputType = Schema.Schema.Type<typeof EnrichmentInput>;
|
|
84
|
+
|
|
85
|
+
export const EnrichmentOutput = Schema.Struct({
|
|
86
|
+
blocks: Schema.Array(EnrichmentBlock),
|
|
87
|
+
});
|
|
88
|
+
export type EnrichmentOutputType = Schema.Schema.Type<typeof EnrichmentOutput>;
|
|
89
|
+
|
|
90
|
+
export const EnrichTranscript = Operation.make({
|
|
91
|
+
meta: {
|
|
92
|
+
key: `${NAMESPACE}.enrich`,
|
|
93
|
+
name: 'Enrich Transcript',
|
|
94
|
+
description: 'Correct Whisper output and surface entity references and candidates over a sliding window.',
|
|
95
|
+
},
|
|
96
|
+
input: EnrichmentInput,
|
|
97
|
+
output: EnrichmentOutput,
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Deterministic stub: echoes the input text as `corrected`, links every block whose text contains
|
|
102
|
+
* a `knownEntities[i].name` substring to that entity's id, and surfaces every capitalised word
|
|
103
|
+
* not already linked as a `proper-noun` candidate. Sentence-end punctuation is added if missing.
|
|
104
|
+
*/
|
|
105
|
+
const enrichHandler: Operation.WithHandler<typeof EnrichTranscript> = EnrichTranscript.pipe(
|
|
106
|
+
Operation.withHandler(
|
|
107
|
+
Effect.fnUntraced(function* ({ window, knownEntities }) {
|
|
108
|
+
const known = knownEntities ?? [];
|
|
109
|
+
const blocks = window.map((entry, index) => {
|
|
110
|
+
const corrected = ensureTerminalPunctuation(entry.text);
|
|
111
|
+
const referenceIds: string[] = [];
|
|
112
|
+
const covered: Array<{ start: number; end: number }> = [];
|
|
113
|
+
for (const entity of known) {
|
|
114
|
+
const haystack = corrected.toLowerCase();
|
|
115
|
+
const needle = entity.name.toLowerCase();
|
|
116
|
+
const idx = haystack.indexOf(needle);
|
|
117
|
+
if (idx >= 0) {
|
|
118
|
+
referenceIds.push(entity.id);
|
|
119
|
+
covered.push({ start: idx, end: idx + entity.name.length });
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const candidates = [...findCapitalisedWords(corrected)].filter(
|
|
124
|
+
({ start, end }) => !covered.some((c) => c.start <= start && end <= c.end),
|
|
125
|
+
);
|
|
126
|
+
|
|
127
|
+
return { index, corrected, referenceIds, candidates };
|
|
128
|
+
});
|
|
129
|
+
return { blocks };
|
|
130
|
+
}),
|
|
131
|
+
),
|
|
132
|
+
);
|
|
133
|
+
|
|
134
|
+
//
|
|
135
|
+
// Pass B — summary (end-of-utterance + periodic).
|
|
136
|
+
//
|
|
137
|
+
|
|
138
|
+
export const SummaryInput = Schema.Struct({
|
|
139
|
+
/** The cumulative summary as of the last invocation (empty string on first call). */
|
|
140
|
+
previousSummary: Schema.String,
|
|
141
|
+
/** New corrected utterances appended since the previous summary. */
|
|
142
|
+
newUtterances: Schema.Array(
|
|
143
|
+
Schema.Struct({
|
|
144
|
+
speaker: Schema.String,
|
|
145
|
+
text: Schema.String,
|
|
146
|
+
}),
|
|
147
|
+
),
|
|
148
|
+
/** Map of speaker label → human-readable display name. Drives "I" resolution. */
|
|
149
|
+
speakerNames: Schema.optional(Schema.Record({ key: Schema.String, value: Schema.String })),
|
|
150
|
+
/** IDs and surface forms of references emitted by recent Pass A invocations. */
|
|
151
|
+
recentReferences: Schema.optional(
|
|
152
|
+
Schema.Array(
|
|
153
|
+
Schema.Struct({
|
|
154
|
+
id: Schema.String,
|
|
155
|
+
name: Schema.String,
|
|
156
|
+
}),
|
|
157
|
+
),
|
|
158
|
+
),
|
|
159
|
+
});
|
|
160
|
+
export type SummaryInputType = Schema.Schema.Type<typeof SummaryInput>;
|
|
161
|
+
|
|
162
|
+
export const ResolvedReferent = Schema.Struct({
|
|
163
|
+
surface: Schema.String.annotations({ description: 'The deictic / anaphoric form, e.g. "I", "there".' }),
|
|
164
|
+
referent: Schema.String.annotations({ description: 'Canonical name or description the surface resolves to.' }),
|
|
165
|
+
refId: Schema.optional(Schema.String).annotations({
|
|
166
|
+
description: 'ECHO object id when the referent maps to a known entity.',
|
|
167
|
+
}),
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
export const SummaryOutput = Schema.Struct({
|
|
171
|
+
summary: Schema.String,
|
|
172
|
+
resolvedReferents: Schema.Array(ResolvedReferent),
|
|
173
|
+
});
|
|
174
|
+
export type SummaryOutputType = Schema.Schema.Type<typeof SummaryOutput>;
|
|
175
|
+
|
|
176
|
+
export const SummarizeConversation = Operation.make({
|
|
177
|
+
meta: {
|
|
178
|
+
key: `${NAMESPACE}.summarize`,
|
|
179
|
+
name: 'Summarize Conversation',
|
|
180
|
+
description: 'Maintain a cumulative summary with referent resolution over a transcript stream.',
|
|
181
|
+
},
|
|
182
|
+
input: SummaryInput,
|
|
183
|
+
output: SummaryOutput,
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Deterministic stub: appends each new utterance to the previous summary on its own line,
|
|
188
|
+
* prefixed with the speaker's display name (resolving the speaker label via `speakerNames`).
|
|
189
|
+
* If any utterance contains "I" as a standalone word, emits a `resolvedReferents` entry binding
|
|
190
|
+
* it to the speaker's display name — this is the minimal demonstration of the Pass-C contract.
|
|
191
|
+
*/
|
|
192
|
+
const summarizeHandler: Operation.WithHandler<typeof SummarizeConversation> = SummarizeConversation.pipe(
|
|
193
|
+
Operation.withHandler(
|
|
194
|
+
Effect.fnUntraced(function* ({ previousSummary, newUtterances, speakerNames, recentReferences }) {
|
|
195
|
+
const names = speakerNames ?? {};
|
|
196
|
+
const lines = newUtterances.map((u) => `${names[u.speaker] ?? u.speaker}: ${u.text}`);
|
|
197
|
+
const summary = [previousSummary.trim(), ...lines].filter(Boolean).join('\n');
|
|
198
|
+
|
|
199
|
+
const resolvedReferents = newUtterances
|
|
200
|
+
.filter((u) => /\bI\b/.test(u.text))
|
|
201
|
+
.map((u) => {
|
|
202
|
+
const displayName = names[u.speaker] ?? u.speaker;
|
|
203
|
+
const ref = (recentReferences ?? []).find((r) => r.name === displayName);
|
|
204
|
+
return { surface: 'I', referent: displayName, refId: ref?.id };
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
return { summary, resolvedReferents };
|
|
208
|
+
}),
|
|
209
|
+
),
|
|
210
|
+
);
|
|
211
|
+
|
|
212
|
+
//
|
|
213
|
+
// Exports.
|
|
214
|
+
//
|
|
215
|
+
|
|
216
|
+
export const EnrichmentHandlers = [enrichHandler, summarizeHandler];
|
|
217
|
+
|
|
218
|
+
//
|
|
219
|
+
// Helpers.
|
|
220
|
+
//
|
|
221
|
+
|
|
222
|
+
const ensureTerminalPunctuation = (text: string): string => {
|
|
223
|
+
const trimmed = text.trim();
|
|
224
|
+
if (trimmed.length === 0) {
|
|
225
|
+
return trimmed;
|
|
226
|
+
}
|
|
227
|
+
return /[.!?]$/.test(trimmed) ? trimmed : `${trimmed}.`;
|
|
228
|
+
};
|
|
229
|
+
|
|
230
|
+
function* findCapitalisedWords(
|
|
231
|
+
text: string,
|
|
232
|
+
): Generator<{ text: string; kind: 'proper-noun'; start: number; end: number }> {
|
|
233
|
+
const re = /\b[A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+)*\b/g;
|
|
234
|
+
let match: RegExpExecArray | null;
|
|
235
|
+
while ((match = re.exec(text)) !== null) {
|
|
236
|
+
// Skip pure sentence-initial single words ("Yes", "I", "But", "They") to keep candidates focused on
|
|
237
|
+
// proper nouns. Heuristic for the stub: drop common stop-words.
|
|
238
|
+
const word = match[0];
|
|
239
|
+
if (STOP_WORDS.has(word)) {
|
|
240
|
+
continue;
|
|
241
|
+
}
|
|
242
|
+
yield { text: word, kind: 'proper-noun', start: match.index, end: match.index + word.length };
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
const STOP_WORDS = new Set([
|
|
247
|
+
'I',
|
|
248
|
+
'Yes',
|
|
249
|
+
'No',
|
|
250
|
+
'But',
|
|
251
|
+
'And',
|
|
252
|
+
'Or',
|
|
253
|
+
'So',
|
|
254
|
+
'They',
|
|
255
|
+
'We',
|
|
256
|
+
'He',
|
|
257
|
+
'She',
|
|
258
|
+
'It',
|
|
259
|
+
'The',
|
|
260
|
+
'A',
|
|
261
|
+
'An',
|
|
262
|
+
'Is',
|
|
263
|
+
'Are',
|
|
264
|
+
'Was',
|
|
265
|
+
'Were',
|
|
266
|
+
'Be',
|
|
267
|
+
'Been',
|
|
268
|
+
]);
|