@dxos/plugin-transcription 0.8.4-main.ae835ea → 0.8.4-main.bc2380dfbc

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (375) hide show
  1. package/LICENSE +102 -5
  2. package/README.md +1 -1
  3. package/dist/lib/neutral/TranscriptionArticle-L6XANBH5.mjs +54 -0
  4. package/dist/lib/neutral/TranscriptionArticle-L6XANBH5.mjs.map +7 -0
  5. package/dist/lib/neutral/TranscriptionPlugin.mjs +34 -0
  6. package/dist/lib/neutral/TranscriptionPlugin.mjs.map +7 -0
  7. package/dist/lib/neutral/TranscriptionPlugin.node.mjs +25 -0
  8. package/dist/lib/neutral/TranscriptionPlugin.node.mjs.map +7 -0
  9. package/dist/lib/neutral/TranscriptionPlugin.workerd.mjs +21 -0
  10. package/dist/lib/neutral/TranscriptionPlugin.workerd.mjs.map +7 -0
  11. package/dist/lib/neutral/blueprint-definition-HOXDG5QP.mjs +15 -0
  12. package/dist/lib/neutral/blueprint-definition-HOXDG5QP.mjs.map +7 -0
  13. package/dist/lib/neutral/blueprints/index.mjs +8 -0
  14. package/dist/lib/neutral/capabilities/index.mjs +17 -0
  15. package/dist/lib/neutral/capabilities/index.mjs.map +7 -0
  16. package/dist/lib/neutral/chunk-2O3ONPRZ.mjs +298 -0
  17. package/dist/lib/neutral/chunk-2O3ONPRZ.mjs.map +7 -0
  18. package/dist/lib/neutral/chunk-5FXU36PN.mjs +120 -0
  19. package/dist/lib/neutral/chunk-5FXU36PN.mjs.map +7 -0
  20. package/dist/lib/{browser/chunk-VBUIO677.mjs → neutral/chunk-GL4RS533.mjs} +2 -9
  21. package/dist/lib/neutral/chunk-GL4RS533.mjs.map +7 -0
  22. package/dist/lib/{browser/chunk-WH36ENNE.mjs → neutral/chunk-ILSA3DU7.mjs} +102 -133
  23. package/dist/lib/neutral/chunk-ILSA3DU7.mjs.map +7 -0
  24. package/dist/lib/neutral/chunk-J5LGTIGS.mjs +10 -0
  25. package/dist/lib/neutral/chunk-KAEXRX3Y.mjs +35 -0
  26. package/dist/lib/neutral/chunk-KAEXRX3Y.mjs.map +7 -0
  27. package/dist/lib/neutral/chunk-MLRJWAQH.mjs +179 -0
  28. package/dist/lib/neutral/chunk-MLRJWAQH.mjs.map +7 -0
  29. package/dist/lib/neutral/chunk-Y5EQAFFR.mjs +8 -0
  30. package/dist/lib/neutral/chunk-Y5EQAFFR.mjs.map +7 -0
  31. package/dist/lib/neutral/chunk-ZR4BH4QQ.mjs +30 -0
  32. package/dist/lib/neutral/chunk-ZR4BH4QQ.mjs.map +7 -0
  33. package/dist/lib/neutral/components/index.mjs +272 -0
  34. package/dist/lib/neutral/components/index.mjs.map +7 -0
  35. package/dist/lib/neutral/containers/index.mjs +9 -0
  36. package/dist/lib/neutral/containers/index.mjs.map +7 -0
  37. package/dist/lib/neutral/create-RRWRIUSR.mjs +21 -0
  38. package/dist/lib/neutral/create-RRWRIUSR.mjs.map +7 -0
  39. package/dist/lib/neutral/hooks/index.mjs +19 -0
  40. package/dist/lib/neutral/index.mjs +44 -0
  41. package/dist/lib/neutral/meta.json +1 -0
  42. package/dist/lib/neutral/meta.mjs +8 -0
  43. package/dist/lib/neutral/meta.mjs.map +7 -0
  44. package/dist/lib/neutral/open-CVMX5KXZ.mjs +27 -0
  45. package/dist/lib/neutral/open-CVMX5KXZ.mjs.map +7 -0
  46. package/dist/lib/neutral/operation-handler-NXMOAXFP.mjs +13 -0
  47. package/dist/lib/neutral/operation-handler-NXMOAXFP.mjs.map +7 -0
  48. package/dist/lib/neutral/operations/index.mjs +8 -0
  49. package/dist/lib/neutral/operations/index.mjs.map +7 -0
  50. package/dist/lib/neutral/plugin.mjs +16 -0
  51. package/dist/lib/neutral/plugin.mjs.map +7 -0
  52. package/dist/lib/neutral/react-surface-YMZRO242.mjs +26 -0
  53. package/dist/lib/neutral/react-surface-YMZRO242.mjs.map +7 -0
  54. package/dist/lib/neutral/summarize-LA7TRW7A.mjs +78 -0
  55. package/dist/lib/neutral/summarize-LA7TRW7A.mjs.map +7 -0
  56. package/dist/lib/neutral/text-content-F4ASMITB.mjs +30 -0
  57. package/dist/lib/neutral/text-content-F4ASMITB.mjs.map +7 -0
  58. package/dist/lib/{node-esm/transcriber-UGB7ECX6.mjs → neutral/transcriber-ABHATBVH.mjs} +20 -18
  59. package/dist/lib/neutral/transcriber-ABHATBVH.mjs.map +7 -0
  60. package/dist/lib/neutral/translations.mjs +24 -0
  61. package/dist/lib/neutral/translations.mjs.map +7 -0
  62. package/dist/lib/neutral/types/index.mjs +10 -0
  63. package/dist/lib/neutral/types/index.mjs.map +7 -0
  64. package/dist/types/src/TranscriptionPlugin.d.ts +3 -1
  65. package/dist/types/src/TranscriptionPlugin.d.ts.map +1 -1
  66. package/dist/types/src/TranscriptionPlugin.node.d.ts +4 -0
  67. package/dist/types/src/TranscriptionPlugin.node.d.ts.map +1 -0
  68. package/dist/types/src/TranscriptionPlugin.test.d.ts +2 -0
  69. package/dist/types/src/TranscriptionPlugin.test.d.ts.map +1 -0
  70. package/dist/types/src/TranscriptionPlugin.workerd.d.ts +4 -0
  71. package/dist/types/src/TranscriptionPlugin.workerd.d.ts.map +1 -0
  72. package/dist/types/src/blueprints/index.d.ts +2 -0
  73. package/dist/types/src/blueprints/index.d.ts.map +1 -0
  74. package/dist/types/src/blueprints/transcription-blueprint.d.ts +4 -0
  75. package/dist/types/src/blueprints/transcription-blueprint.d.ts.map +1 -0
  76. package/dist/types/src/capabilities/blueprint-definition.d.ts +5 -4
  77. package/dist/types/src/capabilities/blueprint-definition.d.ts.map +1 -1
  78. package/dist/types/src/capabilities/index.d.ts +10 -5
  79. package/dist/types/src/capabilities/index.d.ts.map +1 -1
  80. package/dist/types/src/capabilities/operation-handler.d.ts +6 -0
  81. package/dist/types/src/capabilities/operation-handler.d.ts.map +1 -0
  82. package/dist/types/src/capabilities/react-surface.d.ts +3 -2
  83. package/dist/types/src/capabilities/react-surface.d.ts.map +1 -1
  84. package/dist/types/src/capabilities/text-content.d.ts +8 -0
  85. package/dist/types/src/capabilities/text-content.d.ts.map +1 -0
  86. package/dist/types/src/capabilities/transcriber.d.ts +4 -3
  87. package/dist/types/src/capabilities/transcriber.d.ts.map +1 -1
  88. package/dist/types/src/components/Oracle/Oracle.d.ts +24 -0
  89. package/dist/types/src/components/Oracle/Oracle.d.ts.map +1 -0
  90. package/dist/types/src/components/Oracle/Oracle.stories.d.ts +32 -0
  91. package/dist/types/src/components/Oracle/Oracle.stories.d.ts.map +1 -0
  92. package/dist/types/src/components/Oracle/index.d.ts +2 -0
  93. package/dist/types/src/components/Oracle/index.d.ts.map +1 -0
  94. package/dist/types/src/components/Transcription/Transcription.d.ts +13 -0
  95. package/dist/types/src/components/Transcription/Transcription.d.ts.map +1 -0
  96. package/dist/types/src/components/Transcription/Transcription.stories.d.ts +37 -0
  97. package/dist/types/src/components/Transcription/Transcription.stories.d.ts.map +1 -0
  98. package/dist/types/src/components/Transcription/index.d.ts +2 -0
  99. package/dist/types/src/components/Transcription/index.d.ts.map +1 -0
  100. package/dist/types/src/components/Transcription/transcription-extension.d.ts +15 -0
  101. package/dist/types/src/components/Transcription/transcription-extension.d.ts.map +1 -0
  102. package/dist/types/src/components/index.d.ts +2 -3
  103. package/dist/types/src/components/index.d.ts.map +1 -1
  104. package/dist/types/src/containers/TranscriptionArticle/TranscriptionArticle.d.ts +6 -0
  105. package/dist/types/src/containers/TranscriptionArticle/TranscriptionArticle.d.ts.map +1 -0
  106. package/dist/types/src/containers/TranscriptionArticle/TranscriptionArticle.stories.d.ts +28 -0
  107. package/dist/types/src/containers/TranscriptionArticle/TranscriptionArticle.stories.d.ts.map +1 -0
  108. package/dist/types/src/containers/TranscriptionArticle/index.d.ts +2 -0
  109. package/dist/types/src/containers/TranscriptionArticle/index.d.ts.map +1 -0
  110. package/dist/types/src/containers/index.d.ts +3 -0
  111. package/dist/types/src/containers/index.d.ts.map +1 -0
  112. package/dist/types/src/enrichment/conversation.test.d.ts +2 -0
  113. package/dist/types/src/enrichment/conversation.test.d.ts.map +1 -0
  114. package/dist/types/src/enrichment/operations.d.ts +200 -0
  115. package/dist/types/src/enrichment/operations.d.ts.map +1 -0
  116. package/dist/types/src/enrichment/parser.d.ts +26 -0
  117. package/dist/types/src/enrichment/parser.d.ts.map +1 -0
  118. package/dist/types/src/enrichment/stream-simulator.d.ts +38 -0
  119. package/dist/types/src/enrichment/stream-simulator.d.ts.map +1 -0
  120. package/dist/types/src/hooks/index.d.ts +2 -1
  121. package/dist/types/src/hooks/index.d.ts.map +1 -1
  122. package/dist/types/src/hooks/useAudioFile.d.ts.map +1 -1
  123. package/dist/types/src/hooks/useAudioTrack.d.ts.map +1 -1
  124. package/dist/types/src/hooks/useFeedModelAdapter.d.ts +12 -0
  125. package/dist/types/src/hooks/useFeedModelAdapter.d.ts.map +1 -0
  126. package/dist/types/src/hooks/useTranscriber.d.ts +2 -2
  127. package/dist/types/src/hooks/useTranscriber.d.ts.map +1 -1
  128. package/dist/types/src/hooks/useTranscriptionRecording.d.ts +12 -0
  129. package/dist/types/src/hooks/useTranscriptionRecording.d.ts.map +1 -0
  130. package/dist/types/src/hooks/useVoiceInput.d.ts.map +1 -1
  131. package/dist/types/src/index.d.ts +2 -3
  132. package/dist/types/src/index.d.ts.map +1 -1
  133. package/dist/types/src/meta.d.ts +2 -2
  134. package/dist/types/src/meta.d.ts.map +1 -1
  135. package/dist/types/src/model/model.d.ts +2 -2
  136. package/dist/types/src/model/model.d.ts.map +1 -1
  137. package/dist/types/src/{segments-normalization → normalization}/index.d.ts +1 -1
  138. package/dist/types/src/normalization/index.d.ts.map +1 -0
  139. package/dist/types/src/{segments-normalization → normalization}/message-normalizer.d.ts +9 -7
  140. package/dist/types/src/normalization/message-normalizer.d.ts.map +1 -0
  141. package/dist/types/src/normalization/normalization.d.ts +543 -0
  142. package/dist/types/src/normalization/normalization.d.ts.map +1 -0
  143. package/dist/types/src/normalization/normalization.test.d.ts.map +1 -0
  144. package/dist/types/src/normalization/utils.d.ts +3 -0
  145. package/dist/types/src/normalization/utils.d.ts.map +1 -0
  146. package/dist/types/src/operations/create.d.ts +5 -0
  147. package/dist/types/src/operations/create.d.ts.map +1 -0
  148. package/dist/types/src/operations/index.d.ts +3 -0
  149. package/dist/types/src/operations/index.d.ts.map +1 -0
  150. package/dist/types/src/operations/open.d.ts +5 -0
  151. package/dist/types/src/operations/open.d.ts.map +1 -0
  152. package/dist/types/src/operations/summarize.d.ts +8 -0
  153. package/dist/types/src/operations/summarize.d.ts.map +1 -0
  154. package/dist/types/src/plugin.d.ts +4 -0
  155. package/dist/types/src/plugin.d.ts.map +1 -0
  156. package/dist/types/src/stories/FileTranscription.stories.d.ts +20 -0
  157. package/dist/types/src/stories/FileTranscription.stories.d.ts.map +1 -0
  158. package/dist/types/src/{components/stories → stories}/KeyWordRecognition.stories.d.ts +2 -2
  159. package/dist/types/src/stories/KeyWordRecognition.stories.d.ts.map +1 -0
  160. package/dist/types/src/{components/stories/MicrophoneTranscription.stories.d.ts → stories/LiveTranscription.stories.d.ts} +7 -7
  161. package/dist/types/src/stories/LiveTranscription.stories.d.ts.map +1 -0
  162. package/dist/types/src/stories/TranscriptionStory.d.ts +21 -0
  163. package/dist/types/src/stories/TranscriptionStory.d.ts.map +1 -0
  164. package/dist/types/src/stories/common.d.ts +46 -0
  165. package/dist/types/src/stories/common.d.ts.map +1 -0
  166. package/dist/types/src/testing/testing.d.ts +13 -14
  167. package/dist/types/src/testing/testing.d.ts.map +1 -1
  168. package/dist/types/src/transcriber/index.d.ts +1 -1
  169. package/dist/types/src/transcriber/index.d.ts.map +1 -1
  170. package/dist/types/src/transcriber/media-stream-recorder.d.ts +2 -2
  171. package/dist/types/src/transcriber/media-stream-recorder.d.ts.map +1 -1
  172. package/dist/types/src/transcriber/transcriber.d.ts +61 -6
  173. package/dist/types/src/transcriber/transcriber.d.ts.map +1 -1
  174. package/dist/types/src/transcriber/transcriber.test.d.ts +2 -0
  175. package/dist/types/src/transcriber/transcriber.test.d.ts.map +1 -0
  176. package/dist/types/src/transcriber/transcription-manager.d.ts +14 -9
  177. package/dist/types/src/transcriber/transcription-manager.d.ts.map +1 -1
  178. package/dist/types/src/translations.d.ts +9 -8
  179. package/dist/types/src/translations.d.ts.map +1 -1
  180. package/dist/types/src/types/TranscriptOperation.d.ts +917 -0
  181. package/dist/types/src/types/TranscriptOperation.d.ts.map +1 -0
  182. package/dist/types/src/types/TranscriptionCapabilities.d.ts +17 -0
  183. package/dist/types/src/types/TranscriptionCapabilities.d.ts.map +1 -0
  184. package/dist/types/src/types/index.d.ts +2 -3
  185. package/dist/types/src/types/index.d.ts.map +1 -1
  186. package/dist/types/src/util/array.d.ts.map +1 -1
  187. package/dist/types/src/util/get-time-str.d.ts.map +1 -1
  188. package/dist/types/src/util/index.d.ts +1 -0
  189. package/dist/types/src/util/index.d.ts.map +1 -1
  190. package/dist/types/src/util/monitor-audio-level.d.ts.map +1 -1
  191. package/dist/types/src/util/render-byline.d.ts +4 -0
  192. package/dist/types/src/util/render-byline.d.ts.map +1 -0
  193. package/dist/types/tsconfig.tsbuildinfo +1 -1
  194. package/package.json +157 -82
  195. package/src/TranscriptionPlugin.node.ts +20 -0
  196. package/src/TranscriptionPlugin.test.ts +31 -0
  197. package/src/TranscriptionPlugin.tsx +20 -64
  198. package/src/TranscriptionPlugin.workerd.ts +18 -0
  199. package/src/blueprints/index.ts +5 -0
  200. package/src/blueprints/transcription-blueprint.ts +29 -0
  201. package/src/capabilities/blueprint-definition.ts +12 -24
  202. package/src/capabilities/index.ts +11 -7
  203. package/src/capabilities/operation-handler.ts +16 -0
  204. package/src/capabilities/react-surface.tsx +23 -14
  205. package/src/capabilities/text-content.ts +33 -0
  206. package/src/capabilities/transcriber.ts +53 -42
  207. package/src/components/Oracle/Oracle.stories.tsx +147 -0
  208. package/src/components/Oracle/Oracle.tsx +141 -0
  209. package/src/components/Oracle/index.ts +5 -0
  210. package/src/components/Transcription/Transcription.stories.tsx +213 -0
  211. package/src/components/Transcription/Transcription.tsx +58 -0
  212. package/src/components/{Transcript → Transcription}/index.ts +1 -1
  213. package/src/components/{Transcript/transcript-extension.ts → Transcription/transcription-extension.ts} +13 -9
  214. package/src/components/index.ts +2 -6
  215. package/src/containers/TranscriptionArticle/TranscriptionArticle.stories.tsx +61 -0
  216. package/src/containers/TranscriptionArticle/TranscriptionArticle.tsx +62 -0
  217. package/src/containers/TranscriptionArticle/index.ts +5 -0
  218. package/src/containers/index.ts +7 -0
  219. package/src/enrichment/conversation.test.ts +162 -0
  220. package/src/enrichment/conversation.txt +23 -0
  221. package/src/enrichment/operations.ts +268 -0
  222. package/src/enrichment/parser.ts +101 -0
  223. package/src/enrichment/stream-simulator.ts +125 -0
  224. package/src/hooks/index.ts +2 -1
  225. package/src/hooks/useFeedModelAdapter.ts +46 -0
  226. package/src/hooks/useTranscriber.ts +12 -4
  227. package/src/hooks/useTranscriptionRecording.ts +87 -0
  228. package/src/hooks/useVoiceInput.ts +2 -3
  229. package/src/index.ts +4 -5
  230. package/src/meta.ts +3 -3
  231. package/src/model/model.test.ts +20 -14
  232. package/src/model/model.ts +3 -3
  233. package/src/{segments-normalization → normalization}/message-normalizer.ts +37 -24
  234. package/src/{segments-normalization → normalization}/normalization.test.ts +8 -47
  235. package/src/normalization/normalization.ts +76 -0
  236. package/src/{segments-normalization → normalization}/utils.ts +2 -2
  237. package/src/operations/create.ts +24 -0
  238. package/src/operations/index.ts +11 -0
  239. package/src/operations/open.ts +28 -0
  240. package/src/operations/summarize.ts +101 -0
  241. package/src/plugin.ts +11 -0
  242. package/src/stories/FileTranscription.stories.tsx +143 -0
  243. package/src/{components/stories → stories}/KeyWordRecognition.stories.tsx +40 -40
  244. package/src/stories/LiveTranscription.stories.tsx +153 -0
  245. package/src/stories/TranscriptionStory.tsx +101 -0
  246. package/src/stories/common.ts +233 -0
  247. package/src/testing/testing.ts +49 -56
  248. package/src/transcriber/index.ts +1 -1
  249. package/src/transcriber/media-stream-recorder.ts +3 -3
  250. package/src/transcriber/transcriber.browser.test.ts +4 -5
  251. package/src/transcriber/transcriber.test.ts +104 -0
  252. package/src/transcriber/transcriber.ts +105 -60
  253. package/src/transcriber/transcription-manager.ts +33 -22
  254. package/src/translations.ts +10 -7
  255. package/src/types/TranscriptOperation.ts +103 -0
  256. package/src/types/TranscriptionCapabilities.ts +39 -0
  257. package/src/types/index.ts +2 -3
  258. package/src/util/index.ts +1 -0
  259. package/src/util/render-byline.ts +33 -0
  260. package/dist/lib/browser/TranscriptContainer-2GSL2XB5.mjs +0 -11
  261. package/dist/lib/browser/blueprint-definition-UAAJYVLZ.mjs +0 -150
  262. package/dist/lib/browser/blueprint-definition-UAAJYVLZ.mjs.map +0 -7
  263. package/dist/lib/browser/chunk-3MKZ2MNQ.mjs +0 -16
  264. package/dist/lib/browser/chunk-3MKZ2MNQ.mjs.map +0 -7
  265. package/dist/lib/browser/chunk-535OJKFO.mjs +0 -8
  266. package/dist/lib/browser/chunk-535OJKFO.mjs.map +0 -7
  267. package/dist/lib/browser/chunk-7DR6HZEF.mjs +0 -732
  268. package/dist/lib/browser/chunk-7DR6HZEF.mjs.map +0 -7
  269. package/dist/lib/browser/chunk-VBUIO677.mjs.map +0 -7
  270. package/dist/lib/browser/chunk-W7WKZTM4.mjs +0 -63
  271. package/dist/lib/browser/chunk-W7WKZTM4.mjs.map +0 -7
  272. package/dist/lib/browser/chunk-WH36ENNE.mjs.map +0 -7
  273. package/dist/lib/browser/index.mjs +0 -130
  274. package/dist/lib/browser/index.mjs.map +0 -7
  275. package/dist/lib/browser/intent-resolver-ZTKTEAQX.mjs +0 -25
  276. package/dist/lib/browser/intent-resolver-ZTKTEAQX.mjs.map +0 -7
  277. package/dist/lib/browser/meta.json +0 -1
  278. package/dist/lib/browser/react-surface-KA32A72I.mjs +0 -34
  279. package/dist/lib/browser/react-surface-KA32A72I.mjs.map +0 -7
  280. package/dist/lib/browser/transcriber-LWOLYQHC.mjs +0 -56
  281. package/dist/lib/browser/transcriber-LWOLYQHC.mjs.map +0 -7
  282. package/dist/lib/browser/types/index.mjs +0 -12
  283. package/dist/lib/node-esm/TranscriptContainer-GRLDCESB.mjs +0 -12
  284. package/dist/lib/node-esm/blueprint-definition-Z3LTZ3ID.mjs +0 -151
  285. package/dist/lib/node-esm/blueprint-definition-Z3LTZ3ID.mjs.map +0 -7
  286. package/dist/lib/node-esm/chunk-K4AZOB5Y.mjs +0 -733
  287. package/dist/lib/node-esm/chunk-K4AZOB5Y.mjs.map +0 -7
  288. package/dist/lib/node-esm/chunk-OLMSTF3M.mjs +0 -64
  289. package/dist/lib/node-esm/chunk-OLMSTF3M.mjs.map +0 -7
  290. package/dist/lib/node-esm/chunk-Q6WD7L2J.mjs +0 -10
  291. package/dist/lib/node-esm/chunk-Q6WD7L2J.mjs.map +0 -7
  292. package/dist/lib/node-esm/chunk-SWUVPFKR.mjs +0 -17
  293. package/dist/lib/node-esm/chunk-SWUVPFKR.mjs.map +0 -7
  294. package/dist/lib/node-esm/chunk-YFN4FJFK.mjs +0 -467
  295. package/dist/lib/node-esm/chunk-YFN4FJFK.mjs.map +0 -7
  296. package/dist/lib/node-esm/chunk-YVNQN2J6.mjs +0 -30
  297. package/dist/lib/node-esm/chunk-YVNQN2J6.mjs.map +0 -7
  298. package/dist/lib/node-esm/index.mjs +0 -131
  299. package/dist/lib/node-esm/index.mjs.map +0 -7
  300. package/dist/lib/node-esm/intent-resolver-55WH2GY6.mjs +0 -26
  301. package/dist/lib/node-esm/intent-resolver-55WH2GY6.mjs.map +0 -7
  302. package/dist/lib/node-esm/meta.json +0 -1
  303. package/dist/lib/node-esm/react-surface-GU7LW2JO.mjs +0 -35
  304. package/dist/lib/node-esm/react-surface-GU7LW2JO.mjs.map +0 -7
  305. package/dist/lib/node-esm/transcriber-UGB7ECX6.mjs.map +0 -7
  306. package/dist/lib/node-esm/types/index.mjs +0 -13
  307. package/dist/types/src/capabilities/capabilities.d.ts +0 -17
  308. package/dist/types/src/capabilities/capabilities.d.ts.map +0 -1
  309. package/dist/types/src/capabilities/intent-resolver.d.ts +0 -4
  310. package/dist/types/src/capabilities/intent-resolver.d.ts.map +0 -1
  311. package/dist/types/src/components/Transcript/Transcript.d.ts +0 -20
  312. package/dist/types/src/components/Transcript/Transcript.d.ts.map +0 -1
  313. package/dist/types/src/components/Transcript/Transcript.stories.d.ts +0 -39
  314. package/dist/types/src/components/Transcript/Transcript.stories.d.ts.map +0 -1
  315. package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts +0 -28
  316. package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts.map +0 -1
  317. package/dist/types/src/components/Transcript/index.d.ts +0 -2
  318. package/dist/types/src/components/Transcript/index.d.ts.map +0 -1
  319. package/dist/types/src/components/Transcript/transcript-extension.d.ts +0 -15
  320. package/dist/types/src/components/Transcript/transcript-extension.d.ts.map +0 -1
  321. package/dist/types/src/components/TranscriptContainer.d.ts +0 -9
  322. package/dist/types/src/components/TranscriptContainer.d.ts.map +0 -1
  323. package/dist/types/src/components/stories/FileTranscription.stories.d.ts +0 -18
  324. package/dist/types/src/components/stories/FileTranscription.stories.d.ts.map +0 -1
  325. package/dist/types/src/components/stories/KeyWordRecognition.stories.d.ts.map +0 -1
  326. package/dist/types/src/components/stories/MicrophoneTranscription.stories.d.ts.map +0 -1
  327. package/dist/types/src/components/stories/TranscriptionStory.d.ts +0 -12
  328. package/dist/types/src/components/stories/TranscriptionStory.d.ts.map +0 -1
  329. package/dist/types/src/components/stories/useIsSpeaking.d.ts +0 -3
  330. package/dist/types/src/components/stories/useIsSpeaking.d.ts.map +0 -1
  331. package/dist/types/src/functions/index.d.ts +0 -3
  332. package/dist/types/src/functions/index.d.ts.map +0 -1
  333. package/dist/types/src/functions/open.d.ts +0 -7
  334. package/dist/types/src/functions/open.d.ts.map +0 -1
  335. package/dist/types/src/functions/summarize.d.ts +0 -11
  336. package/dist/types/src/functions/summarize.d.ts.map +0 -1
  337. package/dist/types/src/hooks/useQueueModelAdapter.d.ts +0 -8
  338. package/dist/types/src/hooks/useQueueModelAdapter.d.ts.map +0 -1
  339. package/dist/types/src/segments-normalization/index.d.ts.map +0 -1
  340. package/dist/types/src/segments-normalization/message-normalizer.d.ts.map +0 -1
  341. package/dist/types/src/segments-normalization/normalization.d.ts +0 -476
  342. package/dist/types/src/segments-normalization/normalization.d.ts.map +0 -1
  343. package/dist/types/src/segments-normalization/normalization.test.d.ts.map +0 -1
  344. package/dist/types/src/segments-normalization/utils.d.ts +0 -3
  345. package/dist/types/src/segments-normalization/utils.d.ts.map +0 -1
  346. package/dist/types/src/types/Transcript.d.ts +0 -30
  347. package/dist/types/src/types/Transcript.d.ts.map +0 -1
  348. package/dist/types/src/types/TranscriptAction.d.ts +0 -23
  349. package/dist/types/src/types/TranscriptAction.d.ts.map +0 -1
  350. package/dist/types/src/types/types.d.ts +0 -5
  351. package/dist/types/src/types/types.d.ts.map +0 -1
  352. package/src/capabilities/capabilities.ts +0 -33
  353. package/src/capabilities/intent-resolver.ts +0 -18
  354. package/src/components/Transcript/Transcript.stories.tsx +0 -245
  355. package/src/components/Transcript/Transcript.tsx +0 -93
  356. package/src/components/Transcript/deprecated/OldTranscript.tsx +0 -260
  357. package/src/components/TranscriptContainer.tsx +0 -36
  358. package/src/components/stories/FileTranscription.stories.tsx +0 -256
  359. package/src/components/stories/MicrophoneTranscription.stories.tsx +0 -285
  360. package/src/components/stories/TranscriptionStory.tsx +0 -45
  361. package/src/components/stories/useIsSpeaking.tsx +0 -40
  362. package/src/functions/index.ts +0 -6
  363. package/src/functions/open.ts +0 -39
  364. package/src/functions/summarize.ts +0 -103
  365. package/src/hooks/useQueueModelAdapter.ts +0 -66
  366. package/src/segments-normalization/normalization.ts +0 -115
  367. package/src/types/Transcript.ts +0 -44
  368. package/src/types/TranscriptAction.ts +0 -24
  369. package/src/types/types.ts +0 -9
  370. package/dist/lib/{browser/types → neutral/blueprints}/index.mjs.map +0 -0
  371. package/dist/lib/{browser/TranscriptContainer-2GSL2XB5.mjs.map → neutral/chunk-J5LGTIGS.mjs.map} +0 -0
  372. package/dist/lib/{node-esm/types → neutral/hooks}/index.mjs.map +0 -0
  373. package/dist/lib/{node-esm/TranscriptContainer-GRLDCESB.mjs.map → neutral/index.mjs.map} +0 -0
  374. package/dist/types/src/{segments-normalization → normalization}/normalization.test.d.ts +0 -0
  375. package/src/{segments-normalization → normalization}/index.ts +1 -1
@@ -2,22 +2,22 @@
2
2
  // Copyright 2025 DXOS.org
3
3
  //
4
4
 
5
+ import * as Effect from 'effect/Effect';
5
6
  import * as Schema from 'effect/Schema';
6
7
  import { useEffect, useMemo, useState } from 'react';
7
8
 
8
9
  import { extractionAnthropicFunction, processTranscriptMessage } from '@dxos/assistant/extraction';
9
10
  import { scheduleTaskInterval } from '@dxos/async';
10
- import { Filter, type Queue } from '@dxos/client/echo';
11
+ import { createFeedServiceLayer, type Space } from '@dxos/client/echo';
11
12
  import { Context } from '@dxos/context';
12
- import { type Key, Obj, Ref, Type } from '@dxos/echo';
13
- import { createQueueDXN } from '@dxos/echo/internal';
14
- import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
13
+ import { Feed, Filter, Obj, Ref, Type } from '@dxos/echo';
14
+ import { runAndForwardErrors } from '@dxos/effect';
15
15
  import { IdentityDid } from '@dxos/keys';
16
16
  import { log } from '@dxos/log';
17
- import { faker } from '@dxos/random';
18
- import { type Space, useQueue } from '@dxos/react-client/echo';
19
- import { DataType } from '@dxos/schema';
20
- import { Testing, seedTestData } from '@dxos/schema/testing';
17
+ import { random } from '@dxos/random';
18
+ import { TestSchema } from '@dxos/schema/testing';
19
+ import { type ContentBlock, Message, Organization, Person } from '@dxos/types';
20
+ import { seedTestData } from '@dxos/types/testing';
21
21
 
22
22
  // TODO(burdon): Reconcile with plugin-markdown. Move to @dxos/schema/testing.
23
23
  export const TestItem = Schema.Struct({
@@ -30,15 +30,15 @@ export const TestItem = Schema.Struct({
30
30
  description: 'Product description',
31
31
  }),
32
32
  }).pipe(
33
- Type.Obj({
34
- typename: 'dxos.org/type/Test',
33
+ Type.object({
34
+ typename: 'org.dxos.type.test',
35
35
  version: '0.1.0',
36
36
  }),
37
37
  );
38
38
 
39
39
  // TODO(wittjosiah): Make builder generic and reuse for all message types.
40
40
  abstract class AbstractMessageBuilder {
41
- abstract createMessage(numSegments?: number): Promise<DataType.Message>;
41
+ abstract createMessage(numSegments?: number): Promise<Message.Message>;
42
42
  }
43
43
 
44
44
  /**
@@ -49,8 +49,8 @@ export class MessageBuilder extends AbstractMessageBuilder {
49
49
 
50
50
  users = Array.from({ length: 5 }, () => ({
51
51
  identityDid: IdentityDid.random().toString(),
52
- name: faker.person.fullName(),
53
- email: faker.internet.email(),
52
+ name: random.person.fullName(),
53
+ email: random.internet.email(),
54
54
  }));
55
55
 
56
56
  start = new Date(Date.now() - 24 * 60 * 60 * 10_000);
@@ -59,19 +59,24 @@ export class MessageBuilder extends AbstractMessageBuilder {
59
59
  super();
60
60
  }
61
61
 
62
- override async createMessage(numSegments = 1): Promise<DataType.Message> {
63
- return Obj.make(DataType.Message, {
62
+ override async createMessage(numSegments = 1): Promise<Message.Message> {
63
+ return Obj.make(Message.Message, {
64
64
  created: this.next().toISOString(),
65
- sender: faker.helpers.arrayElement(this.users),
65
+ sender: random.helpers.arrayElement(this.users),
66
66
  blocks: Array.from({ length: numSegments }).map(() => this.createBlock()),
67
67
  });
68
68
  }
69
69
 
70
- createBlock(): DataType.MessageBlock.Transcript {
71
- let text = faker.lorem.paragraph();
70
+ createBlock(): ContentBlock.Transcript {
71
+ let text = random.lorem.paragraph();
72
72
  if (this._space) {
73
- const label = faker.commerce.productName();
74
- const obj = this._space.db.add(Obj.make(TestItem, { title: label, description: faker.lorem.paragraph() }));
73
+ const label = random.commerce.productName();
74
+ const obj = this._space.db.add(
75
+ Obj.make(TestItem, {
76
+ title: label,
77
+ description: random.lorem.paragraph(),
78
+ }),
79
+ );
75
80
  const dxn = Ref.make(obj).dxn.toString();
76
81
  const words = text.split(' ');
77
82
  words.splice(Math.floor(Math.random() * words.length), 0, `[${label}](${dxn})`);
@@ -93,17 +98,9 @@ export class MessageBuilder extends AbstractMessageBuilder {
93
98
 
94
99
  // TODO(burdon): Reconcile with BlockBuilder.
95
100
  class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
96
- executor = new FunctionExecutor(
97
- new ServiceContainer().setServices({
98
- // ai: {
99
- // client: this.AiService,
100
- // },
101
- }),
102
- );
103
-
104
101
  space: Space | undefined;
105
102
  currentMessage: number = 0;
106
- transcriptMessages: DataType.Message[] = [];
103
+ transcriptMessages: Message.Message[] = [];
107
104
 
108
105
  async connect(space: Space): Promise<void> {
109
106
  this.space = space;
@@ -111,14 +108,18 @@ class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
111
108
  this.transcriptMessages = transcriptMessages;
112
109
  }
113
110
 
114
- override async createMessage(): Promise<DataType.Message> {
111
+ override async createMessage(): Promise<Message.Message> {
115
112
  if (!this.space) {
116
113
  throw new Error('Space not connected');
117
114
  }
118
115
 
119
- const { objects } = await this.space.db
116
+ const objects = await this.space.db
120
117
  .query(
121
- Filter.or(Filter.type(DataType.Person), Filter.type(DataType.Organization), Filter.type(Testing.DocumentType)),
118
+ Filter.or(
119
+ Filter.type(Person.Person),
120
+ Filter.type(Organization.Organization),
121
+ Filter.type(TestSchema.DocumentType),
122
+ ),
122
123
  )
123
124
  .run();
124
125
 
@@ -129,7 +130,6 @@ class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
129
130
 
130
131
  const { message: enhancedMessage } = await processTranscriptMessage({
131
132
  input: { message },
132
- executor: this.executor,
133
133
  function: extractionAnthropicFunction,
134
134
  });
135
135
 
@@ -139,71 +139,64 @@ class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
139
139
 
140
140
  type UseTestTranscriptionQueue = (
141
141
  space: Space | undefined,
142
- queueId?: Key.ObjectId,
143
142
  running?: boolean,
144
143
  interval?: number,
145
- ) => Queue<DataType.Message> | undefined;
144
+ ) => Feed.Feed | undefined;
146
145
 
147
146
  /**
148
- * Test transcriptionqueue.
147
+ * Test transcription feed.
149
148
  */
150
149
  export const useTestTranscriptionQueue: UseTestTranscriptionQueue = (
151
150
  space: Space | undefined,
152
- queueId?: Key.ObjectId,
153
151
  running = true,
154
152
  interval = 1_000,
155
153
  ) => {
156
- // TODO(dmaretskyi): Use space.queues.create() instead.
157
- const queueDxn = useMemo(() => (space ? createQueueDXN(space.id, queueId) : undefined), [space, queueId]);
158
- const queue = useQueue<DataType.Message>(queueDxn);
154
+ const feed = useMemo(() => (space ? space.db.add(Feed.make({ name: 'transcription' })) : undefined), [space]);
159
155
  const builder = useMemo(() => new MessageBuilder(space), [space]);
160
156
 
161
157
  useEffect(() => {
162
- if (!queue || !running) {
158
+ if (!space || !feed || !running) {
163
159
  return;
164
160
  }
161
+ const feedServiceLayer = createFeedServiceLayer(space.queues);
165
162
 
166
163
  const i = setInterval(() => {
167
164
  void builder.createMessage(Math.ceil(Math.random() * 3)).then(async (message) => {
168
- await queue.append([Obj.make(DataType.Message, message)]);
165
+ await Feed.append(feed, [message]).pipe(Effect.provide(feedServiceLayer), runAndForwardErrors);
169
166
  });
170
167
  }, interval);
171
168
  return () => clearInterval(i);
172
- }, [queue, running, interval]);
169
+ }, [space, feed, running, interval]);
173
170
 
174
- return queue;
171
+ return feed;
175
172
  };
176
173
 
177
174
  /**
178
- * Test transcription queue.
175
+ * Test transcription feed with entity extraction.
179
176
  */
180
177
  // TODO(burdon): Reconcile with useTestTranscriptionQueue.
181
178
  export const useTestTranscriptionQueueWithEntityExtraction: UseTestTranscriptionQueue = (
182
179
  space: Space | undefined,
183
- queueId?: Key.ObjectId,
184
180
  running = true,
185
181
  interval = 1_000,
186
182
  ) => {
187
- // TODO(dmaretskyi): Use space.queues.create() instead.
188
- const queueDxn = useMemo(() => (space ? createQueueDXN(space.id, queueId) : undefined), [space, queueId]);
189
- const queue = useQueue<DataType.Message>(queueDxn);
183
+ const feed = useMemo(() => (space ? space.db.add(Feed.make({ name: 'transcription' })) : undefined), [space]);
190
184
  const [builder] = useState(() => new EntityExtractionMessageBuilder());
191
185
 
192
186
  useEffect(() => {
193
- if (!queue || !running) {
187
+ if (!space || !feed || !running) {
194
188
  return;
195
189
  }
196
190
 
197
- if (space) {
198
- void builder.connect(space);
199
- }
191
+ void builder.connect(space);
200
192
 
193
+ const feedServiceLayer = createFeedServiceLayer(space.queues);
201
194
  const ctx = new Context();
202
195
  scheduleTaskInterval(
203
196
  ctx,
204
197
  async () => {
205
198
  const message = await builder.createMessage();
206
- void queue.append([Obj.make(DataType.Message, message)]);
199
+ await Feed.append(feed, [message]).pipe(Effect.provide(feedServiceLayer), runAndForwardErrors);
207
200
  },
208
201
  interval,
209
202
  );
@@ -211,7 +204,7 @@ export const useTestTranscriptionQueueWithEntityExtraction: UseTestTranscription
211
204
  return () => {
212
205
  void ctx.dispose();
213
206
  };
214
- }, [space, queue, running, interval]);
207
+ }, [space, feed, running, interval]);
215
208
 
216
- return queue;
209
+ return feed;
217
210
  };
@@ -2,7 +2,7 @@
2
2
  // Copyright 2025 DXOS.org
3
3
  //
4
4
 
5
- export type * from './audio-recorder';
5
+ export * from './audio-recorder';
6
6
  export * from './media-stream-recorder';
7
7
  export * from './transcriber';
8
8
  export * from './transcription-manager';
@@ -19,7 +19,7 @@ const initializeExtendableMediaRecorder = async () => {
19
19
  await register(await connect());
20
20
  };
21
21
 
22
- export type MediaStreamRecorderParams = {
22
+ export type MediaStreamRecorderProps = {
23
23
  mediaStreamTrack: MediaStreamTrack;
24
24
  config: {
25
25
  interval: number;
@@ -37,12 +37,12 @@ export class MediaStreamRecorder implements AudioRecorder {
37
37
  /**
38
38
  * Default MediaRecorder implementation do not support wav encoding.
39
39
  */
40
- private readonly _config: MediaStreamRecorderParams['config'];
40
+ private readonly _config: MediaStreamRecorderProps['config'];
41
41
  private _mediaRecorder?: IMediaRecorder = undefined;
42
42
  private _header?: Uint8Array = undefined;
43
43
  private _onChunk?: (chunk: AudioChunk) => void;
44
44
 
45
- constructor({ mediaStreamTrack, config }: MediaStreamRecorderParams) {
45
+ constructor({ mediaStreamTrack, config }: MediaStreamRecorderProps) {
46
46
  this._mediaStreamTrack = mediaStreamTrack;
47
47
  this._config = config;
48
48
  }
@@ -4,15 +4,14 @@
4
4
 
5
5
  import fs from 'node:fs';
6
6
  import path from 'node:path';
7
-
8
7
  import { describe, expect, test } from 'vitest';
9
8
  import { WaveFile } from 'wavefile';
10
9
 
11
10
  import { Trigger } from '@dxos/async';
12
11
  import { log } from '@dxos/log';
13
- import { type DataType } from '@dxos/schema';
14
12
  import { openAndClose } from '@dxos/test-utils';
15
13
  import { TRACE_PROCESSOR, trace } from '@dxos/tracing';
14
+ import { type ContentBlock } from '@dxos/types';
16
15
 
17
16
  import { type AudioChunk, type AudioRecorder, Transcriber } from '../transcriber';
18
17
  import { mergeFloat64Arrays } from '../util';
@@ -94,7 +93,7 @@ describe.skip('Transcriber', () => {
94
93
  });
95
94
 
96
95
  test.skip('transcription of audio recording', { timeout: 10_000 }, async () => {
97
- const trigger = new Trigger<DataType.MessageBlock.Transcript[]>({ autoReset: true });
96
+ const trigger = new Trigger<ContentBlock.Transcript[]>({ autoReset: true });
98
97
  const recorder = new MockAudioRecorder({
99
98
  buffer: await readFile('test.wav'),
100
99
  chunkDuration: 3_000,
@@ -128,7 +127,7 @@ describe.skip('Transcriber', () => {
128
127
  });
129
128
 
130
129
  test.skip('transcription of audio recording with overlapping chunks', { timeout: 20_000 }, async () => {
131
- const trigger = new Trigger<DataType.MessageBlock.Transcript[]>({ autoReset: true });
130
+ const trigger = new Trigger<ContentBlock.Transcript[]>({ autoReset: true });
132
131
  const recorder = new MockAudioRecorder({
133
132
  buffer: await readFile('test.wav'),
134
133
  chunkDuration: 3_000,
@@ -213,7 +212,7 @@ class MockAudioRecorder implements AudioRecorder {
213
212
 
214
213
  @trace.span()
215
214
  async start(): Promise<void> {
216
- log.info('start mock audio recorder', { wavParams: this.wav.fmt });
215
+ log.info('start mock audio recorder', { wavProps: this.wav.fmt });
217
216
  const now = Date.now();
218
217
  const samples = this.wav.getSamples();
219
218
  const sampleRate = this.wavConfig.sampleRate;
@@ -0,0 +1,104 @@
1
+ //
2
+ // Copyright 2026 DXOS.org
3
+ //
4
+
5
+ import { describe, test } from 'vitest';
6
+
7
+ import { type AudioChunk } from './audio-recorder';
8
+ import { alignWhisperSegments, type WhisperSegment } from './transcriber';
9
+
10
+ const T0 = Date.parse('2026-01-01T00:00:00.000Z');
11
+
12
+ const chunk = (offsetMs = 0, samples = 16): AudioChunk => ({
13
+ timestamp: T0 + offsetMs,
14
+ data: new Float64Array(samples),
15
+ });
16
+
17
+ const segment = (overrides: Partial<WhisperSegment>): WhisperSegment => ({
18
+ text: '',
19
+ start: 0,
20
+ end: 0,
21
+ no_speech_prob: 0,
22
+ words: [],
23
+ ...overrides,
24
+ });
25
+
26
+ describe('alignWhisperSegments', () => {
27
+ test('returns empty when no chunks were transcribed', ({ expect }) => {
28
+ const result = alignWhisperSegments(
29
+ [segment({ text: 'x', end: 1, words: [{ word: 'x', start: 0, end: 1 }] })],
30
+ [],
31
+ 0,
32
+ );
33
+ expect(result.transcripts).toEqual([]);
34
+ expect(result.lastTimestamp).toBeUndefined();
35
+ });
36
+
37
+ test('absolute-timestamps and deduplicates segments against lastTimestamp', ({ expect }) => {
38
+ const segments: WhisperSegment[] = [
39
+ segment({
40
+ text: 'hello world',
41
+ start: 0,
42
+ end: 1,
43
+ words: [
44
+ { word: 'hello ', start: 0, end: 0.5 },
45
+ { word: 'world', start: 0.5, end: 1 },
46
+ ],
47
+ }),
48
+ ];
49
+
50
+ const first = alignWhisperSegments(segments, [chunk(0)], 0);
51
+ expect(first.transcripts).toHaveLength(1);
52
+ expect(first.transcripts[0]._tag).toBe('transcript');
53
+ expect(first.transcripts[0].text).toBe('hello world');
54
+ expect(new Date(first.transcripts[0].started).getTime()).toBe(T0);
55
+ expect(first.lastTimestamp).toBe(T0 + 1_000);
56
+
57
+ // Same segments, same chunk start: every word's absolute start is < lastTimestamp -> filtered out.
58
+ const second = alignWhisperSegments(segments, [chunk(0)], first.lastTimestamp!);
59
+ expect(second.transcripts).toEqual([]);
60
+ expect(second.lastTimestamp).toBeUndefined();
61
+ });
62
+
63
+ test('keeps later words when earlier words fall before lastTimestamp', ({ expect }) => {
64
+ const segments: WhisperSegment[] = [
65
+ segment({
66
+ text: 'old new',
67
+ start: 0,
68
+ end: 2,
69
+ words: [
70
+ { word: 'old ', start: 0, end: 1 },
71
+ { word: 'new', start: 1.5, end: 2 },
72
+ ],
73
+ }),
74
+ ];
75
+
76
+ // First word starts at T0, second at T0+1500 — only the second survives a cursor at T0+1000.
77
+ const result = alignWhisperSegments(segments, [chunk(0)], T0 + 1_000);
78
+ expect(result.transcripts).toHaveLength(1);
79
+ expect(result.transcripts[0].text).toBe('new');
80
+ expect(new Date(result.transcripts[0].started).getTime()).toBe(T0 + 1_500);
81
+ expect(result.lastTimestamp).toBe(T0 + 2_000);
82
+ });
83
+
84
+ test('drops segments that end before lastTimestamp', ({ expect }) => {
85
+ const segments: WhisperSegment[] = [
86
+ segment({
87
+ text: 'stale',
88
+ start: 0,
89
+ end: 0.5,
90
+ words: [{ word: 'stale', start: 0, end: 0.5 }],
91
+ }),
92
+ segment({
93
+ text: 'fresh',
94
+ start: 1,
95
+ end: 2,
96
+ words: [{ word: 'fresh', start: 1.5, end: 2 }],
97
+ }),
98
+ ];
99
+
100
+ const result = alignWhisperSegments(segments, [chunk(0)], T0 + 1_000);
101
+ expect(result.transcripts).toHaveLength(1);
102
+ expect(result.transcripts[0].text).toBe('fresh');
103
+ });
104
+ });
@@ -7,15 +7,18 @@ import { WaveFile } from 'wavefile';
7
7
  import { DeferredTask, Trigger } from '@dxos/async';
8
8
  import { type Context, LifecycleState, Resource } from '@dxos/context';
9
9
  import { log } from '@dxos/log';
10
- import { type DataType } from '@dxos/schema';
11
10
  import { trace } from '@dxos/tracing';
11
+ import { type ContentBlock } from '@dxos/types';
12
12
 
13
- import { TRANSCRIPTION_URL } from '../types';
14
13
  import { mergeFloat64Arrays } from '../util';
15
-
16
14
  import { type AudioChunk, type AudioRecorder } from './audio-recorder';
17
15
 
18
- type WhisperWord = {
16
+ /**
17
+ * Endpoint to the calls service.
18
+ */
19
+ const DEFAULT_TRANSCRIPTION_URL = 'https://calls-service.dxos.workers.dev';
20
+
21
+ export type WhisperWord = {
19
22
  word: string;
20
23
 
21
24
  /**
@@ -29,7 +32,7 @@ type WhisperWord = {
29
32
  end: number;
30
33
  };
31
34
 
32
- type WhisperSegment = {
35
+ export type WhisperSegment = {
33
36
  text: string;
34
37
 
35
38
  /**
@@ -61,16 +64,32 @@ export type TranscribeConfig = {
61
64
  * This is needed to catch the beginning of the user's speech.
62
65
  */
63
66
  prefixBufferChunksAmount: number;
67
+
68
+ /**
69
+ * Override the transcription endpoint base URL.
70
+ * Defaults to the DXOS calls service.
71
+ */
72
+ endpoint?: string;
64
73
  };
65
74
 
66
- export type TranscriberParams = {
75
+ /**
76
+ * Function that converts a base64-encoded WAV payload into Whisper segments.
77
+ * Allows callers to swap in alternative providers or mock the transport.
78
+ */
79
+ export type TranscribeFn = (audio: string) => Promise<WhisperSegment[]>;
80
+
81
+ export type TranscriberProps = {
67
82
  config: TranscribeConfig;
68
83
  recorder: AudioRecorder;
69
84
  /**
70
85
  * Callback to handle the transcribed segments, after all segment transformers are applied.
71
86
  * @param segments - The transcribed segments.
72
87
  */
73
- onSegments: (segments: DataType.MessageBlock.Transcript[]) => Promise<void>;
88
+ onSegments: (segments: ContentBlock.Transcript[]) => Promise<void>;
89
+ /**
90
+ * Optional override of the transcription transport. When provided, supersedes `config.endpoint`.
91
+ */
92
+ transcribe?: TranscribeFn;
74
93
  };
75
94
 
76
95
  /**
@@ -80,22 +99,24 @@ export type TranscriberParams = {
80
99
  */
81
100
  export class Transcriber extends Resource {
82
101
  private _audioChunks: AudioChunk[] = [];
83
- private _lastUsedTimestamp = 0;
102
+ private _lastTimestamp = 0;
84
103
 
85
104
  private readonly _openTrigger = new Trigger({ autoReset: true });
86
105
 
87
106
  private readonly _config: TranscribeConfig;
88
107
  private readonly _recorder: AudioRecorder;
89
- private readonly _onSegments: TranscriberParams['onSegments'];
108
+ private readonly _onSegments: TranscriberProps['onSegments'];
109
+ private readonly _transcribeFn?: TranscribeFn;
90
110
 
91
111
  private _recording = false;
92
112
  private _transcribeTask?: DeferredTask = undefined;
93
113
 
94
- constructor({ config, recorder, onSegments }: TranscriberParams) {
114
+ constructor({ config, recorder, onSegments, transcribe }: TranscriberProps) {
95
115
  super();
96
116
  this._config = config;
97
117
  this._recorder = recorder;
98
118
  this._onSegments = onSegments;
119
+ this._transcribeFn = transcribe;
99
120
  }
100
121
 
101
122
  protected override async _open(ctx: Context): Promise<void> {
@@ -192,67 +213,91 @@ export class Transcriber extends Resource {
192
213
  @trace.span({ showInBrowserTimeline: true })
193
214
  private async _fetchTranscription(audio: string): Promise<WhisperSegment[]> {
194
215
  if (audio.length === 0) {
195
- this._audioChunks = [];
196
216
  throw new Error('No audio to send for transcribing');
197
217
  }
198
218
 
199
- const response = await fetch(`${TRANSCRIPTION_URL}/transcribe`, {
200
- method: 'POST',
201
- body: JSON.stringify({ audio }),
202
- headers: {
203
- 'Content-Type': 'application/json',
204
- },
205
- });
206
-
207
- if (!response.ok) {
208
- this._audioChunks = [];
209
- throw new Error(`Transcription failed: ${response.statusText}`);
219
+ let segments: unknown;
220
+ if (this._transcribeFn) {
221
+ segments = await this._transcribeFn(audio);
222
+ } else {
223
+ // TODO(burdon): Create separate endpoint?
224
+ const endpoint = this._config.endpoint ?? DEFAULT_TRANSCRIPTION_URL;
225
+ const response = await fetch(`${endpoint}/transcribe`, {
226
+ method: 'POST',
227
+ body: JSON.stringify({ audio }),
228
+ headers: {
229
+ 'Content-Type': 'application/json',
230
+ },
231
+ });
232
+
233
+ if (!response.ok) {
234
+ throw new Error(`Transcription failed: ${response.statusText}`);
235
+ }
236
+
237
+ ({ segments } = (await response.json()) as { segments?: unknown });
210
238
  }
211
239
 
212
- const { segments } = (await response.json()) as {
213
- segments: WhisperSegment[];
214
- };
215
-
216
- log.info('transcription response', {
217
- segments: segments.length,
218
- string: segments.map((segments) => segments.text).join(' '),
219
- });
240
+ if (!Array.isArray(segments)) {
241
+ throw new Error('Transcription response payload is invalid');
242
+ }
220
243
 
221
- return segments;
244
+ log.info('transcription response', { segments: segments.length });
245
+ return segments as WhisperSegment[];
222
246
  }
223
247
 
224
- private _alignSegments(segments: WhisperSegment[], originalChunks: AudioChunk[]): DataType.MessageBlock.Transcript[] {
225
- // Absolute zero for all relative timestamps in the segments.
226
- const zeroTimestamp = originalChunks.at(0)!.timestamp;
227
-
228
- const filteredSegments = segments
229
- // Use segments that end after the last used timestamp.
230
- // Segments could overlap, so we need to filter words that starts after the last used timestamp.
231
- .filter((segment) => zeroTimestamp + segment.end * 1_000 >= this._lastUsedTimestamp)
232
- // Use words that starts after the last used timestamp.
233
- .map((segment) => {
234
- const words = segment.words.filter((word) => zeroTimestamp + word.start * 1_000 >= this._lastUsedTimestamp);
235
- return {
236
- ...segment,
237
- words,
238
- text: words.map((word) => word.word).join(''),
239
- start: words.at(0)?.start ?? segment.end,
240
- };
241
- })
242
- .filter((segment) => segment.words.length > 0);
243
-
244
- if (filteredSegments.length === 0) {
245
- return [];
248
+ private _alignSegments(segments: WhisperSegment[], originalChunks: AudioChunk[]): ContentBlock.Transcript[] {
249
+ const result = alignWhisperSegments(segments, originalChunks, this._lastTimestamp);
250
+ if (result.lastTimestamp !== undefined) {
251
+ this._lastTimestamp = result.lastTimestamp;
246
252
  }
247
253
 
248
- // Update last timestamp.
249
- this._lastUsedTimestamp = zeroTimestamp + filteredSegments.at(-1)!.end * 1_000;
254
+ return result.transcripts;
255
+ }
256
+ }
257
+
258
+ /**
259
+ * Pure helper that filters and absolute-timestamps Whisper segments against a chunk timeline.
260
+ * Exported for testing.
261
+ *
262
+ * @param segments - Raw segments returned by the Whisper service for `originalChunks`.
263
+ * @param originalChunks - Chunks fed into the transcription request (used for the zero timestamp).
264
+ * @param lastTimestamp - Most recent absolute timestamp already emitted; segments whose end
265
+ * falls strictly before this point are deduped out, as are individual words whose start does.
266
+ */
267
+ export const alignWhisperSegments = (
268
+ segments: WhisperSegment[],
269
+ originalChunks: AudioChunk[],
270
+ lastTimestamp: number,
271
+ ): { transcripts: ContentBlock.Transcript[]; lastTimestamp?: number } => {
272
+ if (originalChunks.length === 0) {
273
+ return { transcripts: [] };
274
+ }
275
+
276
+ const zeroTimestamp = originalChunks[0].timestamp;
277
+
278
+ const filteredSegments = segments
279
+ .filter((segment) => zeroTimestamp + segment.end * 1_000 >= lastTimestamp)
280
+ .map((segment) => {
281
+ const words = segment.words.filter((word) => zeroTimestamp + word.start * 1_000 >= lastTimestamp);
282
+ return {
283
+ ...segment,
284
+ words,
285
+ text: words.map((word) => word.word).join(''),
286
+ start: words.at(0)?.start ?? segment.end,
287
+ };
288
+ })
289
+ .filter((segment) => segment.words.length > 0);
290
+
291
+ if (filteredSegments.length === 0) {
292
+ return { transcripts: [] };
293
+ }
250
294
 
251
- // Add absolute timestamp to each segment.
252
- return filteredSegments.map((segment) => ({
295
+ return {
296
+ lastTimestamp: zeroTimestamp + filteredSegments.at(-1)!.end * 1_000,
297
+ transcripts: filteredSegments.map((segment) => ({
253
298
  _tag: 'transcript',
254
299
  started: new Date(zeroTimestamp + segment.start * 1_000).toISOString(),
255
300
  text: segment.text.trim(),
256
- }));
257
- }
258
- }
301
+ })),
302
+ };
303
+ };