@dxos/plugin-transcription 0.8.1-main.ba2dec9 → 0.8.1-staging.31c3ee1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. package/dist/lib/browser/{TranscriptContainer-N4KQHMY4.mjs → TranscriptContainer-JQ3HFHLQ.mjs} +3 -3
  2. package/dist/lib/browser/app-graph-builder-VDQOVX7P.mjs +191 -0
  3. package/dist/lib/browser/app-graph-builder-VDQOVX7P.mjs.map +7 -0
  4. package/dist/lib/browser/chunk-6ZR5MDK3.mjs +19 -0
  5. package/dist/lib/browser/chunk-6ZR5MDK3.mjs.map +7 -0
  6. package/dist/lib/browser/chunk-G4B6CPC2.mjs +253 -0
  7. package/dist/lib/browser/chunk-G4B6CPC2.mjs.map +7 -0
  8. package/dist/lib/browser/{chunk-GHTRJTEB.mjs → chunk-GU2BNYKZ.mjs} +6 -6
  9. package/dist/lib/browser/chunk-GU2BNYKZ.mjs.map +7 -0
  10. package/dist/lib/browser/chunk-K7FOYAXO.mjs +8 -0
  11. package/dist/lib/browser/{chunk-QIHUPVQD.mjs → chunk-OGNXNGJB.mjs} +21 -28
  12. package/dist/lib/browser/chunk-OGNXNGJB.mjs.map +7 -0
  13. package/dist/lib/browser/chunk-YCZISX63.mjs +464 -0
  14. package/dist/lib/browser/chunk-YCZISX63.mjs.map +7 -0
  15. package/dist/lib/browser/chunk-YEPT334A.mjs +16 -0
  16. package/dist/lib/browser/chunk-YEPT334A.mjs.map +7 -0
  17. package/dist/lib/browser/index.mjs +69 -512
  18. package/dist/lib/browser/index.mjs.map +4 -4
  19. package/dist/lib/browser/{intent-resolver-RW6DEJC2.mjs → intent-resolver-4R66ZCKT.mjs} +8 -7
  20. package/dist/lib/browser/intent-resolver-4R66ZCKT.mjs.map +7 -0
  21. package/dist/lib/browser/meeting-transcription-state-DLFHQK72.mjs +18 -0
  22. package/dist/lib/browser/meeting-transcription-state-DLFHQK72.mjs.map +7 -0
  23. package/dist/lib/browser/meta.json +1 -1
  24. package/dist/lib/browser/{react-surface-UPTZBHVB.mjs → react-surface-FQUHU5G7.mjs} +15 -9
  25. package/dist/lib/{node-esm/react-surface-2WJQ67LB.mjs.map → browser/react-surface-FQUHU5G7.mjs.map} +2 -2
  26. package/dist/lib/browser/transcriber-LGMIWDYI.mjs +36 -0
  27. package/dist/lib/browser/transcriber-LGMIWDYI.mjs.map +7 -0
  28. package/dist/lib/browser/types/index.mjs +2 -1
  29. package/dist/lib/node/{TranscriptContainer-4BWXOVML.cjs → TranscriptContainer-QEYTTT2V.cjs} +8 -8
  30. package/dist/lib/node/{TranscriptContainer-4BWXOVML.cjs.map → TranscriptContainer-QEYTTT2V.cjs.map} +2 -2
  31. package/dist/lib/node/app-graph-builder-IFSIVNUR.cjs +200 -0
  32. package/dist/lib/node/app-graph-builder-IFSIVNUR.cjs.map +7 -0
  33. package/dist/lib/node/{chunk-HDRXDMK2.cjs → chunk-3KHUSQNW.cjs} +6 -6
  34. package/dist/lib/node/chunk-3KHUSQNW.cjs.map +7 -0
  35. package/dist/lib/node/chunk-CZMLKKWT.cjs +35 -0
  36. package/dist/lib/node/chunk-CZMLKKWT.cjs.map +7 -0
  37. package/dist/lib/node/chunk-FCCJALOT.cjs +488 -0
  38. package/dist/lib/node/chunk-FCCJALOT.cjs.map +7 -0
  39. package/dist/lib/node/{chunk-I35KHI2G.cjs → chunk-LP4FG423.cjs} +5 -5
  40. package/dist/lib/node/chunk-NVZWGFJL.cjs +42 -0
  41. package/dist/lib/node/chunk-NVZWGFJL.cjs.map +7 -0
  42. package/dist/lib/node/{chunk-UIHEGU6Q.cjs → chunk-P4VVD6ZO.cjs} +21 -29
  43. package/dist/lib/node/chunk-P4VVD6ZO.cjs.map +7 -0
  44. package/dist/lib/node/chunk-UOFHT4GZ.cjs +280 -0
  45. package/dist/lib/node/chunk-UOFHT4GZ.cjs.map +7 -0
  46. package/dist/lib/node/index.cjs +75 -513
  47. package/dist/lib/node/index.cjs.map +4 -4
  48. package/dist/lib/node/{intent-resolver-KLBZ25XJ.cjs → intent-resolver-WMMCOUEH.cjs} +12 -11
  49. package/dist/lib/node/intent-resolver-WMMCOUEH.cjs.map +7 -0
  50. package/dist/lib/node/{app-graph-builder-YFBX3OTP.cjs → meeting-transcription-state-URIYILEQ.cjs} +14 -6
  51. package/dist/lib/node/meeting-transcription-state-URIYILEQ.cjs.map +7 -0
  52. package/dist/lib/node/meta.json +1 -1
  53. package/dist/lib/node/{react-surface-NCKXB7EQ.cjs → react-surface-WI2MGWCC.cjs} +17 -12
  54. package/dist/lib/node/{react-surface-NCKXB7EQ.cjs.map → react-surface-WI2MGWCC.cjs.map} +2 -2
  55. package/dist/lib/node/transcriber-QG5WPKX4.cjs +49 -0
  56. package/dist/lib/node/transcriber-QG5WPKX4.cjs.map +7 -0
  57. package/dist/lib/node/types/index.cjs +8 -7
  58. package/dist/lib/node/types/index.cjs.map +2 -2
  59. package/dist/lib/node-esm/{TranscriptContainer-6IKRASR6.mjs → TranscriptContainer-ZXUCOWGY.mjs} +3 -3
  60. package/dist/lib/node-esm/app-graph-builder-UL6XLIZY.mjs +192 -0
  61. package/dist/lib/node-esm/app-graph-builder-UL6XLIZY.mjs.map +7 -0
  62. package/dist/lib/node-esm/chunk-6X7X2UZF.mjs +254 -0
  63. package/dist/lib/node-esm/chunk-6X7X2UZF.mjs.map +7 -0
  64. package/dist/lib/node-esm/{chunk-KEG3ME4O.mjs → chunk-GIO6J6FE.mjs} +20 -28
  65. package/dist/lib/node-esm/chunk-GIO6J6FE.mjs.map +7 -0
  66. package/dist/lib/node-esm/{chunk-LORUIVS4.mjs → chunk-HKATMD34.mjs} +6 -6
  67. package/dist/lib/node-esm/chunk-HKATMD34.mjs.map +7 -0
  68. package/dist/lib/node-esm/{chunk-2CPKXRN6.mjs → chunk-J4D2RBBH.mjs} +3 -3
  69. package/dist/lib/node-esm/chunk-PTZHDZ36.mjs +17 -0
  70. package/dist/lib/node-esm/chunk-PTZHDZ36.mjs.map +7 -0
  71. package/dist/lib/node-esm/chunk-X4DDJIVL.mjs +21 -0
  72. package/dist/lib/node-esm/chunk-X4DDJIVL.mjs.map +7 -0
  73. package/dist/lib/node-esm/chunk-YQYBLFNI.mjs +465 -0
  74. package/dist/lib/node-esm/chunk-YQYBLFNI.mjs.map +7 -0
  75. package/dist/lib/node-esm/index.mjs +69 -512
  76. package/dist/lib/node-esm/index.mjs.map +4 -4
  77. package/dist/lib/node-esm/{intent-resolver-4FMISUPW.mjs → intent-resolver-QN2LO5TX.mjs} +8 -7
  78. package/dist/lib/node-esm/intent-resolver-QN2LO5TX.mjs.map +7 -0
  79. package/dist/lib/node-esm/meeting-transcription-state-ZYVRAW6Y.mjs +19 -0
  80. package/dist/lib/node-esm/meeting-transcription-state-ZYVRAW6Y.mjs.map +7 -0
  81. package/dist/lib/node-esm/meta.json +1 -1
  82. package/dist/lib/node-esm/{react-surface-2WJQ67LB.mjs → react-surface-PCH6NTVB.mjs} +15 -9
  83. package/dist/lib/{browser/react-surface-UPTZBHVB.mjs.map → node-esm/react-surface-PCH6NTVB.mjs.map} +2 -2
  84. package/dist/lib/node-esm/transcriber-TMPYZ5L4.mjs +37 -0
  85. package/dist/lib/node-esm/transcriber-TMPYZ5L4.mjs.map +7 -0
  86. package/dist/lib/node-esm/types/index.mjs +2 -1
  87. package/dist/types/src/TranscriptionPlugin.d.ts.map +1 -1
  88. package/dist/types/src/capabilities/app-graph-builder.d.ts +1 -0
  89. package/dist/types/src/capabilities/app-graph-builder.d.ts.map +1 -1
  90. package/dist/types/src/capabilities/capabilities.d.ts +16 -1
  91. package/dist/types/src/capabilities/capabilities.d.ts.map +1 -1
  92. package/dist/types/src/capabilities/index.d.ts +5 -0
  93. package/dist/types/src/capabilities/index.d.ts.map +1 -1
  94. package/dist/types/src/capabilities/intent-resolver.d.ts.map +1 -1
  95. package/dist/types/src/capabilities/meeting-transcription-state.d.ts +6 -0
  96. package/dist/types/src/capabilities/meeting-transcription-state.d.ts.map +1 -0
  97. package/dist/types/src/capabilities/transcriber.d.ts +7 -0
  98. package/dist/types/src/capabilities/transcriber.d.ts.map +1 -0
  99. package/dist/types/src/components/Transcript/Transcript.d.ts +5 -2
  100. package/dist/types/src/components/Transcript/Transcript.d.ts.map +1 -1
  101. package/dist/types/src/components/Transcript/Transcript.stories.d.ts.map +1 -1
  102. package/dist/types/src/components/TranscriptContainer.d.ts +1 -0
  103. package/dist/types/src/components/TranscriptContainer.d.ts.map +1 -1
  104. package/dist/types/src/components/index.d.ts +1 -0
  105. package/dist/types/src/components/index.d.ts.map +1 -1
  106. package/dist/types/src/components/transcription.stories.d.ts +1 -1
  107. package/dist/types/src/components/transcription.stories.d.ts.map +1 -1
  108. package/dist/types/src/hooks/useTranscriber.d.ts +3 -6
  109. package/dist/types/src/hooks/useTranscriber.d.ts.map +1 -1
  110. package/dist/types/src/index.d.ts +1 -2
  111. package/dist/types/src/index.d.ts.map +1 -1
  112. package/dist/types/src/meta.d.ts +2 -9
  113. package/dist/types/src/meta.d.ts.map +1 -1
  114. package/dist/types/src/transcriber/index.d.ts +1 -0
  115. package/dist/types/src/transcriber/index.d.ts.map +1 -1
  116. package/dist/types/src/{transcription-manager.d.ts → transcriber/transcription-manager.d.ts} +5 -2
  117. package/dist/types/src/transcriber/transcription-manager.d.ts.map +1 -0
  118. package/dist/types/src/translations.d.ts +14 -18
  119. package/dist/types/src/translations.d.ts.map +1 -1
  120. package/dist/types/src/types/schema.d.ts +13 -19
  121. package/dist/types/src/types/schema.d.ts.map +1 -1
  122. package/dist/types/src/types/types.d.ts +2 -6
  123. package/dist/types/src/types/types.d.ts.map +1 -1
  124. package/dist/types/src/util/random-queue-dxn.d.ts +2 -2
  125. package/dist/types/src/util/random-queue-dxn.d.ts.map +1 -1
  126. package/package.json +40 -36
  127. package/src/TranscriptionPlugin.tsx +24 -7
  128. package/src/capabilities/app-graph-builder.ts +147 -2
  129. package/src/capabilities/capabilities.ts +16 -2
  130. package/src/capabilities/index.ts +2 -0
  131. package/src/capabilities/intent-resolver.ts +4 -3
  132. package/src/capabilities/meeting-transcription-state.ts +16 -0
  133. package/src/capabilities/react-surface.tsx +3 -3
  134. package/src/capabilities/transcriber.ts +48 -0
  135. package/src/components/Transcript/Transcript.stories.tsx +46 -27
  136. package/src/components/Transcript/Transcript.tsx +206 -50
  137. package/src/components/TranscriptContainer.tsx +9 -147
  138. package/src/components/transcription.stories.tsx +11 -13
  139. package/src/hooks/useTranscriber.ts +12 -37
  140. package/src/index.ts +2 -2
  141. package/src/meta.ts +2 -2
  142. package/src/transcriber/index.ts +1 -0
  143. package/src/{transcription-manager.ts → transcriber/transcription-manager.ts} +16 -5
  144. package/src/translations.ts +6 -6
  145. package/src/types/schema.ts +7 -5
  146. package/src/types/types.ts +4 -2
  147. package/src/util/random-queue-dxn.ts +2 -2
  148. package/dist/lib/browser/app-graph-builder-ZSSJIFY7.mjs +0 -7
  149. package/dist/lib/browser/app-graph-builder-ZSSJIFY7.mjs.map +0 -7
  150. package/dist/lib/browser/chunk-BDVUIRBP.mjs +0 -8
  151. package/dist/lib/browser/chunk-GHTRJTEB.mjs.map +0 -7
  152. package/dist/lib/browser/chunk-QIHUPVQD.mjs.map +0 -7
  153. package/dist/lib/browser/chunk-YX447BJ4.mjs +0 -241
  154. package/dist/lib/browser/chunk-YX447BJ4.mjs.map +0 -7
  155. package/dist/lib/browser/intent-resolver-RW6DEJC2.mjs.map +0 -7
  156. package/dist/lib/node/app-graph-builder-YFBX3OTP.cjs.map +0 -7
  157. package/dist/lib/node/chunk-ANSWGFMS.cjs +0 -267
  158. package/dist/lib/node/chunk-ANSWGFMS.cjs.map +0 -7
  159. package/dist/lib/node/chunk-HDRXDMK2.cjs.map +0 -7
  160. package/dist/lib/node/chunk-UIHEGU6Q.cjs.map +0 -7
  161. package/dist/lib/node/intent-resolver-KLBZ25XJ.cjs.map +0 -7
  162. package/dist/lib/node-esm/app-graph-builder-77GQG6UF.mjs +0 -9
  163. package/dist/lib/node-esm/app-graph-builder-77GQG6UF.mjs.map +0 -7
  164. package/dist/lib/node-esm/chunk-KEG3ME4O.mjs.map +0 -7
  165. package/dist/lib/node-esm/chunk-LORUIVS4.mjs.map +0 -7
  166. package/dist/lib/node-esm/chunk-ZMK2QLW2.mjs +0 -242
  167. package/dist/lib/node-esm/chunk-ZMK2QLW2.mjs.map +0 -7
  168. package/dist/lib/node-esm/intent-resolver-4FMISUPW.mjs.map +0 -7
  169. package/dist/types/src/transcription-manager.d.ts.map +0 -1
  170. /package/dist/lib/browser/{TranscriptContainer-N4KQHMY4.mjs.map → TranscriptContainer-JQ3HFHLQ.mjs.map} +0 -0
  171. /package/dist/lib/browser/{chunk-BDVUIRBP.mjs.map → chunk-K7FOYAXO.mjs.map} +0 -0
  172. /package/dist/lib/node/{chunk-I35KHI2G.cjs.map → chunk-LP4FG423.cjs.map} +0 -0
  173. /package/dist/lib/node-esm/{TranscriptContainer-6IKRASR6.mjs.map → TranscriptContainer-ZXUCOWGY.mjs.map} +0 -0
  174. /package/dist/lib/node-esm/{chunk-2CPKXRN6.mjs.map → chunk-J4D2RBBH.mjs.map} +0 -0
@@ -1,40 +1,40 @@
1
- import {
2
- mergeFloat64Arrays
3
- } from "./chunk-GHTRJTEB.mjs";
4
1
  import {
5
2
  TranscriptContainer
6
- } from "./chunk-BDVUIRBP.mjs";
3
+ } from "./chunk-K7FOYAXO.mjs";
7
4
  import {
8
5
  Transcript,
9
6
  TranscriptionContainer
10
- } from "./chunk-YX447BJ4.mjs";
7
+ } from "./chunk-G4B6CPC2.mjs";
8
+ import {
9
+ MediaStreamRecorder,
10
+ Transcriber,
11
+ TranscriptionManager
12
+ } from "./chunk-YCZISX63.mjs";
13
+ import "./chunk-GU2BNYKZ.mjs";
14
+ import {
15
+ TranscriptionCapabilities
16
+ } from "./chunk-YEPT334A.mjs";
11
17
  import {
12
- TRANSCRIPTION_PLUGIN,
13
- TRANSCRIPTION_URL,
14
18
  TranscriptBlock,
15
- TranscriptType,
19
+ TranscriptType
20
+ } from "./chunk-OGNXNGJB.mjs";
21
+ import {
22
+ TRANSCRIPTION_PLUGIN,
16
23
  meta
17
- } from "./chunk-QIHUPVQD.mjs";
24
+ } from "./chunk-6ZR5MDK3.mjs";
18
25
 
19
- // packages/plugins/experimental/plugin-transcription/src/capabilities/index.ts
26
+ // packages/plugins/plugin-transcription/src/capabilities/index.ts
20
27
  import { lazy } from "@dxos/app-framework";
28
+ var AppGraphBuilder = lazy(() => import("./app-graph-builder-VDQOVX7P.mjs"));
29
+ var IntentResolver = lazy(() => import("./intent-resolver-4R66ZCKT.mjs"));
30
+ var MeetingTranscriptionState = lazy(() => import("./meeting-transcription-state-DLFHQK72.mjs"));
31
+ var ReactSurface = lazy(() => import("./react-surface-FQUHU5G7.mjs"));
32
+ var Transcriber2 = lazy(() => import("./transcriber-LGMIWDYI.mjs"));
21
33
 
22
- // packages/plugins/experimental/plugin-transcription/src/capabilities/capabilities.ts
23
- import { defineCapability } from "@dxos/app-framework";
24
- var TranscriptionCapabilities;
25
- (function(TranscriptionCapabilities2) {
26
- TranscriptionCapabilities2.Transcription = defineCapability(`${TRANSCRIPTION_PLUGIN}/capability/transcription`);
27
- })(TranscriptionCapabilities || (TranscriptionCapabilities = {}));
28
-
29
- // packages/plugins/experimental/plugin-transcription/src/capabilities/index.ts
30
- var AppGraphBuilder = lazy(() => import("./app-graph-builder-ZSSJIFY7.mjs"));
31
- var IntentResolver = lazy(() => import("./intent-resolver-RW6DEJC2.mjs"));
32
- var ReactSurface = lazy(() => import("./react-surface-UPTZBHVB.mjs"));
33
-
34
- // packages/plugins/experimental/plugin-transcription/src/hooks/useAudioFile.ts
34
+ // packages/plugins/plugin-transcription/src/hooks/useAudioFile.ts
35
35
  import { useEffect, useState } from "react";
36
36
  import { log } from "@dxos/log";
37
- var __dxlog_file = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/hooks/useAudioFile.ts";
37
+ var __dxlog_file = "/home/runner/work/dxos/dxos/packages/plugins/plugin-transcription/src/hooks/useAudioFile.ts";
38
38
  var useAudioFile = (audioUrl) => {
39
39
  const [{ audio, stream, track }, setStream] = useState({});
40
40
  useEffect(() => {
@@ -92,7 +92,7 @@ var useAudioFile = (audioUrl) => {
92
92
  };
93
93
  };
94
94
 
95
- // packages/plugins/experimental/plugin-transcription/src/hooks/useAudioTrack.ts
95
+ // packages/plugins/plugin-transcription/src/hooks/useAudioTrack.ts
96
96
  import { useEffect as useEffect2, useState as useState2 } from "react";
97
97
  var useAudioTrack = (active) => {
98
98
  const [track, setTrack] = useState2();
@@ -121,310 +121,23 @@ var useAudioTrack = (active) => {
121
121
  return track;
122
122
  };
123
123
 
124
- // packages/plugins/experimental/plugin-transcription/src/hooks/useTranscriber.ts
124
+ // packages/plugins/plugin-transcription/src/hooks/useTranscriber.ts
125
125
  import { useEffect as useEffect3, useMemo } from "react";
126
-
127
- // packages/plugins/experimental/plugin-transcription/src/transcriber/media-stream-recorder.ts
128
- import { MediaRecorder, register } from "extendable-media-recorder";
129
- import { WaveFile } from "wavefile";
130
- import { synchronized } from "@dxos/async";
131
- import { invariant } from "@dxos/invariant";
132
- import { log as log2 } from "@dxos/log";
133
- import { trace } from "@dxos/tracing";
134
- function _ts_decorate(decorators, target, key, desc) {
135
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
136
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
137
- else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
138
- return c > 3 && r && Object.defineProperty(target, key, r), r;
139
- }
140
- var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/transcriber/media-stream-recorder.ts";
141
- var initializingPromise;
142
- var initializeExtendableMediaRecorder = async () => {
143
- const { connect } = await import("extendable-media-recorder-wav-encoder");
144
- await register(await connect());
145
- };
146
- var MediaStreamRecorder = class {
147
- constructor({ mediaStreamTrack, interval }) {
148
- this._mediaRecorder = void 0;
149
- this._header = void 0;
150
- this._mediaStreamTrack = mediaStreamTrack;
151
- this._interval = interval;
152
- }
153
- get wavConfig() {
154
- const settings = this._mediaStreamTrack.getSettings();
155
- return {
156
- channels: settings.channelCount ?? 1,
157
- sampleRate: settings.sampleRate ?? 16e3,
158
- bitDepthCode: settings.sampleSize ? String(settings.sampleSize) : "16"
159
- };
160
- }
161
- get mediaStreamTrack() {
162
- return this._mediaStreamTrack;
163
- }
164
- setOnChunk(onChunk) {
165
- this._onChunk = onChunk;
166
- }
167
- async start() {
168
- await (initializingPromise ??= initializeExtendableMediaRecorder()).catch((err) => log2.info("initializeExtendableMediaRecorder", {
169
- err
170
- }, {
171
- F: __dxlog_file2,
172
- L: 62,
173
- S: this,
174
- C: (f, a) => f(...a)
175
- }));
176
- if (!this._mediaRecorder) {
177
- const stream = new MediaStream([
178
- this._mediaStreamTrack
179
- ]);
180
- this._mediaRecorder = new MediaRecorder(stream, {
181
- mimeType: "audio/wav"
182
- });
183
- }
184
- if (this._mediaRecorder.state === "recording") {
185
- return;
186
- }
187
- invariant(this._onChunk, "MediaStreamRecorder: onChunk is not set", {
188
- F: __dxlog_file2,
189
- L: 73,
190
- S: this,
191
- A: [
192
- "this._onChunk",
193
- "'MediaStreamRecorder: onChunk is not set'"
194
- ]
195
- });
196
- this._mediaRecorder.ondataavailable = (event) => this._ondataavailable(event);
197
- this._mediaRecorder.start(this._interval);
198
- }
199
- async stop() {
200
- if (this._mediaRecorder?.state !== "recording") {
201
- return;
202
- }
203
- this._mediaRecorder.stop();
204
- }
205
- async _ondataavailable(event) {
206
- const blob = event.data;
207
- const uint8Array = new Uint8Array(await blob.arrayBuffer());
208
- const isHeader = uint8Array[0] === 82 && // R
209
- uint8Array[1] === 73 && // I
210
- uint8Array[2] === 70 && // F
211
- uint8Array[3] === 70;
212
- let wav;
213
- if (isHeader) {
214
- wav = new WaveFile(uint8Array);
215
- this._header = uint8Array.slice(0, 44);
216
- } else if (this._header) {
217
- wav = new WaveFile(new Uint8Array([
218
- ...this._header,
219
- ...uint8Array
220
- ]));
221
- } else {
222
- log2.warn("MediaStreamRecorder: no header", void 0, {
223
- F: __dxlog_file2,
224
- L: 104,
225
- S: this,
226
- C: (f, a) => f(...a)
227
- });
228
- return;
229
- }
230
- this._onChunk({
231
- timestamp: Date.now(),
232
- data: wav.getSamples()
233
- });
234
- }
235
- };
236
- _ts_decorate([
237
- synchronized
238
- ], MediaStreamRecorder.prototype, "_ondataavailable", null);
239
- MediaStreamRecorder = _ts_decorate([
240
- trace.resource()
241
- ], MediaStreamRecorder);
242
-
243
- // packages/plugins/experimental/plugin-transcription/src/transcriber/transcriber.ts
244
- import { WaveFile as WaveFile2 } from "wavefile";
245
- import { DeferredTask, Trigger } from "@dxos/async";
246
- import { LifecycleState, Resource } from "@dxos/context";
247
- import { log as log3 } from "@dxos/log";
248
- import { trace as trace2 } from "@dxos/tracing";
249
- function _ts_decorate2(decorators, target, key, desc) {
250
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
251
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
252
- else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
253
- return c > 3 && r && Object.defineProperty(target, key, r), r;
254
- }
255
- var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/transcriber/transcriber.ts";
256
- var Transcriber = class extends Resource {
257
- constructor({ config, recorder, onSegments }) {
258
- super();
259
- this._audioChunks = [];
260
- this._lastUsedTimestamp = 0;
261
- this._openTrigger = new Trigger({
262
- autoReset: true
263
- });
264
- this._recording = false;
265
- this._transcribeTask = void 0;
266
- this._config = config;
267
- this._recorder = recorder;
268
- this._onSegments = onSegments;
269
- }
270
- async _open(ctx) {
271
- this._recorder.setOnChunk((chunk) => this._saveAudioChunk(chunk));
272
- await this._recorder.start();
273
- this._transcribeTask = new DeferredTask(ctx, async () => this._transcribe());
274
- this._openTrigger.wake();
275
- }
276
- async _close() {
277
- this._recording = false;
278
- this._transcribeTask = void 0;
279
- await this._recorder.stop();
280
- }
281
- startChunksRecording() {
282
- if (this._lifecycleState !== LifecycleState.OPEN) {
283
- return;
284
- }
285
- this._recording = true;
286
- }
287
- stopChunksRecording() {
288
- if (this._lifecycleState !== LifecycleState.OPEN || !this._recording) {
289
- return;
290
- }
291
- this._recording = false;
292
- this._transcribeTask?.schedule();
293
- }
294
- _saveAudioChunk(chunk) {
295
- log3("saving audio chunk", {
296
- chunk
297
- }, {
298
- F: __dxlog_file3,
299
- L: 124,
300
- S: this,
301
- C: (f, a) => f(...a)
302
- });
303
- this._audioChunks.push(chunk);
304
- if (!this._recording && (!this._transcribeTask || !this._transcribeTask.scheduled) && this._audioChunks.length >= this._config.prefixBufferChunksAmount) {
305
- this._audioChunks = this._dropOldChunks();
306
- } else if (this._audioChunks.length >= this._config.transcribeAfterChunksAmount && this._recording) {
307
- this._transcribeTask?.schedule();
308
- }
309
- }
310
- async _transcribe() {
311
- log3("transcribing", {
312
- chunks: this._audioChunks.length
313
- }, {
314
- F: __dxlog_file3,
315
- L: 140,
316
- S: this,
317
- C: (f, a) => f(...a)
318
- });
319
- const chunks = this._audioChunks;
320
- this._audioChunks = this._dropOldChunks();
321
- if (chunks.length === 0) {
322
- return;
323
- }
324
- const audio = await this._mergeAudioChunks(chunks);
325
- const segments = await this._fetchTranscription(audio);
326
- if (!Array.isArray(segments) || segments.length === 0) {
327
- return;
328
- }
329
- const alignedSegments = this._alignSegments(segments, chunks);
330
- if (alignedSegments.length > 0) {
331
- await this._onSegments(alignedSegments);
332
- }
333
- }
334
- _dropOldChunks() {
335
- return this._config.prefixBufferChunksAmount > 0 ? this._audioChunks.slice(-this._config.prefixBufferChunksAmount) : [];
336
- }
337
- async _mergeAudioChunks(chunks) {
338
- const file = new WaveFile2();
339
- const wavConfig = this._recorder.wavConfig;
340
- file.fromScratch(wavConfig.channels, wavConfig.sampleRate, wavConfig.bitDepthCode, mergeFloat64Arrays(chunks.map(({ data }) => data)));
341
- return file.toBase64();
342
- }
343
- async _fetchTranscription(audio) {
344
- if (audio.length === 0) {
345
- this._audioChunks = [];
346
- throw new Error("No audio to send for transcribing");
347
- }
348
- const response = await fetch(`${TRANSCRIPTION_URL}/transcribe`, {
349
- method: "POST",
350
- body: JSON.stringify({
351
- audio
352
- }),
353
- headers: {
354
- "Content-Type": "application/json"
355
- }
356
- });
357
- if (!response.ok) {
358
- this._audioChunks = [];
359
- throw new Error("Failed to transcribe");
360
- }
361
- const { segments } = await response.json();
362
- log3.info("transcription response", {
363
- segments,
364
- string: segments.map((segments2) => segments2.text).join(" ")
365
- }, {
366
- F: __dxlog_file3,
367
- L: 206,
368
- S: this,
369
- C: (f, a) => f(...a)
370
- });
371
- return segments;
372
- }
373
- _alignSegments(segments, originalChunks) {
374
- const zeroTimestamp = originalChunks.at(0).timestamp;
375
- const filteredSegments = segments.filter((segment) => zeroTimestamp + segment.end * 1e3 >= this._lastUsedTimestamp).map((segment) => {
376
- const words = segment.words.filter((word) => zeroTimestamp + word.start * 1e3 >= this._lastUsedTimestamp);
377
- return {
378
- ...segment,
379
- words,
380
- text: words.map((word) => word.word).join(""),
381
- start: words.at(0)?.start ?? segment.end
382
- };
383
- }).filter((segment) => segment.words.length > 0);
384
- if (filteredSegments.length === 0) {
385
- return [];
386
- }
387
- this._lastUsedTimestamp = zeroTimestamp + filteredSegments.at(-1).end * 1e3;
388
- return filteredSegments.map((segment) => ({
389
- started: new Date(zeroTimestamp + segment.start * 1e3),
390
- text: segment.text
391
- }));
392
- }
393
- };
394
- _ts_decorate2([
395
- trace2.span({
396
- showInBrowserTimeline: true
397
- })
398
- ], Transcriber.prototype, "_mergeAudioChunks", null);
399
- _ts_decorate2([
400
- trace2.span({
401
- showInBrowserTimeline: true
402
- })
403
- ], Transcriber.prototype, "_fetchTranscription", null);
404
-
405
- // packages/plugins/experimental/plugin-transcription/src/hooks/useTranscriber.ts
406
- var RECORD_INTERVAL = 200;
407
- var PREFIXED_CHUNKS_AMOUNT = 10;
408
- var TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
126
+ import { useCapabilities } from "@dxos/app-framework";
409
127
  var useTranscriber = ({ audioStreamTrack, onSegments }) => {
128
+ const [getTranscriber] = useCapabilities(TranscriptionCapabilities.Transcriber);
410
129
  const transcriber = useMemo(() => {
411
- if (!onSegments || !audioStreamTrack) {
130
+ if (!onSegments || !audioStreamTrack || !getTranscriber) {
412
131
  return void 0;
413
132
  }
414
- return new Transcriber({
415
- config: {
416
- transcribeAfterChunksAmount: TRANSCRIBE_AFTER_CHUNKS_AMOUNT,
417
- prefixBufferChunksAmount: PREFIXED_CHUNKS_AMOUNT
418
- },
419
- recorder: new MediaStreamRecorder({
420
- mediaStreamTrack: audioStreamTrack,
421
- interval: RECORD_INTERVAL
422
- }),
133
+ return getTranscriber({
134
+ audioStreamTrack,
423
135
  onSegments
424
136
  });
425
137
  }, [
426
138
  audioStreamTrack,
427
- onSegments
139
+ onSegments,
140
+ getTranscriber
428
141
  ]);
429
142
  useEffect3(() => {
430
143
  return () => {
@@ -436,13 +149,13 @@ var useTranscriber = ({ audioStreamTrack, onSegments }) => {
436
149
  return transcriber;
437
150
  };
438
151
 
439
- // packages/plugins/experimental/plugin-transcription/src/hooks/useVoiceInput.ts
152
+ // packages/plugins/plugin-transcription/src/hooks/useVoiceInput.ts
440
153
  import { useState as useState3, useEffect as useEffect4, useCallback } from "react";
441
154
  import { scheduleMicroTask } from "@dxos/async";
442
155
  import { Context } from "@dxos/context";
443
- import { log as log4 } from "@dxos/log";
156
+ import { log as log2 } from "@dxos/log";
444
157
  import { useSoundEffect } from "@dxos/react-ui-sfx";
445
- var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/hooks/useVoiceInput.ts";
158
+ var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/plugins/plugin-transcription/src/hooks/useVoiceInput.ts";
446
159
  var useVoiceInput = ({ active, onUpdate }) => {
447
160
  const soundStart = useSoundEffect("StartRecording");
448
161
  const soundStop = useSoundEffect("StopRecording");
@@ -458,14 +171,14 @@ var useVoiceInput = ({ active, onUpdate }) => {
458
171
  });
459
172
  useEffect4(() => {
460
173
  const ctx = new Context(void 0, {
461
- F: __dxlog_file4,
174
+ F: __dxlog_file2,
462
175
  L: 43
463
176
  });
464
177
  scheduleMicroTask(ctx, async () => {
465
178
  if (active && transcriber) {
466
179
  await transcriber.open();
467
- log4.info("starting...", void 0, {
468
- F: __dxlog_file4,
180
+ log2.info("starting...", void 0, {
181
+ F: __dxlog_file2,
469
182
  L: 47,
470
183
  S: void 0,
471
184
  C: (f, a) => f(...a)
@@ -476,8 +189,8 @@ var useVoiceInput = ({ active, onUpdate }) => {
476
189
  } else if (!active && transcriber?.isOpen) {
477
190
  transcriber?.stopChunksRecording();
478
191
  await transcriber?.close();
479
- log4.info("stopped", void 0, {
480
- F: __dxlog_file4,
192
+ log2.info("stopped", void 0, {
193
+ F: __dxlog_file2,
481
194
  L: 54,
482
195
  S: void 0,
483
196
  C: (f, a) => f(...a)
@@ -498,194 +211,34 @@ var useVoiceInput = ({ active, onUpdate }) => {
498
211
  };
499
212
  };
500
213
 
501
- // packages/plugins/experimental/plugin-transcription/src/transcription-manager.ts
502
- import { synchronized as synchronized2 } from "@dxos/async";
503
- import { Resource as Resource2 } from "@dxos/context";
504
- import { QueueImpl } from "@dxos/echo-db";
505
- import { createStatic } from "@dxos/echo-schema";
506
- import { invariant as invariant2 } from "@dxos/invariant";
507
- import { DXN } from "@dxos/keys";
508
- function _ts_decorate3(decorators, target, key, desc) {
509
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
510
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
511
- else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
512
- return c > 3 && r && Object.defineProperty(target, key, r), r;
513
- }
514
- var __dxlog_file5 = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/transcription-manager.ts";
515
- var RECORD_INTERVAL2 = 200;
516
- var PREFIXED_CHUNKS_AMOUNT2 = 10;
517
- var TRANSCRIBE_AFTER_CHUNKS_AMOUNT2 = 50;
518
- var TranscriptionManager = class extends Resource2 {
519
- constructor(edgeClient) {
520
- super();
521
- this._audioStreamTrack = void 0;
522
- this._name = void 0;
523
- this._mediaRecorder = void 0;
524
- this._transcriber = void 0;
525
- this._enabled = false;
526
- this._queue = void 0;
527
- this._edgeClient = edgeClient;
528
- }
529
- async _open() {
530
- await this._toggleTranscriber();
531
- }
532
- async _close() {
533
- void this._transcriber?.close();
534
- }
535
- setRecording(recording) {
536
- if (!this.isOpen || !this._enabled) {
537
- return;
538
- }
539
- if (recording) {
540
- this._transcriber?.startChunksRecording();
541
- } else {
542
- this._transcriber?.stopChunksRecording();
543
- }
544
- }
545
- /**
546
- * Enable or disable transcription.
547
- * @param enabled - Whether to enable transcription.
548
- */
549
- async setEnabled(enabled) {
550
- if (this._enabled === enabled) {
551
- return;
552
- }
553
- this._enabled = enabled ?? false;
554
- this.isOpen && await this._toggleTranscriber();
555
- }
556
- async setAudioTrack(track) {
557
- if (this._audioStreamTrack === track) {
558
- return;
559
- }
560
- this._audioStreamTrack = track;
561
- this.isOpen && await this._toggleTranscriber();
562
- }
563
- /**
564
- * Set the queue to save the transcription to.
565
- * @param queue - The queue to save the transcription to or the DXN of the queue.
566
- */
567
- setQueue(queue) {
568
- switch (typeof queue) {
569
- case "string":
570
- if (this._queue?.dxn.toString() === queue) {
571
- return;
572
- }
573
- this._queue = new QueueImpl(this._edgeClient, DXN.parse(queue));
574
- break;
575
- case "object":
576
- if (this._queue === queue) {
577
- return;
578
- }
579
- invariant2(queue instanceof QueueImpl, void 0, {
580
- F: __dxlog_file5,
581
- L: 110,
582
- S: this,
583
- A: [
584
- "queue instanceof QueueImpl",
585
- ""
586
- ]
587
- });
588
- this._queue = queue;
589
- break;
590
- case "undefined":
591
- this._queue = void 0;
592
- break;
593
- }
594
- }
595
- setName(name) {
596
- if (this._name === name) {
597
- return;
598
- }
599
- this._name = name;
600
- }
601
- async _toggleTranscriber() {
602
- await this._maybeReinitTranscriber();
603
- if (this._enabled) {
604
- await this._transcriber?.open();
605
- } else {
606
- await this._transcriber?.close();
607
- }
608
- }
609
- async _maybeReinitTranscriber() {
610
- if (!this._audioStreamTrack || !this._enabled) {
611
- return;
612
- }
613
- let needReinit = false;
614
- if (this._audioStreamTrack !== this._mediaRecorder?.mediaStreamTrack) {
615
- this._mediaRecorder = new MediaStreamRecorder({
616
- mediaStreamTrack: this._audioStreamTrack,
617
- interval: RECORD_INTERVAL2
618
- });
619
- needReinit = true;
620
- }
621
- if (needReinit) {
622
- await this._transcriber?.close();
623
- this._transcriber = new Transcriber({
624
- config: {
625
- transcribeAfterChunksAmount: TRANSCRIBE_AFTER_CHUNKS_AMOUNT2,
626
- prefixBufferChunksAmount: PREFIXED_CHUNKS_AMOUNT2
627
- },
628
- recorder: this._mediaRecorder,
629
- onSegments: (segments) => this._onSegments(segments)
630
- });
631
- }
632
- }
633
- async _onSegments(segments) {
634
- if (!this.isOpen || !this._queue) {
635
- return;
636
- }
637
- const block = createStatic(TranscriptBlock, {
638
- author: this._name,
639
- segments
640
- });
641
- this._queue.append([
642
- block
643
- ]);
644
- }
645
- };
646
- _ts_decorate3([
647
- synchronized2
648
- ], TranscriptionManager.prototype, "setRecording", null);
649
- _ts_decorate3([
650
- synchronized2
651
- ], TranscriptionManager.prototype, "setEnabled", null);
652
- _ts_decorate3([
653
- synchronized2
654
- ], TranscriptionManager.prototype, "setAudioTrack", null);
655
- _ts_decorate3([
656
- synchronized2
657
- ], TranscriptionManager.prototype, "setQueue", null);
658
- _ts_decorate3([
659
- synchronized2
660
- ], TranscriptionManager.prototype, "setName", null);
661
-
662
- // packages/plugins/experimental/plugin-transcription/src/TranscriptionPlugin.tsx
214
+ // packages/plugins/plugin-transcription/src/TranscriptionPlugin.tsx
663
215
  import { Capabilities, Events, contributes, defineModule, definePlugin } from "@dxos/app-framework";
216
+ import { QueueImpl } from "@dxos/echo-db";
217
+ import { isInstanceOf } from "@dxos/echo-schema";
664
218
  import { ClientCapabilities, ClientEvents } from "@dxos/plugin-client";
665
219
 
666
- // packages/plugins/experimental/plugin-transcription/src/translations.ts
220
+ // packages/plugins/plugin-transcription/src/translations.ts
667
221
  var translations_default = [
668
222
  {
669
223
  "en-US": {
670
224
  [TranscriptType.typename]: {
671
- "typename label": "Transcript"
225
+ "typename label": "Transcript",
226
+ "object name placeholder": "New transcript"
672
227
  },
673
228
  [TRANSCRIPTION_PLUGIN]: {
674
229
  "plugin name": "Transcription",
675
- "calls label": "Transcript",
676
- "calls panel label": "Transcription",
677
- "create calls label": "Create transcription",
678
- "delete calls label": "Delete",
230
+ "transcript companion label": "Transcript",
231
+ "start transcription label": "Start transcription",
232
+ "stop transcription label": "Stop transcription",
679
233
  "delete button": "Delete",
680
234
  "bookmark button": "Bookmark",
681
- "summary button": "Create summary",
682
- "summarizing label": "Summarizing..."
235
+ "scroll to end label": "Scroll to latest"
683
236
  }
684
237
  }
685
238
  }
686
239
  ];
687
240
 
688
- // packages/plugins/experimental/plugin-transcription/src/TranscriptionPlugin.tsx
241
+ // packages/plugins/plugin-transcription/src/TranscriptionPlugin.tsx
689
242
  var TranscriptionPlugin = () => definePlugin(meta, [
690
243
  defineModule({
691
244
  id: `${meta.id}/module/translations`,
@@ -694,22 +247,29 @@ var TranscriptionPlugin = () => definePlugin(meta, [
694
247
  }),
695
248
  defineModule({
696
249
  id: `${meta.id}/module/transcription`,
697
- activatesOn: Events.Startup,
698
- activate: () => contributes(TranscriptionCapabilities.Transcription, null)
250
+ activatesOn: Events.SetupAppGraph,
251
+ activate: Transcriber2
252
+ }),
253
+ defineModule({
254
+ id: `${meta.id}/module/meeting-transcription-state`,
255
+ activatesOn: Events.SetupAppGraph,
256
+ activate: MeetingTranscriptionState
699
257
  }),
700
258
  defineModule({
701
259
  id: `${meta.id}/module/metadata`,
702
260
  activatesOn: Events.SetupMetadata,
703
- activate: () => contributes(Capabilities.Metadata, {
261
+ activate: (context) => contributes(Capabilities.Metadata, {
704
262
  id: TranscriptType.typename,
705
263
  metadata: {
706
- placeholder: [
707
- "transcript title placeholder",
708
- {
709
- ns: TRANSCRIPTION_PLUGIN
710
- }
711
- ],
712
- icon: "ph--subtitles--regular"
264
+ icon: "ph--subtitles--regular",
265
+ // TODO(wittjosiah): Factor out. Artifact? Separate capability?
266
+ getTextContent: async (transcript) => {
267
+ const client = context.requestCapability(ClientCapabilities.Client);
268
+ const queue = new QueueImpl(client.edge, transcript.queue.dxn);
269
+ await queue.refresh();
270
+ const content = queue.items.filter((block) => isInstanceOf(TranscriptBlock, block)).map((block) => `${block.authorName}: ${block.segments.map((segment) => segment.text).join("\n")}`).join("\n\n");
271
+ return content;
272
+ }
713
273
  }
714
274
  })
715
275
  }),
@@ -739,10 +299,7 @@ var TranscriptionPlugin = () => definePlugin(meta, [
739
299
  })
740
300
  ]);
741
301
  export {
742
- AppGraphBuilder,
743
- IntentResolver,
744
302
  MediaStreamRecorder,
745
- ReactSurface,
746
303
  TRANSCRIPTION_PLUGIN,
747
304
  Transcriber,
748
305
  Transcript,