@dxos/plugin-transcription 0.8.0 → 0.8.1-main.013e445

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/dist/lib/browser/{TranscriptContainer-SX6PNFEC.mjs → TranscriptContainer-TJXP6ICP.mjs} +3 -3
  2. package/dist/lib/browser/app-graph-builder-ZIIDYMHP.mjs +39 -0
  3. package/dist/lib/browser/app-graph-builder-ZIIDYMHP.mjs.map +7 -0
  4. package/dist/lib/browser/{chunk-AS364WZZ.mjs → chunk-3NWYXRZS.mjs} +2 -2
  5. package/dist/lib/browser/chunk-MD4NQO6W.mjs +252 -0
  6. package/dist/lib/browser/chunk-MD4NQO6W.mjs.map +7 -0
  7. package/dist/lib/browser/chunk-N3VKF2J3.mjs +415 -0
  8. package/dist/lib/browser/chunk-N3VKF2J3.mjs.map +7 -0
  9. package/dist/lib/browser/chunk-YEVVZZTC.mjs +19 -0
  10. package/dist/lib/browser/chunk-YEVVZZTC.mjs.map +7 -0
  11. package/dist/lib/browser/{chunk-QIHUPVQD.mjs → chunk-YRW4MNRE.mjs} +23 -24
  12. package/dist/lib/browser/chunk-YRW4MNRE.mjs.map +7 -0
  13. package/dist/lib/browser/index.mjs +203 -366
  14. package/dist/lib/browser/index.mjs.map +4 -4
  15. package/dist/lib/browser/intent-resolver-AC6LFBE4.mjs +73 -0
  16. package/dist/lib/browser/intent-resolver-AC6LFBE4.mjs.map +7 -0
  17. package/dist/lib/browser/meta.json +1 -1
  18. package/dist/lib/browser/{react-surface-FOEWLLTZ.mjs → react-surface-OM2EECBI.mjs} +15 -8
  19. package/dist/lib/browser/{react-surface-FOEWLLTZ.mjs.map → react-surface-OM2EECBI.mjs.map} +2 -2
  20. package/dist/lib/browser/types/index.mjs +2 -1
  21. package/dist/lib/node/{TranscriptContainer-HU763SN3.cjs → TranscriptContainer-KNTN76RZ.cjs} +8 -8
  22. package/dist/lib/node/{TranscriptContainer-HU763SN3.cjs.map → TranscriptContainer-KNTN76RZ.cjs.map} +2 -2
  23. package/dist/lib/node/app-graph-builder-CJSNQYJZ.cjs +52 -0
  24. package/dist/lib/node/app-graph-builder-CJSNQYJZ.cjs.map +7 -0
  25. package/dist/lib/node/chunk-EKSRBQX7.cjs +437 -0
  26. package/dist/lib/node/chunk-EKSRBQX7.cjs.map +7 -0
  27. package/dist/lib/node/{chunk-QWXLQPP5.cjs → chunk-I6CUIAQH.cjs} +5 -5
  28. package/dist/lib/node/{app-graph-builder-YFBX3OTP.cjs → chunk-LEHTJHKP.cjs} +23 -7
  29. package/dist/lib/node/chunk-LEHTJHKP.cjs.map +7 -0
  30. package/dist/lib/node/chunk-T4W4AKGM.cjs +279 -0
  31. package/dist/lib/node/chunk-T4W4AKGM.cjs.map +7 -0
  32. package/dist/lib/node/{chunk-UIHEGU6Q.cjs → chunk-ZAE2IOZC.cjs} +25 -27
  33. package/dist/lib/node/chunk-ZAE2IOZC.cjs.map +7 -0
  34. package/dist/lib/node/index.cjs +220 -378
  35. package/dist/lib/node/index.cjs.map +4 -4
  36. package/dist/lib/node/intent-resolver-ZSQ2MHJJ.cjs +84 -0
  37. package/dist/lib/node/intent-resolver-ZSQ2MHJJ.cjs.map +7 -0
  38. package/dist/lib/node/meta.json +1 -1
  39. package/dist/lib/node/{react-surface-AGGGDOUC.cjs → react-surface-ZENSCF6S.cjs} +18 -12
  40. package/dist/lib/node/{react-surface-AGGGDOUC.cjs.map → react-surface-ZENSCF6S.cjs.map} +2 -2
  41. package/dist/lib/node/types/index.cjs +8 -7
  42. package/dist/lib/node/types/index.cjs.map +2 -2
  43. package/dist/lib/node-esm/{TranscriptContainer-DVBXQIQ4.mjs → TranscriptContainer-MOWZQSEO.mjs} +3 -3
  44. package/dist/lib/node-esm/app-graph-builder-ARTLU4U5.mjs +40 -0
  45. package/dist/lib/node-esm/app-graph-builder-ARTLU4U5.mjs.map +7 -0
  46. package/dist/lib/node-esm/chunk-4A6BWPRB.mjs +253 -0
  47. package/dist/lib/node-esm/chunk-4A6BWPRB.mjs.map +7 -0
  48. package/dist/lib/node-esm/{chunk-KEG3ME4O.mjs → chunk-DLXORY7Y.mjs} +22 -24
  49. package/dist/lib/node-esm/chunk-DLXORY7Y.mjs.map +7 -0
  50. package/dist/lib/node-esm/{chunk-WEZGRUG6.mjs → chunk-NFPDIQYG.mjs} +2 -2
  51. package/dist/lib/node-esm/chunk-SB5TLXBU.mjs +21 -0
  52. package/dist/lib/node-esm/chunk-SB5TLXBU.mjs.map +7 -0
  53. package/dist/lib/node-esm/chunk-SWV74BYU.mjs +416 -0
  54. package/dist/lib/node-esm/chunk-SWV74BYU.mjs.map +7 -0
  55. package/dist/lib/node-esm/index.mjs +203 -366
  56. package/dist/lib/node-esm/index.mjs.map +4 -4
  57. package/dist/lib/node-esm/intent-resolver-XU4BLUSF.mjs +74 -0
  58. package/dist/lib/node-esm/intent-resolver-XU4BLUSF.mjs.map +7 -0
  59. package/dist/lib/node-esm/meta.json +1 -1
  60. package/dist/lib/node-esm/{react-surface-6AYPLJ2B.mjs → react-surface-5CBEXLXJ.mjs} +15 -8
  61. package/dist/lib/node-esm/{react-surface-6AYPLJ2B.mjs.map → react-surface-5CBEXLXJ.mjs.map} +2 -2
  62. package/dist/lib/node-esm/types/index.mjs +2 -1
  63. package/dist/types/src/TranscriptionPlugin.d.ts.map +1 -1
  64. package/dist/types/src/capabilities/app-graph-builder.d.ts +1 -0
  65. package/dist/types/src/capabilities/app-graph-builder.d.ts.map +1 -1
  66. package/dist/types/src/capabilities/index.d.ts.map +1 -1
  67. package/dist/types/src/capabilities/intent-resolver.d.ts.map +1 -1
  68. package/dist/types/src/components/Transcript/Transcript.d.ts +2 -0
  69. package/dist/types/src/components/Transcript/Transcript.d.ts.map +1 -1
  70. package/dist/types/src/components/Transcript/Transcript.stories.d.ts.map +1 -1
  71. package/dist/types/src/components/TranscriptContainer.d.ts +1 -0
  72. package/dist/types/src/components/TranscriptContainer.d.ts.map +1 -1
  73. package/dist/types/src/components/index.d.ts +1 -0
  74. package/dist/types/src/components/index.d.ts.map +1 -1
  75. package/dist/types/src/components/transcription.stories.d.ts +3 -2
  76. package/dist/types/src/components/transcription.stories.d.ts.map +1 -1
  77. package/dist/types/src/hooks/index.d.ts +0 -1
  78. package/dist/types/src/hooks/index.d.ts.map +1 -1
  79. package/dist/types/src/index.d.ts +1 -0
  80. package/dist/types/src/index.d.ts.map +1 -1
  81. package/dist/types/src/meta.d.ts +2 -9
  82. package/dist/types/src/meta.d.ts.map +1 -1
  83. package/dist/types/src/transcriber/index.d.ts +1 -0
  84. package/dist/types/src/transcriber/index.d.ts.map +1 -1
  85. package/dist/types/src/transcriber/media-stream-recorder.d.ts +1 -0
  86. package/dist/types/src/transcriber/media-stream-recorder.d.ts.map +1 -1
  87. package/dist/types/src/transcriber/summarize.d.ts +5 -0
  88. package/dist/types/src/transcriber/summarize.d.ts.map +1 -0
  89. package/dist/types/src/transcription-manager.d.ts +39 -0
  90. package/dist/types/src/transcription-manager.d.ts.map +1 -0
  91. package/dist/types/src/translations.d.ts +5 -6
  92. package/dist/types/src/translations.d.ts.map +1 -1
  93. package/dist/types/src/types/schema.d.ts +9 -15
  94. package/dist/types/src/types/schema.d.ts.map +1 -1
  95. package/dist/types/src/types/types.d.ts +13 -6
  96. package/dist/types/src/types/types.d.ts.map +1 -1
  97. package/dist/types/src/util/random-queue-dxn.d.ts +2 -2
  98. package/dist/types/src/util/random-queue-dxn.d.ts.map +1 -1
  99. package/package.json +40 -37
  100. package/src/TranscriptionPlugin.tsx +1 -2
  101. package/src/capabilities/app-graph-builder.ts +25 -2
  102. package/src/capabilities/intent-resolver.ts +24 -2
  103. package/src/capabilities/react-surface.tsx +2 -2
  104. package/src/components/Transcript/Transcript.stories.tsx +46 -27
  105. package/src/components/Transcript/Transcript.tsx +205 -49
  106. package/src/components/TranscriptContainer.tsx +12 -147
  107. package/src/components/transcription.stories.tsx +11 -13
  108. package/src/hooks/index.ts +0 -1
  109. package/src/index.ts +1 -0
  110. package/src/meta.ts +2 -2
  111. package/src/transcriber/index.ts +1 -0
  112. package/src/transcriber/media-stream-recorder.ts +4 -0
  113. package/src/transcriber/summarize.ts +77 -0
  114. package/src/transcription-manager.ts +183 -0
  115. package/src/translations.ts +2 -2
  116. package/src/types/schema.ts +5 -3
  117. package/src/types/types.ts +12 -2
  118. package/src/util/random-queue-dxn.ts +2 -2
  119. package/dist/lib/browser/app-graph-builder-ZSSJIFY7.mjs +0 -7
  120. package/dist/lib/browser/app-graph-builder-ZSSJIFY7.mjs.map +0 -7
  121. package/dist/lib/browser/chunk-QIHUPVQD.mjs.map +0 -7
  122. package/dist/lib/browser/chunk-R4HHS6FA.mjs +0 -241
  123. package/dist/lib/browser/chunk-R4HHS6FA.mjs.map +0 -7
  124. package/dist/lib/browser/chunk-ZMES3U2D.mjs +0 -78
  125. package/dist/lib/browser/chunk-ZMES3U2D.mjs.map +0 -7
  126. package/dist/lib/browser/intent-resolver-EAR2TFK2.mjs +0 -32
  127. package/dist/lib/browser/intent-resolver-EAR2TFK2.mjs.map +0 -7
  128. package/dist/lib/node/app-graph-builder-YFBX3OTP.cjs.map +0 -7
  129. package/dist/lib/node/chunk-JBN3P6ZF.cjs +0 -97
  130. package/dist/lib/node/chunk-JBN3P6ZF.cjs.map +0 -7
  131. package/dist/lib/node/chunk-UIHEGU6Q.cjs.map +0 -7
  132. package/dist/lib/node/chunk-YV74Z74I.cjs +0 -267
  133. package/dist/lib/node/chunk-YV74Z74I.cjs.map +0 -7
  134. package/dist/lib/node/intent-resolver-3KJZGGUQ.cjs +0 -44
  135. package/dist/lib/node/intent-resolver-3KJZGGUQ.cjs.map +0 -7
  136. package/dist/lib/node-esm/app-graph-builder-77GQG6UF.mjs +0 -9
  137. package/dist/lib/node-esm/app-graph-builder-77GQG6UF.mjs.map +0 -7
  138. package/dist/lib/node-esm/chunk-36EQXBWR.mjs +0 -242
  139. package/dist/lib/node-esm/chunk-36EQXBWR.mjs.map +0 -7
  140. package/dist/lib/node-esm/chunk-KEG3ME4O.mjs.map +0 -7
  141. package/dist/lib/node-esm/chunk-XYU6B73V.mjs +0 -80
  142. package/dist/lib/node-esm/chunk-XYU6B73V.mjs.map +0 -7
  143. package/dist/lib/node-esm/intent-resolver-UGLYRD4X.mjs +0 -33
  144. package/dist/lib/node-esm/intent-resolver-UGLYRD4X.mjs.map +0 -7
  145. package/dist/types/src/hooks/useIsSpeaking.d.ts +0 -2
  146. package/dist/types/src/hooks/useIsSpeaking.d.ts.map +0 -1
  147. package/src/hooks/useIsSpeaking.ts +0 -77
  148. /package/dist/lib/browser/{TranscriptContainer-SX6PNFEC.mjs.map → TranscriptContainer-TJXP6ICP.mjs.map} +0 -0
  149. /package/dist/lib/browser/{chunk-AS364WZZ.mjs.map → chunk-3NWYXRZS.mjs.map} +0 -0
  150. /package/dist/lib/node/{chunk-QWXLQPP5.cjs.map → chunk-I6CUIAQH.cjs.map} +0 -0
  151. /package/dist/lib/node-esm/{TranscriptContainer-DVBXQIQ4.mjs.map → TranscriptContainer-MOWZQSEO.mjs.map} +0 -0
  152. /package/dist/lib/node-esm/{chunk-WEZGRUG6.mjs.map → chunk-NFPDIQYG.mjs.map} +0 -0
@@ -1,20 +1,23 @@
1
1
  import {
2
- mergeFloat64Arrays,
3
- monitorAudioLevel
4
- } from "./chunk-ZMES3U2D.mjs";
2
+ MediaStreamRecorder,
3
+ Transcriber,
4
+ summarizeTranscript
5
+ } from "./chunk-N3VKF2J3.mjs";
5
6
  import {
6
7
  TranscriptContainer
7
- } from "./chunk-AS364WZZ.mjs";
8
+ } from "./chunk-3NWYXRZS.mjs";
9
+ import {
10
+ TranscriptBlock,
11
+ TranscriptType
12
+ } from "./chunk-YRW4MNRE.mjs";
8
13
  import {
9
14
  Transcript,
10
15
  TranscriptionContainer
11
- } from "./chunk-R4HHS6FA.mjs";
16
+ } from "./chunk-MD4NQO6W.mjs";
12
17
  import {
13
18
  TRANSCRIPTION_PLUGIN,
14
- TRANSCRIPTION_URL,
15
- TranscriptType,
16
19
  meta
17
- } from "./chunk-QIHUPVQD.mjs";
20
+ } from "./chunk-YEVVZZTC.mjs";
18
21
 
19
22
  // packages/plugins/experimental/plugin-transcription/src/capabilities/index.ts
20
23
  import { lazy } from "@dxos/app-framework";
@@ -27,9 +30,9 @@ var TranscriptionCapabilities;
27
30
  })(TranscriptionCapabilities || (TranscriptionCapabilities = {}));
28
31
 
29
32
  // packages/plugins/experimental/plugin-transcription/src/capabilities/index.ts
30
- var AppGraphBuilder = lazy(() => import("./app-graph-builder-ZSSJIFY7.mjs"));
31
- var IntentResolver = lazy(() => import("./intent-resolver-EAR2TFK2.mjs"));
32
- var ReactSurface = lazy(() => import("./react-surface-FOEWLLTZ.mjs"));
33
+ var AppGraphBuilder = lazy(() => import("./app-graph-builder-ZIIDYMHP.mjs"));
34
+ var IntentResolver = lazy(() => import("./intent-resolver-AC6LFBE4.mjs"));
35
+ var ReactSurface = lazy(() => import("./react-surface-OM2EECBI.mjs"));
33
36
 
34
37
  // packages/plugins/experimental/plugin-transcription/src/hooks/useAudioFile.ts
35
38
  import { useEffect, useState } from "react";
@@ -121,341 +124,8 @@ var useAudioTrack = (active) => {
121
124
  return track;
122
125
  };
123
126
 
124
- // packages/plugins/experimental/plugin-transcription/src/hooks/useIsSpeaking.ts
125
- import { useEffect as useEffect3, useRef, useState as useState3 } from "react";
126
- var useIsSpeaking = (mediaStreamTrack) => {
127
- const [isSpeaking, setIsSpeaking] = useState3(false);
128
- const isSpeakingRef = useRef(isSpeaking);
129
- useEffect3(() => {
130
- if (!mediaStreamTrack) {
131
- setIsSpeaking(false);
132
- return;
133
- }
134
- isSpeakingRef.current = isSpeaking;
135
- const interval = window.setInterval(() => {
136
- if (isSpeaking === isSpeakingRef.current) {
137
- return;
138
- }
139
- setIsSpeaking(isSpeakingRef.current);
140
- }, 50);
141
- return () => {
142
- clearInterval(interval);
143
- };
144
- }, [
145
- isSpeaking,
146
- isSpeakingRef,
147
- mediaStreamTrack
148
- ]);
149
- useEffect3(() => {
150
- if (!mediaStreamTrack) {
151
- return;
152
- }
153
- let timeout = -1;
154
- const cleanup = monitorAudioLevel({
155
- mediaStreamTrack,
156
- onMeasure: (vol) => {
157
- const audioLevelAboveThreshold = vol > (isSpeakingRef.current ? 0.02 : 0.05);
158
- if (audioLevelAboveThreshold) {
159
- clearTimeout(timeout);
160
- timeout = -1;
161
- isSpeakingRef.current = true;
162
- } else if (timeout === -1) {
163
- timeout = window.setTimeout(() => {
164
- isSpeakingRef.current = false;
165
- timeout = -1;
166
- }, 1e3);
167
- }
168
- }
169
- });
170
- return () => {
171
- cleanup();
172
- };
173
- }, [
174
- isSpeaking,
175
- mediaStreamTrack
176
- ]);
177
- return isSpeaking;
178
- };
179
-
180
- // packages/plugins/experimental/plugin-transcription/src/hooks/useTranscriber.ts
181
- import { useEffect as useEffect4, useMemo } from "react";
182
-
183
- // packages/plugins/experimental/plugin-transcription/src/transcriber/media-stream-recorder.ts
184
- import { MediaRecorder, register } from "extendable-media-recorder";
185
- import { WaveFile } from "wavefile";
186
- import { synchronized } from "@dxos/async";
187
- import { invariant } from "@dxos/invariant";
188
- import { log as log2 } from "@dxos/log";
189
- import { trace } from "@dxos/tracing";
190
- function _ts_decorate(decorators, target, key, desc) {
191
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
192
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
193
- else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
194
- return c > 3 && r && Object.defineProperty(target, key, r), r;
195
- }
196
- var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/transcriber/media-stream-recorder.ts";
197
- var initializingPromise;
198
- var initializeExtendableMediaRecorder = async () => {
199
- const { connect } = await import("extendable-media-recorder-wav-encoder");
200
- await register(await connect());
201
- };
202
- var MediaStreamRecorder = class {
203
- constructor({ mediaStreamTrack, interval }) {
204
- this._mediaRecorder = void 0;
205
- this._header = void 0;
206
- this._mediaStreamTrack = mediaStreamTrack;
207
- this._interval = interval;
208
- }
209
- get wavConfig() {
210
- const settings = this._mediaStreamTrack.getSettings();
211
- return {
212
- channels: settings.channelCount ?? 1,
213
- sampleRate: settings.sampleRate ?? 16e3,
214
- bitDepthCode: settings.sampleSize ? String(settings.sampleSize) : "16"
215
- };
216
- }
217
- setOnChunk(onChunk) {
218
- this._onChunk = onChunk;
219
- }
220
- async start() {
221
- await (initializingPromise ??= initializeExtendableMediaRecorder()).catch((err) => log2.info("initializeExtendableMediaRecorder", {
222
- err
223
- }, {
224
- F: __dxlog_file2,
225
- L: 58,
226
- S: this,
227
- C: (f, a) => f(...a)
228
- }));
229
- if (!this._mediaRecorder) {
230
- const stream = new MediaStream([
231
- this._mediaStreamTrack
232
- ]);
233
- this._mediaRecorder = new MediaRecorder(stream, {
234
- mimeType: "audio/wav"
235
- });
236
- }
237
- if (this._mediaRecorder.state === "recording") {
238
- return;
239
- }
240
- invariant(this._onChunk, "MediaStreamRecorder: onChunk is not set", {
241
- F: __dxlog_file2,
242
- L: 69,
243
- S: this,
244
- A: [
245
- "this._onChunk",
246
- "'MediaStreamRecorder: onChunk is not set'"
247
- ]
248
- });
249
- this._mediaRecorder.ondataavailable = (event) => this._ondataavailable(event);
250
- this._mediaRecorder.start(this._interval);
251
- }
252
- async stop() {
253
- if (this._mediaRecorder?.state !== "recording") {
254
- return;
255
- }
256
- this._mediaRecorder.stop();
257
- }
258
- async _ondataavailable(event) {
259
- const blob = event.data;
260
- const uint8Array = new Uint8Array(await blob.arrayBuffer());
261
- const isHeader = uint8Array[0] === 82 && // R
262
- uint8Array[1] === 73 && // I
263
- uint8Array[2] === 70 && // F
264
- uint8Array[3] === 70;
265
- let wav;
266
- if (isHeader) {
267
- wav = new WaveFile(uint8Array);
268
- this._header = uint8Array.slice(0, 44);
269
- } else if (this._header) {
270
- wav = new WaveFile(new Uint8Array([
271
- ...this._header,
272
- ...uint8Array
273
- ]));
274
- } else {
275
- log2.warn("MediaStreamRecorder: no header", void 0, {
276
- F: __dxlog_file2,
277
- L: 100,
278
- S: this,
279
- C: (f, a) => f(...a)
280
- });
281
- return;
282
- }
283
- this._onChunk({
284
- timestamp: Date.now(),
285
- data: wav.getSamples()
286
- });
287
- }
288
- };
289
- _ts_decorate([
290
- synchronized
291
- ], MediaStreamRecorder.prototype, "_ondataavailable", null);
292
- MediaStreamRecorder = _ts_decorate([
293
- trace.resource()
294
- ], MediaStreamRecorder);
295
-
296
- // packages/plugins/experimental/plugin-transcription/src/transcriber/transcriber.ts
297
- import { WaveFile as WaveFile2 } from "wavefile";
298
- import { DeferredTask, Trigger } from "@dxos/async";
299
- import { LifecycleState, Resource } from "@dxos/context";
300
- import { log as log3 } from "@dxos/log";
301
- import { trace as trace2 } from "@dxos/tracing";
302
- function _ts_decorate2(decorators, target, key, desc) {
303
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
304
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
305
- else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
306
- return c > 3 && r && Object.defineProperty(target, key, r), r;
307
- }
308
- var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/transcriber/transcriber.ts";
309
- var Transcriber = class extends Resource {
310
- constructor({ config, recorder, onSegments }) {
311
- super();
312
- this._audioChunks = [];
313
- this._lastUsedTimestamp = 0;
314
- this._openTrigger = new Trigger({
315
- autoReset: true
316
- });
317
- this._recording = false;
318
- this._transcribeTask = void 0;
319
- this._config = config;
320
- this._recorder = recorder;
321
- this._onSegments = onSegments;
322
- }
323
- async _open(ctx) {
324
- this._recorder.setOnChunk((chunk) => this._saveAudioChunk(chunk));
325
- await this._recorder.start();
326
- this._transcribeTask = new DeferredTask(ctx, async () => this._transcribe());
327
- this._openTrigger.wake();
328
- }
329
- async _close() {
330
- this._recording = false;
331
- this._transcribeTask = void 0;
332
- await this._recorder.stop();
333
- }
334
- startChunksRecording() {
335
- if (this._lifecycleState !== LifecycleState.OPEN) {
336
- return;
337
- }
338
- this._recording = true;
339
- }
340
- stopChunksRecording() {
341
- if (this._lifecycleState !== LifecycleState.OPEN || !this._recording) {
342
- return;
343
- }
344
- this._recording = false;
345
- this._transcribeTask?.schedule();
346
- }
347
- _saveAudioChunk(chunk) {
348
- log3("saving audio chunk", {
349
- chunk
350
- }, {
351
- F: __dxlog_file3,
352
- L: 124,
353
- S: this,
354
- C: (f, a) => f(...a)
355
- });
356
- this._audioChunks.push(chunk);
357
- if (!this._recording && (!this._transcribeTask || !this._transcribeTask.scheduled) && this._audioChunks.length >= this._config.prefixBufferChunksAmount) {
358
- this._audioChunks = this._dropOldChunks();
359
- } else if (this._audioChunks.length >= this._config.transcribeAfterChunksAmount && this._recording) {
360
- this._transcribeTask?.schedule();
361
- }
362
- }
363
- async _transcribe() {
364
- log3("transcribing", {
365
- chunks: this._audioChunks.length
366
- }, {
367
- F: __dxlog_file3,
368
- L: 140,
369
- S: this,
370
- C: (f, a) => f(...a)
371
- });
372
- const chunks = this._audioChunks;
373
- this._audioChunks = this._dropOldChunks();
374
- if (chunks.length === 0) {
375
- return;
376
- }
377
- const audio = await this._mergeAudioChunks(chunks);
378
- const segments = await this._fetchTranscription(audio);
379
- if (!Array.isArray(segments) || segments.length === 0) {
380
- return;
381
- }
382
- const alignedSegments = this._alignSegments(segments, chunks);
383
- if (alignedSegments.length > 0) {
384
- await this._onSegments(alignedSegments);
385
- }
386
- }
387
- _dropOldChunks() {
388
- return this._config.prefixBufferChunksAmount > 0 ? this._audioChunks.slice(-this._config.prefixBufferChunksAmount) : [];
389
- }
390
- async _mergeAudioChunks(chunks) {
391
- const file = new WaveFile2();
392
- const wavConfig = this._recorder.wavConfig;
393
- file.fromScratch(wavConfig.channels, wavConfig.sampleRate, wavConfig.bitDepthCode, mergeFloat64Arrays(chunks.map(({ data }) => data)));
394
- return file.toBase64();
395
- }
396
- async _fetchTranscription(audio) {
397
- if (audio.length === 0) {
398
- this._audioChunks = [];
399
- throw new Error("No audio to send for transcribing");
400
- }
401
- const response = await fetch(`${TRANSCRIPTION_URL}/transcribe`, {
402
- method: "POST",
403
- body: JSON.stringify({
404
- audio
405
- }),
406
- headers: {
407
- "Content-Type": "application/json"
408
- }
409
- });
410
- if (!response.ok) {
411
- this._audioChunks = [];
412
- throw new Error("Failed to transcribe");
413
- }
414
- const { segments } = await response.json();
415
- log3.info("transcription response", {
416
- segments,
417
- string: segments.map((segments2) => segments2.text).join(" ")
418
- }, {
419
- F: __dxlog_file3,
420
- L: 206,
421
- S: this,
422
- C: (f, a) => f(...a)
423
- });
424
- return segments;
425
- }
426
- _alignSegments(segments, originalChunks) {
427
- const zeroTimestamp = originalChunks.at(0).timestamp;
428
- const filteredSegments = segments.filter((segment) => zeroTimestamp + segment.end * 1e3 >= this._lastUsedTimestamp).map((segment) => {
429
- const words = segment.words.filter((word) => zeroTimestamp + word.start * 1e3 >= this._lastUsedTimestamp);
430
- return {
431
- ...segment,
432
- words,
433
- text: words.map((word) => word.word).join(""),
434
- start: words.at(0)?.start ?? segment.end
435
- };
436
- }).filter((segment) => segment.words.length > 0);
437
- if (filteredSegments.length === 0) {
438
- return [];
439
- }
440
- this._lastUsedTimestamp = zeroTimestamp + filteredSegments.at(-1).end * 1e3;
441
- return filteredSegments.map((segment) => ({
442
- started: new Date(zeroTimestamp + segment.start * 1e3),
443
- text: segment.text
444
- }));
445
- }
446
- };
447
- _ts_decorate2([
448
- trace2.span({
449
- showInBrowserTimeline: true
450
- })
451
- ], Transcriber.prototype, "_mergeAudioChunks", null);
452
- _ts_decorate2([
453
- trace2.span({
454
- showInBrowserTimeline: true
455
- })
456
- ], Transcriber.prototype, "_fetchTranscription", null);
457
-
458
127
  // packages/plugins/experimental/plugin-transcription/src/hooks/useTranscriber.ts
128
+ import { useEffect as useEffect3, useMemo } from "react";
459
129
  var RECORD_INTERVAL = 200;
460
130
  var PREFIXED_CHUNKS_AMOUNT = 10;
461
131
  var TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
@@ -479,7 +149,7 @@ var useTranscriber = ({ audioStreamTrack, onSegments }) => {
479
149
  audioStreamTrack,
480
150
  onSegments
481
151
  ]);
482
- useEffect4(() => {
152
+ useEffect3(() => {
483
153
  return () => {
484
154
  void transcriber?.close();
485
155
  };
@@ -490,12 +160,12 @@ var useTranscriber = ({ audioStreamTrack, onSegments }) => {
490
160
  };
491
161
 
492
162
  // packages/plugins/experimental/plugin-transcription/src/hooks/useVoiceInput.ts
493
- import { useState as useState4, useEffect as useEffect5, useCallback } from "react";
163
+ import { useState as useState3, useEffect as useEffect4, useCallback } from "react";
494
164
  import { scheduleMicroTask } from "@dxos/async";
495
165
  import { Context } from "@dxos/context";
496
- import { log as log4 } from "@dxos/log";
166
+ import { log as log2 } from "@dxos/log";
497
167
  import { useSoundEffect } from "@dxos/react-ui-sfx";
498
- var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/hooks/useVoiceInput.ts";
168
+ var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/hooks/useVoiceInput.ts";
499
169
  var useVoiceInput = ({ active, onUpdate }) => {
500
170
  const soundStart = useSoundEffect("StartRecording");
501
171
  const soundStop = useSoundEffect("StopRecording");
@@ -503,22 +173,22 @@ var useVoiceInput = ({ active, onUpdate }) => {
503
173
  const text = segments.map((str) => str.text.trim().replace(/[^\w\s]/g, "")).join(" ");
504
174
  onUpdate(text);
505
175
  }, []);
506
- const [recording, setRecording] = useState4(false);
176
+ const [recording, setRecording] = useState3(false);
507
177
  const track = useAudioTrack(active);
508
178
  const transcriber = useTranscriber({
509
179
  audioStreamTrack: track,
510
180
  onSegments: handleSegments
511
181
  });
512
- useEffect5(() => {
182
+ useEffect4(() => {
513
183
  const ctx = new Context(void 0, {
514
- F: __dxlog_file4,
184
+ F: __dxlog_file2,
515
185
  L: 43
516
186
  });
517
187
  scheduleMicroTask(ctx, async () => {
518
188
  if (active && transcriber) {
519
189
  await transcriber.open();
520
- log4.info("starting...", void 0, {
521
- F: __dxlog_file4,
190
+ log2.info("starting...", void 0, {
191
+ F: __dxlog_file2,
522
192
  L: 47,
523
193
  S: void 0,
524
194
  C: (f, a) => f(...a)
@@ -529,8 +199,8 @@ var useVoiceInput = ({ active, onUpdate }) => {
529
199
  } else if (!active && transcriber?.isOpen) {
530
200
  transcriber?.stopChunksRecording();
531
201
  await transcriber?.close();
532
- log4.info("stopped", void 0, {
533
- F: __dxlog_file4,
202
+ log2.info("stopped", void 0, {
203
+ F: __dxlog_file2,
534
204
  L: 54,
535
205
  S: void 0,
536
206
  C: (f, a) => f(...a)
@@ -551,6 +221,178 @@ var useVoiceInput = ({ active, onUpdate }) => {
551
221
  };
552
222
  };
553
223
 
224
+ // packages/plugins/experimental/plugin-transcription/src/transcription-manager.ts
225
+ import { synchronized } from "@dxos/async";
226
+ import { Resource } from "@dxos/context";
227
+ import { QueueImpl } from "@dxos/echo-db";
228
+ import { createStatic } from "@dxos/echo-schema";
229
+ import { invariant } from "@dxos/invariant";
230
+ import { DXN } from "@dxos/keys";
231
+ function _ts_decorate(decorators, target, key, desc) {
232
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
233
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
234
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
235
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
236
+ }
237
+ var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/transcription-manager.ts";
238
+ var RECORD_INTERVAL2 = 200;
239
+ var PREFIXED_CHUNKS_AMOUNT2 = 10;
240
+ var TRANSCRIBE_AFTER_CHUNKS_AMOUNT2 = 50;
241
+ var TranscriptionManager = class extends Resource {
242
+ constructor(edgeClient) {
243
+ super();
244
+ this._audioStreamTrack = void 0;
245
+ this._name = void 0;
246
+ this._hue = void 0;
247
+ this._mediaRecorder = void 0;
248
+ this._transcriber = void 0;
249
+ this._enabled = false;
250
+ this._queue = void 0;
251
+ this._edgeClient = edgeClient;
252
+ }
253
+ async _open() {
254
+ await this._toggleTranscriber();
255
+ }
256
+ async _close() {
257
+ void this._transcriber?.close();
258
+ }
259
+ setRecording(recording) {
260
+ if (!this.isOpen || !this._enabled) {
261
+ return;
262
+ }
263
+ if (recording) {
264
+ this._transcriber?.startChunksRecording();
265
+ } else {
266
+ this._transcriber?.stopChunksRecording();
267
+ }
268
+ }
269
+ /**
270
+ * Enable or disable transcription.
271
+ * @param enabled - Whether to enable transcription.
272
+ */
273
+ async setEnabled(enabled) {
274
+ if (this._enabled === enabled) {
275
+ return;
276
+ }
277
+ this._enabled = enabled ?? false;
278
+ this.isOpen && await this._toggleTranscriber();
279
+ }
280
+ async setAudioTrack(track) {
281
+ if (this._audioStreamTrack === track) {
282
+ return;
283
+ }
284
+ this._audioStreamTrack = track;
285
+ this.isOpen && await this._toggleTranscriber();
286
+ }
287
+ /**
288
+ * Set the queue to save the transcription to.
289
+ * @param queue - The queue to save the transcription to or the DXN of the queue.
290
+ */
291
+ setQueue(queue) {
292
+ switch (typeof queue) {
293
+ case "string":
294
+ if (this._queue?.dxn.toString() === queue) {
295
+ return;
296
+ }
297
+ this._queue = new QueueImpl(this._edgeClient, DXN.parse(queue));
298
+ break;
299
+ case "object":
300
+ if (this._queue === queue) {
301
+ return;
302
+ }
303
+ invariant(queue instanceof QueueImpl, void 0, {
304
+ F: __dxlog_file3,
305
+ L: 112,
306
+ S: this,
307
+ A: [
308
+ "queue instanceof QueueImpl",
309
+ ""
310
+ ]
311
+ });
312
+ this._queue = queue;
313
+ break;
314
+ case "undefined":
315
+ this._queue = void 0;
316
+ break;
317
+ }
318
+ }
319
+ setName(name) {
320
+ if (this._name === name) {
321
+ return;
322
+ }
323
+ this._name = name;
324
+ }
325
+ setHue(hue) {
326
+ if (this._hue === hue) {
327
+ return;
328
+ }
329
+ this._hue = hue;
330
+ }
331
+ async _toggleTranscriber() {
332
+ await this._maybeReinitTranscriber();
333
+ if (this._enabled) {
334
+ await this._transcriber?.open();
335
+ } else {
336
+ await this._transcriber?.close();
337
+ }
338
+ }
339
+ async _maybeReinitTranscriber() {
340
+ if (!this._audioStreamTrack || !this._enabled) {
341
+ return;
342
+ }
343
+ let needReinit = false;
344
+ if (this._audioStreamTrack !== this._mediaRecorder?.mediaStreamTrack) {
345
+ this._mediaRecorder = new MediaStreamRecorder({
346
+ mediaStreamTrack: this._audioStreamTrack,
347
+ interval: RECORD_INTERVAL2
348
+ });
349
+ needReinit = true;
350
+ }
351
+ if (needReinit) {
352
+ await this._transcriber?.close();
353
+ this._transcriber = new Transcriber({
354
+ config: {
355
+ transcribeAfterChunksAmount: TRANSCRIBE_AFTER_CHUNKS_AMOUNT2,
356
+ prefixBufferChunksAmount: PREFIXED_CHUNKS_AMOUNT2
357
+ },
358
+ recorder: this._mediaRecorder,
359
+ onSegments: (segments) => this._onSegments(segments)
360
+ });
361
+ }
362
+ }
363
+ async _onSegments(segments) {
364
+ if (!this.isOpen || !this._queue) {
365
+ return;
366
+ }
367
+ const block = createStatic(TranscriptBlock, {
368
+ authorName: this._name,
369
+ authorHue: this._hue,
370
+ segments
371
+ });
372
+ this._queue.append([
373
+ block
374
+ ]);
375
+ }
376
+ };
377
+ _ts_decorate([
378
+ synchronized
379
+ ], TranscriptionManager.prototype, "setRecording", null);
380
+ _ts_decorate([
381
+ synchronized
382
+ ], TranscriptionManager.prototype, "setEnabled", null);
383
+ _ts_decorate([
384
+ synchronized
385
+ ], TranscriptionManager.prototype, "setAudioTrack", null);
386
+ _ts_decorate([
387
+ synchronized
388
+ ], TranscriptionManager.prototype, "setQueue", null);
389
+ _ts_decorate([
390
+ synchronized
391
+ ], TranscriptionManager.prototype, "setName", null);
392
+ _ts_decorate([
393
+ synchronized
394
+ ], TranscriptionManager.prototype, "setHue", null);
395
+
554
396
  // packages/plugins/experimental/plugin-transcription/src/TranscriptionPlugin.tsx
555
397
  import { Capabilities, Events, contributes, defineModule, definePlugin } from "@dxos/app-framework";
556
398
  import { ClientCapabilities, ClientEvents } from "@dxos/plugin-client";
@@ -560,7 +402,8 @@ var translations_default = [
560
402
  {
561
403
  "en-US": {
562
404
  [TranscriptType.typename]: {
563
- "typename label": "Transcript"
405
+ "typename label": "Transcript",
406
+ "object name placeholder": "New transcript"
564
407
  },
565
408
  [TRANSCRIPTION_PLUGIN]: {
566
409
  "plugin name": "Transcription",
@@ -570,8 +413,7 @@ var translations_default = [
570
413
  "delete calls label": "Delete",
571
414
  "delete button": "Delete",
572
415
  "bookmark button": "Bookmark",
573
- "summary button": "Create summary",
574
- "summarizing label": "Summarizing..."
416
+ "scroll to end label": "Scroll to latest"
575
417
  }
576
418
  }
577
419
  }
@@ -595,12 +437,6 @@ var TranscriptionPlugin = () => definePlugin(meta, [
595
437
  activate: () => contributes(Capabilities.Metadata, {
596
438
  id: TranscriptType.typename,
597
439
  metadata: {
598
- placeholder: [
599
- "transcript title placeholder",
600
- {
601
- ns: TRANSCRIPTION_PLUGIN
602
- }
603
- ],
604
440
  icon: "ph--subtitles--regular"
605
441
  }
606
442
  })
@@ -641,11 +477,12 @@ export {
641
477
  TranscriptContainer,
642
478
  TranscriptionCapabilities,
643
479
  TranscriptionContainer,
480
+ TranscriptionManager,
644
481
  TranscriptionPlugin,
645
482
  meta,
483
+ summarizeTranscript,
646
484
  useAudioFile,
647
485
  useAudioTrack,
648
- useIsSpeaking,
649
486
  useTranscriber,
650
487
  useVoiceInput
651
488
  };