@dxos/plugin-transcription 0.8.0 → 0.8.1-main.013e445

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/dist/lib/browser/{TranscriptContainer-SX6PNFEC.mjs → TranscriptContainer-TJXP6ICP.mjs} +3 -3
  2. package/dist/lib/browser/app-graph-builder-ZIIDYMHP.mjs +39 -0
  3. package/dist/lib/browser/app-graph-builder-ZIIDYMHP.mjs.map +7 -0
  4. package/dist/lib/browser/{chunk-AS364WZZ.mjs → chunk-3NWYXRZS.mjs} +2 -2
  5. package/dist/lib/browser/chunk-MD4NQO6W.mjs +252 -0
  6. package/dist/lib/browser/chunk-MD4NQO6W.mjs.map +7 -0
  7. package/dist/lib/browser/chunk-N3VKF2J3.mjs +415 -0
  8. package/dist/lib/browser/chunk-N3VKF2J3.mjs.map +7 -0
  9. package/dist/lib/browser/chunk-YEVVZZTC.mjs +19 -0
  10. package/dist/lib/browser/chunk-YEVVZZTC.mjs.map +7 -0
  11. package/dist/lib/browser/{chunk-QIHUPVQD.mjs → chunk-YRW4MNRE.mjs} +23 -24
  12. package/dist/lib/browser/chunk-YRW4MNRE.mjs.map +7 -0
  13. package/dist/lib/browser/index.mjs +203 -366
  14. package/dist/lib/browser/index.mjs.map +4 -4
  15. package/dist/lib/browser/intent-resolver-AC6LFBE4.mjs +73 -0
  16. package/dist/lib/browser/intent-resolver-AC6LFBE4.mjs.map +7 -0
  17. package/dist/lib/browser/meta.json +1 -1
  18. package/dist/lib/browser/{react-surface-FOEWLLTZ.mjs → react-surface-OM2EECBI.mjs} +15 -8
  19. package/dist/lib/browser/{react-surface-FOEWLLTZ.mjs.map → react-surface-OM2EECBI.mjs.map} +2 -2
  20. package/dist/lib/browser/types/index.mjs +2 -1
  21. package/dist/lib/node/{TranscriptContainer-HU763SN3.cjs → TranscriptContainer-KNTN76RZ.cjs} +8 -8
  22. package/dist/lib/node/{TranscriptContainer-HU763SN3.cjs.map → TranscriptContainer-KNTN76RZ.cjs.map} +2 -2
  23. package/dist/lib/node/app-graph-builder-CJSNQYJZ.cjs +52 -0
  24. package/dist/lib/node/app-graph-builder-CJSNQYJZ.cjs.map +7 -0
  25. package/dist/lib/node/chunk-EKSRBQX7.cjs +437 -0
  26. package/dist/lib/node/chunk-EKSRBQX7.cjs.map +7 -0
  27. package/dist/lib/node/{chunk-QWXLQPP5.cjs → chunk-I6CUIAQH.cjs} +5 -5
  28. package/dist/lib/node/{app-graph-builder-YFBX3OTP.cjs → chunk-LEHTJHKP.cjs} +23 -7
  29. package/dist/lib/node/chunk-LEHTJHKP.cjs.map +7 -0
  30. package/dist/lib/node/chunk-T4W4AKGM.cjs +279 -0
  31. package/dist/lib/node/chunk-T4W4AKGM.cjs.map +7 -0
  32. package/dist/lib/node/{chunk-UIHEGU6Q.cjs → chunk-ZAE2IOZC.cjs} +25 -27
  33. package/dist/lib/node/chunk-ZAE2IOZC.cjs.map +7 -0
  34. package/dist/lib/node/index.cjs +220 -378
  35. package/dist/lib/node/index.cjs.map +4 -4
  36. package/dist/lib/node/intent-resolver-ZSQ2MHJJ.cjs +84 -0
  37. package/dist/lib/node/intent-resolver-ZSQ2MHJJ.cjs.map +7 -0
  38. package/dist/lib/node/meta.json +1 -1
  39. package/dist/lib/node/{react-surface-AGGGDOUC.cjs → react-surface-ZENSCF6S.cjs} +18 -12
  40. package/dist/lib/node/{react-surface-AGGGDOUC.cjs.map → react-surface-ZENSCF6S.cjs.map} +2 -2
  41. package/dist/lib/node/types/index.cjs +8 -7
  42. package/dist/lib/node/types/index.cjs.map +2 -2
  43. package/dist/lib/node-esm/{TranscriptContainer-DVBXQIQ4.mjs → TranscriptContainer-MOWZQSEO.mjs} +3 -3
  44. package/dist/lib/node-esm/app-graph-builder-ARTLU4U5.mjs +40 -0
  45. package/dist/lib/node-esm/app-graph-builder-ARTLU4U5.mjs.map +7 -0
  46. package/dist/lib/node-esm/chunk-4A6BWPRB.mjs +253 -0
  47. package/dist/lib/node-esm/chunk-4A6BWPRB.mjs.map +7 -0
  48. package/dist/lib/node-esm/{chunk-KEG3ME4O.mjs → chunk-DLXORY7Y.mjs} +22 -24
  49. package/dist/lib/node-esm/chunk-DLXORY7Y.mjs.map +7 -0
  50. package/dist/lib/node-esm/{chunk-WEZGRUG6.mjs → chunk-NFPDIQYG.mjs} +2 -2
  51. package/dist/lib/node-esm/chunk-SB5TLXBU.mjs +21 -0
  52. package/dist/lib/node-esm/chunk-SB5TLXBU.mjs.map +7 -0
  53. package/dist/lib/node-esm/chunk-SWV74BYU.mjs +416 -0
  54. package/dist/lib/node-esm/chunk-SWV74BYU.mjs.map +7 -0
  55. package/dist/lib/node-esm/index.mjs +203 -366
  56. package/dist/lib/node-esm/index.mjs.map +4 -4
  57. package/dist/lib/node-esm/intent-resolver-XU4BLUSF.mjs +74 -0
  58. package/dist/lib/node-esm/intent-resolver-XU4BLUSF.mjs.map +7 -0
  59. package/dist/lib/node-esm/meta.json +1 -1
  60. package/dist/lib/node-esm/{react-surface-6AYPLJ2B.mjs → react-surface-5CBEXLXJ.mjs} +15 -8
  61. package/dist/lib/node-esm/{react-surface-6AYPLJ2B.mjs.map → react-surface-5CBEXLXJ.mjs.map} +2 -2
  62. package/dist/lib/node-esm/types/index.mjs +2 -1
  63. package/dist/types/src/TranscriptionPlugin.d.ts.map +1 -1
  64. package/dist/types/src/capabilities/app-graph-builder.d.ts +1 -0
  65. package/dist/types/src/capabilities/app-graph-builder.d.ts.map +1 -1
  66. package/dist/types/src/capabilities/index.d.ts.map +1 -1
  67. package/dist/types/src/capabilities/intent-resolver.d.ts.map +1 -1
  68. package/dist/types/src/components/Transcript/Transcript.d.ts +2 -0
  69. package/dist/types/src/components/Transcript/Transcript.d.ts.map +1 -1
  70. package/dist/types/src/components/Transcript/Transcript.stories.d.ts.map +1 -1
  71. package/dist/types/src/components/TranscriptContainer.d.ts +1 -0
  72. package/dist/types/src/components/TranscriptContainer.d.ts.map +1 -1
  73. package/dist/types/src/components/index.d.ts +1 -0
  74. package/dist/types/src/components/index.d.ts.map +1 -1
  75. package/dist/types/src/components/transcription.stories.d.ts +3 -2
  76. package/dist/types/src/components/transcription.stories.d.ts.map +1 -1
  77. package/dist/types/src/hooks/index.d.ts +0 -1
  78. package/dist/types/src/hooks/index.d.ts.map +1 -1
  79. package/dist/types/src/index.d.ts +1 -0
  80. package/dist/types/src/index.d.ts.map +1 -1
  81. package/dist/types/src/meta.d.ts +2 -9
  82. package/dist/types/src/meta.d.ts.map +1 -1
  83. package/dist/types/src/transcriber/index.d.ts +1 -0
  84. package/dist/types/src/transcriber/index.d.ts.map +1 -1
  85. package/dist/types/src/transcriber/media-stream-recorder.d.ts +1 -0
  86. package/dist/types/src/transcriber/media-stream-recorder.d.ts.map +1 -1
  87. package/dist/types/src/transcriber/summarize.d.ts +5 -0
  88. package/dist/types/src/transcriber/summarize.d.ts.map +1 -0
  89. package/dist/types/src/transcription-manager.d.ts +39 -0
  90. package/dist/types/src/transcription-manager.d.ts.map +1 -0
  91. package/dist/types/src/translations.d.ts +5 -6
  92. package/dist/types/src/translations.d.ts.map +1 -1
  93. package/dist/types/src/types/schema.d.ts +9 -15
  94. package/dist/types/src/types/schema.d.ts.map +1 -1
  95. package/dist/types/src/types/types.d.ts +13 -6
  96. package/dist/types/src/types/types.d.ts.map +1 -1
  97. package/dist/types/src/util/random-queue-dxn.d.ts +2 -2
  98. package/dist/types/src/util/random-queue-dxn.d.ts.map +1 -1
  99. package/package.json +40 -37
  100. package/src/TranscriptionPlugin.tsx +1 -2
  101. package/src/capabilities/app-graph-builder.ts +25 -2
  102. package/src/capabilities/intent-resolver.ts +24 -2
  103. package/src/capabilities/react-surface.tsx +2 -2
  104. package/src/components/Transcript/Transcript.stories.tsx +46 -27
  105. package/src/components/Transcript/Transcript.tsx +205 -49
  106. package/src/components/TranscriptContainer.tsx +12 -147
  107. package/src/components/transcription.stories.tsx +11 -13
  108. package/src/hooks/index.ts +0 -1
  109. package/src/index.ts +1 -0
  110. package/src/meta.ts +2 -2
  111. package/src/transcriber/index.ts +1 -0
  112. package/src/transcriber/media-stream-recorder.ts +4 -0
  113. package/src/transcriber/summarize.ts +77 -0
  114. package/src/transcription-manager.ts +183 -0
  115. package/src/translations.ts +2 -2
  116. package/src/types/schema.ts +5 -3
  117. package/src/types/types.ts +12 -2
  118. package/src/util/random-queue-dxn.ts +2 -2
  119. package/dist/lib/browser/app-graph-builder-ZSSJIFY7.mjs +0 -7
  120. package/dist/lib/browser/app-graph-builder-ZSSJIFY7.mjs.map +0 -7
  121. package/dist/lib/browser/chunk-QIHUPVQD.mjs.map +0 -7
  122. package/dist/lib/browser/chunk-R4HHS6FA.mjs +0 -241
  123. package/dist/lib/browser/chunk-R4HHS6FA.mjs.map +0 -7
  124. package/dist/lib/browser/chunk-ZMES3U2D.mjs +0 -78
  125. package/dist/lib/browser/chunk-ZMES3U2D.mjs.map +0 -7
  126. package/dist/lib/browser/intent-resolver-EAR2TFK2.mjs +0 -32
  127. package/dist/lib/browser/intent-resolver-EAR2TFK2.mjs.map +0 -7
  128. package/dist/lib/node/app-graph-builder-YFBX3OTP.cjs.map +0 -7
  129. package/dist/lib/node/chunk-JBN3P6ZF.cjs +0 -97
  130. package/dist/lib/node/chunk-JBN3P6ZF.cjs.map +0 -7
  131. package/dist/lib/node/chunk-UIHEGU6Q.cjs.map +0 -7
  132. package/dist/lib/node/chunk-YV74Z74I.cjs +0 -267
  133. package/dist/lib/node/chunk-YV74Z74I.cjs.map +0 -7
  134. package/dist/lib/node/intent-resolver-3KJZGGUQ.cjs +0 -44
  135. package/dist/lib/node/intent-resolver-3KJZGGUQ.cjs.map +0 -7
  136. package/dist/lib/node-esm/app-graph-builder-77GQG6UF.mjs +0 -9
  137. package/dist/lib/node-esm/app-graph-builder-77GQG6UF.mjs.map +0 -7
  138. package/dist/lib/node-esm/chunk-36EQXBWR.mjs +0 -242
  139. package/dist/lib/node-esm/chunk-36EQXBWR.mjs.map +0 -7
  140. package/dist/lib/node-esm/chunk-KEG3ME4O.mjs.map +0 -7
  141. package/dist/lib/node-esm/chunk-XYU6B73V.mjs +0 -80
  142. package/dist/lib/node-esm/chunk-XYU6B73V.mjs.map +0 -7
  143. package/dist/lib/node-esm/intent-resolver-UGLYRD4X.mjs +0 -33
  144. package/dist/lib/node-esm/intent-resolver-UGLYRD4X.mjs.map +0 -7
  145. package/dist/types/src/hooks/useIsSpeaking.d.ts +0 -2
  146. package/dist/types/src/hooks/useIsSpeaking.d.ts.map +0 -1
  147. package/src/hooks/useIsSpeaking.ts +0 -77
  148. /package/dist/lib/browser/{TranscriptContainer-SX6PNFEC.mjs.map → TranscriptContainer-TJXP6ICP.mjs.map} +0 -0
  149. /package/dist/lib/browser/{chunk-AS364WZZ.mjs.map → chunk-3NWYXRZS.mjs.map} +0 -0
  150. /package/dist/lib/node/{chunk-QWXLQPP5.cjs.map → chunk-I6CUIAQH.cjs.map} +0 -0
  151. /package/dist/lib/node-esm/{TranscriptContainer-DVBXQIQ4.mjs.map → TranscriptContainer-MOWZQSEO.mjs.map} +0 -0
  152. /package/dist/lib/node-esm/{chunk-WEZGRUG6.mjs.map → chunk-NFPDIQYG.mjs.map} +0 -0
@@ -1,21 +1,24 @@
1
1
  import { createRequire } from 'node:module';const require = createRequire(import.meta.url);
2
2
  import {
3
- mergeFloat64Arrays,
4
- monitorAudioLevel
5
- } from "./chunk-XYU6B73V.mjs";
3
+ MediaStreamRecorder,
4
+ Transcriber,
5
+ summarizeTranscript
6
+ } from "./chunk-SWV74BYU.mjs";
6
7
  import {
7
8
  TranscriptContainer
8
- } from "./chunk-WEZGRUG6.mjs";
9
+ } from "./chunk-NFPDIQYG.mjs";
10
+ import {
11
+ TranscriptBlock,
12
+ TranscriptType
13
+ } from "./chunk-DLXORY7Y.mjs";
9
14
  import {
10
15
  Transcript,
11
16
  TranscriptionContainer
12
- } from "./chunk-36EQXBWR.mjs";
17
+ } from "./chunk-4A6BWPRB.mjs";
13
18
  import {
14
19
  TRANSCRIPTION_PLUGIN,
15
- TRANSCRIPTION_URL,
16
- TranscriptType,
17
20
  meta
18
- } from "./chunk-KEG3ME4O.mjs";
21
+ } from "./chunk-SB5TLXBU.mjs";
19
22
 
20
23
  // packages/plugins/experimental/plugin-transcription/src/capabilities/index.ts
21
24
  import { lazy } from "@dxos/app-framework";
@@ -28,9 +31,9 @@ var TranscriptionCapabilities;
28
31
  })(TranscriptionCapabilities || (TranscriptionCapabilities = {}));
29
32
 
30
33
  // packages/plugins/experimental/plugin-transcription/src/capabilities/index.ts
31
- var AppGraphBuilder = lazy(() => import("./app-graph-builder-77GQG6UF.mjs"));
32
- var IntentResolver = lazy(() => import("./intent-resolver-UGLYRD4X.mjs"));
33
- var ReactSurface = lazy(() => import("./react-surface-6AYPLJ2B.mjs"));
34
+ var AppGraphBuilder = lazy(() => import("./app-graph-builder-ARTLU4U5.mjs"));
35
+ var IntentResolver = lazy(() => import("./intent-resolver-XU4BLUSF.mjs"));
36
+ var ReactSurface = lazy(() => import("./react-surface-5CBEXLXJ.mjs"));
34
37
 
35
38
  // packages/plugins/experimental/plugin-transcription/src/hooks/useAudioFile.ts
36
39
  import { useEffect, useState } from "react";
@@ -122,341 +125,8 @@ var useAudioTrack = (active) => {
122
125
  return track;
123
126
  };
124
127
 
125
- // packages/plugins/experimental/plugin-transcription/src/hooks/useIsSpeaking.ts
126
- import { useEffect as useEffect3, useRef, useState as useState3 } from "react";
127
- var useIsSpeaking = (mediaStreamTrack) => {
128
- const [isSpeaking, setIsSpeaking] = useState3(false);
129
- const isSpeakingRef = useRef(isSpeaking);
130
- useEffect3(() => {
131
- if (!mediaStreamTrack) {
132
- setIsSpeaking(false);
133
- return;
134
- }
135
- isSpeakingRef.current = isSpeaking;
136
- const interval = window.setInterval(() => {
137
- if (isSpeaking === isSpeakingRef.current) {
138
- return;
139
- }
140
- setIsSpeaking(isSpeakingRef.current);
141
- }, 50);
142
- return () => {
143
- clearInterval(interval);
144
- };
145
- }, [
146
- isSpeaking,
147
- isSpeakingRef,
148
- mediaStreamTrack
149
- ]);
150
- useEffect3(() => {
151
- if (!mediaStreamTrack) {
152
- return;
153
- }
154
- let timeout = -1;
155
- const cleanup = monitorAudioLevel({
156
- mediaStreamTrack,
157
- onMeasure: (vol) => {
158
- const audioLevelAboveThreshold = vol > (isSpeakingRef.current ? 0.02 : 0.05);
159
- if (audioLevelAboveThreshold) {
160
- clearTimeout(timeout);
161
- timeout = -1;
162
- isSpeakingRef.current = true;
163
- } else if (timeout === -1) {
164
- timeout = window.setTimeout(() => {
165
- isSpeakingRef.current = false;
166
- timeout = -1;
167
- }, 1e3);
168
- }
169
- }
170
- });
171
- return () => {
172
- cleanup();
173
- };
174
- }, [
175
- isSpeaking,
176
- mediaStreamTrack
177
- ]);
178
- return isSpeaking;
179
- };
180
-
181
- // packages/plugins/experimental/plugin-transcription/src/hooks/useTranscriber.ts
182
- import { useEffect as useEffect4, useMemo } from "react";
183
-
184
- // packages/plugins/experimental/plugin-transcription/src/transcriber/media-stream-recorder.ts
185
- import { MediaRecorder, register } from "extendable-media-recorder";
186
- import { WaveFile } from "wavefile";
187
- import { synchronized } from "@dxos/async";
188
- import { invariant } from "@dxos/invariant";
189
- import { log as log2 } from "@dxos/log";
190
- import { trace } from "@dxos/tracing";
191
- function _ts_decorate(decorators, target, key, desc) {
192
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
193
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
194
- else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
195
- return c > 3 && r && Object.defineProperty(target, key, r), r;
196
- }
197
- var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/transcriber/media-stream-recorder.ts";
198
- var initializingPromise;
199
- var initializeExtendableMediaRecorder = async () => {
200
- const { connect } = await import("extendable-media-recorder-wav-encoder");
201
- await register(await connect());
202
- };
203
- var MediaStreamRecorder = class {
204
- constructor({ mediaStreamTrack, interval }) {
205
- this._mediaRecorder = void 0;
206
- this._header = void 0;
207
- this._mediaStreamTrack = mediaStreamTrack;
208
- this._interval = interval;
209
- }
210
- get wavConfig() {
211
- const settings = this._mediaStreamTrack.getSettings();
212
- return {
213
- channels: settings.channelCount ?? 1,
214
- sampleRate: settings.sampleRate ?? 16e3,
215
- bitDepthCode: settings.sampleSize ? String(settings.sampleSize) : "16"
216
- };
217
- }
218
- setOnChunk(onChunk) {
219
- this._onChunk = onChunk;
220
- }
221
- async start() {
222
- await (initializingPromise ??= initializeExtendableMediaRecorder()).catch((err) => log2.info("initializeExtendableMediaRecorder", {
223
- err
224
- }, {
225
- F: __dxlog_file2,
226
- L: 58,
227
- S: this,
228
- C: (f, a) => f(...a)
229
- }));
230
- if (!this._mediaRecorder) {
231
- const stream = new MediaStream([
232
- this._mediaStreamTrack
233
- ]);
234
- this._mediaRecorder = new MediaRecorder(stream, {
235
- mimeType: "audio/wav"
236
- });
237
- }
238
- if (this._mediaRecorder.state === "recording") {
239
- return;
240
- }
241
- invariant(this._onChunk, "MediaStreamRecorder: onChunk is not set", {
242
- F: __dxlog_file2,
243
- L: 69,
244
- S: this,
245
- A: [
246
- "this._onChunk",
247
- "'MediaStreamRecorder: onChunk is not set'"
248
- ]
249
- });
250
- this._mediaRecorder.ondataavailable = (event) => this._ondataavailable(event);
251
- this._mediaRecorder.start(this._interval);
252
- }
253
- async stop() {
254
- if (this._mediaRecorder?.state !== "recording") {
255
- return;
256
- }
257
- this._mediaRecorder.stop();
258
- }
259
- async _ondataavailable(event) {
260
- const blob = event.data;
261
- const uint8Array = new Uint8Array(await blob.arrayBuffer());
262
- const isHeader = uint8Array[0] === 82 && // R
263
- uint8Array[1] === 73 && // I
264
- uint8Array[2] === 70 && // F
265
- uint8Array[3] === 70;
266
- let wav;
267
- if (isHeader) {
268
- wav = new WaveFile(uint8Array);
269
- this._header = uint8Array.slice(0, 44);
270
- } else if (this._header) {
271
- wav = new WaveFile(new Uint8Array([
272
- ...this._header,
273
- ...uint8Array
274
- ]));
275
- } else {
276
- log2.warn("MediaStreamRecorder: no header", void 0, {
277
- F: __dxlog_file2,
278
- L: 100,
279
- S: this,
280
- C: (f, a) => f(...a)
281
- });
282
- return;
283
- }
284
- this._onChunk({
285
- timestamp: Date.now(),
286
- data: wav.getSamples()
287
- });
288
- }
289
- };
290
- _ts_decorate([
291
- synchronized
292
- ], MediaStreamRecorder.prototype, "_ondataavailable", null);
293
- MediaStreamRecorder = _ts_decorate([
294
- trace.resource()
295
- ], MediaStreamRecorder);
296
-
297
- // packages/plugins/experimental/plugin-transcription/src/transcriber/transcriber.ts
298
- import { WaveFile as WaveFile2 } from "wavefile";
299
- import { DeferredTask, Trigger } from "@dxos/async";
300
- import { LifecycleState, Resource } from "@dxos/context";
301
- import { log as log3 } from "@dxos/log";
302
- import { trace as trace2 } from "@dxos/tracing";
303
- function _ts_decorate2(decorators, target, key, desc) {
304
- var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
305
- if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
306
- else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
307
- return c > 3 && r && Object.defineProperty(target, key, r), r;
308
- }
309
- var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/transcriber/transcriber.ts";
310
- var Transcriber = class extends Resource {
311
- constructor({ config, recorder, onSegments }) {
312
- super();
313
- this._audioChunks = [];
314
- this._lastUsedTimestamp = 0;
315
- this._openTrigger = new Trigger({
316
- autoReset: true
317
- });
318
- this._recording = false;
319
- this._transcribeTask = void 0;
320
- this._config = config;
321
- this._recorder = recorder;
322
- this._onSegments = onSegments;
323
- }
324
- async _open(ctx) {
325
- this._recorder.setOnChunk((chunk) => this._saveAudioChunk(chunk));
326
- await this._recorder.start();
327
- this._transcribeTask = new DeferredTask(ctx, async () => this._transcribe());
328
- this._openTrigger.wake();
329
- }
330
- async _close() {
331
- this._recording = false;
332
- this._transcribeTask = void 0;
333
- await this._recorder.stop();
334
- }
335
- startChunksRecording() {
336
- if (this._lifecycleState !== LifecycleState.OPEN) {
337
- return;
338
- }
339
- this._recording = true;
340
- }
341
- stopChunksRecording() {
342
- if (this._lifecycleState !== LifecycleState.OPEN || !this._recording) {
343
- return;
344
- }
345
- this._recording = false;
346
- this._transcribeTask?.schedule();
347
- }
348
- _saveAudioChunk(chunk) {
349
- log3("saving audio chunk", {
350
- chunk
351
- }, {
352
- F: __dxlog_file3,
353
- L: 124,
354
- S: this,
355
- C: (f, a) => f(...a)
356
- });
357
- this._audioChunks.push(chunk);
358
- if (!this._recording && (!this._transcribeTask || !this._transcribeTask.scheduled) && this._audioChunks.length >= this._config.prefixBufferChunksAmount) {
359
- this._audioChunks = this._dropOldChunks();
360
- } else if (this._audioChunks.length >= this._config.transcribeAfterChunksAmount && this._recording) {
361
- this._transcribeTask?.schedule();
362
- }
363
- }
364
- async _transcribe() {
365
- log3("transcribing", {
366
- chunks: this._audioChunks.length
367
- }, {
368
- F: __dxlog_file3,
369
- L: 140,
370
- S: this,
371
- C: (f, a) => f(...a)
372
- });
373
- const chunks = this._audioChunks;
374
- this._audioChunks = this._dropOldChunks();
375
- if (chunks.length === 0) {
376
- return;
377
- }
378
- const audio = await this._mergeAudioChunks(chunks);
379
- const segments = await this._fetchTranscription(audio);
380
- if (!Array.isArray(segments) || segments.length === 0) {
381
- return;
382
- }
383
- const alignedSegments = this._alignSegments(segments, chunks);
384
- if (alignedSegments.length > 0) {
385
- await this._onSegments(alignedSegments);
386
- }
387
- }
388
- _dropOldChunks() {
389
- return this._config.prefixBufferChunksAmount > 0 ? this._audioChunks.slice(-this._config.prefixBufferChunksAmount) : [];
390
- }
391
- async _mergeAudioChunks(chunks) {
392
- const file = new WaveFile2();
393
- const wavConfig = this._recorder.wavConfig;
394
- file.fromScratch(wavConfig.channels, wavConfig.sampleRate, wavConfig.bitDepthCode, mergeFloat64Arrays(chunks.map(({ data }) => data)));
395
- return file.toBase64();
396
- }
397
- async _fetchTranscription(audio) {
398
- if (audio.length === 0) {
399
- this._audioChunks = [];
400
- throw new Error("No audio to send for transcribing");
401
- }
402
- const response = await fetch(`${TRANSCRIPTION_URL}/transcribe`, {
403
- method: "POST",
404
- body: JSON.stringify({
405
- audio
406
- }),
407
- headers: {
408
- "Content-Type": "application/json"
409
- }
410
- });
411
- if (!response.ok) {
412
- this._audioChunks = [];
413
- throw new Error("Failed to transcribe");
414
- }
415
- const { segments } = await response.json();
416
- log3.info("transcription response", {
417
- segments,
418
- string: segments.map((segments2) => segments2.text).join(" ")
419
- }, {
420
- F: __dxlog_file3,
421
- L: 206,
422
- S: this,
423
- C: (f, a) => f(...a)
424
- });
425
- return segments;
426
- }
427
- _alignSegments(segments, originalChunks) {
428
- const zeroTimestamp = originalChunks.at(0).timestamp;
429
- const filteredSegments = segments.filter((segment) => zeroTimestamp + segment.end * 1e3 >= this._lastUsedTimestamp).map((segment) => {
430
- const words = segment.words.filter((word) => zeroTimestamp + word.start * 1e3 >= this._lastUsedTimestamp);
431
- return {
432
- ...segment,
433
- words,
434
- text: words.map((word) => word.word).join(""),
435
- start: words.at(0)?.start ?? segment.end
436
- };
437
- }).filter((segment) => segment.words.length > 0);
438
- if (filteredSegments.length === 0) {
439
- return [];
440
- }
441
- this._lastUsedTimestamp = zeroTimestamp + filteredSegments.at(-1).end * 1e3;
442
- return filteredSegments.map((segment) => ({
443
- started: new Date(zeroTimestamp + segment.start * 1e3),
444
- text: segment.text
445
- }));
446
- }
447
- };
448
- _ts_decorate2([
449
- trace2.span({
450
- showInBrowserTimeline: true
451
- })
452
- ], Transcriber.prototype, "_mergeAudioChunks", null);
453
- _ts_decorate2([
454
- trace2.span({
455
- showInBrowserTimeline: true
456
- })
457
- ], Transcriber.prototype, "_fetchTranscription", null);
458
-
459
128
  // packages/plugins/experimental/plugin-transcription/src/hooks/useTranscriber.ts
129
+ import { useEffect as useEffect3, useMemo } from "react";
460
130
  var RECORD_INTERVAL = 200;
461
131
  var PREFIXED_CHUNKS_AMOUNT = 10;
462
132
  var TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
@@ -480,7 +150,7 @@ var useTranscriber = ({ audioStreamTrack, onSegments }) => {
480
150
  audioStreamTrack,
481
151
  onSegments
482
152
  ]);
483
- useEffect4(() => {
153
+ useEffect3(() => {
484
154
  return () => {
485
155
  void transcriber?.close();
486
156
  };
@@ -491,12 +161,12 @@ var useTranscriber = ({ audioStreamTrack, onSegments }) => {
491
161
  };
492
162
 
493
163
  // packages/plugins/experimental/plugin-transcription/src/hooks/useVoiceInput.ts
494
- import { useState as useState4, useEffect as useEffect5, useCallback } from "react";
164
+ import { useState as useState3, useEffect as useEffect4, useCallback } from "react";
495
165
  import { scheduleMicroTask } from "@dxos/async";
496
166
  import { Context } from "@dxos/context";
497
- import { log as log4 } from "@dxos/log";
167
+ import { log as log2 } from "@dxos/log";
498
168
  import { useSoundEffect } from "@dxos/react-ui-sfx";
499
- var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/hooks/useVoiceInput.ts";
169
+ var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/hooks/useVoiceInput.ts";
500
170
  var useVoiceInput = ({ active, onUpdate }) => {
501
171
  const soundStart = useSoundEffect("StartRecording");
502
172
  const soundStop = useSoundEffect("StopRecording");
@@ -504,22 +174,22 @@ var useVoiceInput = ({ active, onUpdate }) => {
504
174
  const text = segments.map((str) => str.text.trim().replace(/[^\w\s]/g, "")).join(" ");
505
175
  onUpdate(text);
506
176
  }, []);
507
- const [recording, setRecording] = useState4(false);
177
+ const [recording, setRecording] = useState3(false);
508
178
  const track = useAudioTrack(active);
509
179
  const transcriber = useTranscriber({
510
180
  audioStreamTrack: track,
511
181
  onSegments: handleSegments
512
182
  });
513
- useEffect5(() => {
183
+ useEffect4(() => {
514
184
  const ctx = new Context(void 0, {
515
- F: __dxlog_file4,
185
+ F: __dxlog_file2,
516
186
  L: 43
517
187
  });
518
188
  scheduleMicroTask(ctx, async () => {
519
189
  if (active && transcriber) {
520
190
  await transcriber.open();
521
- log4.info("starting...", void 0, {
522
- F: __dxlog_file4,
191
+ log2.info("starting...", void 0, {
192
+ F: __dxlog_file2,
523
193
  L: 47,
524
194
  S: void 0,
525
195
  C: (f, a) => f(...a)
@@ -530,8 +200,8 @@ var useVoiceInput = ({ active, onUpdate }) => {
530
200
  } else if (!active && transcriber?.isOpen) {
531
201
  transcriber?.stopChunksRecording();
532
202
  await transcriber?.close();
533
- log4.info("stopped", void 0, {
534
- F: __dxlog_file4,
203
+ log2.info("stopped", void 0, {
204
+ F: __dxlog_file2,
535
205
  L: 54,
536
206
  S: void 0,
537
207
  C: (f, a) => f(...a)
@@ -552,6 +222,178 @@ var useVoiceInput = ({ active, onUpdate }) => {
552
222
  };
553
223
  };
554
224
 
225
+ // packages/plugins/experimental/plugin-transcription/src/transcription-manager.ts
226
+ import { synchronized } from "@dxos/async";
227
+ import { Resource } from "@dxos/context";
228
+ import { QueueImpl } from "@dxos/echo-db";
229
+ import { createStatic } from "@dxos/echo-schema";
230
+ import { invariant } from "@dxos/invariant";
231
+ import { DXN } from "@dxos/keys";
232
+ function _ts_decorate(decorators, target, key, desc) {
233
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
234
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
235
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
236
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
237
+ }
238
+ var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/transcription-manager.ts";
239
+ var RECORD_INTERVAL2 = 200;
240
+ var PREFIXED_CHUNKS_AMOUNT2 = 10;
241
+ var TRANSCRIBE_AFTER_CHUNKS_AMOUNT2 = 50;
242
+ var TranscriptionManager = class extends Resource {
243
+ constructor(edgeClient) {
244
+ super();
245
+ this._audioStreamTrack = void 0;
246
+ this._name = void 0;
247
+ this._hue = void 0;
248
+ this._mediaRecorder = void 0;
249
+ this._transcriber = void 0;
250
+ this._enabled = false;
251
+ this._queue = void 0;
252
+ this._edgeClient = edgeClient;
253
+ }
254
+ async _open() {
255
+ await this._toggleTranscriber();
256
+ }
257
+ async _close() {
258
+ void this._transcriber?.close();
259
+ }
260
+ setRecording(recording) {
261
+ if (!this.isOpen || !this._enabled) {
262
+ return;
263
+ }
264
+ if (recording) {
265
+ this._transcriber?.startChunksRecording();
266
+ } else {
267
+ this._transcriber?.stopChunksRecording();
268
+ }
269
+ }
270
+ /**
271
+ * Enable or disable transcription.
272
+ * @param enabled - Whether to enable transcription.
273
+ */
274
+ async setEnabled(enabled) {
275
+ if (this._enabled === enabled) {
276
+ return;
277
+ }
278
+ this._enabled = enabled ?? false;
279
+ this.isOpen && await this._toggleTranscriber();
280
+ }
281
+ async setAudioTrack(track) {
282
+ if (this._audioStreamTrack === track) {
283
+ return;
284
+ }
285
+ this._audioStreamTrack = track;
286
+ this.isOpen && await this._toggleTranscriber();
287
+ }
288
+ /**
289
+ * Set the queue to save the transcription to.
290
+ * @param queue - The queue to save the transcription to or the DXN of the queue.
291
+ */
292
+ setQueue(queue) {
293
+ switch (typeof queue) {
294
+ case "string":
295
+ if (this._queue?.dxn.toString() === queue) {
296
+ return;
297
+ }
298
+ this._queue = new QueueImpl(this._edgeClient, DXN.parse(queue));
299
+ break;
300
+ case "object":
301
+ if (this._queue === queue) {
302
+ return;
303
+ }
304
+ invariant(queue instanceof QueueImpl, void 0, {
305
+ F: __dxlog_file3,
306
+ L: 112,
307
+ S: this,
308
+ A: [
309
+ "queue instanceof QueueImpl",
310
+ ""
311
+ ]
312
+ });
313
+ this._queue = queue;
314
+ break;
315
+ case "undefined":
316
+ this._queue = void 0;
317
+ break;
318
+ }
319
+ }
320
+ setName(name) {
321
+ if (this._name === name) {
322
+ return;
323
+ }
324
+ this._name = name;
325
+ }
326
+ setHue(hue) {
327
+ if (this._hue === hue) {
328
+ return;
329
+ }
330
+ this._hue = hue;
331
+ }
332
+ async _toggleTranscriber() {
333
+ await this._maybeReinitTranscriber();
334
+ if (this._enabled) {
335
+ await this._transcriber?.open();
336
+ } else {
337
+ await this._transcriber?.close();
338
+ }
339
+ }
340
+ async _maybeReinitTranscriber() {
341
+ if (!this._audioStreamTrack || !this._enabled) {
342
+ return;
343
+ }
344
+ let needReinit = false;
345
+ if (this._audioStreamTrack !== this._mediaRecorder?.mediaStreamTrack) {
346
+ this._mediaRecorder = new MediaStreamRecorder({
347
+ mediaStreamTrack: this._audioStreamTrack,
348
+ interval: RECORD_INTERVAL2
349
+ });
350
+ needReinit = true;
351
+ }
352
+ if (needReinit) {
353
+ await this._transcriber?.close();
354
+ this._transcriber = new Transcriber({
355
+ config: {
356
+ transcribeAfterChunksAmount: TRANSCRIBE_AFTER_CHUNKS_AMOUNT2,
357
+ prefixBufferChunksAmount: PREFIXED_CHUNKS_AMOUNT2
358
+ },
359
+ recorder: this._mediaRecorder,
360
+ onSegments: (segments) => this._onSegments(segments)
361
+ });
362
+ }
363
+ }
364
+ async _onSegments(segments) {
365
+ if (!this.isOpen || !this._queue) {
366
+ return;
367
+ }
368
+ const block = createStatic(TranscriptBlock, {
369
+ authorName: this._name,
370
+ authorHue: this._hue,
371
+ segments
372
+ });
373
+ this._queue.append([
374
+ block
375
+ ]);
376
+ }
377
+ };
378
+ _ts_decorate([
379
+ synchronized
380
+ ], TranscriptionManager.prototype, "setRecording", null);
381
+ _ts_decorate([
382
+ synchronized
383
+ ], TranscriptionManager.prototype, "setEnabled", null);
384
+ _ts_decorate([
385
+ synchronized
386
+ ], TranscriptionManager.prototype, "setAudioTrack", null);
387
+ _ts_decorate([
388
+ synchronized
389
+ ], TranscriptionManager.prototype, "setQueue", null);
390
+ _ts_decorate([
391
+ synchronized
392
+ ], TranscriptionManager.prototype, "setName", null);
393
+ _ts_decorate([
394
+ synchronized
395
+ ], TranscriptionManager.prototype, "setHue", null);
396
+
555
397
  // packages/plugins/experimental/plugin-transcription/src/TranscriptionPlugin.tsx
556
398
  import { Capabilities, Events, contributes, defineModule, definePlugin } from "@dxos/app-framework";
557
399
  import { ClientCapabilities, ClientEvents } from "@dxos/plugin-client";
@@ -561,7 +403,8 @@ var translations_default = [
561
403
  {
562
404
  "en-US": {
563
405
  [TranscriptType.typename]: {
564
- "typename label": "Transcript"
406
+ "typename label": "Transcript",
407
+ "object name placeholder": "New transcript"
565
408
  },
566
409
  [TRANSCRIPTION_PLUGIN]: {
567
410
  "plugin name": "Transcription",
@@ -571,8 +414,7 @@ var translations_default = [
571
414
  "delete calls label": "Delete",
572
415
  "delete button": "Delete",
573
416
  "bookmark button": "Bookmark",
574
- "summary button": "Create summary",
575
- "summarizing label": "Summarizing..."
417
+ "scroll to end label": "Scroll to latest"
576
418
  }
577
419
  }
578
420
  }
@@ -596,12 +438,6 @@ var TranscriptionPlugin = () => definePlugin(meta, [
596
438
  activate: () => contributes(Capabilities.Metadata, {
597
439
  id: TranscriptType.typename,
598
440
  metadata: {
599
- placeholder: [
600
- "transcript title placeholder",
601
- {
602
- ns: TRANSCRIPTION_PLUGIN
603
- }
604
- ],
605
441
  icon: "ph--subtitles--regular"
606
442
  }
607
443
  })
@@ -642,11 +478,12 @@ export {
642
478
  TranscriptContainer,
643
479
  TranscriptionCapabilities,
644
480
  TranscriptionContainer,
481
+ TranscriptionManager,
645
482
  TranscriptionPlugin,
646
483
  meta,
484
+ summarizeTranscript,
647
485
  useAudioFile,
648
486
  useAudioTrack,
649
- useIsSpeaking,
650
487
  useTranscriber,
651
488
  useVoiceInput
652
489
  };