@dxos/plugin-transcription 0.7.5-main.b19bfc8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (172) hide show
  1. package/LICENSE +8 -0
  2. package/README.md +15 -0
  3. package/dist/lib/browser/TranscriptContainer-G7NUUBPJ.mjs +10 -0
  4. package/dist/lib/browser/TranscriptContainer-G7NUUBPJ.mjs.map +7 -0
  5. package/dist/lib/browser/app-graph-builder-ZSSJIFY7.mjs +7 -0
  6. package/dist/lib/browser/app-graph-builder-ZSSJIFY7.mjs.map +7 -0
  7. package/dist/lib/browser/chunk-7V6ZINS5.mjs +8 -0
  8. package/dist/lib/browser/chunk-7V6ZINS5.mjs.map +7 -0
  9. package/dist/lib/browser/chunk-D4ZMHRVK.mjs +241 -0
  10. package/dist/lib/browser/chunk-D4ZMHRVK.mjs.map +7 -0
  11. package/dist/lib/browser/chunk-QIHUPVQD.mjs +80 -0
  12. package/dist/lib/browser/chunk-QIHUPVQD.mjs.map +7 -0
  13. package/dist/lib/browser/chunk-ZMES3U2D.mjs +78 -0
  14. package/dist/lib/browser/chunk-ZMES3U2D.mjs.map +7 -0
  15. package/dist/lib/browser/index.mjs +652 -0
  16. package/dist/lib/browser/index.mjs.map +7 -0
  17. package/dist/lib/browser/intent-resolver-EAR2TFK2.mjs +32 -0
  18. package/dist/lib/browser/intent-resolver-EAR2TFK2.mjs.map +7 -0
  19. package/dist/lib/browser/meta.json +1 -0
  20. package/dist/lib/browser/react-surface-IXF7SETX.mjs +26 -0
  21. package/dist/lib/browser/react-surface-IXF7SETX.mjs.map +7 -0
  22. package/dist/lib/browser/types/index.mjs +17 -0
  23. package/dist/lib/browser/types/index.mjs.map +7 -0
  24. package/dist/lib/node/TranscriptContainer-IKMCPYQ2.cjs +31 -0
  25. package/dist/lib/node/TranscriptContainer-IKMCPYQ2.cjs.map +7 -0
  26. package/dist/lib/node/app-graph-builder-YFBX3OTP.cjs +26 -0
  27. package/dist/lib/node/app-graph-builder-YFBX3OTP.cjs.map +7 -0
  28. package/dist/lib/node/chunk-JBN3P6ZF.cjs +97 -0
  29. package/dist/lib/node/chunk-JBN3P6ZF.cjs.map +7 -0
  30. package/dist/lib/node/chunk-SQHF7N3B.cjs +40 -0
  31. package/dist/lib/node/chunk-SQHF7N3B.cjs.map +7 -0
  32. package/dist/lib/node/chunk-UIHEGU6Q.cjs +103 -0
  33. package/dist/lib/node/chunk-UIHEGU6Q.cjs.map +7 -0
  34. package/dist/lib/node/chunk-Y2GCR535.cjs +267 -0
  35. package/dist/lib/node/chunk-Y2GCR535.cjs.map +7 -0
  36. package/dist/lib/node/index.cjs +661 -0
  37. package/dist/lib/node/index.cjs.map +7 -0
  38. package/dist/lib/node/intent-resolver-3KJZGGUQ.cjs +44 -0
  39. package/dist/lib/node/intent-resolver-3KJZGGUQ.cjs.map +7 -0
  40. package/dist/lib/node/meta.json +1 -0
  41. package/dist/lib/node/react-surface-L4NQQWN4.cjs +49 -0
  42. package/dist/lib/node/react-surface-L4NQQWN4.cjs.map +7 -0
  43. package/dist/lib/node/types/index.cjs +39 -0
  44. package/dist/lib/node/types/index.cjs.map +7 -0
  45. package/dist/lib/node-esm/TranscriptContainer-HJTGGLU5.mjs +11 -0
  46. package/dist/lib/node-esm/TranscriptContainer-HJTGGLU5.mjs.map +7 -0
  47. package/dist/lib/node-esm/app-graph-builder-77GQG6UF.mjs +9 -0
  48. package/dist/lib/node-esm/app-graph-builder-77GQG6UF.mjs.map +7 -0
  49. package/dist/lib/node-esm/chunk-F4J7FV6Q.mjs +10 -0
  50. package/dist/lib/node-esm/chunk-F4J7FV6Q.mjs.map +7 -0
  51. package/dist/lib/node-esm/chunk-KEG3ME4O.mjs +82 -0
  52. package/dist/lib/node-esm/chunk-KEG3ME4O.mjs.map +7 -0
  53. package/dist/lib/node-esm/chunk-TNJUQCWM.mjs +242 -0
  54. package/dist/lib/node-esm/chunk-TNJUQCWM.mjs.map +7 -0
  55. package/dist/lib/node-esm/chunk-XYU6B73V.mjs +80 -0
  56. package/dist/lib/node-esm/chunk-XYU6B73V.mjs.map +7 -0
  57. package/dist/lib/node-esm/index.mjs +653 -0
  58. package/dist/lib/node-esm/index.mjs.map +7 -0
  59. package/dist/lib/node-esm/intent-resolver-UGLYRD4X.mjs +33 -0
  60. package/dist/lib/node-esm/intent-resolver-UGLYRD4X.mjs.map +7 -0
  61. package/dist/lib/node-esm/meta.json +1 -0
  62. package/dist/lib/node-esm/react-surface-TX4ULXGK.mjs +27 -0
  63. package/dist/lib/node-esm/react-surface-TX4ULXGK.mjs.map +7 -0
  64. package/dist/lib/node-esm/types/index.mjs +18 -0
  65. package/dist/lib/node-esm/types/index.mjs.map +7 -0
  66. package/dist/types/src/TranscriptionPlugin.d.ts +2 -0
  67. package/dist/types/src/TranscriptionPlugin.d.ts.map +1 -0
  68. package/dist/types/src/capabilities/app-graph-builder.d.ts +180 -0
  69. package/dist/types/src/capabilities/app-graph-builder.d.ts.map +1 -0
  70. package/dist/types/src/capabilities/capabilities.d.ts +4 -0
  71. package/dist/types/src/capabilities/capabilities.d.ts.map +1 -0
  72. package/dist/types/src/capabilities/index.d.ts +181 -0
  73. package/dist/types/src/capabilities/index.d.ts.map +1 -0
  74. package/dist/types/src/capabilities/intent-resolver.d.ts +4 -0
  75. package/dist/types/src/capabilities/intent-resolver.d.ts.map +1 -0
  76. package/dist/types/src/capabilities/react-surface.d.ts +4 -0
  77. package/dist/types/src/capabilities/react-surface.d.ts.map +1 -0
  78. package/dist/types/src/components/Transcript/Transcript.d.ts +7 -0
  79. package/dist/types/src/components/Transcript/Transcript.d.ts.map +1 -0
  80. package/dist/types/src/components/Transcript/Transcript.stories.d.ts +9 -0
  81. package/dist/types/src/components/Transcript/Transcript.stories.d.ts.map +1 -0
  82. package/dist/types/src/components/Transcript/index.d.ts +2 -0
  83. package/dist/types/src/components/Transcript/index.d.ts.map +1 -0
  84. package/dist/types/src/components/TranscriptContainer.d.ts +7 -0
  85. package/dist/types/src/components/TranscriptContainer.d.ts.map +1 -0
  86. package/dist/types/src/components/index.d.ts +6 -0
  87. package/dist/types/src/components/index.d.ts.map +1 -0
  88. package/dist/types/src/components/transcription.stories.d.ts +14 -0
  89. package/dist/types/src/components/transcription.stories.d.ts.map +1 -0
  90. package/dist/types/src/hooks/index.d.ts +6 -0
  91. package/dist/types/src/hooks/index.d.ts.map +1 -0
  92. package/dist/types/src/hooks/useAudioFile.d.ts +7 -0
  93. package/dist/types/src/hooks/useAudioFile.d.ts.map +1 -0
  94. package/dist/types/src/hooks/useAudioTrack.d.ts +5 -0
  95. package/dist/types/src/hooks/useAudioTrack.d.ts.map +1 -0
  96. package/dist/types/src/hooks/useIsSpeaking.d.ts +2 -0
  97. package/dist/types/src/hooks/useIsSpeaking.d.ts.map +1 -0
  98. package/dist/types/src/hooks/useTranscriber.d.ts +10 -0
  99. package/dist/types/src/hooks/useTranscriber.d.ts.map +1 -0
  100. package/dist/types/src/hooks/useVoiceInput.d.ts +11 -0
  101. package/dist/types/src/hooks/useVoiceInput.d.ts.map +1 -0
  102. package/dist/types/src/index.d.ts +7 -0
  103. package/dist/types/src/index.d.ts.map +1 -0
  104. package/dist/types/src/meta.d.ts +11 -0
  105. package/dist/types/src/meta.d.ts.map +1 -0
  106. package/dist/types/src/sanity.test.d.ts +2 -0
  107. package/dist/types/src/sanity.test.d.ts.map +1 -0
  108. package/dist/types/src/transcriber/audio-recorder.d.ts +47 -0
  109. package/dist/types/src/transcriber/audio-recorder.d.ts.map +1 -0
  110. package/dist/types/src/transcriber/index.d.ts +4 -0
  111. package/dist/types/src/transcriber/index.d.ts.map +1 -0
  112. package/dist/types/src/transcriber/media-stream-recorder.d.ts +26 -0
  113. package/dist/types/src/transcriber/media-stream-recorder.d.ts.map +1 -0
  114. package/dist/types/src/transcriber/transcriber.d.ts +46 -0
  115. package/dist/types/src/transcriber/transcriber.d.ts.map +1 -0
  116. package/dist/types/src/transcriber/transcriber.test.d.ts +2 -0
  117. package/dist/types/src/transcriber/transcriber.test.d.ts.map +1 -0
  118. package/dist/types/src/translations.d.ts +40 -0
  119. package/dist/types/src/translations.d.ts.map +1 -0
  120. package/dist/types/src/types/index.d.ts +3 -0
  121. package/dist/types/src/types/index.d.ts.map +1 -0
  122. package/dist/types/src/types/schema.d.ts +63 -0
  123. package/dist/types/src/types/schema.d.ts.map +1 -0
  124. package/dist/types/src/types/types.d.ts +28 -0
  125. package/dist/types/src/types/types.d.ts.map +1 -0
  126. package/dist/types/src/util/array.d.ts +2 -0
  127. package/dist/types/src/util/array.d.ts.map +1 -0
  128. package/dist/types/src/util/get-time-str.d.ts +2 -0
  129. package/dist/types/src/util/get-time-str.d.ts.map +1 -0
  130. package/dist/types/src/util/index.d.ts +5 -0
  131. package/dist/types/src/util/index.d.ts.map +1 -0
  132. package/dist/types/src/util/monitor-audio-level.d.ts +6 -0
  133. package/dist/types/src/util/monitor-audio-level.d.ts.map +1 -0
  134. package/dist/types/src/util/random-queue-dxn.d.ts +3 -0
  135. package/dist/types/src/util/random-queue-dxn.d.ts.map +1 -0
  136. package/dist/types/tsconfig.tsbuildinfo +1 -0
  137. package/package.json +93 -0
  138. package/src/TranscriptionPlugin.tsx +57 -0
  139. package/src/capabilities/app-graph-builder.ts +7 -0
  140. package/src/capabilities/capabilities.ts +12 -0
  141. package/src/capabilities/index.ts +11 -0
  142. package/src/capabilities/intent-resolver.ts +24 -0
  143. package/src/capabilities/react-surface.tsx +21 -0
  144. package/src/components/Transcript/Transcript.stories.tsx +65 -0
  145. package/src/components/Transcript/Transcript.tsx +69 -0
  146. package/src/components/Transcript/index.ts +5 -0
  147. package/src/components/TranscriptContainer.tsx +167 -0
  148. package/src/components/index.ts +10 -0
  149. package/src/components/transcription.stories.tsx +198 -0
  150. package/src/hooks/index.ts +9 -0
  151. package/src/hooks/useAudioFile.ts +62 -0
  152. package/src/hooks/useAudioTrack.ts +35 -0
  153. package/src/hooks/useIsSpeaking.ts +77 -0
  154. package/src/hooks/useTranscriber.ts +62 -0
  155. package/src/hooks/useVoiceInput.ts +66 -0
  156. package/src/index.ts +11 -0
  157. package/src/meta.ts +17 -0
  158. package/src/sanity.test.ts +11 -0
  159. package/src/transcriber/audio-recorder.ts +56 -0
  160. package/src/transcriber/index.ts +7 -0
  161. package/src/transcriber/media-stream-recorder.ts +109 -0
  162. package/src/transcriber/transcriber.test.ts +230 -0
  163. package/src/transcriber/transcriber.ts +247 -0
  164. package/src/translations.ts +28 -0
  165. package/src/types/index.ts +6 -0
  166. package/src/types/schema.ts +60 -0
  167. package/src/types/types.ts +34 -0
  168. package/src/util/array.ts +14 -0
  169. package/src/util/get-time-str.ts +12 -0
  170. package/src/util/index.ts +8 -0
  171. package/src/util/monitor-audio-level.ts +58 -0
  172. package/src/util/random-queue-dxn.ts +9 -0
@@ -0,0 +1,661 @@
1
+ "use strict";
2
+ var __create = Object.create;
3
+ var __defProp = Object.defineProperty;
4
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
5
+ var __getOwnPropNames = Object.getOwnPropertyNames;
6
+ var __getProtoOf = Object.getPrototypeOf;
7
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
8
+ var __export = (target, all) => {
9
+ for (var name in all)
10
+ __defProp(target, name, { get: all[name], enumerable: true });
11
+ };
12
+ var __copyProps = (to, from, except, desc) => {
13
+ if (from && typeof from === "object" || typeof from === "function") {
14
+ for (let key of __getOwnPropNames(from))
15
+ if (!__hasOwnProp.call(to, key) && key !== except)
16
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
17
+ }
18
+ return to;
19
+ };
20
+ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
21
+ // If the importer is in node compatibility mode or this is not an ESM
22
+ // file that has been converted to a CommonJS file using a Babel-
23
+ // compatible transform (i.e. "__esModule" has not been set), then set
24
+ // "default" to the CommonJS "module.exports" for node compatibility.
25
+ isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
26
+ mod
27
+ ));
28
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
29
+ var node_exports = {};
30
+ __export(node_exports, {
31
+ AppGraphBuilder: () => AppGraphBuilder,
32
+ IntentResolver: () => IntentResolver,
33
+ MediaStreamRecorder: () => MediaStreamRecorder,
34
+ ReactSurface: () => ReactSurface,
35
+ TRANSCRIPTION_PLUGIN: () => import_chunk_UIHEGU6Q.TRANSCRIPTION_PLUGIN,
36
+ Transcriber: () => Transcriber,
37
+ Transcript: () => import_chunk_Y2GCR535.Transcript,
38
+ TranscriptContainer: () => import_chunk_SQHF7N3B.TranscriptContainer,
39
+ TranscriptionCapabilities: () => TranscriptionCapabilities,
40
+ TranscriptionContainer: () => import_chunk_Y2GCR535.TranscriptionContainer,
41
+ TranscriptionPlugin: () => TranscriptionPlugin,
42
+ meta: () => import_chunk_UIHEGU6Q.meta,
43
+ useAudioFile: () => useAudioFile,
44
+ useAudioTrack: () => useAudioTrack,
45
+ useIsSpeaking: () => useIsSpeaking,
46
+ useTranscriber: () => useTranscriber,
47
+ useVoiceInput: () => useVoiceInput
48
+ });
49
+ module.exports = __toCommonJS(node_exports);
50
+ var import_chunk_JBN3P6ZF = require("./chunk-JBN3P6ZF.cjs");
51
+ var import_chunk_SQHF7N3B = require("./chunk-SQHF7N3B.cjs");
52
+ var import_chunk_Y2GCR535 = require("./chunk-Y2GCR535.cjs");
53
+ var import_chunk_UIHEGU6Q = require("./chunk-UIHEGU6Q.cjs");
54
+ var import_app_framework = require("@dxos/app-framework");
55
+ var import_app_framework2 = require("@dxos/app-framework");
56
+ var import_react = require("react");
57
+ var import_log = require("@dxos/log");
58
+ var import_react2 = require("react");
59
+ var import_react3 = require("react");
60
+ var import_react4 = require("react");
61
+ var import_extendable_media_recorder = require("extendable-media-recorder");
62
+ var import_wavefile = require("wavefile");
63
+ var import_async = require("@dxos/async");
64
+ var import_invariant = require("@dxos/invariant");
65
+ var import_log2 = require("@dxos/log");
66
+ var import_tracing = require("@dxos/tracing");
67
+ var import_wavefile2 = require("wavefile");
68
+ var import_async2 = require("@dxos/async");
69
+ var import_context = require("@dxos/context");
70
+ var import_log3 = require("@dxos/log");
71
+ var import_tracing2 = require("@dxos/tracing");
72
+ var import_react5 = require("react");
73
+ var import_async3 = require("@dxos/async");
74
+ var import_context2 = require("@dxos/context");
75
+ var import_log4 = require("@dxos/log");
76
+ var import_react_ui_sfx = require("@dxos/react-ui-sfx");
77
+ var import_app_framework3 = require("@dxos/app-framework");
78
+ var import_plugin_client = require("@dxos/plugin-client");
79
+ var TranscriptionCapabilities;
80
+ (function(TranscriptionCapabilities2) {
81
+ TranscriptionCapabilities2.Transcription = (0, import_app_framework2.defineCapability)(`${import_chunk_UIHEGU6Q.TRANSCRIPTION_PLUGIN}/capability/transcription`);
82
+ })(TranscriptionCapabilities || (TranscriptionCapabilities = {}));
83
+ var AppGraphBuilder = (0, import_app_framework.lazy)(() => import("./app-graph-builder-YFBX3OTP.cjs"));
84
+ var IntentResolver = (0, import_app_framework.lazy)(() => import("./intent-resolver-3KJZGGUQ.cjs"));
85
+ var ReactSurface = (0, import_app_framework.lazy)(() => import("./react-surface-L4NQQWN4.cjs"));
86
+ var __dxlog_file = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/hooks/useAudioFile.ts";
87
+ var useAudioFile = (audioUrl) => {
88
+ const [{ audio, stream, track }, setStream] = (0, import_react.useState)({});
89
+ (0, import_react.useEffect)(() => {
90
+ const t = setTimeout(async () => {
91
+ const response = await fetch(audioUrl);
92
+ const blob = await response.blob();
93
+ const audio2 = new Audio();
94
+ audio2.src = URL.createObjectURL(blob);
95
+ await new Promise((resolve, reject) => {
96
+ audio2.addEventListener("error", (err) => {
97
+ import_log.log.error("error", {
98
+ err
99
+ }, {
100
+ F: __dxlog_file,
101
+ L: 27,
102
+ S: void 0,
103
+ C: (f, a) => f(...a)
104
+ });
105
+ reject(err);
106
+ }, {
107
+ once: true
108
+ });
109
+ audio2.addEventListener("canplay", () => {
110
+ import_log.log.info("starting...", void 0, {
111
+ F: __dxlog_file,
112
+ L: 35,
113
+ S: void 0,
114
+ C: (f, a) => f(...a)
115
+ });
116
+ resolve();
117
+ }, {
118
+ once: true
119
+ });
120
+ audio2.load();
121
+ });
122
+ const ctx = new AudioContext();
123
+ const destination = ctx.createMediaStreamDestination();
124
+ destination.channelCount = 1;
125
+ const source = ctx.createMediaElementSource(audio2);
126
+ source.connect(destination);
127
+ setStream({
128
+ audio: audio2,
129
+ stream: destination.stream,
130
+ track: destination.stream.getAudioTracks()[0]
131
+ });
132
+ });
133
+ return () => clearTimeout(t);
134
+ }, [
135
+ audioUrl
136
+ ]);
137
+ return {
138
+ audio,
139
+ stream,
140
+ track
141
+ };
142
+ };
143
+ var useAudioTrack = (active) => {
144
+ const [track, setTrack] = (0, import_react2.useState)();
145
+ (0, import_react2.useEffect)(() => {
146
+ if (!active) {
147
+ track?.stop();
148
+ setTrack(void 0);
149
+ return;
150
+ }
151
+ const initAudio = async () => {
152
+ const audio = new Audio();
153
+ audio.srcObject = await navigator.mediaDevices.getUserMedia({
154
+ audio: true
155
+ });
156
+ const [track2] = audio.srcObject.getAudioTracks();
157
+ if (track2) {
158
+ setTrack(track2);
159
+ }
160
+ };
161
+ if (active) {
162
+ void initAudio();
163
+ }
164
+ }, [
165
+ active
166
+ ]);
167
+ return track;
168
+ };
169
+ var useIsSpeaking = (mediaStreamTrack) => {
170
+ const [isSpeaking, setIsSpeaking] = (0, import_react3.useState)(false);
171
+ const isSpeakingRef = (0, import_react3.useRef)(isSpeaking);
172
+ (0, import_react3.useEffect)(() => {
173
+ if (!mediaStreamTrack) {
174
+ setIsSpeaking(false);
175
+ return;
176
+ }
177
+ isSpeakingRef.current = isSpeaking;
178
+ const interval = window.setInterval(() => {
179
+ if (isSpeaking === isSpeakingRef.current) {
180
+ return;
181
+ }
182
+ setIsSpeaking(isSpeakingRef.current);
183
+ }, 50);
184
+ return () => {
185
+ clearInterval(interval);
186
+ };
187
+ }, [
188
+ isSpeaking,
189
+ isSpeakingRef,
190
+ mediaStreamTrack
191
+ ]);
192
+ (0, import_react3.useEffect)(() => {
193
+ if (!mediaStreamTrack) {
194
+ return;
195
+ }
196
+ let timeout = -1;
197
+ const cleanup = (0, import_chunk_JBN3P6ZF.monitorAudioLevel)({
198
+ mediaStreamTrack,
199
+ onMeasure: (vol) => {
200
+ const audioLevelAboveThreshold = vol > (isSpeakingRef.current ? 0.02 : 0.05);
201
+ if (audioLevelAboveThreshold) {
202
+ clearTimeout(timeout);
203
+ timeout = -1;
204
+ isSpeakingRef.current = true;
205
+ } else if (timeout === -1) {
206
+ timeout = window.setTimeout(() => {
207
+ isSpeakingRef.current = false;
208
+ timeout = -1;
209
+ }, 1e3);
210
+ }
211
+ }
212
+ });
213
+ return () => {
214
+ cleanup();
215
+ };
216
+ }, [
217
+ isSpeaking,
218
+ mediaStreamTrack
219
+ ]);
220
+ return isSpeaking;
221
+ };
222
+ function _ts_decorate(decorators, target, key, desc) {
223
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
224
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
225
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
226
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
227
+ }
228
+ var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/transcriber/media-stream-recorder.ts";
229
+ var initializingPromise;
230
+ var initializeExtendableMediaRecorder = async () => {
231
+ const { connect } = await import("extendable-media-recorder-wav-encoder");
232
+ await (0, import_extendable_media_recorder.register)(await connect());
233
+ };
234
+ var MediaStreamRecorder = class {
235
+ constructor({ mediaStreamTrack, interval }) {
236
+ this._mediaRecorder = void 0;
237
+ this._header = void 0;
238
+ this._mediaStreamTrack = mediaStreamTrack;
239
+ this._interval = interval;
240
+ }
241
+ get wavConfig() {
242
+ const settings = this._mediaStreamTrack.getSettings();
243
+ return {
244
+ channels: settings.channelCount ?? 1,
245
+ sampleRate: settings.sampleRate ?? 16e3,
246
+ bitDepthCode: settings.sampleSize ? String(settings.sampleSize) : "16"
247
+ };
248
+ }
249
+ setOnChunk(onChunk) {
250
+ this._onChunk = onChunk;
251
+ }
252
+ async start() {
253
+ await (initializingPromise ??= initializeExtendableMediaRecorder()).catch((err) => import_log2.log.info("initializeExtendableMediaRecorder", {
254
+ err
255
+ }, {
256
+ F: __dxlog_file2,
257
+ L: 58,
258
+ S: this,
259
+ C: (f, a) => f(...a)
260
+ }));
261
+ if (!this._mediaRecorder) {
262
+ const stream = new MediaStream([
263
+ this._mediaStreamTrack
264
+ ]);
265
+ this._mediaRecorder = new import_extendable_media_recorder.MediaRecorder(stream, {
266
+ mimeType: "audio/wav"
267
+ });
268
+ }
269
+ if (this._mediaRecorder.state === "recording") {
270
+ return;
271
+ }
272
+ (0, import_invariant.invariant)(this._onChunk, "MediaStreamRecorder: onChunk is not set", {
273
+ F: __dxlog_file2,
274
+ L: 69,
275
+ S: this,
276
+ A: [
277
+ "this._onChunk",
278
+ "'MediaStreamRecorder: onChunk is not set'"
279
+ ]
280
+ });
281
+ this._mediaRecorder.ondataavailable = (event) => this._ondataavailable(event);
282
+ this._mediaRecorder.start(this._interval);
283
+ }
284
+ async stop() {
285
+ if (this._mediaRecorder?.state !== "recording") {
286
+ return;
287
+ }
288
+ this._mediaRecorder.stop();
289
+ }
290
+ async _ondataavailable(event) {
291
+ const blob = event.data;
292
+ const uint8Array = new Uint8Array(await blob.arrayBuffer());
293
+ const isHeader = uint8Array[0] === 82 && // R
294
+ uint8Array[1] === 73 && // I
295
+ uint8Array[2] === 70 && // F
296
+ uint8Array[3] === 70;
297
+ let wav;
298
+ if (isHeader) {
299
+ wav = new import_wavefile.WaveFile(uint8Array);
300
+ this._header = uint8Array.slice(0, 44);
301
+ } else if (this._header) {
302
+ wav = new import_wavefile.WaveFile(new Uint8Array([
303
+ ...this._header,
304
+ ...uint8Array
305
+ ]));
306
+ } else {
307
+ import_log2.log.warn("MediaStreamRecorder: no header", void 0, {
308
+ F: __dxlog_file2,
309
+ L: 100,
310
+ S: this,
311
+ C: (f, a) => f(...a)
312
+ });
313
+ return;
314
+ }
315
+ this._onChunk({
316
+ timestamp: Date.now(),
317
+ data: wav.getSamples()
318
+ });
319
+ }
320
+ };
321
+ _ts_decorate([
322
+ import_async.synchronized
323
+ ], MediaStreamRecorder.prototype, "_ondataavailable", null);
324
+ MediaStreamRecorder = _ts_decorate([
325
+ import_tracing.trace.resource()
326
+ ], MediaStreamRecorder);
327
+ function _ts_decorate2(decorators, target, key, desc) {
328
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
329
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
330
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
331
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
332
+ }
333
+ var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/transcriber/transcriber.ts";
334
+ var Transcriber = class extends import_context.Resource {
335
+ constructor({ config, recorder, onSegments }) {
336
+ super();
337
+ this._audioChunks = [];
338
+ this._lastUsedTimestamp = 0;
339
+ this._openTrigger = new import_async2.Trigger({
340
+ autoReset: true
341
+ });
342
+ this._recording = false;
343
+ this._transcribeTask = void 0;
344
+ this._config = config;
345
+ this._recorder = recorder;
346
+ this._onSegments = onSegments;
347
+ }
348
+ async _open(ctx) {
349
+ this._recorder.setOnChunk((chunk) => this._saveAudioChunk(chunk));
350
+ await this._recorder.start();
351
+ this._transcribeTask = new import_async2.DeferredTask(ctx, async () => this._transcribe());
352
+ this._openTrigger.wake();
353
+ }
354
+ async _close() {
355
+ this._recording = false;
356
+ this._transcribeTask = void 0;
357
+ await this._recorder.stop();
358
+ }
359
+ startChunksRecording() {
360
+ if (this._lifecycleState !== import_context.LifecycleState.OPEN) {
361
+ return;
362
+ }
363
+ this._recording = true;
364
+ }
365
+ stopChunksRecording() {
366
+ if (this._lifecycleState !== import_context.LifecycleState.OPEN || !this._recording) {
367
+ return;
368
+ }
369
+ this._recording = false;
370
+ this._transcribeTask?.schedule();
371
+ }
372
+ _saveAudioChunk(chunk) {
373
+ (0, import_log3.log)("saving audio chunk", {
374
+ chunk
375
+ }, {
376
+ F: __dxlog_file3,
377
+ L: 124,
378
+ S: this,
379
+ C: (f, a) => f(...a)
380
+ });
381
+ this._audioChunks.push(chunk);
382
+ if (!this._recording && (!this._transcribeTask || !this._transcribeTask.scheduled) && this._audioChunks.length >= this._config.prefixBufferChunksAmount) {
383
+ this._audioChunks = this._dropOldChunks();
384
+ } else if (this._audioChunks.length >= this._config.transcribeAfterChunksAmount && this._recording) {
385
+ this._transcribeTask?.schedule();
386
+ }
387
+ }
388
+ async _transcribe() {
389
+ (0, import_log3.log)("transcribing", {
390
+ chunks: this._audioChunks.length
391
+ }, {
392
+ F: __dxlog_file3,
393
+ L: 140,
394
+ S: this,
395
+ C: (f, a) => f(...a)
396
+ });
397
+ const chunks = this._audioChunks;
398
+ this._audioChunks = this._dropOldChunks();
399
+ if (chunks.length === 0) {
400
+ return;
401
+ }
402
+ const audio = await this._mergeAudioChunks(chunks);
403
+ const segments = await this._fetchTranscription(audio);
404
+ if (!Array.isArray(segments) || segments.length === 0) {
405
+ return;
406
+ }
407
+ const alignedSegments = this._alignSegments(segments, chunks);
408
+ if (alignedSegments.length > 0) {
409
+ await this._onSegments(alignedSegments);
410
+ }
411
+ }
412
+ _dropOldChunks() {
413
+ return this._config.prefixBufferChunksAmount > 0 ? this._audioChunks.slice(-this._config.prefixBufferChunksAmount) : [];
414
+ }
415
+ async _mergeAudioChunks(chunks) {
416
+ const file = new import_wavefile2.WaveFile();
417
+ const wavConfig = this._recorder.wavConfig;
418
+ file.fromScratch(wavConfig.channels, wavConfig.sampleRate, wavConfig.bitDepthCode, (0, import_chunk_JBN3P6ZF.mergeFloat64Arrays)(chunks.map(({ data }) => data)));
419
+ return file.toBase64();
420
+ }
421
+ async _fetchTranscription(audio) {
422
+ if (audio.length === 0) {
423
+ this._audioChunks = [];
424
+ throw new Error("No audio to send for transcribing");
425
+ }
426
+ const response = await fetch(`${import_chunk_UIHEGU6Q.TRANSCRIPTION_URL}/transcribe`, {
427
+ method: "POST",
428
+ body: JSON.stringify({
429
+ audio
430
+ }),
431
+ headers: {
432
+ "Content-Type": "application/json"
433
+ }
434
+ });
435
+ if (!response.ok) {
436
+ this._audioChunks = [];
437
+ throw new Error("Failed to transcribe");
438
+ }
439
+ const { segments } = await response.json();
440
+ import_log3.log.info("transcription response", {
441
+ segments,
442
+ string: segments.map((segments2) => segments2.text).join(" ")
443
+ }, {
444
+ F: __dxlog_file3,
445
+ L: 206,
446
+ S: this,
447
+ C: (f, a) => f(...a)
448
+ });
449
+ return segments;
450
+ }
451
+ _alignSegments(segments, originalChunks) {
452
+ const zeroTimestamp = originalChunks.at(0).timestamp;
453
+ const filteredSegments = segments.filter((segment) => zeroTimestamp + segment.end * 1e3 >= this._lastUsedTimestamp).map((segment) => {
454
+ const words = segment.words.filter((word) => zeroTimestamp + word.start * 1e3 >= this._lastUsedTimestamp);
455
+ return {
456
+ ...segment,
457
+ words,
458
+ text: words.map((word) => word.word).join(""),
459
+ start: words.at(0)?.start ?? segment.end
460
+ };
461
+ }).filter((segment) => segment.words.length > 0);
462
+ if (filteredSegments.length === 0) {
463
+ return [];
464
+ }
465
+ this._lastUsedTimestamp = zeroTimestamp + filteredSegments.at(-1).end * 1e3;
466
+ return filteredSegments.map((segment) => ({
467
+ started: new Date(zeroTimestamp + segment.start * 1e3),
468
+ text: segment.text
469
+ }));
470
+ }
471
+ };
472
+ _ts_decorate2([
473
+ import_tracing2.trace.span({
474
+ showInBrowserTimeline: true
475
+ })
476
+ ], Transcriber.prototype, "_mergeAudioChunks", null);
477
+ _ts_decorate2([
478
+ import_tracing2.trace.span({
479
+ showInBrowserTimeline: true
480
+ })
481
+ ], Transcriber.prototype, "_fetchTranscription", null);
482
+ var RECORD_INTERVAL = 200;
483
+ var PREFIXED_CHUNKS_AMOUNT = 10;
484
+ var TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
485
+ var useTranscriber = ({ audioStreamTrack, onSegments }) => {
486
+ const transcriber = (0, import_react4.useMemo)(() => {
487
+ if (!onSegments || !audioStreamTrack) {
488
+ return void 0;
489
+ }
490
+ return new Transcriber({
491
+ config: {
492
+ transcribeAfterChunksAmount: TRANSCRIBE_AFTER_CHUNKS_AMOUNT,
493
+ prefixBufferChunksAmount: PREFIXED_CHUNKS_AMOUNT
494
+ },
495
+ recorder: new MediaStreamRecorder({
496
+ mediaStreamTrack: audioStreamTrack,
497
+ interval: RECORD_INTERVAL
498
+ }),
499
+ onSegments
500
+ });
501
+ }, [
502
+ audioStreamTrack,
503
+ onSegments
504
+ ]);
505
+ (0, import_react4.useEffect)(() => {
506
+ return () => {
507
+ void transcriber?.close();
508
+ };
509
+ }, [
510
+ transcriber
511
+ ]);
512
+ return transcriber;
513
+ };
514
+ var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/plugins/experimental/plugin-transcription/src/hooks/useVoiceInput.ts";
515
+ var useVoiceInput = ({ active, onUpdate }) => {
516
+ const soundStart = (0, import_react_ui_sfx.useSoundEffect)("StartRecording");
517
+ const soundStop = (0, import_react_ui_sfx.useSoundEffect)("StopRecording");
518
+ const handleSegments = (0, import_react5.useCallback)(async (segments) => {
519
+ const text = segments.map((str) => str.text.trim().replace(/[^\w\s]/g, "")).join(" ");
520
+ onUpdate(text);
521
+ }, []);
522
+ const [recording, setRecording] = (0, import_react5.useState)(false);
523
+ const track = useAudioTrack(active);
524
+ const transcriber = useTranscriber({
525
+ audioStreamTrack: track,
526
+ onSegments: handleSegments
527
+ });
528
+ (0, import_react5.useEffect)(() => {
529
+ const ctx = new import_context2.Context(void 0, {
530
+ F: __dxlog_file4,
531
+ L: 43
532
+ });
533
+ (0, import_async3.scheduleMicroTask)(ctx, async () => {
534
+ if (active && transcriber) {
535
+ await transcriber.open();
536
+ import_log4.log.info("starting...", void 0, {
537
+ F: __dxlog_file4,
538
+ L: 47,
539
+ S: void 0,
540
+ C: (f, a) => f(...a)
541
+ });
542
+ setRecording(true);
543
+ void soundStart.play();
544
+ transcriber.startChunksRecording();
545
+ } else if (!active && transcriber?.isOpen) {
546
+ transcriber?.stopChunksRecording();
547
+ await transcriber?.close();
548
+ import_log4.log.info("stopped", void 0, {
549
+ F: __dxlog_file4,
550
+ L: 54,
551
+ S: void 0,
552
+ C: (f, a) => f(...a)
553
+ });
554
+ setRecording(false);
555
+ void soundStop.play();
556
+ }
557
+ });
558
+ return () => {
559
+ void ctx.dispose();
560
+ };
561
+ }, [
562
+ active,
563
+ transcriber
564
+ ]);
565
+ return {
566
+ recording
567
+ };
568
+ };
569
+ var translations_default = [
570
+ {
571
+ "en-US": {
572
+ [import_chunk_UIHEGU6Q.TranscriptType.typename]: {
573
+ "typename label": "Transcript"
574
+ },
575
+ [import_chunk_UIHEGU6Q.TRANSCRIPTION_PLUGIN]: {
576
+ "plugin name": "Transcription",
577
+ "calls label": "Transcript",
578
+ "calls panel label": "Transcription",
579
+ "create calls label": "Create transcription",
580
+ "delete calls label": "Delete",
581
+ "delete button": "Delete",
582
+ "bookmark button": "Bookmark",
583
+ "summary button": "Create summary",
584
+ "summarizing label": "Summarizing..."
585
+ }
586
+ }
587
+ }
588
+ ];
589
+ var TranscriptionPlugin = () => (0, import_app_framework3.definePlugin)(import_chunk_UIHEGU6Q.meta, [
590
+ (0, import_app_framework3.defineModule)({
591
+ id: `${import_chunk_UIHEGU6Q.meta.id}/module/translations`,
592
+ activatesOn: import_app_framework3.Events.SetupTranslations,
593
+ activate: () => (0, import_app_framework3.contributes)(import_app_framework3.Capabilities.Translations, translations_default)
594
+ }),
595
+ (0, import_app_framework3.defineModule)({
596
+ id: `${import_chunk_UIHEGU6Q.meta.id}/module/transcription`,
597
+ activatesOn: import_app_framework3.Events.Startup,
598
+ activate: () => (0, import_app_framework3.contributes)(TranscriptionCapabilities.Transcription, null)
599
+ }),
600
+ (0, import_app_framework3.defineModule)({
601
+ id: `${import_chunk_UIHEGU6Q.meta.id}/module/metadata`,
602
+ activatesOn: import_app_framework3.Events.SetupMetadata,
603
+ activate: () => (0, import_app_framework3.contributes)(import_app_framework3.Capabilities.Metadata, {
604
+ id: import_chunk_UIHEGU6Q.TranscriptType.typename,
605
+ metadata: {
606
+ placeholder: [
607
+ "transcript title placeholder",
608
+ {
609
+ ns: import_chunk_UIHEGU6Q.TRANSCRIPTION_PLUGIN
610
+ }
611
+ ],
612
+ icon: "ph--subtitles--regular"
613
+ }
614
+ })
615
+ }),
616
+ (0, import_app_framework3.defineModule)({
617
+ id: `${import_chunk_UIHEGU6Q.meta.id}/module/schema`,
618
+ activatesOn: import_plugin_client.ClientEvents.SetupSchema,
619
+ activate: () => [
620
+ (0, import_app_framework3.contributes)(import_plugin_client.ClientCapabilities.Schema, [
621
+ import_chunk_UIHEGU6Q.TranscriptType
622
+ ])
623
+ ]
624
+ }),
625
+ (0, import_app_framework3.defineModule)({
626
+ id: `${import_chunk_UIHEGU6Q.meta.id}/module/react-surface`,
627
+ activatesOn: import_app_framework3.Events.SetupReactSurface,
628
+ activate: ReactSurface
629
+ }),
630
+ (0, import_app_framework3.defineModule)({
631
+ id: `${import_chunk_UIHEGU6Q.meta.id}/module/intent-resolver`,
632
+ activatesOn: import_app_framework3.Events.SetupIntentResolver,
633
+ activate: IntentResolver
634
+ }),
635
+ (0, import_app_framework3.defineModule)({
636
+ id: `${import_chunk_UIHEGU6Q.meta.id}/module/app-graph-builder`,
637
+ activatesOn: import_app_framework3.Events.SetupAppGraph,
638
+ activate: AppGraphBuilder
639
+ })
640
+ ]);
641
+ // Annotate the CommonJS export names for ESM import in node:
642
+ 0 && (module.exports = {
643
+ AppGraphBuilder,
644
+ IntentResolver,
645
+ MediaStreamRecorder,
646
+ ReactSurface,
647
+ TRANSCRIPTION_PLUGIN,
648
+ Transcriber,
649
+ Transcript,
650
+ TranscriptContainer,
651
+ TranscriptionCapabilities,
652
+ TranscriptionContainer,
653
+ TranscriptionPlugin,
654
+ meta,
655
+ useAudioFile,
656
+ useAudioTrack,
657
+ useIsSpeaking,
658
+ useTranscriber,
659
+ useVoiceInput
660
+ });
661
+ //# sourceMappingURL=index.cjs.map