@dxos/plugin-transformer 0.8.4-main.ead640a → 0.8.4-main.effb148878
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +102 -5
- package/README.md +1 -1
- package/dist/lib/neutral/TransformerPlugin.mjs +25 -0
- package/dist/lib/neutral/TransformerPlugin.mjs.map +7 -0
- package/dist/lib/neutral/chunk-YTFCIBAZ.mjs +34 -0
- package/dist/lib/neutral/chunk-YTFCIBAZ.mjs.map +7 -0
- package/dist/lib/neutral/components/index.mjs +136 -0
- package/dist/lib/neutral/components/index.mjs.map +7 -0
- package/dist/lib/neutral/hooks/index.mjs +339 -0
- package/dist/lib/neutral/hooks/index.mjs.map +7 -0
- package/dist/lib/neutral/index.mjs +7 -0
- package/dist/lib/neutral/meta.json +1 -0
- package/dist/lib/neutral/meta.mjs +7 -0
- package/dist/lib/neutral/plugin.mjs +11 -0
- package/dist/lib/neutral/plugin.mjs.map +7 -0
- package/dist/lib/neutral/translations.mjs +15 -0
- package/dist/lib/neutral/translations.mjs.map +7 -0
- package/dist/types/src/TransformerPlugin.d.ts +3 -1
- package/dist/types/src/TransformerPlugin.d.ts.map +1 -1
- package/dist/types/src/TransformerPlugin.test.d.ts +2 -0
- package/dist/types/src/TransformerPlugin.test.d.ts.map +1 -0
- package/dist/types/src/components/Voice/DebugInfo.d.ts.map +1 -0
- package/dist/types/src/components/Voice/Voice.d.ts.map +1 -0
- package/dist/types/src/components/Voice/Voice.stories.d.ts.map +1 -0
- package/dist/types/src/components/Voice/index.d.ts +3 -0
- package/dist/types/src/components/Voice/index.d.ts.map +1 -0
- package/dist/types/src/components/index.d.ts +2 -0
- package/dist/types/src/components/index.d.ts.map +1 -0
- package/dist/types/src/hooks/useAudioStream.d.ts.map +1 -1
- package/dist/types/src/hooks/usePipeline.d.ts +1 -16
- package/dist/types/src/hooks/usePipeline.d.ts.map +1 -1
- package/dist/types/src/index.d.ts +0 -1
- package/dist/types/src/index.d.ts.map +1 -1
- package/dist/types/src/meta.d.ts +2 -2
- package/dist/types/src/meta.d.ts.map +1 -1
- package/dist/types/src/plugin.d.ts +3 -0
- package/dist/types/src/plugin.d.ts.map +1 -0
- package/dist/types/src/testing/node-pipeline.d.ts +1 -1
- package/dist/types/src/testing/node-pipeline.d.ts.map +1 -1
- package/dist/types/src/testing/pipeline.d.ts.map +1 -1
- package/dist/types/src/testing/web-pipeline.d.ts +1 -1
- package/dist/types/src/testing/web-pipeline.d.ts.map +1 -1
- package/dist/types/src/translations.d.ts +2 -3
- package/dist/types/src/translations.d.ts.map +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +77 -50
- package/src/TransformerPlugin.test.ts +25 -0
- package/src/TransformerPlugin.tsx +13 -23
- package/src/components/{DebugInfo.tsx → Voice/DebugInfo.tsx} +3 -3
- package/src/components/{Voice.stories.tsx → Voice/Voice.stories.tsx} +2 -2
- package/src/components/{Voice.tsx → Voice/Voice.tsx} +1 -1
- package/src/components/Voice/index.ts +6 -0
- package/src/{capabilities → components}/index.ts +2 -0
- package/src/hooks/usePipeline.ts +9 -33
- package/src/index.ts +0 -2
- package/src/meta.ts +22 -4
- package/src/plugin.ts +9 -0
- package/src/translations.ts +2 -2
- package/dist/lib/browser/index.mjs +0 -50
- package/dist/lib/browser/index.mjs.map +0 -7
- package/dist/lib/browser/meta.json +0 -1
- package/dist/lib/browser/types/index.mjs +0 -1
- package/dist/lib/node-esm/index.mjs +0 -52
- package/dist/lib/node-esm/index.mjs.map +0 -7
- package/dist/lib/node-esm/meta.json +0 -1
- package/dist/lib/node-esm/types/index.mjs +0 -2
- package/dist/types/src/capabilities/index.d.ts +0 -1
- package/dist/types/src/capabilities/index.d.ts.map +0 -1
- package/dist/types/src/components/DebugInfo.d.ts.map +0 -1
- package/dist/types/src/components/Voice.d.ts.map +0 -1
- package/dist/types/src/components/Voice.stories.d.ts.map +0 -1
- /package/dist/lib/{browser/types → neutral}/index.mjs.map +0 -0
- /package/dist/lib/{node-esm/types/index.mjs.map → neutral/meta.mjs.map} +0 -0
- /package/dist/types/src/components/{DebugInfo.d.ts → Voice/DebugInfo.d.ts} +0 -0
- /package/dist/types/src/components/{Voice.d.ts → Voice/Voice.d.ts} +0 -0
- /package/dist/types/src/components/{Voice.stories.d.ts → Voice/Voice.stories.d.ts} +0 -0
|
@@ -0,0 +1,339 @@
|
|
|
1
|
+
// src/hooks/useAudioStream.ts
|
|
2
|
+
import { useCallback, useEffect, useRef, useState } from "react";
|
|
3
|
+
import { log } from "@dxos/log";
|
|
4
|
+
var __dxlog_file = "/__w/dxos/dxos/packages/plugins/plugin-transformer/src/hooks/useAudioStream.ts";
|
|
5
|
+
var useAudioStream = ({ active, debug, onAudioData }) => {
|
|
6
|
+
const [state, setState] = useState({
|
|
7
|
+
stream: null,
|
|
8
|
+
error: null,
|
|
9
|
+
audioLevel: 0
|
|
10
|
+
});
|
|
11
|
+
const audioContextRef = useRef(null);
|
|
12
|
+
const analyserRef = useRef(null);
|
|
13
|
+
const animationFrameRef = useRef(void 0);
|
|
14
|
+
const workletNodeRef = useRef(null);
|
|
15
|
+
const isProcessingRef = useRef(false);
|
|
16
|
+
const mediaStreamRef = useRef(null);
|
|
17
|
+
const audioBufferRef = useRef([]);
|
|
18
|
+
const updateAudioLevel = useCallback(() => {
|
|
19
|
+
if (analyserRef.current) {
|
|
20
|
+
const dataArray = new Uint8Array(analyserRef.current.frequencyBinCount);
|
|
21
|
+
analyserRef.current.getByteFrequencyData(dataArray);
|
|
22
|
+
const average = dataArray.reduce((acc, val) => acc + val, 0) / dataArray.length;
|
|
23
|
+
setState((prev) => ({
|
|
24
|
+
...prev,
|
|
25
|
+
audioLevel: average
|
|
26
|
+
}));
|
|
27
|
+
animationFrameRef.current = requestAnimationFrame(updateAudioLevel);
|
|
28
|
+
}
|
|
29
|
+
}, []);
|
|
30
|
+
const cleanup = useCallback(() => {
|
|
31
|
+
log("cleaning up audio resources", void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 34, S: void 0 });
|
|
32
|
+
if (mediaStreamRef.current) {
|
|
33
|
+
mediaStreamRef.current.getTracks().forEach((track) => {
|
|
34
|
+
track.stop();
|
|
35
|
+
track.enabled = false;
|
|
36
|
+
});
|
|
37
|
+
mediaStreamRef.current = null;
|
|
38
|
+
}
|
|
39
|
+
if (workletNodeRef.current) {
|
|
40
|
+
workletNodeRef.current.disconnect();
|
|
41
|
+
workletNodeRef.current = null;
|
|
42
|
+
}
|
|
43
|
+
if (analyserRef.current) {
|
|
44
|
+
analyserRef.current.disconnect();
|
|
45
|
+
analyserRef.current = null;
|
|
46
|
+
}
|
|
47
|
+
if (audioContextRef.current) {
|
|
48
|
+
void audioContextRef.current.close();
|
|
49
|
+
audioContextRef.current = null;
|
|
50
|
+
}
|
|
51
|
+
if (animationFrameRef.current) {
|
|
52
|
+
cancelAnimationFrame(animationFrameRef.current);
|
|
53
|
+
animationFrameRef.current = void 0;
|
|
54
|
+
}
|
|
55
|
+
audioBufferRef.current = [];
|
|
56
|
+
setState({
|
|
57
|
+
stream: null,
|
|
58
|
+
error: null,
|
|
59
|
+
audioLevel: 0
|
|
60
|
+
});
|
|
61
|
+
}, [
|
|
62
|
+
debug
|
|
63
|
+
]);
|
|
64
|
+
useEffect(() => {
|
|
65
|
+
let mounted = true;
|
|
66
|
+
const startStream = async () => {
|
|
67
|
+
try {
|
|
68
|
+
if (active) {
|
|
69
|
+
cleanup();
|
|
70
|
+
log.info("initializing audio stream...", void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 75, S: void 0 });
|
|
71
|
+
const stream = await navigator.mediaDevices.getUserMedia({
|
|
72
|
+
audio: {
|
|
73
|
+
channelCount: 1,
|
|
74
|
+
sampleRate: 16e3,
|
|
75
|
+
echoCancellation: true,
|
|
76
|
+
noiseSuppression: true,
|
|
77
|
+
autoGainControl: true
|
|
78
|
+
},
|
|
79
|
+
video: false
|
|
80
|
+
});
|
|
81
|
+
if (!mounted || !active) {
|
|
82
|
+
stream.getTracks().forEach((track) => {
|
|
83
|
+
track.stop();
|
|
84
|
+
track.enabled = false;
|
|
85
|
+
});
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
mediaStreamRef.current = stream;
|
|
89
|
+
const context = new AudioContext({
|
|
90
|
+
sampleRate: 16e3
|
|
91
|
+
});
|
|
92
|
+
await context.audioWorklet.addModule(URL.createObjectURL(new Blob([
|
|
93
|
+
`class AudioProcessor extends AudioWorkletProcessor {
|
|
94
|
+
constructor() {
|
|
95
|
+
super();
|
|
96
|
+
this._buffer = [];
|
|
97
|
+
this._samplesProcessed = 0;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
process(inputs, outputs) {
|
|
101
|
+
const input = inputs[0];
|
|
102
|
+
const channel = input[0];
|
|
103
|
+
|
|
104
|
+
if (channel) {
|
|
105
|
+
this._buffer.push(new Float32Array(channel));
|
|
106
|
+
this._samplesProcessed += channel.length;
|
|
107
|
+
|
|
108
|
+
// Process every 2 seconds (32000 samples at 16kHz).
|
|
109
|
+
if (this._samplesProcessed >= 32000) {
|
|
110
|
+
const combinedLength = this._buffer.reduce((acc, curr) => acc + curr.length, 0);
|
|
111
|
+
const combinedAudio = new Float32Array(combinedLength);
|
|
112
|
+
let offset = 0;
|
|
113
|
+
|
|
114
|
+
for (const buffer of this._buffer) {
|
|
115
|
+
combinedAudio.set(buffer, offset);
|
|
116
|
+
offset += buffer.length;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
this.port.postMessage({ type: 'audio-data', data: combinedAudio });
|
|
120
|
+
|
|
121
|
+
// Reset buffer and counter.
|
|
122
|
+
this._buffer = [];
|
|
123
|
+
this._samplesProcessed = 0;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
return true;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
registerProcessor('audio-processor', AudioProcessor);`
|
|
131
|
+
], {
|
|
132
|
+
type: "application/javascript"
|
|
133
|
+
})));
|
|
134
|
+
const source = context.createMediaStreamSource(stream);
|
|
135
|
+
const analyser = context.createAnalyser();
|
|
136
|
+
analyserRef.current = analyser;
|
|
137
|
+
const workletNode = new AudioWorkletNode(context, "audio-processor");
|
|
138
|
+
workletNodeRef.current = workletNode;
|
|
139
|
+
workletNode.port.onmessage = async (event) => {
|
|
140
|
+
if (!mounted || !active) {
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
if (event.data.type === "audio-data") {
|
|
144
|
+
isProcessingRef.current = true;
|
|
145
|
+
try {
|
|
146
|
+
log("processing audio", {
|
|
147
|
+
sampleRate: context.sampleRate,
|
|
148
|
+
length: event.data.data.length,
|
|
149
|
+
min: Math.min(...event.data.data),
|
|
150
|
+
max: Math.max(...event.data.data)
|
|
151
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 154, S: void 0 });
|
|
152
|
+
await onAudioData?.(event.data.data);
|
|
153
|
+
} catch (err) {
|
|
154
|
+
if (mounted) {
|
|
155
|
+
setState((prev) => ({
|
|
156
|
+
...prev,
|
|
157
|
+
error: "Error processing audio: " + err.message
|
|
158
|
+
}));
|
|
159
|
+
}
|
|
160
|
+
log.error("audio processing error", {
|
|
161
|
+
err
|
|
162
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 168, S: void 0 });
|
|
163
|
+
} finally {
|
|
164
|
+
isProcessingRef.current = false;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
};
|
|
168
|
+
source.connect(analyser);
|
|
169
|
+
analyser.connect(workletNode);
|
|
170
|
+
workletNode.connect(context.destination);
|
|
171
|
+
if (debug) {
|
|
172
|
+
analyser.fftSize = 256;
|
|
173
|
+
updateAudioLevel();
|
|
174
|
+
}
|
|
175
|
+
audioContextRef.current = context;
|
|
176
|
+
if (mounted && active) {
|
|
177
|
+
setState({
|
|
178
|
+
stream,
|
|
179
|
+
error: null,
|
|
180
|
+
audioLevel: 0
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
} catch (err) {
|
|
185
|
+
if (mounted) {
|
|
186
|
+
setState((prev) => ({
|
|
187
|
+
...prev,
|
|
188
|
+
error: "Error accessing microphone: " + err.message,
|
|
189
|
+
stream: null
|
|
190
|
+
}));
|
|
191
|
+
}
|
|
192
|
+
log.error("microphone error", {
|
|
193
|
+
err
|
|
194
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 201, S: void 0 });
|
|
195
|
+
cleanup();
|
|
196
|
+
}
|
|
197
|
+
};
|
|
198
|
+
void startStream();
|
|
199
|
+
return () => {
|
|
200
|
+
mounted = false;
|
|
201
|
+
cleanup();
|
|
202
|
+
};
|
|
203
|
+
}, [
|
|
204
|
+
active,
|
|
205
|
+
debug,
|
|
206
|
+
onAudioData,
|
|
207
|
+
updateAudioLevel,
|
|
208
|
+
cleanup
|
|
209
|
+
]);
|
|
210
|
+
useEffect(() => {
|
|
211
|
+
if (!active) {
|
|
212
|
+
cleanup();
|
|
213
|
+
}
|
|
214
|
+
}, [
|
|
215
|
+
active,
|
|
216
|
+
cleanup
|
|
217
|
+
]);
|
|
218
|
+
return state;
|
|
219
|
+
};
|
|
220
|
+
|
|
221
|
+
// src/hooks/usePipeline.ts
|
|
222
|
+
import { env, pipeline } from "@xenova/transformers";
|
|
223
|
+
import { useEffect as useEffect2, useRef as useRef2, useState as useState2 } from "react";
|
|
224
|
+
import { invariant } from "@dxos/invariant";
|
|
225
|
+
import { log as log2 } from "@dxos/log";
|
|
226
|
+
var __dxlog_file2 = "/__w/dxos/dxos/packages/plugins/plugin-transformer/src/hooks/usePipeline.ts";
|
|
227
|
+
env.cacheDir = "./.cache";
|
|
228
|
+
env.allowLocalModels = true;
|
|
229
|
+
env.backends.onnx.wasm.numThreads = 1;
|
|
230
|
+
env.backends.onnx.provider = "webgpu";
|
|
231
|
+
env.backends.onnx.webgpu = {
|
|
232
|
+
profilingMode: true
|
|
233
|
+
};
|
|
234
|
+
var usePipeline = ({ active, model, debug }) => {
|
|
235
|
+
const [state, setState] = useState2({
|
|
236
|
+
gpuInfo: "",
|
|
237
|
+
isLoaded: false,
|
|
238
|
+
isLoading: false,
|
|
239
|
+
error: null
|
|
240
|
+
});
|
|
241
|
+
const pipelineRef = useRef2(null);
|
|
242
|
+
useEffect2(() => {
|
|
243
|
+
const loadModel = async () => {
|
|
244
|
+
try {
|
|
245
|
+
setState((prev) => ({
|
|
246
|
+
...prev,
|
|
247
|
+
isLoading: true,
|
|
248
|
+
error: null
|
|
249
|
+
}));
|
|
250
|
+
if (!navigator.gpu) {
|
|
251
|
+
log2.warn("WebGPU is not supported, falling back to CPU", void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 36, S: void 0 });
|
|
252
|
+
setState((prev) => ({
|
|
253
|
+
...prev,
|
|
254
|
+
gpuInfo: "WebGPU not supported (using CPU)"
|
|
255
|
+
}));
|
|
256
|
+
} else {
|
|
257
|
+
try {
|
|
258
|
+
const adapter = await navigator.gpu.requestAdapter();
|
|
259
|
+
if (!adapter) {
|
|
260
|
+
throw new Error("No GPU adapter found");
|
|
261
|
+
}
|
|
262
|
+
const adapterInfo = adapter.info;
|
|
263
|
+
if (adapterInfo) {
|
|
264
|
+
setState((prev) => ({
|
|
265
|
+
...prev,
|
|
266
|
+
gpuInfo: `${adapterInfo.description || "GPU"} (${adapterInfo.vendor || "Unknown"})`
|
|
267
|
+
}));
|
|
268
|
+
} else {
|
|
269
|
+
setState((prev) => ({
|
|
270
|
+
...prev,
|
|
271
|
+
gpuInfo: "GPU Available (details unknown)"
|
|
272
|
+
}));
|
|
273
|
+
}
|
|
274
|
+
} catch (err) {
|
|
275
|
+
log2.warn("WebGPU initialization failed", {
|
|
276
|
+
err
|
|
277
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 61, S: void 0 });
|
|
278
|
+
setState((prev) => ({
|
|
279
|
+
...prev,
|
|
280
|
+
gpuInfo: "GPU initialization failed (using CPU)"
|
|
281
|
+
}));
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
const pipe = await pipeline("automatic-speech-recognition", model, {
|
|
285
|
+
quantized: true,
|
|
286
|
+
progress_callback: (progress) => {
|
|
287
|
+
if (debug) {
|
|
288
|
+
log2(`loading model: ${Math.round(progress.progress * 100)}%`, void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 74, S: void 0 });
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
});
|
|
292
|
+
pipelineRef.current = pipe;
|
|
293
|
+
setState((prev) => ({
|
|
294
|
+
...prev,
|
|
295
|
+
isLoaded: true,
|
|
296
|
+
isLoading: false
|
|
297
|
+
}));
|
|
298
|
+
log2.info("model loaded successfully", void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 84, S: void 0 });
|
|
299
|
+
} catch (err) {
|
|
300
|
+
log2.error("error loading model", {
|
|
301
|
+
err
|
|
302
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 86, S: void 0 });
|
|
303
|
+
setState((prev) => ({
|
|
304
|
+
...prev,
|
|
305
|
+
isLoading: false,
|
|
306
|
+
error: "error loading model: " + err.message
|
|
307
|
+
}));
|
|
308
|
+
}
|
|
309
|
+
};
|
|
310
|
+
if (active) {
|
|
311
|
+
void loadModel();
|
|
312
|
+
}
|
|
313
|
+
return () => {
|
|
314
|
+
pipelineRef.current = null;
|
|
315
|
+
setState((prev) => ({
|
|
316
|
+
...prev,
|
|
317
|
+
isLoaded: false
|
|
318
|
+
}));
|
|
319
|
+
};
|
|
320
|
+
}, [
|
|
321
|
+
active,
|
|
322
|
+
debug,
|
|
323
|
+
model
|
|
324
|
+
]);
|
|
325
|
+
const transcribe = async (audioData, options) => {
|
|
326
|
+
invariant(pipelineRef.current, "pipeline not initialized", { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 112, S: void 0, A: ["pipelineRef.current", "'pipeline not initialized'"] });
|
|
327
|
+
const result = await pipelineRef.current(audioData, options);
|
|
328
|
+
return result;
|
|
329
|
+
};
|
|
330
|
+
return {
|
|
331
|
+
...state,
|
|
332
|
+
transcribe
|
|
333
|
+
};
|
|
334
|
+
};
|
|
335
|
+
export {
|
|
336
|
+
useAudioStream,
|
|
337
|
+
usePipeline
|
|
338
|
+
};
|
|
339
|
+
//# sourceMappingURL=index.mjs.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../../../src/hooks/useAudioStream.ts", "../../../../src/hooks/usePipeline.ts"],
|
|
4
|
+
"sourcesContent": ["//\n// Copyright 2025 DXOS.org\n//\n\nimport { useCallback, useEffect, useRef, useState } from 'react';\n\nimport { log } from '@dxos/log';\n\nexport type AudioStreamConfig = {\n active?: boolean;\n debug?: boolean;\n onAudioData?: (audioData: Float32Array) => Promise<void>;\n};\n\nexport type AudioStreamState = {\n stream: MediaStream | null;\n error: string | null;\n audioLevel: number;\n};\n\nexport const useAudioStream = ({ active, debug, onAudioData }: AudioStreamConfig) => {\n const [state, setState] = useState<AudioStreamState>({\n stream: null,\n error: null,\n audioLevel: 0,\n });\n\n // TODO(burdon): Convert to class.\n const audioContextRef = useRef<AudioContext | null>(null);\n const analyserRef = useRef<AnalyserNode | null>(null);\n const animationFrameRef = useRef<number>(undefined);\n const workletNodeRef = useRef<AudioWorkletNode | null>(null);\n const isProcessingRef = useRef(false);\n const mediaStreamRef = useRef<MediaStream | null>(null);\n const audioBufferRef = useRef<Float32Array[]>([]);\n\n // Stats for visualization.\n const updateAudioLevel = useCallback(() => {\n if (analyserRef.current) {\n const dataArray = new Uint8Array(analyserRef.current.frequencyBinCount);\n analyserRef.current.getByteFrequencyData(dataArray);\n const average = dataArray.reduce((acc, val) => acc + val, 0) / dataArray.length;\n setState((prev) => ({ ...prev, audioLevel: average }));\n animationFrameRef.current = requestAnimationFrame(updateAudioLevel);\n }\n }, []);\n\n const cleanup = useCallback(() => {\n log('cleaning up audio resources');\n\n // Stop all tracks.\n if (mediaStreamRef.current) {\n mediaStreamRef.current.getTracks().forEach((track) => {\n track.stop();\n track.enabled = false;\n });\n mediaStreamRef.current = null;\n }\n\n // Disconnect and cleanup audio nodes.\n if (workletNodeRef.current) {\n workletNodeRef.current.disconnect();\n workletNodeRef.current = null;\n }\n if (analyserRef.current) {\n analyserRef.current.disconnect();\n analyserRef.current = null;\n }\n if (audioContextRef.current) {\n void audioContextRef.current.close();\n audioContextRef.current = null;\n }\n if (animationFrameRef.current) {\n cancelAnimationFrame(animationFrameRef.current);\n animationFrameRef.current = undefined;\n }\n\n audioBufferRef.current = [];\n setState({\n stream: null,\n error: null,\n audioLevel: 0,\n });\n }, [debug]);\n\n useEffect(() => {\n let mounted = true;\n\n const startStream = async () => {\n try {\n if (active) {\n cleanup();\n log.info('initializing audio stream...');\n const stream = await navigator.mediaDevices.getUserMedia({\n audio: {\n channelCount: 1,\n sampleRate: 16_000,\n echoCancellation: true,\n noiseSuppression: true,\n autoGainControl: true,\n },\n video: false,\n });\n\n if (!mounted || !active) {\n stream.getTracks().forEach((track) => {\n track.stop();\n track.enabled = false;\n });\n return;\n }\n\n mediaStreamRef.current = stream;\n\n // Create AudioContext for proper audio format.\n const context = new AudioContext({ sampleRate: 16_000 });\n\n // Add the audio worklet module.\n await context.audioWorklet.addModule(\n URL.createObjectURL(\n new Blob(\n [\n `class AudioProcessor extends AudioWorkletProcessor {\n constructor() {\n super();\n this._buffer = [];\n this._samplesProcessed = 0;\n }\n\n process(inputs, outputs) {\n const input = inputs[0];\n const channel = input[0];\n \n if (channel) {\n this._buffer.push(new Float32Array(channel));\n this._samplesProcessed += channel.length;\n\n // Process every 2 seconds (32000 samples at 16kHz).\n if (this._samplesProcessed >= 32000) {\n const combinedLength = this._buffer.reduce((acc, curr) => acc + curr.length, 0);\n const combinedAudio = new Float32Array(combinedLength);\n let offset = 0;\n\n for (const buffer of this._buffer) {\n combinedAudio.set(buffer, offset);\n offset += buffer.length;\n }\n\n this.port.postMessage({ type: 'audio-data', data: combinedAudio });\n \n // Reset buffer and counter.\n this._buffer = [];\n this._samplesProcessed = 0;\n }\n }\n return true;\n }\n }\n\n registerProcessor('audio-processor', AudioProcessor);`,\n ],\n { type: 'application/javascript' },\n ),\n ),\n );\n\n const source = context.createMediaStreamSource(stream);\n const analyser = context.createAnalyser();\n analyserRef.current = analyser;\n\n // Create and connect the audio worklet node.\n const workletNode = new AudioWorkletNode(context, 'audio-processor');\n workletNodeRef.current = workletNode;\n\n workletNode.port.onmessage = async (event) => {\n if (!mounted || !active) {\n return;\n }\n\n if (event.data.type === 'audio-data') {\n isProcessingRef.current = true;\n try {\n log('processing audio', {\n sampleRate: context.sampleRate,\n length: event.data.data.length,\n min: Math.min(...event.data.data),\n max: Math.max(...event.data.data),\n });\n\n await onAudioData?.(event.data.data);\n } catch (err) {\n if (mounted) {\n setState((prev) => ({\n ...prev,\n error: 'Error processing audio: ' + (err as Error).message,\n }));\n }\n log.error('audio processing error', { err });\n } finally {\n isProcessingRef.current = false;\n }\n }\n };\n\n // Connect the audio nodes.\n source.connect(analyser);\n analyser.connect(workletNode);\n workletNode.connect(context.destination);\n\n if (debug) {\n analyser.fftSize = 256;\n updateAudioLevel();\n }\n\n audioContextRef.current = context;\n if (mounted && active) {\n setState({\n stream,\n error: null,\n audioLevel: 0,\n });\n }\n }\n } catch (err) {\n if (mounted) {\n setState((prev) => ({\n ...prev,\n error: 'Error accessing microphone: ' + (err as Error).message,\n stream: null,\n }));\n }\n log.error('microphone error', { err });\n cleanup();\n }\n };\n\n void startStream();\n\n return () => {\n mounted = false;\n cleanup();\n };\n }, [active, debug, onAudioData, updateAudioLevel, cleanup]);\n\n useEffect(() => {\n if (!active) {\n cleanup();\n }\n }, [active, cleanup]);\n\n return state;\n};\n", "//\n// Copyright 2025 DXOS.org\n//\n\nimport { type Pipeline, env, pipeline } from '@xenova/transformers';\nimport { useEffect, useRef, useState } from 'react';\n\nimport { invariant } from '@dxos/invariant';\nimport { log } from '@dxos/log';\n\n// Configure cache and runtime settings.\nenv.cacheDir = './.cache';\nenv.allowLocalModels = true;\n\n// Configure ONNX runtime for WebGPU.\n(env.backends.onnx as any).wasm.numThreads = 1;\n(env.backends.onnx as any).provider = 'webgpu';\n(env.backends.onnx as any).webgpu = {\n profilingMode: true,\n};\n\nexport type PipelineConfig = {\n active?: boolean;\n debug?: boolean;\n model: string;\n};\n\nexport type PipelineState = {\n gpuInfo: string;\n isLoaded: boolean;\n isLoading: boolean;\n error: string | null;\n};\n\n// TODO(burdon): Document external API.\nexport type TranscriptionOptions = {\n sampling_rate: number;\n chunk_length_s: number;\n stride_length_s: number;\n return_timestamps: boolean;\n language: string;\n};\n\nexport const usePipeline = ({ active, model, debug }: PipelineConfig) => {\n const [state, setState] = useState<PipelineState>({\n gpuInfo: '',\n isLoaded: false,\n isLoading: false,\n error: null,\n });\n\n const pipelineRef = useRef<Pipeline | null>(null);\n\n // TODO(burdon): Factor out loading model. Separate tests.\n useEffect(() => {\n const loadModel = async () => {\n try {\n setState((prev) => ({ ...prev, isLoading: true, error: null }));\n\n // Check WebGPU support.\n if (!navigator.gpu) {\n log.warn('WebGPU is not supported, falling back to CPU');\n setState((prev) => ({ ...prev, gpuInfo: 'WebGPU not supported (using CPU)' }));\n } else {\n try {\n const adapter = await navigator.gpu.requestAdapter();\n if (!adapter) {\n throw new Error('No GPU adapter found');\n }\n\n // Get adapter info from the info property (modern WebGPU API).\n const adapterInfo = adapter.info;\n if (adapterInfo) {\n setState((prev) => ({\n ...prev,\n gpuInfo: `${adapterInfo.description || 'GPU'} (${adapterInfo.vendor || 'Unknown'})`,\n }));\n } else {\n setState((prev) => ({ ...prev, gpuInfo: 'GPU Available (details unknown)' }));\n }\n } catch (err) {\n log.warn('WebGPU initialization failed', { err });\n setState((prev) => ({ ...prev, gpuInfo: 'GPU initialization failed (using CPU)' }));\n }\n }\n\n const pipe = await pipeline('automatic-speech-recognition', model, {\n quantized: true,\n progress_callback: (progress: any) => {\n if (debug) {\n log(`loading model: ${Math.round(progress.progress * 100)}%`);\n }\n },\n });\n\n pipelineRef.current = pipe;\n setState((prev) => ({ ...prev, isLoaded: true, isLoading: false }));\n log.info('model loaded successfully');\n } catch (err) {\n log.error('error loading model', { err });\n setState((prev) => ({\n ...prev,\n isLoading: false,\n error: 'error loading model: ' + (err as Error).message,\n }));\n }\n };\n\n if (active) {\n void loadModel();\n }\n\n return () => {\n pipelineRef.current = null;\n setState((prev) => ({ ...prev, isLoaded: false }));\n };\n }, [active, debug, model]);\n\n const transcribe = async (audioData: Float32Array, options: TranscriptionOptions) => {\n invariant(pipelineRef.current, 'pipeline not initialized');\n const result = await pipelineRef.current(audioData, options);\n return result;\n };\n\n return {\n ...state,\n transcribe,\n };\n};\n"],
|
|
5
|
+
"mappings": ";AAIA,SAASA,aAAaC,WAAWC,QAAQC,gBAAgB;AAEzD,SAASC,WAAW;AAcpB,IAAA,eAAaC;IAETC,iBAAQ,CAAA,EAAA,QAAA,OAAA,YAAA,MAAA;QACRC,CAAAA,OAAO,QAAA,IAAA,SAAA;IACPC,QAAAA;IACF,OAAA;IAEA,YAAA;EACA,CAAA;AAEA,QAAMC,kBAAAA,OAAoBP,IAAeQ;AACzC,QAAMC,cAAAA,OAAiBT,IAAgC;AACvD,QAAMU,oBAAkBV,OAAO,MAAA;AAC/B,QAAMW,iBAAiBX,OAA2B,IAAA;AAClD,QAAMY,kBAAiBZ,OAAuB,KAAE;AAEhD,QAAA,iBAAA,OAA2B,IAAA;AAC3B,QAAMa,iBAAAA,OAAmBf,CAAAA,CAAAA;2BAEfgB,YAAgBC,MAAWC;QACjCA,YAAYC,SAAQC;AACpB,YAAMC,YAAUL,IAAAA,WAAiB,YAAcM,QAAMC,iBAAUP;AAC/DQ,kBAAUC,QAAU,qBAAA,SAAA;sBAAS,UAAA,OAAA,CAAA,KAAA,QAAA,MAAA,KAAA,CAAA,IAAA,UAAA;eAAEjB,CAAAA,UAAAA;QAAoB,GAAA;QACnDC,YAAkBU;MACpB,EAAA;AACG,wBAAA,UAAA,sBAAA,gBAAA;IAEL;MACEf,CAAAA;QAEA,UAAA,YAAmB,MAAA;AACnB,QAAIS,+BAAwB,QAAA,EAAA,YAAA,YAAA,GAAA,cAAA,GAAA,IAAA,GAAA,OAAA,CAAA;uBAEd,SAAA;qBACJa,QAAU,UAAA,EAAA,QAAA,CAAA,UAAA;AAClB,cAAA,KAAA;AACAb,cAAAA,UAAeM;MACjB,CAAA;AAEA,qBAAA,UAAA;IACA;QAEER,eAAeQ,SAAO;AACxB,qBAAA,QAAA,WAAA;AACID,qBAAYC,UAAS;;QAEvBD,YAAYC,SAAO;AACrB,kBAAA,QAAA,WAAA;AACIQ,kBAAAA,UAAgBR;;QAElBQ,gBAAgBR,SAAO;AACzB,WAAA,gBAAA,QAAA,MAAA;AACIV,sBAAAA,UAAyB;;QAE3BA,kBAAkBU,SAAO;AAC3B,2BAAA,kBAAA,OAAA;AAEAL,wBAAsB,UAAK;IAC3BU;mBACU,UAAA,CAAA;aACRjB;MACAC,QAAAA;MACF,OAAA;MACC,YAAA;IAACoB,CAAAA;KAAM;IAEV3B;;YAGQ4B,MAAAA;QACJ,UAAI;wBACU,YAAA;;YAEVzB,QAAQ;AACR,kBAAME;cACJwB,KAAAA,gCAAO,QAAA,EAAA,YAAA,YAAA,GAAA,cAAA,GAAA,IAAA,GAAA,OAAA,CAAA;yBACLC,MAAAA,UAAc,aAAA,aAAA;mBACdC;cACAC,cAAAA;cACAC,YAAAA;cACAC,kBAAiB;cACnB,kBAAA;cACAC,iBAAO;YACT;YAEI,OAACC;;0BAEKC,CAAAA,QAAI;mBACVC,UAAa,EAAA,QAAG,CAAA,UAAA;AAClB,oBAAA,KAAA;AACA,oBAAA,UAAA;YACF,CAAA;AAEA1B;UAEA;AACA,yBAAM2B,UAAcC;AAAkC,gBAAA,UAAA,IAAA,aAAA;YAEtD,YAAA;UACA,CAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;UA2CuC,GAAA;YAKvC,MAAMC;UACN,CAAA,CAAA,CAAA;AACAxB,gBAAAA,SAAYC,QAAUwB,wBAAAA,MAAAA;AAEtB,gBAAA,WAAA,QAAA,eAAA;AACA,sBAAMC,UAAc;AAGpBA,gBAAAA,cAAiBC,IAAAA,iBAAmBC,SAAAA,iBAAAA;yBAC7BT,UAAYU;sBACf,KAAA,YAAA,OAAA,UAAA;AACF,gBAAA,CAAA,WAAA,CAAA,QAAA;AAEID;;gBAEF,MAAI,KAAA,SAAA,cAAA;8BACE,UAAA;;oBAEFE,oBAAmBC;kBACnBC,YAAUA,QAAOJ;kBACjBK,QAAKC,MAAQ,KAAIN,KAAMG;kBACzB,KAAA,KAAA,IAAA,GAAA,MAAA,KAAA,IAAA;kBAEA,KAAMI,KAAAA,IAAAA,GAAcP,MAAMG,KAAKA,IAAI;gBACnC,GAAA,EAAA,YAAY,YAAA,GAAA,cAAA,GAAA,KAAA,GAAA,OAAA,CAAA;AACZ,sBAAIZ,cAAS,MAAA,KAAA,IAAA;uBACXb,KAAAA;6BACE;2BACAjB,CAAAA,UAAO;oBACT,GAAA;oBACF,OAAA,6BAAA,IAAA;kBACIA,EAAM;;AAAgC,oBAAA,MAAA,0BAAA;kBAC5C;gBACEK,GAAAA,EAAAA,YAAAA,YAAuB,GAAG,cAAA,GAAA,KAAA,GAAA,OAAA,CAAA;cAC5B,UAAA;AACF,gCAAA,UAAA;cACF;YAEA;UACA8B;AAEAE,iBAAAA,QAAYU,QAAQd;AAEpB,mBAAIZ,QAAO,WAAA;sBACTe,QAAgB,QAAG,WAAA;cACnB5B,OAAAA;AACF,qBAAA,UAAA;AAEAY,6BAAgBR;UAChB;0BACW,UAAA;yBACPb,QAAAA;qBACAC;cACAC;cACF,OAAA;cACF,YAAA;YACF,CAAA;UACA;QACA;eACEgB,KAAAA;qBACE;mBACAjB,CAAAA,UAAO;YACPD,GAAAA;YACF,OAAA,iCAAA,IAAA;YACF,QAAA;UACIC,EAAM;;AAA0B,YAAA,MAAA,oBAAA;UACpCgD;QACF,GAAA,EAAA,YAAA,YAAA,GAAA,cAAA,GAAA,KAAA,GAAA,OAAA,CAAA;AACF,gBAAA;MAEA;IAEA;SACElB,YAAU;WACVkB,MAAAA;AACF,gBAAA;AACC,cAAA;IAACR;;IAAeM;IAAatC;IAAkBwC;IAAQ;IAE1DtD;;YAEIsD,MAAAA;AACF,QAAA,CAAA,QAAA;AACC,cAAA;IAACR;;IAAgB;IAEpB;EACA,CAAA;;;;;ACvPF,SAAwBS,KAAKC,gBAAgB;AAC7C,SAASC,aAAAA,YAAWC,UAAAA,SAAQC,YAAAA,iBAAgB;AAE5C,SAASC,iBAAiB;AAC1B,SAASC,OAAAA,YAAW;AAEpB,IAAAC,gBAAA;AAIA,IAAA,WAAA;AACCP,IAAIQ,mBAA2BC;AAE/BT,IAAIQ,SAASE,KAAaC,KAAAA,aAAS;IAClCC,SAAAA,KAAAA,WAAe;AACjB,IAAA,SAAA,KAAA,SAAA;EAwBA,eAAaC;;IAETC,cAAS,CAAA,EAAA,QAAA,OAAA,MAAA,MAAA;QACTC,CAAAA,OAAU,QAAA,IAAAX,UAAA;IACVY,SAAAA;IACAC,UAAO;IACT,WAAA;IAEA,OAAMC;EAEN,CAAA;AACAhB,QAAAA,cAAUC,QAAA,IAAA;aAEN,MAAI;sBACQgB,YAAU;;iBAAWH,CAAAA,UAAAA;UAAiBC,GAAAA;UAAY,WAAA;UAE5D,OAAA;QACI,EAACG;YAEHC,CAAAA,UAAUF,KAAAA;oBAAY,gDAAO,QAAA,EAAA,YAAA,YAAA,GAAAZ,eAAA,GAAA,IAAA,GAAA,OAAA,CAAA;mBAAEO,CAAAA,UAAS;YAAmC,GAAA;YACtE,SAAA;UACD,EAAA;;cAEF;kBACE,UAAUQ,MAAM,UAAA,IAAA,eAAA;AAClB,gBAAA,CAAA,SAAA;AAEA,oBAAA,IAAA,MAAA,sBAAA;YACA;kBAEED,cAAUF,QAAU;6BACfA;uBACHL,CAAAA,UAAS;gBACX,GAAA;gBACK,SAAA,GAAA,YAAA,eAAA,KAAA,KAAA,YAAA,UAAA,SAAA;cACLO,EAAAA;;uBAA+BP,CAAAA,UAAS;gBAAkC,GAAA;gBAC5E,SAAA;cACOS,EAAK;YACZjB;mBAA2CiB,KAAAA;AAAI,YAAAjB,KAAA,KAAA,gCAAA;cAC/Ce;6BAAyBF,YAAI,GAAAZ,eAAA,GAAA,IAAA,GAAA,OAAA,CAAA;qBAAEO,CAAAA,UAAS;cAAwC,GAAA;cAClF,SAAA;YACF,EAAA;UAEA;;cAEEU,OAAAA,MAAAA,SAAoBC,gCAAAA,OAAAA;qBACdC;6BACG,CAAA,aAAiBC;AACxB,gBAAA,OAAA;AACF,cAAArB,KAAA,kBAAA,KAAA,MAAA,SAAA,WAAA,GAAA,CAAA,KAAA,QAAA,EAAA,YAAA,YAAA,GAAAC,eAAA,GAAA,IAAA,GAAA,OAAA,CAAA;YACF;UAEAW;QACAG,CAAAA;oBAAyBF,UAAI;iBAAEJ,CAAAA,UAAU;UAAMC,GAAAA;UAAiB,UAAA;UACxD,WAAC;QACT,EAAOO;AACPjB,QAAAA,KAAIW,KAAK,6BAAwB,QAAA,EAAA,YAAA,YAAA,GAAAV,eAAA,GAAA,IAAA,GAAA,OAAA,CAAA;eAAEgB,KAAAA;AAAI,QAAAjB,KAAA,MAAA,uBAAA;UACvCe;yBACKF,YAAI,GAAAZ,eAAA,GAAA,IAAA,GAAA,OAAA,CAAA;iBACPS,CAAAA,UAAAA;UACAC,GAAAA;UACF,WAAA;UACF,OAAA,0BAAA,IAAA;QACF,EAAA;MAEIW;;AAEJ,QAAA,QAAA;AAEA,WAAO,UAAA;;WAELP,MAAAA;kBAAyBF,UAAI;eAAEJ,CAAAA,UAAU;QAAM,GAAA;QACjD,UAAA;MACC,EAAA;IAACa;;IAAeC;IAAM;IAEzB;;QAEE,aAAe,OAAMX,WAAYY,YAAQC;AACzC,cAAOC,YAAAA,SAAAA,4BAAAA,EAAAA,YAAAA,YAAAA,GAAAA,eAAAA,GAAAA,KAAAA,GAAAA,QAAAA,GAAAA,CAAAA,uBAAAA,4BAAAA,EAAAA,CAAAA;AACT,UAAA,SAAA,MAAA,YAAA,QAAA,WAAA,OAAA;AAEA,WAAO;;SAELC;IACF,GAAA;IACA;;;",
|
|
6
|
+
"names": ["useCallback", "useEffect", "useRef", "useState", "log", "useAudioStream", "stream", "error", "audioLevel", "animationFrameRef", "undefined", "workletNodeRef", "isProcessingRef", "mediaStreamRef", "audioBufferRef", "updateAudioLevel", "dataArray", "Uint8Array", "analyserRef", "current", "getByteFrequencyData", "average", "acc", "val", "setState", "prev", "enabled", "audioContextRef", "debug", "startStream", "audio", "channelCount", "sampleRate", "echoCancellation", "noiseSuppression", "autoGainControl", "video", "mounted", "stop", "track", "context", "AudioContext", "source", "analyser", "workletNode", "onmessage", "event", "active", "length", "data", "min", "max", "Math", "onAudioData", "connect", "cleanup", "env", "pipeline", "useEffect", "useRef", "useState", "invariant", "log", "__dxlog_file", "backends", "numThreads", "onnx", "webgpu", "profilingMode", "usePipeline", "gpuInfo", "isLoaded", "isLoading", "error", "pipelineRef", "prev", "navigator", "setState", "Error", "err", "progress_callback", "progress", "debug", "Math", "active", "model", "current", "audioData", "result", "transcribe"]
|
|
7
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"inputs":{"src/meta.ts":{"bytes":3584,"imports":[{"path":"@dxos/util","kind":"import-statement","external":true}],"format":"esm"},"src/index.ts":{"bytes":368,"imports":[{"path":"src/meta.ts","kind":"import-statement","original":"./meta"}],"format":"esm"},"src/TransformerPlugin.tsx":{"bytes":2327,"imports":[{"path":"@dxos/app-framework","kind":"import-statement","external":true},{"path":"@dxos/app-toolkit","kind":"import-statement","external":true},{"path":"#meta","kind":"import-statement","external":true},{"path":"#translations","kind":"import-statement","external":true}],"format":"esm"},"src/components/Voice/DebugInfo.tsx":{"bytes":7253,"imports":[{"path":"react","kind":"import-statement","external":true}],"format":"esm"},"src/components/Voice/Voice.tsx":{"bytes":9591,"imports":[{"path":"react","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"#hooks","kind":"import-statement","external":true},{"path":"src/components/Voice/DebugInfo.tsx","kind":"import-statement","original":"./DebugInfo"}],"format":"esm"},"src/components/Voice/index.ts":{"bytes":458,"imports":[{"path":"src/components/Voice/DebugInfo.tsx","kind":"import-statement","original":"./DebugInfo"},{"path":"src/components/Voice/Voice.tsx","kind":"import-statement","original":"./Voice"}],"format":"esm"},"src/components/index.ts":{"bytes":369,"imports":[{"path":"src/components/Voice/index.ts","kind":"import-statement","original":"./Voice"}],"format":"esm"},"src/hooks/useAudioStream.ts":{"bytes":26320,"imports":[{"path":"react","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true}],"format":"esm"},"src/hooks/usePipeline.ts":{"bytes":13877,"imports":[{"path":"@xenova/transformers","kind":"import-statement","external":true},{"path":"react","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true}],"format":"esm"},"src/hooks/index.ts":{"bytes":489,"imports":[{"path":"src/hooks/useAudioStream.ts","kind":"import-statement","original":"./useAudioStream"},{"path":"src/hooks/usePipeline.ts","kind":"import-statement","original":"./usePipeline"}],"format":"esm"},"src/plugin.ts":{"bytes":876,"imports":[{"path":"@dxos/app-framework","kind":"import-statement","external":true},{"path":"src/meta.ts","kind":"import-statement","original":"./meta"},{"path":"#plugin","kind":"dynamic-import","external":true}],"format":"esm"},"src/translations.ts":{"bytes":1041,"imports":[{"path":"#meta","kind":"import-statement","external":true}],"format":"esm"}},"outputs":{"dist/lib/neutral/index.mjs.map":{"imports":[],"exports":[],"inputs":{},"bytes":93},"dist/lib/neutral/index.mjs":{"imports":[{"path":"dist/lib/neutral/chunk-YTFCIBAZ.mjs","kind":"import-statement"}],"exports":["meta"],"entryPoint":"src/index.ts","inputs":{"src/index.ts":{"bytesInOutput":0}},"bytes":101},"dist/lib/neutral/TransformerPlugin.mjs.map":{"imports":[],"exports":[],"inputs":{},"bytes":1149},"dist/lib/neutral/TransformerPlugin.mjs":{"imports":[{"path":"@dxos/app-framework","kind":"import-statement","external":true},{"path":"@dxos/app-toolkit","kind":"import-statement","external":true},{"path":"#meta","kind":"import-statement","external":true},{"path":"#translations","kind":"import-statement","external":true}],"exports":["TransformerPlugin","default"],"entryPoint":"src/TransformerPlugin.tsx","inputs":{"src/TransformerPlugin.tsx":{"bytesInOutput":541}},"bytes":689},"dist/lib/neutral/components/index.mjs.map":{"imports":[],"exports":[],"inputs":{},"bytes":8435},"dist/lib/neutral/components/index.mjs":{"imports":[{"path":"react","kind":"import-statement","external":true},{"path":"react","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"#hooks","kind":"import-statement","external":true}],"exports":["DebugInfo","Voice"],"entryPoint":"src/components/index.ts","inputs":{"src/components/Voice/DebugInfo.tsx":{"bytesInOutput":2426},"src/components/Voice/index.ts":{"bytesInOutput":0},"src/components/Voice/Voice.tsx":{"bytesInOutput":2573},"src/components/index.ts":{"bytesInOutput":0}},"bytes":5140},"dist/lib/neutral/hooks/index.mjs.map":{"imports":[],"exports":[],"inputs":{},"bytes":18099},"dist/lib/neutral/hooks/index.mjs":{"imports":[{"path":"react","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true},{"path":"@xenova/transformers","kind":"import-statement","external":true},{"path":"react","kind":"import-statement","external":true},{"path":"@dxos/invariant","kind":"import-statement","external":true},{"path":"@dxos/log","kind":"import-statement","external":true}],"exports":["useAudioStream","usePipeline"],"entryPoint":"src/hooks/index.ts","inputs":{"src/hooks/useAudioStream.ts":{"bytesInOutput":7364},"src/hooks/index.ts":{"bytesInOutput":0},"src/hooks/usePipeline.ts":{"bytesInOutput":3729}},"bytes":11232},"dist/lib/neutral/meta.mjs.map":{"imports":[],"exports":[],"inputs":{},"bytes":93},"dist/lib/neutral/meta.mjs":{"imports":[{"path":"dist/lib/neutral/chunk-YTFCIBAZ.mjs","kind":"import-statement"}],"exports":["meta"],"entryPoint":"src/meta.ts","inputs":{},"bytes":100},"dist/lib/neutral/plugin.mjs.map":{"imports":[],"exports":[],"inputs":{},"bytes":451},"dist/lib/neutral/plugin.mjs":{"imports":[{"path":"dist/lib/neutral/chunk-YTFCIBAZ.mjs","kind":"import-statement"},{"path":"@dxos/app-framework","kind":"import-statement","external":true},{"path":"#plugin","kind":"dynamic-import","external":true}],"exports":["TransformerPlugin"],"entryPoint":"src/plugin.ts","inputs":{"src/plugin.ts":{"bytesInOutput":114}},"bytes":247},"dist/lib/neutral/chunk-YTFCIBAZ.mjs.map":{"imports":[],"exports":[],"inputs":{},"bytes":1677},"dist/lib/neutral/chunk-YTFCIBAZ.mjs":{"imports":[{"path":"@dxos/util","kind":"import-statement","external":true}],"exports":["meta"],"inputs":{"src/meta.ts":{"bytesInOutput":1189}},"bytes":1268},"dist/lib/neutral/translations.mjs.map":{"imports":[],"exports":[],"inputs":{},"bytes":548},"dist/lib/neutral/translations.mjs":{"imports":[{"path":"#meta","kind":"import-statement","external":true}],"exports":["translations"],"entryPoint":"src/translations.ts","inputs":{"src/translations.ts":{"bytesInOutput":148}},"bytes":240}}}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import {
|
|
2
|
+
meta
|
|
3
|
+
} from "./chunk-YTFCIBAZ.mjs";
|
|
4
|
+
|
|
5
|
+
// src/plugin.ts
|
|
6
|
+
import { Plugin } from "@dxos/app-framework";
|
|
7
|
+
var TransformerPlugin = Plugin.lazy(meta, () => import("#plugin"));
|
|
8
|
+
export {
|
|
9
|
+
TransformerPlugin
|
|
10
|
+
};
|
|
11
|
+
//# sourceMappingURL=plugin.mjs.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../../src/plugin.ts"],
|
|
4
|
+
"sourcesContent": ["//\n// Copyright 2025 DXOS.org\n//\n\nimport { Plugin } from '@dxos/app-framework';\n\nimport { meta } from './meta';\n\nexport const TransformerPlugin = Plugin.lazy(meta, () => import('#plugin'));\n"],
|
|
5
|
+
"mappings": ";;;;;AAIA,SAASA,cAAc;AAIhB,IAAMC,oBAAoBC,OAAOC,KAAKC,MAAM,MAAM,OAAO,SAAA,CAAA;",
|
|
6
|
+
"names": ["Plugin", "TransformerPlugin", "Plugin", "lazy", "meta"]
|
|
7
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../../src/translations.ts"],
|
|
4
|
+
"sourcesContent": ["//\n// Copyright 2023 DXOS.org\n//\n\nimport { type Resource } from '@dxos/react-ui';\n\nimport { meta } from '#meta';\n\nexport const translations = [\n {\n 'en-US': {\n [meta.id]: {\n 'plugin.name': 'Transformers',\n },\n },\n },\n] as const satisfies Resource[];\n"],
|
|
5
|
+
"mappings": ";AAMA,SAASA,YAAY;AAEd,IAAMC,eAAe;EAC1B;IACE,SAAS;MACP,CAACD,KAAKE,EAAE,GAAG;QACT,eAAe;MACjB;IACF;EACF;;",
|
|
6
|
+
"names": ["meta", "translations", "id"]
|
|
7
|
+
}
|
|
@@ -1,2 +1,4 @@
|
|
|
1
|
-
|
|
1
|
+
import { Plugin } from '@dxos/app-framework';
|
|
2
|
+
export declare const TransformerPlugin: Plugin.PluginFactory<void>;
|
|
3
|
+
export default TransformerPlugin;
|
|
2
4
|
//# sourceMappingURL=TransformerPlugin.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"TransformerPlugin.d.ts","sourceRoot":"","sources":["../../../src/TransformerPlugin.tsx"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"TransformerPlugin.d.ts","sourceRoot":"","sources":["../../../src/TransformerPlugin.tsx"],"names":[],"mappings":"AAIA,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAO7C,eAAO,MAAM,iBAAiB,4BAS7B,CAAC;eAEa,iBAAiB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"TransformerPlugin.test.d.ts","sourceRoot":"","sources":["../../../src/TransformerPlugin.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"DebugInfo.d.ts","sourceRoot":"","sources":["../../../../../src/components/Voice/DebugInfo.tsx"],"names":[],"mappings":"AAIA,OAAc,EAAE,KAAK,EAAE,EAAE,MAAM,OAAO,CAAC;AAEvC,MAAM,MAAM,cAAc,GAAG;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,OAAO,CAAC;IACxB,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;IAC3B,cAAc,EAAE,OAAO,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,OAAO,CAAC;CAChB,CAAC;AAEF,eAAO,MAAM,SAAS,EAAE,EAAE,CAAC,OAAO,CAAC,cAAc,CAAC,CA4DjD,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Voice.d.ts","sourceRoot":"","sources":["../../../../../src/components/Voice/Voice.tsx"],"names":[],"mappings":"AAIA,OAAO,KAA2C,MAAM,OAAO,CAAC;AAQhE,MAAM,MAAM,UAAU,GAAG;IACvB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,eAAO,MAAM,KAAK,6BAAsD,UAAU,sBA4FjF,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Voice.stories.d.ts","sourceRoot":"","sources":["../../../../../src/components/Voice/Voice.stories.tsx"],"names":[],"mappings":"AAIA,OAAO,EAAa,KAAK,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAIjE,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAEhC,QAAA,MAAM,IAAI;;;;;QAKN,MAAM;;CAEoB,CAAC;eAEhB,IAAI;AAEnB,KAAK,KAAK,GAAG,QAAQ,CAAC,OAAO,KAAK,CAAC,CAAC;AAEpC,eAAO,MAAM,OAAO,EAAE,KAMrB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/components/Voice/index.ts"],"names":[],"mappings":"AAIA,cAAc,aAAa,CAAC;AAC5B,cAAc,SAAS,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/components/index.ts"],"names":[],"mappings":"AAIA,cAAc,SAAS,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"useAudioStream.d.ts","sourceRoot":"","sources":["../../../../src/hooks/useAudioStream.ts"],"names":[],"mappings":"AAQA,MAAM,MAAM,iBAAiB,GAAG;IAC9B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,WAAW,CAAC,EAAE,CAAC,SAAS,EAAE,YAAY,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1D,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;IAC3B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,eAAO,MAAM,cAAc,
|
|
1
|
+
{"version":3,"file":"useAudioStream.d.ts","sourceRoot":"","sources":["../../../../src/hooks/useAudioStream.ts"],"names":[],"mappings":"AAQA,MAAM,MAAM,iBAAiB,GAAG;IAC9B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,WAAW,CAAC,EAAE,CAAC,SAAS,EAAE,YAAY,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1D,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;IAC3B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,eAAO,MAAM,cAAc,mCAAoC,iBAAiB,qBAuO/E,CAAC"}
|
|
@@ -1,18 +1,3 @@
|
|
|
1
|
-
declare global {
|
|
2
|
-
interface Navigator {
|
|
3
|
-
gpu?: {
|
|
4
|
-
requestAdapter(): Promise<GPUAdapter | null>;
|
|
5
|
-
};
|
|
6
|
-
}
|
|
7
|
-
interface GPUAdapter {
|
|
8
|
-
requestAdapterInfo(): Promise<GPUAdapterInfo>;
|
|
9
|
-
}
|
|
10
|
-
interface GPUAdapterInfo {
|
|
11
|
-
vendor: string;
|
|
12
|
-
architecture: string;
|
|
13
|
-
description: string;
|
|
14
|
-
}
|
|
15
|
-
}
|
|
16
1
|
export type PipelineConfig = {
|
|
17
2
|
active?: boolean;
|
|
18
3
|
debug?: boolean;
|
|
@@ -32,10 +17,10 @@ export type TranscriptionOptions = {
|
|
|
32
17
|
language: string;
|
|
33
18
|
};
|
|
34
19
|
export declare const usePipeline: ({ active, model, debug }: PipelineConfig) => {
|
|
35
|
-
transcribe: (audioData: Float32Array, options: TranscriptionOptions) => Promise<any>;
|
|
36
20
|
gpuInfo: string;
|
|
37
21
|
isLoaded: boolean;
|
|
38
22
|
isLoading: boolean;
|
|
39
23
|
error: string | null;
|
|
24
|
+
transcribe: (audioData: Float32Array, options: TranscriptionOptions) => Promise<any>;
|
|
40
25
|
};
|
|
41
26
|
//# sourceMappingURL=usePipeline.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"usePipeline.d.ts","sourceRoot":"","sources":["../../../../src/hooks/usePipeline.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"usePipeline.d.ts","sourceRoot":"","sources":["../../../../src/hooks/usePipeline.ts"],"names":[],"mappings":"AAqBA,MAAM,MAAM,cAAc,GAAG;IAC3B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;IACnB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB,CAAC;AAGF,MAAM,MAAM,oBAAoB,GAAG;IACjC,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,eAAO,MAAM,WAAW,6BAA8B,cAAc;aAfzD,MAAM;cACL,OAAO;eACN,OAAO;WACX,MAAM,GAAG,IAAI;4BAuFiB,YAAY,WAAW,oBAAoB;CAUjF,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/index.ts"],"names":[],"mappings":"AAIA,cAAc,QAAQ,CAAC
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/index.ts"],"names":[],"mappings":"AAIA,cAAc,QAAQ,CAAC"}
|
package/dist/types/src/meta.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import { type
|
|
2
|
-
export declare const meta:
|
|
1
|
+
import { type Plugin } from '@dxos/app-framework';
|
|
2
|
+
export declare const meta: Plugin.Meta;
|
|
3
3
|
//# sourceMappingURL=meta.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"meta.d.ts","sourceRoot":"","sources":["../../../src/meta.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,
|
|
1
|
+
{"version":3,"file":"meta.d.ts","sourceRoot":"","sources":["../../../src/meta.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,KAAK,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAGlD,eAAO,MAAM,IAAI,EAAE,MAAM,CAAC,IAwBzB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"plugin.d.ts","sourceRoot":"","sources":["../../../src/plugin.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAI7C,eAAO,MAAM,iBAAiB,4BAA6C,CAAC"}
|
|
@@ -7,6 +7,6 @@ export declare class NodeRagPipeline extends RagPipeline {
|
|
|
7
7
|
private textModel;
|
|
8
8
|
constructor(embeddingModel?: string, textModel?: string);
|
|
9
9
|
protected generateEmbeddings(text: string | string[]): Promise<EmbeddingOutput>;
|
|
10
|
-
protected generateText(prompt: string): Promise<import("@xenova/transformers").Text2TextGenerationOutput | import("@xenova/transformers").Text2TextGenerationOutput
|
|
10
|
+
protected generateText(prompt: string): Promise<import("@xenova/transformers").Text2TextGenerationOutput[] | import("@xenova/transformers").Text2TextGenerationOutput>;
|
|
11
11
|
}
|
|
12
12
|
//# sourceMappingURL=node-pipeline.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"node-pipeline.d.ts","sourceRoot":"","sources":["../../../../src/testing/node-pipeline.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,KAAK,eAAe,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE/D;;GAEG;AACH,qBAAa,eAAgB,SAAQ,WAAW;IAC9C,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;
|
|
1
|
+
{"version":3,"file":"node-pipeline.d.ts","sourceRoot":"","sources":["../../../../src/testing/node-pipeline.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,KAAK,eAAe,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE/D;;GAEG;AACH,qBAAa,eAAgB,SAAQ,WAAW;IAC9C,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAS;IAE1B,YAAY,cAAc,SAA4B,EAAE,SAAS,SAAoB,EAIpF;IAED,UAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC,CAIpF;IAED,UAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,kIAM1C;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../../../src/testing/pipeline.ts"],"names":[],"mappings":"AAMA;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB,CAAC;AAEF;;GAEG;AACH,8BAAsB,WAAW;IAC/B;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../../../src/testing/pipeline.ts"],"names":[],"mappings":"AAMA;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB,CAAC;AAEF;;GAEG;AACH,8BAAsB,WAAW;IAC/B;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;IAEzF;;OAEG;IACH,SAAS,CAAC,QAAQ,CAAC,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IAE9D;;OAEG;IACH,SAAS,CAAC,gBAAgB,CAAC,CAAC,EAAE,YAAY,GAAG,MAAM,EAAE,EAAE,CAAC,EAAE,YAAY,GAAG,MAAM,EAAE,GAAG,MAAM,CAOzF;IAED;;OAEG;IACG,mBAAmB,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,gBA8B/D;CACF"}
|
|
@@ -7,6 +7,6 @@ export declare class WebRagPipeline extends RagPipeline {
|
|
|
7
7
|
private textModel;
|
|
8
8
|
constructor(embeddingModel?: string, textModel?: string);
|
|
9
9
|
protected generateEmbeddings(text: string | string[]): Promise<EmbeddingOutput>;
|
|
10
|
-
protected generateText(prompt: string): Promise<import("@huggingface/transformers").TextGenerationOutput | import("@huggingface/transformers").TextGenerationOutput
|
|
10
|
+
protected generateText(prompt: string): Promise<import("@huggingface/transformers").TextGenerationOutput[] | import("@huggingface/transformers").TextGenerationOutput>;
|
|
11
11
|
}
|
|
12
12
|
//# sourceMappingURL=web-pipeline.d.ts.map
|