@pexip/media-processor 16.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1047 -0
- package/LICENSE +188 -0
- package/README.md +193 -0
- package/dist/index.d.ts +1129 -0
- package/dist/index.mjs +2441 -0
- package/dist/worklets/denoise.worklet.js +2 -0
- package/dist/worklets/denoise.worklet.js.map +7 -0
- package/package.json +53 -0
package/dist/index.mjs
ADDED
|
@@ -0,0 +1,2441 @@
|
|
|
1
|
+
// src/math.ts
|
|
2
|
+
var sum = (nums) => nums.reduce((accm, num) => accm + num, 0);
|
|
3
|
+
var avg = (nums) => {
|
|
4
|
+
if (nums.length === 0) {
|
|
5
|
+
return 0;
|
|
6
|
+
}
|
|
7
|
+
if (nums.length === 1) {
|
|
8
|
+
return nums[0] ?? 0;
|
|
9
|
+
}
|
|
10
|
+
return sum(nums) / nums.length;
|
|
11
|
+
};
|
|
12
|
+
var pow = (exponent) => (base) => Math.pow(base, exponent);
|
|
13
|
+
var rms = (nums) => {
|
|
14
|
+
if (!nums || !Array.isArray(nums) || !nums.length) {
|
|
15
|
+
return 0;
|
|
16
|
+
}
|
|
17
|
+
if (nums.length === 1 && nums[0] !== void 0) {
|
|
18
|
+
return Math.abs(nums[0]);
|
|
19
|
+
}
|
|
20
|
+
return Math.sqrt(sum(nums.map(pow(2))) / nums.length);
|
|
21
|
+
};
|
|
22
|
+
var round = (num) => num ? Math.round(num) : -Math.round(-num);
|
|
23
|
+
|
|
24
|
+
// src/utils.ts
|
|
25
|
+
var hasAudioContext = () => typeof AudioContext !== "undefined";
|
|
26
|
+
var hasCreateGain = (context) => typeof context.createGain !== "undefined";
|
|
27
|
+
var stopTrack = (track) => track.stop();
|
|
28
|
+
var stopStreamTracks = (stream) => stream?.getTracks().forEach(stopTrack);
|
|
29
|
+
var createMediaStreamAudioSourceNode = (context, options) => {
|
|
30
|
+
try {
|
|
31
|
+
const source = new MediaStreamAudioSourceNode(context, options);
|
|
32
|
+
return source;
|
|
33
|
+
} catch {
|
|
34
|
+
return context.createMediaStreamSource(options.mediaStream);
|
|
35
|
+
}
|
|
36
|
+
};
|
|
37
|
+
var createMediaElementSourceNode = (context, options) => {
|
|
38
|
+
try {
|
|
39
|
+
const source = new MediaElementAudioSourceNode(context, options);
|
|
40
|
+
return source;
|
|
41
|
+
} catch {
|
|
42
|
+
return context.createMediaElementSource(options.mediaElement);
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
var setAudioNodeOptions = (node, options) => {
|
|
46
|
+
if (options?.channelCount) {
|
|
47
|
+
node.channelCount = options.channelCount;
|
|
48
|
+
}
|
|
49
|
+
if (options?.channelCountMode) {
|
|
50
|
+
node.channelCountMode = options.channelCountMode;
|
|
51
|
+
}
|
|
52
|
+
if (options?.channelInterpretation) {
|
|
53
|
+
node.channelInterpretation = options.channelInterpretation;
|
|
54
|
+
}
|
|
55
|
+
};
|
|
56
|
+
var createAnalyserNode = (audioContext, options) => {
|
|
57
|
+
try {
|
|
58
|
+
const analyser = new AnalyserNode(audioContext, options);
|
|
59
|
+
return analyser;
|
|
60
|
+
} catch {
|
|
61
|
+
const analyser = audioContext.createAnalyser();
|
|
62
|
+
options?.fftSize && (analyser.fftSize = options.fftSize);
|
|
63
|
+
options?.maxDecibels && (analyser.maxDecibels = options.maxDecibels);
|
|
64
|
+
options?.minDecibels && (analyser.minDecibels = options.minDecibels);
|
|
65
|
+
options?.smoothingTimeConstant && (analyser.smoothingTimeConstant = options.smoothingTimeConstant);
|
|
66
|
+
setAudioNodeOptions(analyser, options);
|
|
67
|
+
return analyser;
|
|
68
|
+
}
|
|
69
|
+
};
|
|
70
|
+
var createGainNode = (context, options) => {
|
|
71
|
+
try {
|
|
72
|
+
const volume = new GainNode(context, options);
|
|
73
|
+
return volume;
|
|
74
|
+
} catch {
|
|
75
|
+
const volume = hasCreateGain(context) ? context.createGain() : context.createGainNode();
|
|
76
|
+
if (options?.gain) {
|
|
77
|
+
volume.gain.setValueAtTime(options.gain, context.currentTime);
|
|
78
|
+
}
|
|
79
|
+
setAudioNodeOptions(volume, options);
|
|
80
|
+
return volume;
|
|
81
|
+
}
|
|
82
|
+
};
|
|
83
|
+
var createMediaStreamAudioClone = (stream) => {
|
|
84
|
+
try {
|
|
85
|
+
const mediaStream = new MediaStream(
|
|
86
|
+
stream.getAudioTracks().map((track) => track.clone())
|
|
87
|
+
);
|
|
88
|
+
return mediaStream;
|
|
89
|
+
} catch {
|
|
90
|
+
return stream.clone();
|
|
91
|
+
}
|
|
92
|
+
};
|
|
93
|
+
var createMediaStreamAudioDestinationNode = (context, options) => {
|
|
94
|
+
try {
|
|
95
|
+
const destination = new MediaStreamAudioDestinationNode(
|
|
96
|
+
context,
|
|
97
|
+
options
|
|
98
|
+
);
|
|
99
|
+
return destination;
|
|
100
|
+
} catch {
|
|
101
|
+
const destination = context.createMediaStreamDestination();
|
|
102
|
+
setAudioNodeOptions(destination, options);
|
|
103
|
+
return destination;
|
|
104
|
+
}
|
|
105
|
+
};
|
|
106
|
+
var createDelayNode = (context, options) => {
|
|
107
|
+
try {
|
|
108
|
+
const delay = new DelayNode(context, options);
|
|
109
|
+
return delay;
|
|
110
|
+
} catch {
|
|
111
|
+
const delay = context.createDelay(options?.maxDelayTime);
|
|
112
|
+
if (options?.delayTime !== void 0) {
|
|
113
|
+
delay.delayTime.setValueAtTime(
|
|
114
|
+
options?.delayTime,
|
|
115
|
+
context.currentTime
|
|
116
|
+
);
|
|
117
|
+
}
|
|
118
|
+
setAudioNodeOptions(delay, options);
|
|
119
|
+
return delay;
|
|
120
|
+
}
|
|
121
|
+
};
|
|
122
|
+
var createChannelSplitterNode = (context, options) => {
|
|
123
|
+
try {
|
|
124
|
+
const node = new ChannelSplitterNode(context, options);
|
|
125
|
+
return node;
|
|
126
|
+
} catch {
|
|
127
|
+
const node = context.createChannelSplitter(options?.numberOfOutputs);
|
|
128
|
+
setAudioNodeOptions(node, options);
|
|
129
|
+
return node;
|
|
130
|
+
}
|
|
131
|
+
};
|
|
132
|
+
var createChannelMergerNode = (context, options) => {
|
|
133
|
+
try {
|
|
134
|
+
const node = new ChannelMergerNode(context, options);
|
|
135
|
+
return node;
|
|
136
|
+
} catch {
|
|
137
|
+
const node = context.createChannelMerger(options?.numberOfInputs);
|
|
138
|
+
setAudioNodeOptions(node, options);
|
|
139
|
+
return node;
|
|
140
|
+
}
|
|
141
|
+
};
|
|
142
|
+
var muteToGain = (mute) => mute ? 0 : 1;
|
|
143
|
+
var calculateNextTimeout = (targetTime, startTime, endTime) => Math.max(targetTime - Math.max(endTime - startTime, 0), 0);
|
|
144
|
+
var createDelayedCallback = (callback, {
|
|
145
|
+
setTimeout: setTimeout2 = window.setTimeout,
|
|
146
|
+
clearTimeout = window.clearTimeout
|
|
147
|
+
} = {}) => {
|
|
148
|
+
const props = {
|
|
149
|
+
timeoutID: 0
|
|
150
|
+
};
|
|
151
|
+
const cancelTimeout = () => {
|
|
152
|
+
if (props.timeoutID) {
|
|
153
|
+
clearTimeout(props.timeoutID);
|
|
154
|
+
props.timeoutID = 0;
|
|
155
|
+
}
|
|
156
|
+
};
|
|
157
|
+
const delayedCallback = async (delayMs, ...params) => {
|
|
158
|
+
const resolved = await new Promise((resolve) => {
|
|
159
|
+
cancelTimeout();
|
|
160
|
+
props.timeoutID = setTimeout2(() => {
|
|
161
|
+
const result = callback(...params);
|
|
162
|
+
if (result instanceof Promise) {
|
|
163
|
+
result.then((resolved2) => resolve(resolved2)).catch((e) => {
|
|
164
|
+
throw e;
|
|
165
|
+
});
|
|
166
|
+
} else {
|
|
167
|
+
resolve(result);
|
|
168
|
+
}
|
|
169
|
+
}, delayMs);
|
|
170
|
+
props.cancel = resolve;
|
|
171
|
+
});
|
|
172
|
+
return resolved;
|
|
173
|
+
};
|
|
174
|
+
const cancel = () => {
|
|
175
|
+
cancelTimeout();
|
|
176
|
+
props.cancel?.();
|
|
177
|
+
};
|
|
178
|
+
return [delayedCallback, cancel];
|
|
179
|
+
};
|
|
180
|
+
var rateToMs = (rate) => Math.ceil(1e3 / rate);
|
|
181
|
+
var createAsyncCallbackLoop = (callback, frameRate, {
|
|
182
|
+
setTimeout: setTimeout2 = window.setTimeout,
|
|
183
|
+
clearTimeout = window.clearTimeout,
|
|
184
|
+
now = () => performance.now()
|
|
185
|
+
} = {}) => {
|
|
186
|
+
const props = {
|
|
187
|
+
frameRate,
|
|
188
|
+
targetMs: rateToMs(frameRate),
|
|
189
|
+
prevCalledMs: 0,
|
|
190
|
+
timeoutID: 0,
|
|
191
|
+
stopped: false
|
|
192
|
+
};
|
|
193
|
+
const [delayedCallback, cancel] = createDelayedCallback(callback, {
|
|
194
|
+
setTimeout: setTimeout2,
|
|
195
|
+
clearTimeout
|
|
196
|
+
});
|
|
197
|
+
const fork = async (...params) => {
|
|
198
|
+
if (props.stopped) {
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
201
|
+
const currentMs = now();
|
|
202
|
+
const nextMs = calculateNextTimeout(
|
|
203
|
+
props.targetMs,
|
|
204
|
+
props.prevCalledMs,
|
|
205
|
+
currentMs
|
|
206
|
+
);
|
|
207
|
+
props.prevCalledMs = currentMs;
|
|
208
|
+
await delayedCallback(nextMs, ...params);
|
|
209
|
+
await fork(...params);
|
|
210
|
+
};
|
|
211
|
+
return {
|
|
212
|
+
start: async (...params) => {
|
|
213
|
+
props.prevCalledMs = now();
|
|
214
|
+
props.stopped = false;
|
|
215
|
+
await delayedCallback(0, ...params);
|
|
216
|
+
void fork(...params);
|
|
217
|
+
},
|
|
218
|
+
stop: () => {
|
|
219
|
+
props.stopped = true;
|
|
220
|
+
cancel();
|
|
221
|
+
},
|
|
222
|
+
get frameRate() {
|
|
223
|
+
return props.frameRate;
|
|
224
|
+
},
|
|
225
|
+
set frameRate(value) {
|
|
226
|
+
props.frameRate = value;
|
|
227
|
+
props.targetMs = rateToMs(value);
|
|
228
|
+
}
|
|
229
|
+
};
|
|
230
|
+
};
|
|
231
|
+
var DEFAULT_THROTTLE_MS = 3e3;
|
|
232
|
+
var throttleProcess = (callback, throttleMs = DEFAULT_THROTTLE_MS, clock = performance) => {
|
|
233
|
+
let lastCall = 0;
|
|
234
|
+
return (...params) => {
|
|
235
|
+
const now = clock.now();
|
|
236
|
+
if (now - lastCall >= throttleMs) {
|
|
237
|
+
callback(...params);
|
|
238
|
+
lastCall = now;
|
|
239
|
+
}
|
|
240
|
+
};
|
|
241
|
+
};
|
|
242
|
+
var subscribeVisibilityChangeEvent = (callback) => {
|
|
243
|
+
const handleEvent = () => {
|
|
244
|
+
callback(document.hidden).catch((error) => {
|
|
245
|
+
throw error;
|
|
246
|
+
});
|
|
247
|
+
};
|
|
248
|
+
document.addEventListener("visibilitychange", handleEvent);
|
|
249
|
+
return () => {
|
|
250
|
+
document.removeEventListener("visibilitychange", handleEvent);
|
|
251
|
+
};
|
|
252
|
+
};
|
|
253
|
+
|
|
254
|
+
// src/process.ts
|
|
255
|
+
var SILENT_THRESHOLD = 1 / 32767;
|
|
256
|
+
var MONO_THRESHOLD = 1 / 65536;
|
|
257
|
+
var LOW_VOLUME_THRESHOLD = -60;
|
|
258
|
+
var CLIP_THRESHOLD = 0.98;
|
|
259
|
+
var VOICE_PROBABILITY_THRESHOLD = 0.3;
|
|
260
|
+
var CLIP_COUNT_THRESHOLD = 6;
|
|
261
|
+
var createAudioStats = (stats = {}, {
|
|
262
|
+
silentThreshold,
|
|
263
|
+
lowVolumeThreshold,
|
|
264
|
+
clipCountThreshold
|
|
265
|
+
} = {}) => {
|
|
266
|
+
return {
|
|
267
|
+
peak: stats.peak ?? 0,
|
|
268
|
+
maxRms: stats.maxRms ?? 0,
|
|
269
|
+
maxClipCount: stats.maxClipCount ?? 0,
|
|
270
|
+
sumSquare: stats.sumSquare ?? 0,
|
|
271
|
+
sumLength: stats.sumLength ?? 0,
|
|
272
|
+
get silent() {
|
|
273
|
+
return isSilent([this.peak], silentThreshold);
|
|
274
|
+
},
|
|
275
|
+
get clipping() {
|
|
276
|
+
return isClipping(this.maxClipCount, clipCountThreshold);
|
|
277
|
+
},
|
|
278
|
+
set clipping(value) {
|
|
279
|
+
this.clipping = value;
|
|
280
|
+
},
|
|
281
|
+
get rms() {
|
|
282
|
+
return this.sumLength && Math.sqrt(this.sumSquare / this.sumLength);
|
|
283
|
+
},
|
|
284
|
+
get lowVolume() {
|
|
285
|
+
return this.rms === void 0 ? false : isLowVolume(this.rms, lowVolumeThreshold);
|
|
286
|
+
}
|
|
287
|
+
};
|
|
288
|
+
};
|
|
289
|
+
var fromByteToFloat = (value) => (value - 128) / 128;
|
|
290
|
+
var fromFloatToByte = (value) => round(value * 128 + 128);
|
|
291
|
+
var copyByteBufferToFloatBuffer = (bytes, floats) => {
|
|
292
|
+
bytes.forEach((value, idx) => {
|
|
293
|
+
floats[idx] = fromByteToFloat(value);
|
|
294
|
+
});
|
|
295
|
+
};
|
|
296
|
+
var toDecibel = (gain) => 20 * Math.log10(Math.abs(gain));
|
|
297
|
+
var processAverageVolume = (data) => data.length ? rms(data) : 0;
|
|
298
|
+
var isSilent = (samples, threshold = SILENT_THRESHOLD) => samples.length === 0 || getFirstSample(samples) <= threshold && getLastSample(samples) <= threshold;
|
|
299
|
+
function getFirstSample(samples) {
|
|
300
|
+
if (samples[0] !== void 0) {
|
|
301
|
+
return Math.abs(samples[0]);
|
|
302
|
+
}
|
|
303
|
+
return 0;
|
|
304
|
+
}
|
|
305
|
+
function getLastSample(samples) {
|
|
306
|
+
const last = samples[samples.length - 1];
|
|
307
|
+
if (last) {
|
|
308
|
+
return Math.abs(last);
|
|
309
|
+
}
|
|
310
|
+
return 0;
|
|
311
|
+
}
|
|
312
|
+
var isLowVolume = (gain, threshold = LOW_VOLUME_THRESHOLD) => toDecibel(gain) < threshold;
|
|
313
|
+
var isClipping = (clipCount, threshold = CLIP_COUNT_THRESHOLD) => clipCount > threshold;
|
|
314
|
+
var isMono = (channels, threshold = MONO_THRESHOLD) => {
|
|
315
|
+
let sampleDiffCount = 0;
|
|
316
|
+
if (channels.length < 2 || channels.filter((channel) => !isSilent(channel)).length < 2) {
|
|
317
|
+
return true;
|
|
318
|
+
}
|
|
319
|
+
if (channels[0]?.length === channels[1]?.length) {
|
|
320
|
+
channels[0]?.forEach((l, idx) => {
|
|
321
|
+
const r = channels[1]?.[idx];
|
|
322
|
+
if (r !== void 0 && Math.abs(l - r) > threshold) {
|
|
323
|
+
sampleDiffCount++;
|
|
324
|
+
}
|
|
325
|
+
});
|
|
326
|
+
} else {
|
|
327
|
+
sampleDiffCount++;
|
|
328
|
+
}
|
|
329
|
+
return sampleDiffCount === 0;
|
|
330
|
+
};
|
|
331
|
+
var getAudioStats = ({
|
|
332
|
+
samples,
|
|
333
|
+
baseStats,
|
|
334
|
+
clipThreshold = CLIP_THRESHOLD
|
|
335
|
+
}) => {
|
|
336
|
+
let rms2 = 0;
|
|
337
|
+
let clipCount = 0;
|
|
338
|
+
let maxClipCount = 0;
|
|
339
|
+
let peak = 0;
|
|
340
|
+
const stats = baseStats || createAudioStats();
|
|
341
|
+
samples.forEach((s) => {
|
|
342
|
+
const absS = Math.abs(s);
|
|
343
|
+
peak = Math.max(peak, absS);
|
|
344
|
+
if (absS >= clipThreshold) {
|
|
345
|
+
clipCount += 1;
|
|
346
|
+
maxClipCount = Math.max(clipCount, maxClipCount);
|
|
347
|
+
} else {
|
|
348
|
+
clipCount = 0;
|
|
349
|
+
}
|
|
350
|
+
rms2 += absS * absS;
|
|
351
|
+
});
|
|
352
|
+
stats.peak = Math.max(stats.peak ?? 0, peak);
|
|
353
|
+
stats.sumSquare += rms2;
|
|
354
|
+
stats.sumLength += samples.length;
|
|
355
|
+
rms2 = samples.length ? Math.sqrt(rms2 / samples.length) : 0;
|
|
356
|
+
stats.maxRms = Math.max(stats.maxRms ?? 0, rms2);
|
|
357
|
+
stats.maxClipCount = Math.max(maxClipCount, stats.maxClipCount ?? 0);
|
|
358
|
+
return stats;
|
|
359
|
+
};
|
|
360
|
+
var isVoiceActivity = ({
|
|
361
|
+
volumeThreshold = 0.05,
|
|
362
|
+
VADTimeThreshold = 500,
|
|
363
|
+
clock = performance
|
|
364
|
+
} = {}) => {
|
|
365
|
+
let lastVADTime = 0;
|
|
366
|
+
return (volume) => {
|
|
367
|
+
if (volume >= volumeThreshold) {
|
|
368
|
+
const now = clock.now();
|
|
369
|
+
if (!lastVADTime) {
|
|
370
|
+
lastVADTime = now;
|
|
371
|
+
return false;
|
|
372
|
+
}
|
|
373
|
+
if (now - lastVADTime >= VADTimeThreshold) {
|
|
374
|
+
return true;
|
|
375
|
+
}
|
|
376
|
+
return false;
|
|
377
|
+
}
|
|
378
|
+
if (lastVADTime) {
|
|
379
|
+
lastVADTime = 0;
|
|
380
|
+
}
|
|
381
|
+
return false;
|
|
382
|
+
};
|
|
383
|
+
};
|
|
384
|
+
var isEqualSize = (widthA, heightA, widthB, heightB) => widthA === widthB && heightA === heightB;
|
|
385
|
+
var fitDestinationSize = (sw, sh, dw, dh) => {
|
|
386
|
+
if (!sw || !sh || !dw || !dh) {
|
|
387
|
+
return { x: 0, y: 0, width: 0, height: 0 };
|
|
388
|
+
}
|
|
389
|
+
if (isEqualSize(sw, sh, dw, dh)) {
|
|
390
|
+
return { x: 0, y: 0, width: sw, height: sh };
|
|
391
|
+
}
|
|
392
|
+
const height = Math.floor(dw * (sh / sw));
|
|
393
|
+
const y = Math.floor((dh - height) / 2);
|
|
394
|
+
return { x: 0, y, width: dw, height };
|
|
395
|
+
};
|
|
396
|
+
var createVoiceDetectorFromTimeData = (options = {}) => {
|
|
397
|
+
const isVoice = isVoiceActivity(options);
|
|
398
|
+
return (timeData) => isVoice(rms(timeData));
|
|
399
|
+
};
|
|
400
|
+
var createVoiceDetectorFromProbability = (voiceThreshold = VOICE_PROBABILITY_THRESHOLD) => (probability) => probability >= voiceThreshold;
|
|
401
|
+
var createVADetector = (onDetected, shouldDetect, options) => (isVoice) => {
|
|
402
|
+
const throttledTrigger = throttleProcess(
|
|
403
|
+
onDetected,
|
|
404
|
+
options?.throttleMs,
|
|
405
|
+
options?.clock
|
|
406
|
+
);
|
|
407
|
+
const process = (data) => {
|
|
408
|
+
if (!shouldDetect()) {
|
|
409
|
+
return;
|
|
410
|
+
}
|
|
411
|
+
if (isVoice(data)) {
|
|
412
|
+
throttledTrigger();
|
|
413
|
+
}
|
|
414
|
+
};
|
|
415
|
+
return process;
|
|
416
|
+
};
|
|
417
|
+
var createAudioSignalDetector = (shouldDetect, onDetected) => (buffer, threshold) => {
|
|
418
|
+
const props = { silent: false, lastCheck: false };
|
|
419
|
+
return (samples) => {
|
|
420
|
+
if (!shouldDetect()) {
|
|
421
|
+
buffer.empty();
|
|
422
|
+
props.silent = false;
|
|
423
|
+
props.lastCheck = false;
|
|
424
|
+
return;
|
|
425
|
+
}
|
|
426
|
+
if (buffer.enqueue(samples) >= buffer.maxSize) {
|
|
427
|
+
props.lastCheck = props.silent;
|
|
428
|
+
props.silent = buffer.dequeueAll().every((samples2) => isSilent(samples2, threshold));
|
|
429
|
+
if (props.lastCheck !== props.silent) {
|
|
430
|
+
onDetected(props.silent);
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
};
|
|
434
|
+
};
|
|
435
|
+
|
|
436
|
+
// src/workletNodes.ts
|
|
437
|
+
var createDenoiseWorkletNode = (context, options) => {
|
|
438
|
+
return new AudioWorkletNode(context, "denoise-processor", options);
|
|
439
|
+
};
|
|
440
|
+
|
|
441
|
+
// src/typeGuards.ts
|
|
442
|
+
var isAudioNode = (t) => {
|
|
443
|
+
if (typeof t === "object" && t !== null) {
|
|
444
|
+
return "connect" in t && "disconnect" in t;
|
|
445
|
+
}
|
|
446
|
+
return false;
|
|
447
|
+
};
|
|
448
|
+
var isAudioParam = (t) => {
|
|
449
|
+
if (typeof t === "object" && t !== null) {
|
|
450
|
+
return "setValueAtTime" in t;
|
|
451
|
+
}
|
|
452
|
+
return false;
|
|
453
|
+
};
|
|
454
|
+
var isAudioNodeInit = (t) => {
|
|
455
|
+
if (typeof t === "object" && t !== null) {
|
|
456
|
+
return "audioNode" in t && "create" in t && "release" in t;
|
|
457
|
+
}
|
|
458
|
+
return false;
|
|
459
|
+
};
|
|
460
|
+
var isAnalyzerNodeInit = (t) => {
|
|
461
|
+
if (isAudioNodeInit(t) && t.audioNode) {
|
|
462
|
+
return "getFloatTimeDomainData" in t.audioNode;
|
|
463
|
+
}
|
|
464
|
+
return false;
|
|
465
|
+
};
|
|
466
|
+
|
|
467
|
+
// src/audio.ts
|
|
468
|
+
var createAudioContext = (options) => {
|
|
469
|
+
if (hasAudioContext()) {
|
|
470
|
+
return new AudioContext(options);
|
|
471
|
+
}
|
|
472
|
+
return new window.webkitAudioContext(options);
|
|
473
|
+
};
|
|
474
|
+
function resumeAudioOnInterruption(audioContext) {
|
|
475
|
+
const resumeInterrupted = () => {
|
|
476
|
+
if (audioContext.state === "interrupted") {
|
|
477
|
+
void audioContext.resume();
|
|
478
|
+
}
|
|
479
|
+
};
|
|
480
|
+
audioContext.addEventListener("statechange", resumeInterrupted);
|
|
481
|
+
return () => {
|
|
482
|
+
audioContext.removeEventListener("statechange", resumeInterrupted);
|
|
483
|
+
};
|
|
484
|
+
}
|
|
485
|
+
var resumeAudioOnUnmute = (context) => (
|
|
486
|
+
/**
|
|
487
|
+
* @param track - The source track to listen on the `unmute` event
|
|
488
|
+
*/
|
|
489
|
+
(track) => {
|
|
490
|
+
const resume = () => {
|
|
491
|
+
if (context.state === "suspended") {
|
|
492
|
+
void context.resume();
|
|
493
|
+
}
|
|
494
|
+
};
|
|
495
|
+
track.addEventListener("unmute", resume);
|
|
496
|
+
return () => {
|
|
497
|
+
track.removeEventListener("unmute", resume);
|
|
498
|
+
};
|
|
499
|
+
}
|
|
500
|
+
);
|
|
501
|
+
var createGainWithMute = (context, muteInit = false) => {
|
|
502
|
+
let mute = muteInit;
|
|
503
|
+
const gain = createGainNode(context, {
|
|
504
|
+
gain: muteToGain(mute),
|
|
505
|
+
channelCountMode: "explicit"
|
|
506
|
+
});
|
|
507
|
+
const disconnect = gain.disconnect.bind(gain);
|
|
508
|
+
const connect = gain.connect.bind(gain);
|
|
509
|
+
return {
|
|
510
|
+
get node() {
|
|
511
|
+
return gain;
|
|
512
|
+
},
|
|
513
|
+
get mute() {
|
|
514
|
+
return mute;
|
|
515
|
+
},
|
|
516
|
+
set mute(shouldMute) {
|
|
517
|
+
mute = shouldMute;
|
|
518
|
+
const time = context.currentTime;
|
|
519
|
+
if (isFinite(time)) {
|
|
520
|
+
gain.gain.setValueAtTime(muteToGain(shouldMute), time);
|
|
521
|
+
} else {
|
|
522
|
+
throw new Error("Set gain with non-finite time value");
|
|
523
|
+
}
|
|
524
|
+
},
|
|
525
|
+
connect,
|
|
526
|
+
disconnect
|
|
527
|
+
};
|
|
528
|
+
};
|
|
529
|
+
var createAnalyzer = (context, options) => {
|
|
530
|
+
const analyser = createAnalyserNode(context, options);
|
|
531
|
+
const getByteFrequencyData = analyser.getByteFrequencyData.bind(analyser);
|
|
532
|
+
const getByteTimeDomainData = analyser.getByteTimeDomainData.bind(analyser);
|
|
533
|
+
const getFloatFrequencyData = analyser.getFloatFrequencyData.bind(analyser);
|
|
534
|
+
const connect = analyser.connect.bind(analyser);
|
|
535
|
+
const disconnect = analyser.disconnect.bind(analyser);
|
|
536
|
+
let byteBuffer;
|
|
537
|
+
const getFloatTimeDomainData = (buffer) => {
|
|
538
|
+
if ("getFloatTimeDomainData" in AnalyserNode.prototype) {
|
|
539
|
+
return analyser.getFloatTimeDomainData(buffer);
|
|
540
|
+
}
|
|
541
|
+
if (!byteBuffer) {
|
|
542
|
+
byteBuffer = new Uint8Array(buffer.length);
|
|
543
|
+
}
|
|
544
|
+
analyser.getByteTimeDomainData(byteBuffer);
|
|
545
|
+
return copyByteBufferToFloatBuffer(byteBuffer, buffer);
|
|
546
|
+
};
|
|
547
|
+
const getAverageVolume = (buffer) => {
|
|
548
|
+
getFloatTimeDomainData(buffer);
|
|
549
|
+
return processAverageVolume(Array.from(buffer));
|
|
550
|
+
};
|
|
551
|
+
return {
|
|
552
|
+
get node() {
|
|
553
|
+
return analyser;
|
|
554
|
+
},
|
|
555
|
+
get frequencyBinCount() {
|
|
556
|
+
return analyser.frequencyBinCount;
|
|
557
|
+
},
|
|
558
|
+
get fftSize() {
|
|
559
|
+
return analyser.fftSize;
|
|
560
|
+
},
|
|
561
|
+
set fftSize(size) {
|
|
562
|
+
analyser.fftSize = size;
|
|
563
|
+
},
|
|
564
|
+
get minDecibels() {
|
|
565
|
+
return analyser.minDecibels;
|
|
566
|
+
},
|
|
567
|
+
set minDecibels(decibels) {
|
|
568
|
+
analyser.minDecibels = decibels;
|
|
569
|
+
},
|
|
570
|
+
get maxDecibels() {
|
|
571
|
+
return analyser.maxDecibels;
|
|
572
|
+
},
|
|
573
|
+
set maxDecibels(decibels) {
|
|
574
|
+
analyser.maxDecibels = decibels;
|
|
575
|
+
},
|
|
576
|
+
get smoothingTimeConstant() {
|
|
577
|
+
return analyser.smoothingTimeConstant;
|
|
578
|
+
},
|
|
579
|
+
set smoothingTimeConstant(constant) {
|
|
580
|
+
analyser.smoothingTimeConstant = constant;
|
|
581
|
+
},
|
|
582
|
+
getByteTimeDomainData,
|
|
583
|
+
getByteFrequencyData,
|
|
584
|
+
getFloatFrequencyData,
|
|
585
|
+
getFloatTimeDomainData,
|
|
586
|
+
getAverageVolume,
|
|
587
|
+
connect,
|
|
588
|
+
disconnect
|
|
589
|
+
};
|
|
590
|
+
};
|
|
591
|
+
var subscribeWorkletNode = (workletNode, { messageHandler, errorHandler } = {}) => {
|
|
592
|
+
const subscribeMessage = (event) => {
|
|
593
|
+
messageHandler?.(event.data);
|
|
594
|
+
};
|
|
595
|
+
const subscribeToPortError = (event) => {
|
|
596
|
+
if (errorHandler) {
|
|
597
|
+
errorHandler(event);
|
|
598
|
+
}
|
|
599
|
+
};
|
|
600
|
+
const subscribeToProcessorError = (event) => {
|
|
601
|
+
if (errorHandler) {
|
|
602
|
+
errorHandler(event);
|
|
603
|
+
}
|
|
604
|
+
};
|
|
605
|
+
workletNode.port.addEventListener("message", subscribeMessage);
|
|
606
|
+
workletNode.port.start();
|
|
607
|
+
workletNode.addEventListener("processorerror", subscribeToProcessorError);
|
|
608
|
+
workletNode.port.addEventListener("messageerror", subscribeToPortError);
|
|
609
|
+
return () => {
|
|
610
|
+
workletNode.port.postMessage({ type: "release" });
|
|
611
|
+
workletNode.port.removeEventListener("message", subscribeMessage);
|
|
612
|
+
workletNode.removeEventListener(
|
|
613
|
+
"processorerror",
|
|
614
|
+
subscribeToProcessorError
|
|
615
|
+
);
|
|
616
|
+
workletNode.port.removeEventListener(
|
|
617
|
+
"messageerror",
|
|
618
|
+
subscribeToPortError
|
|
619
|
+
);
|
|
620
|
+
};
|
|
621
|
+
};
|
|
622
|
+
var TIMEOUT_DELAY_ADJUSTMENT = 0.5;
|
|
623
|
+
var subscribeTimeoutAnalyzerNode = (analyzer, { messageHandler, updateFrequency = 2 }) => {
|
|
624
|
+
const timeoutMs = updateFrequency * 1e3 * TIMEOUT_DELAY_ADJUSTMENT;
|
|
625
|
+
let timeoutId = 0;
|
|
626
|
+
let stopTimeout = false;
|
|
627
|
+
const clearTimeout = () => {
|
|
628
|
+
if (timeoutId) {
|
|
629
|
+
window.clearTimeout(timeoutId);
|
|
630
|
+
timeoutId = 0;
|
|
631
|
+
}
|
|
632
|
+
};
|
|
633
|
+
const process = () => {
|
|
634
|
+
messageHandler(analyzer);
|
|
635
|
+
clearTimeout();
|
|
636
|
+
if (!stopTimeout) {
|
|
637
|
+
timeoutId = window.setTimeout(process, timeoutMs);
|
|
638
|
+
}
|
|
639
|
+
};
|
|
640
|
+
process();
|
|
641
|
+
return () => {
|
|
642
|
+
stopTimeout = true;
|
|
643
|
+
clearTimeout();
|
|
644
|
+
};
|
|
645
|
+
};
|
|
646
|
+
function createBaseAudioNode(name, create, release) {
|
|
647
|
+
const props = {
|
|
648
|
+
name,
|
|
649
|
+
node: void 0,
|
|
650
|
+
audioNode: void 0,
|
|
651
|
+
outputs: /* @__PURE__ */ new WeakSet()
|
|
652
|
+
};
|
|
653
|
+
const createNode = (context, prevNode) => {
|
|
654
|
+
const [audioNode, node] = create(context, prevNode);
|
|
655
|
+
props.audioNode = audioNode;
|
|
656
|
+
props.node = node;
|
|
657
|
+
return [audioNode, node];
|
|
658
|
+
};
|
|
659
|
+
const releaseNode = () => {
|
|
660
|
+
release?.();
|
|
661
|
+
props.outputs = /* @__PURE__ */ new WeakSet();
|
|
662
|
+
props.audioNode?.disconnect();
|
|
663
|
+
props.audioNode = void 0;
|
|
664
|
+
};
|
|
665
|
+
const connectNode = (param) => {
|
|
666
|
+
if (!param) {
|
|
667
|
+
return;
|
|
668
|
+
}
|
|
669
|
+
if (!props.audioNode) {
|
|
670
|
+
throw new Error("Source AudioNode is not initialized");
|
|
671
|
+
}
|
|
672
|
+
const [destination, output, input] = Array.isArray(param) ? param : [param];
|
|
673
|
+
if (isAudioParam(destination)) {
|
|
674
|
+
props.outputs.add(destination);
|
|
675
|
+
return props.audioNode.connect(destination, output);
|
|
676
|
+
}
|
|
677
|
+
if (isAudioNodeInit(destination)) {
|
|
678
|
+
if (!destination.audioNode) {
|
|
679
|
+
throw new Error("Destination AudioNode is not initialized");
|
|
680
|
+
}
|
|
681
|
+
props.outputs.add(destination);
|
|
682
|
+
return props.audioNode.connect(
|
|
683
|
+
destination.audioNode,
|
|
684
|
+
output,
|
|
685
|
+
input
|
|
686
|
+
);
|
|
687
|
+
}
|
|
688
|
+
};
|
|
689
|
+
const disconnectNode = (destInit) => {
|
|
690
|
+
if (!destInit) {
|
|
691
|
+
return;
|
|
692
|
+
}
|
|
693
|
+
if (!props.audioNode) {
|
|
694
|
+
throw new Error("AudioNode is not initialized");
|
|
695
|
+
}
|
|
696
|
+
const [destination, output, input] = Array.isArray(destInit) ? destInit : [destInit];
|
|
697
|
+
if (destination && !props.outputs.has(destination)) {
|
|
698
|
+
return;
|
|
699
|
+
}
|
|
700
|
+
if (isAudioParam(destination)) {
|
|
701
|
+
props.outputs.delete(destination);
|
|
702
|
+
if (output !== void 0) {
|
|
703
|
+
return props.audioNode.disconnect(destination, output);
|
|
704
|
+
}
|
|
705
|
+
return props.audioNode.disconnect(destination);
|
|
706
|
+
}
|
|
707
|
+
if (isAudioNodeInit(destination)) {
|
|
708
|
+
if (!destination.audioNode) {
|
|
709
|
+
throw new Error("Destination AudioNode is not initialized");
|
|
710
|
+
}
|
|
711
|
+
props.outputs.delete(destination);
|
|
712
|
+
if (output !== void 0) {
|
|
713
|
+
if (input !== void 0) {
|
|
714
|
+
return props.audioNode.disconnect(
|
|
715
|
+
destination.audioNode,
|
|
716
|
+
output,
|
|
717
|
+
input
|
|
718
|
+
);
|
|
719
|
+
}
|
|
720
|
+
return props.audioNode.disconnect(
|
|
721
|
+
destination.audioNode,
|
|
722
|
+
output
|
|
723
|
+
);
|
|
724
|
+
}
|
|
725
|
+
return props.audioNode.disconnect(destination.audioNode);
|
|
726
|
+
}
|
|
727
|
+
props.outputs = /* @__PURE__ */ new WeakSet();
|
|
728
|
+
props.audioNode.disconnect();
|
|
729
|
+
};
|
|
730
|
+
const hasConnectedTo = (init) => {
|
|
731
|
+
return props.outputs.has(init);
|
|
732
|
+
};
|
|
733
|
+
return Object.assign(props, {
|
|
734
|
+
create: createNode,
|
|
735
|
+
connect: connectNode,
|
|
736
|
+
disconnect: disconnectNode,
|
|
737
|
+
release: releaseNode,
|
|
738
|
+
hasConnectedTo,
|
|
739
|
+
toJSON: () => ({
|
|
740
|
+
name: props.name,
|
|
741
|
+
node: props.node,
|
|
742
|
+
audioNode: props.audioNode
|
|
743
|
+
})
|
|
744
|
+
});
|
|
745
|
+
}
|
|
746
|
+
var createStreamSourceGraphNode = (mediaStream, shouldResetEnabled = true) => {
|
|
747
|
+
let stream = void 0;
|
|
748
|
+
let unsubscribeUnmutes = void 0;
|
|
749
|
+
return createBaseAudioNode(
|
|
750
|
+
"source",
|
|
751
|
+
(context) => {
|
|
752
|
+
stream = createMediaStreamAudioClone(mediaStream);
|
|
753
|
+
if (shouldResetEnabled) {
|
|
754
|
+
stream.getAudioTracks().forEach((track) => track.enabled = true);
|
|
755
|
+
}
|
|
756
|
+
unsubscribeUnmutes = mediaStream.getAudioTracks().map(resumeAudioOnUnmute(context));
|
|
757
|
+
const node = createMediaStreamAudioSourceNode(context, {
|
|
758
|
+
mediaStream: stream
|
|
759
|
+
});
|
|
760
|
+
return [node, node];
|
|
761
|
+
},
|
|
762
|
+
() => {
|
|
763
|
+
stopStreamTracks(stream);
|
|
764
|
+
unsubscribeUnmutes?.forEach((unsubscribe) => unsubscribe());
|
|
765
|
+
stream = void 0;
|
|
766
|
+
}
|
|
767
|
+
);
|
|
768
|
+
};
|
|
769
|
+
var createMediaElementSourceGraphNode = (mediaElement) => {
|
|
770
|
+
return createBaseAudioNode("source", (context) => {
|
|
771
|
+
const node = createMediaElementSourceNode(context, {
|
|
772
|
+
mediaElement
|
|
773
|
+
});
|
|
774
|
+
return [node, node];
|
|
775
|
+
});
|
|
776
|
+
};
|
|
777
|
+
var createAnalyzerSubscribableGraphNode = ({
|
|
778
|
+
messageHandler,
|
|
779
|
+
updateFrequency = 2,
|
|
780
|
+
...analyserOptions
|
|
781
|
+
}) => {
|
|
782
|
+
let unsubscribe;
|
|
783
|
+
return createBaseAudioNode(
|
|
784
|
+
"analyzer",
|
|
785
|
+
(context) => {
|
|
786
|
+
const node = createAnalyzer(context, analyserOptions);
|
|
787
|
+
unsubscribe = subscribeTimeoutAnalyzerNode(node, {
|
|
788
|
+
messageHandler,
|
|
789
|
+
updateFrequency
|
|
790
|
+
});
|
|
791
|
+
return [node.node, node];
|
|
792
|
+
},
|
|
793
|
+
() => {
|
|
794
|
+
unsubscribe?.();
|
|
795
|
+
unsubscribe = void 0;
|
|
796
|
+
}
|
|
797
|
+
);
|
|
798
|
+
};
|
|
799
|
+
var createDenoiseWorkletGraphNode = (data, messageHandler) => {
|
|
800
|
+
let unsubscribe;
|
|
801
|
+
return createBaseAudioNode(
|
|
802
|
+
"denoise",
|
|
803
|
+
(context, prevNode) => {
|
|
804
|
+
const sampleRate = context.sampleRate;
|
|
805
|
+
const channelCount = prevNode?.channelCount ?? 1;
|
|
806
|
+
const node = createDenoiseWorkletNode(context, {
|
|
807
|
+
outputChannelCount: [channelCount],
|
|
808
|
+
processorOptions: {
|
|
809
|
+
data,
|
|
810
|
+
sampleRate,
|
|
811
|
+
shouldSendVAD: !!messageHandler
|
|
812
|
+
}
|
|
813
|
+
});
|
|
814
|
+
unsubscribe = subscribeWorkletNode(node, { messageHandler });
|
|
815
|
+
return [node, node];
|
|
816
|
+
},
|
|
817
|
+
() => {
|
|
818
|
+
unsubscribe?.();
|
|
819
|
+
unsubscribe = void 0;
|
|
820
|
+
}
|
|
821
|
+
);
|
|
822
|
+
};
|
|
823
|
+
var createGainGraphNode = (mute) => {
|
|
824
|
+
return createBaseAudioNode("gain", (context) => {
|
|
825
|
+
const node = createGainWithMute(context, mute);
|
|
826
|
+
return [node.node, node];
|
|
827
|
+
});
|
|
828
|
+
};
|
|
829
|
+
var createAnalyzerGraphNode = (options) => {
|
|
830
|
+
return createBaseAudioNode("analyzer", (context) => {
|
|
831
|
+
const node = createAnalyzer(context, options);
|
|
832
|
+
return [node.node, node];
|
|
833
|
+
});
|
|
834
|
+
};
|
|
835
|
+
var createStreamDestinationGraphNode = (options) => {
|
|
836
|
+
let node = void 0;
|
|
837
|
+
return createBaseAudioNode(
|
|
838
|
+
"destination",
|
|
839
|
+
(context) => {
|
|
840
|
+
node = createMediaStreamAudioDestinationNode(context, options);
|
|
841
|
+
return [node, node];
|
|
842
|
+
},
|
|
843
|
+
() => {
|
|
844
|
+
stopStreamTracks(node?.stream);
|
|
845
|
+
node = void 0;
|
|
846
|
+
}
|
|
847
|
+
);
|
|
848
|
+
};
|
|
849
|
+
var createAudioDestinationGraphNode = () => {
|
|
850
|
+
return createBaseAudioNode("destination", (context) => {
|
|
851
|
+
const node = context.destination;
|
|
852
|
+
return [node, node];
|
|
853
|
+
});
|
|
854
|
+
};
|
|
855
|
+
var createDelayGraphNode = (options) => {
|
|
856
|
+
return createBaseAudioNode("delay", (context) => {
|
|
857
|
+
const node = createDelayNode(context, options);
|
|
858
|
+
return [node, node];
|
|
859
|
+
});
|
|
860
|
+
};
|
|
861
|
+
var createChannelSplitterGraphNode = (options) => {
|
|
862
|
+
return createBaseAudioNode("splitter", (context) => {
|
|
863
|
+
const node = createChannelSplitterNode(context, options);
|
|
864
|
+
return [node, node];
|
|
865
|
+
});
|
|
866
|
+
};
|
|
867
|
+
var createChannelMergerGraphNode = (options) => {
|
|
868
|
+
return createBaseAudioNode("merger", (context) => {
|
|
869
|
+
const node = createChannelMergerNode(context, options);
|
|
870
|
+
return [node, node];
|
|
871
|
+
});
|
|
872
|
+
};
|
|
873
|
+
var createNodeInitDisconnector = () => (srcInit, destInit) => {
|
|
874
|
+
if (!srcInit) {
|
|
875
|
+
return;
|
|
876
|
+
}
|
|
877
|
+
const [source, outputIdx, inputIdx] = Array.isArray(srcInit) ? srcInit : [srcInit];
|
|
878
|
+
if (!source) {
|
|
879
|
+
return;
|
|
880
|
+
}
|
|
881
|
+
if (isAudioParam(source)) {
|
|
882
|
+
throw new Error(
|
|
883
|
+
"AudioParam cannot be used as the source of disconnect"
|
|
884
|
+
);
|
|
885
|
+
}
|
|
886
|
+
const [destination] = Array.isArray(destInit) ? destInit : [destInit];
|
|
887
|
+
return source.disconnect([destination, outputIdx, inputIdx]);
|
|
888
|
+
};
|
|
889
|
+
var createNodeInitConnector = (context) => (srcInit, destInit) => {
|
|
890
|
+
if (!srcInit || !destInit) {
|
|
891
|
+
return;
|
|
892
|
+
}
|
|
893
|
+
const [source, outputIdx, inputIdx] = Array.isArray(srcInit) ? srcInit : [srcInit];
|
|
894
|
+
const [destination] = Array.isArray(destInit) ? destInit : [destInit];
|
|
895
|
+
if (!source || !destination) {
|
|
896
|
+
return;
|
|
897
|
+
}
|
|
898
|
+
if (isAudioParam(source)) {
|
|
899
|
+
throw new Error(
|
|
900
|
+
"AudioParam cannot be used as the source of connect"
|
|
901
|
+
);
|
|
902
|
+
}
|
|
903
|
+
const [sourceNode] = source.audioNode ? [source.audioNode] : source.create(context);
|
|
904
|
+
if (isAudioNodeInit(destination) && !destination.audioNode) {
|
|
905
|
+
destination.create(context, sourceNode);
|
|
906
|
+
}
|
|
907
|
+
return source.connect([destination, outputIdx, inputIdx]);
|
|
908
|
+
};
|
|
909
|
+
var createAudioGraph = (initialConnections, options = {}) => {
|
|
910
|
+
const props = {
|
|
911
|
+
closing: false,
|
|
912
|
+
inits: /* @__PURE__ */ new Set()
|
|
913
|
+
};
|
|
914
|
+
const audioContext = options.context ?? createAudioContext(options.contextOptions);
|
|
915
|
+
const unsubscribeStateChanged = resumeAudioOnInterruption(audioContext);
|
|
916
|
+
const connectInit = createNodeInitConnector(audioContext);
|
|
917
|
+
const disconnectInit = createNodeInitDisconnector();
|
|
918
|
+
const connect = (sequence) => {
|
|
919
|
+
sequence.forEach((initParam, idx, initParams) => {
|
|
920
|
+
const prevParam = initParams[idx - 1];
|
|
921
|
+
const [currentInit] = Array.isArray(initParam) ? initParam : [initParam];
|
|
922
|
+
connectInit(prevParam, currentInit);
|
|
923
|
+
if (isAudioNodeInit(currentInit)) {
|
|
924
|
+
props.inits.add(currentInit);
|
|
925
|
+
}
|
|
926
|
+
});
|
|
927
|
+
};
|
|
928
|
+
const disconnect = (sequence) => {
|
|
929
|
+
sequence.forEach((initParam, idx, initParams) => {
|
|
930
|
+
const prevParam = initParams[idx - 1];
|
|
931
|
+
const [currentInit] = Array.isArray(initParam) ? initParam : [initParam];
|
|
932
|
+
disconnectInit(prevParam, currentInit);
|
|
933
|
+
});
|
|
934
|
+
};
|
|
935
|
+
const addWorklet = async (moduleURL, options2) => {
|
|
936
|
+
if (!props.workletModule) {
|
|
937
|
+
await audioContext.audioWorklet.addModule(moduleURL, options2);
|
|
938
|
+
props.workletModule = { moduleURL, options: options2 };
|
|
939
|
+
}
|
|
940
|
+
};
|
|
941
|
+
initialConnections.forEach(connect);
|
|
942
|
+
const releaseInit = (init) => {
|
|
943
|
+
if (props.inits.has(init)) {
|
|
944
|
+
init.release();
|
|
945
|
+
props.inits.delete(init);
|
|
946
|
+
}
|
|
947
|
+
};
|
|
948
|
+
return {
|
|
949
|
+
get inits() {
|
|
950
|
+
return Array.from(props.inits);
|
|
951
|
+
},
|
|
952
|
+
get context() {
|
|
953
|
+
return audioContext;
|
|
954
|
+
},
|
|
955
|
+
get state() {
|
|
956
|
+
return props.closing ? "closing" : audioContext.state;
|
|
957
|
+
},
|
|
958
|
+
connect,
|
|
959
|
+
disconnect,
|
|
960
|
+
addWorklet,
|
|
961
|
+
releaseInit,
|
|
962
|
+
release: () => {
|
|
963
|
+
return new Promise((resolve) => {
|
|
964
|
+
if (!props.closing && audioContext.state !== "closed") {
|
|
965
|
+
props.closing = true;
|
|
966
|
+
props.workletModule = void 0;
|
|
967
|
+
unsubscribeStateChanged();
|
|
968
|
+
void audioContext.close().then(() => {
|
|
969
|
+
props.closing = false;
|
|
970
|
+
props.inits.forEach(releaseInit);
|
|
971
|
+
resolve();
|
|
972
|
+
});
|
|
973
|
+
} else {
|
|
974
|
+
resolve();
|
|
975
|
+
}
|
|
976
|
+
});
|
|
977
|
+
}
|
|
978
|
+
};
|
|
979
|
+
};
|
|
980
|
+
var createAudioGraphProxy = (audioGraph, handlers) => new Proxy(audioGraph, {
|
|
981
|
+
get: (target, p) => {
|
|
982
|
+
switch (p) {
|
|
983
|
+
case "connect": {
|
|
984
|
+
const r = target[p];
|
|
985
|
+
return (...args) => {
|
|
986
|
+
handlers.connect?.(target, args);
|
|
987
|
+
return r.apply(target, args);
|
|
988
|
+
};
|
|
989
|
+
}
|
|
990
|
+
case "disconnect": {
|
|
991
|
+
const r = target[p];
|
|
992
|
+
return (...args) => {
|
|
993
|
+
handlers.disconnect?.(target, args);
|
|
994
|
+
return r.apply(target, args);
|
|
995
|
+
};
|
|
996
|
+
}
|
|
997
|
+
default: {
|
|
998
|
+
return target[p];
|
|
999
|
+
}
|
|
1000
|
+
}
|
|
1001
|
+
},
|
|
1002
|
+
set: () => false
|
|
1003
|
+
});
|
|
1004
|
+
|
|
1005
|
+
// src/video/load.ts
|
|
1006
|
+
var policy = {
|
|
1007
|
+
createScriptURL: (url) => {
|
|
1008
|
+
if (new URL(url, document.baseURI).origin !== window.location.origin) {
|
|
1009
|
+
throw new Error(
|
|
1010
|
+
`Trying to create script url not on same origin (${url} not on ${window.location.origin})`
|
|
1011
|
+
);
|
|
1012
|
+
}
|
|
1013
|
+
return url;
|
|
1014
|
+
}
|
|
1015
|
+
};
|
|
1016
|
+
var sameOriginPolicy = typeof window.trustedTypes !== "undefined" && window.trustedTypes.createPolicy ? window.trustedTypes.createPolicy("same-origin", policy) : policy;
|
|
1017
|
+
var loadScript = (path, id) => new Promise((resolve, reject) => {
|
|
1018
|
+
const scriptExist = document.getElementById(id);
|
|
1019
|
+
if (scriptExist) {
|
|
1020
|
+
return;
|
|
1021
|
+
}
|
|
1022
|
+
const removeScript = () => {
|
|
1023
|
+
document.head.removeChild(script);
|
|
1024
|
+
};
|
|
1025
|
+
const script = document.createElement("script");
|
|
1026
|
+
script.src = sameOriginPolicy.createScriptURL(path);
|
|
1027
|
+
script.id = id;
|
|
1028
|
+
script.onload = () => {
|
|
1029
|
+
removeScript();
|
|
1030
|
+
resolve();
|
|
1031
|
+
};
|
|
1032
|
+
script.onerror = (ev) => {
|
|
1033
|
+
removeScript();
|
|
1034
|
+
reject(ev);
|
|
1035
|
+
};
|
|
1036
|
+
document.head.appendChild(script);
|
|
1037
|
+
});
|
|
1038
|
+
var loadWasms = async (paths) => {
|
|
1039
|
+
await Promise.all(paths.map(([path]) => loadScript(path, path)));
|
|
1040
|
+
};
|
|
1041
|
+
var loadTfjsCore = async (prodMode) => {
|
|
1042
|
+
const tfjs = await import("@tensorflow/tfjs-core");
|
|
1043
|
+
if (prodMode) {
|
|
1044
|
+
tfjs.enableProdMode();
|
|
1045
|
+
}
|
|
1046
|
+
return tfjs;
|
|
1047
|
+
};
|
|
1048
|
+
var loadTfjsBackendWebGl = () => import("@tensorflow/tfjs-backend-webgl");
|
|
1049
|
+
|
|
1050
|
+
// src/video/constants.ts
|
|
1051
|
+
var PROCESSING_WIDTH = 768;
|
|
1052
|
+
var PROCESSING_HEIGHT = 432;
|
|
1053
|
+
var FOREGROUND_THRESHOLD = 0.5;
|
|
1054
|
+
var BACKGROUND_BLUR_AMOUNT = 3;
|
|
1055
|
+
var EDGE_BLUR_AMOUNT = 3;
|
|
1056
|
+
var FLIP_HORIZONTAL = false;
|
|
1057
|
+
var FRAME_RATE = 20;
|
|
1058
|
+
var AbortReason = /* @__PURE__ */ ((AbortReason2) => {
|
|
1059
|
+
AbortReason2["Close"] = "close";
|
|
1060
|
+
return AbortReason2;
|
|
1061
|
+
})(AbortReason || {});
|
|
1062
|
+
|
|
1063
|
+
// src/video/video.ts
|
|
1064
|
+
var createVideoProcessor = (transformers, processTrack) => {
|
|
1065
|
+
const props = {
|
|
1066
|
+
processing: false
|
|
1067
|
+
};
|
|
1068
|
+
const open = async () => {
|
|
1069
|
+
await Promise.all(transformers.map((transformer) => transformer.init()));
|
|
1070
|
+
};
|
|
1071
|
+
const process = async (source) => {
|
|
1072
|
+
const [track] = source.getVideoTracks();
|
|
1073
|
+
if (!track) {
|
|
1074
|
+
return source;
|
|
1075
|
+
}
|
|
1076
|
+
if (props.processing) {
|
|
1077
|
+
throw new Error("Cannot process when it is already processing");
|
|
1078
|
+
}
|
|
1079
|
+
props.abortController = new AbortController();
|
|
1080
|
+
const trackGenerated = await processTrack(track, transformers, {
|
|
1081
|
+
signal: props.abortController.signal
|
|
1082
|
+
});
|
|
1083
|
+
props.outputStream = new MediaStream([
|
|
1084
|
+
trackGenerated,
|
|
1085
|
+
...source.getAudioTracks().map((track2) => track2.clone())
|
|
1086
|
+
]);
|
|
1087
|
+
props.processing = true;
|
|
1088
|
+
return props.outputStream;
|
|
1089
|
+
};
|
|
1090
|
+
const close = () => {
|
|
1091
|
+
if (!props.processing) {
|
|
1092
|
+
return;
|
|
1093
|
+
}
|
|
1094
|
+
props.abortController?.abort("close" /* Close */);
|
|
1095
|
+
stopStreamTracks(props.outputStream);
|
|
1096
|
+
props.outputStream = void 0;
|
|
1097
|
+
props.processing = false;
|
|
1098
|
+
};
|
|
1099
|
+
const destroy = async () => {
|
|
1100
|
+
close();
|
|
1101
|
+
await Promise.all(
|
|
1102
|
+
transformers.map((transformer) => transformer.destroy())
|
|
1103
|
+
);
|
|
1104
|
+
};
|
|
1105
|
+
return {
|
|
1106
|
+
open,
|
|
1107
|
+
process,
|
|
1108
|
+
close,
|
|
1109
|
+
destroy
|
|
1110
|
+
};
|
|
1111
|
+
};
|
|
1112
|
+
|
|
1113
|
+
// src/video/types.ts
|
|
1114
|
+
var RENDER_EFFECTS = [
|
|
1115
|
+
/**
|
|
1116
|
+
* Pass through, and thus there is no effects applied
|
|
1117
|
+
*/
|
|
1118
|
+
"none",
|
|
1119
|
+
/**
|
|
1120
|
+
* Background blur effects
|
|
1121
|
+
*/
|
|
1122
|
+
"blur",
|
|
1123
|
+
/**
|
|
1124
|
+
* Background overlay effects
|
|
1125
|
+
*/
|
|
1126
|
+
"overlay"
|
|
1127
|
+
];
|
|
1128
|
+
var SEG_MODELS = [
|
|
1129
|
+
/**
|
|
1130
|
+
* Mediapipe's Selfie Segmentation model
|
|
1131
|
+
*/
|
|
1132
|
+
"mediapipeSelfie"
|
|
1133
|
+
];
|
|
1134
|
+
|
|
1135
|
+
// src/video/utils.ts
|
|
1136
|
+
import { Tensor, browser } from "@tensorflow/tfjs-core/dist/base.js";
|
|
1137
|
+
var createCanvas = (width, height) => {
|
|
1138
|
+
const canvas = document.createElement("canvas");
|
|
1139
|
+
canvas.width = width;
|
|
1140
|
+
canvas.height = height;
|
|
1141
|
+
return canvas;
|
|
1142
|
+
};
|
|
1143
|
+
var createOffscreenCanvas = (width, height) => {
|
|
1144
|
+
try {
|
|
1145
|
+
const offscreen = new OffscreenCanvas(width, height);
|
|
1146
|
+
return offscreen;
|
|
1147
|
+
} catch {
|
|
1148
|
+
return createCanvas(width, height);
|
|
1149
|
+
}
|
|
1150
|
+
};
|
|
1151
|
+
var createVideoElement = (width, height) => {
|
|
1152
|
+
const video = document.createElement("video");
|
|
1153
|
+
video.width = width;
|
|
1154
|
+
video.height = height;
|
|
1155
|
+
video.muted = true;
|
|
1156
|
+
return video;
|
|
1157
|
+
};
|
|
1158
|
+
var setVideoElementSrc = (video, src) => {
|
|
1159
|
+
let url = "";
|
|
1160
|
+
const revokeObjectURL = () => {
|
|
1161
|
+
if (url) {
|
|
1162
|
+
URL.revokeObjectURL(url);
|
|
1163
|
+
}
|
|
1164
|
+
};
|
|
1165
|
+
if (src instanceof MediaStream) {
|
|
1166
|
+
src.getVideoTracks().forEach((track) => {
|
|
1167
|
+
track.addEventListener("ended", revokeObjectURL);
|
|
1168
|
+
});
|
|
1169
|
+
}
|
|
1170
|
+
if ("MediaSource" in window && src instanceof MediaSource) {
|
|
1171
|
+
src.addEventListener("sourceended", revokeObjectURL);
|
|
1172
|
+
}
|
|
1173
|
+
try {
|
|
1174
|
+
video.srcObject = src;
|
|
1175
|
+
} catch (error) {
|
|
1176
|
+
if (error instanceof Error) {
|
|
1177
|
+
if (error.name === "TypeError") {
|
|
1178
|
+
throw error;
|
|
1179
|
+
}
|
|
1180
|
+
if ("MediaSource" in window && src instanceof MediaSource || src instanceof Blob) {
|
|
1181
|
+
url = URL.createObjectURL(src);
|
|
1182
|
+
}
|
|
1183
|
+
video.src = url;
|
|
1184
|
+
}
|
|
1185
|
+
}
|
|
1186
|
+
};
|
|
1187
|
+
var toVideoElement = (input, width, height) => {
|
|
1188
|
+
const [settings = {}] = input.getVideoTracks().map((track) => track.getSettings());
|
|
1189
|
+
const video = createVideoElement(
|
|
1190
|
+
settings.width ?? width,
|
|
1191
|
+
settings.height ?? height
|
|
1192
|
+
);
|
|
1193
|
+
video.playsInline = true;
|
|
1194
|
+
setVideoElementSrc(video, input);
|
|
1195
|
+
return video;
|
|
1196
|
+
};
|
|
1197
|
+
var playVideo = async (video) => {
|
|
1198
|
+
if (video.autoplay) {
|
|
1199
|
+
return;
|
|
1200
|
+
}
|
|
1201
|
+
if (video.readyState < 2 /* HaveCurrentData */) {
|
|
1202
|
+
await Promise.race([
|
|
1203
|
+
new Promise((resolve) => {
|
|
1204
|
+
const waitForEvent = () => {
|
|
1205
|
+
video.removeEventListener("loadeddata", waitForEvent);
|
|
1206
|
+
resolve();
|
|
1207
|
+
};
|
|
1208
|
+
video.addEventListener("loadeddata", waitForEvent);
|
|
1209
|
+
}),
|
|
1210
|
+
// Use setTimeout to prevent a race to `loadeddata` event
|
|
1211
|
+
new Promise((resolve) => {
|
|
1212
|
+
setTimeout(() => {
|
|
1213
|
+
resolve();
|
|
1214
|
+
}, 3e3);
|
|
1215
|
+
})
|
|
1216
|
+
]);
|
|
1217
|
+
}
|
|
1218
|
+
return await video.play();
|
|
1219
|
+
};
|
|
1220
|
+
var loadImage = async (image, src) => {
|
|
1221
|
+
image.src = src;
|
|
1222
|
+
return new Promise((resolve, reject) => {
|
|
1223
|
+
image.onload = () => resolve();
|
|
1224
|
+
image.onerror = reject;
|
|
1225
|
+
});
|
|
1226
|
+
};
|
|
1227
|
+
var toNumber = (value) => value instanceof SVGAnimatedLength ? value.baseVal.value : value;
|
|
1228
|
+
var getCanvasRenderingContext2D = (canvas, options) => {
|
|
1229
|
+
const context = canvas.getContext("2d", options);
|
|
1230
|
+
if (!context) {
|
|
1231
|
+
throw new Error("Cannot get CanvasRenderingContext2D");
|
|
1232
|
+
}
|
|
1233
|
+
return context;
|
|
1234
|
+
};
|
|
1235
|
+
var getImageSize = (image) => {
|
|
1236
|
+
if (image instanceof Tensor) {
|
|
1237
|
+
const [height = 0, width = 0] = image.shape.slice(0, 2);
|
|
1238
|
+
return { height, width };
|
|
1239
|
+
}
|
|
1240
|
+
if ("VideoFrame" in window && image instanceof VideoFrame) {
|
|
1241
|
+
return { height: image.displayHeight, width: image.displayWidth };
|
|
1242
|
+
}
|
|
1243
|
+
if ("offsetHeight" in image && image.offsetHeight !== 0 && "offsetWidth" in image && image.offsetWidth !== 0) {
|
|
1244
|
+
return { height: image.offsetHeight, width: image.offsetWidth };
|
|
1245
|
+
}
|
|
1246
|
+
if ("height" in image && image.height !== 0 && "width" in image && image.width !== 0) {
|
|
1247
|
+
return { height: toNumber(image.height), width: toNumber(image.width) };
|
|
1248
|
+
}
|
|
1249
|
+
throw new Error("Unknown input image");
|
|
1250
|
+
};
|
|
1251
|
+
var toHTMLCanvasElementLossy = async (image) => {
|
|
1252
|
+
const { width, height } = getImageSize(image);
|
|
1253
|
+
const canvas = createCanvas(width, height);
|
|
1254
|
+
if (image instanceof Tensor) {
|
|
1255
|
+
await browser.toPixels(image, canvas);
|
|
1256
|
+
return canvas;
|
|
1257
|
+
}
|
|
1258
|
+
const context = getCanvasRenderingContext2D(canvas);
|
|
1259
|
+
if (image instanceof ImageData) {
|
|
1260
|
+
context.putImageData(image, 0, 0);
|
|
1261
|
+
} else {
|
|
1262
|
+
context.drawImage(image, 0, 0);
|
|
1263
|
+
}
|
|
1264
|
+
return canvas;
|
|
1265
|
+
};
|
|
1266
|
+
var toImageDataLossy = async (image) => {
|
|
1267
|
+
const { width, height } = getImageSize(image);
|
|
1268
|
+
if (image instanceof Tensor) {
|
|
1269
|
+
return new ImageData(await browser.toPixels(image), width, height);
|
|
1270
|
+
}
|
|
1271
|
+
const canvas = createOffscreenCanvas(width, height);
|
|
1272
|
+
const context = getCanvasRenderingContext2D(canvas);
|
|
1273
|
+
context.drawImage(image, 0, 0);
|
|
1274
|
+
return context.getImageData(0, 0, canvas.width, canvas.height);
|
|
1275
|
+
};
|
|
1276
|
+
var toTensorLossy = async (image) => {
|
|
1277
|
+
const pixelsInput = image instanceof SVGImageElement || image instanceof OffscreenCanvas ? await toHTMLCanvasElementLossy(image) : image;
|
|
1278
|
+
return browser.fromPixels(pixelsInput, 4);
|
|
1279
|
+
};
|
|
1280
|
+
var toSegmentation = (data, type, maskValueToLabel) => {
|
|
1281
|
+
const mask = {
|
|
1282
|
+
toCanvasImageSource: async () => {
|
|
1283
|
+
if (data instanceof HTMLCanvasElement) {
|
|
1284
|
+
return data;
|
|
1285
|
+
}
|
|
1286
|
+
if (data instanceof HTMLVideoElement || data instanceof ImageBitmap || data instanceof HTMLImageElement) {
|
|
1287
|
+
const canvas = createCanvas(data.width, data.height);
|
|
1288
|
+
const context = canvas.getContext("2d");
|
|
1289
|
+
context?.drawImage(data, 0, 0, data.width, data.height);
|
|
1290
|
+
return canvas;
|
|
1291
|
+
}
|
|
1292
|
+
return await toHTMLCanvasElementLossy(data);
|
|
1293
|
+
},
|
|
1294
|
+
toImageData: async () => {
|
|
1295
|
+
if (data instanceof ImageData) {
|
|
1296
|
+
return data;
|
|
1297
|
+
}
|
|
1298
|
+
return await toImageDataLossy(data);
|
|
1299
|
+
},
|
|
1300
|
+
toTensor: async () => {
|
|
1301
|
+
if (data instanceof Tensor) {
|
|
1302
|
+
return data;
|
|
1303
|
+
}
|
|
1304
|
+
return await toTensorLossy(data);
|
|
1305
|
+
},
|
|
1306
|
+
getUnderlyingType: () => type
|
|
1307
|
+
};
|
|
1308
|
+
return {
|
|
1309
|
+
maskValueToLabel,
|
|
1310
|
+
mask
|
|
1311
|
+
};
|
|
1312
|
+
};
|
|
1313
|
+
var flipCanvasHorizontal = (canvas) => {
|
|
1314
|
+
const ctx = getCanvasRenderingContext2D(canvas);
|
|
1315
|
+
ctx.scale(-1, 1);
|
|
1316
|
+
ctx.translate(-canvas.width, 0);
|
|
1317
|
+
};
|
|
1318
|
+
var drawStroke = (bytes, row, column, width, radius, color = {
|
|
1319
|
+
r: 0,
|
|
1320
|
+
g: 255,
|
|
1321
|
+
b: 255,
|
|
1322
|
+
a: 255
|
|
1323
|
+
}) => {
|
|
1324
|
+
for (let i = -radius; i <= radius; i++) {
|
|
1325
|
+
for (let j = -radius; j <= radius; j++) {
|
|
1326
|
+
if (i !== 0 && j !== 0) {
|
|
1327
|
+
const n = (row + i) * width + (column + j);
|
|
1328
|
+
bytes[4 * n + 0] = color.r;
|
|
1329
|
+
bytes[4 * n + 1] = color.g;
|
|
1330
|
+
bytes[4 * n + 2] = color.b;
|
|
1331
|
+
bytes[4 * n + 3] = color.a;
|
|
1332
|
+
}
|
|
1333
|
+
}
|
|
1334
|
+
}
|
|
1335
|
+
};
|
|
1336
|
+
var isSegmentationBoundary = (data, row, column, width, isForegroundId, alphaCutoff, radius = 1) => {
|
|
1337
|
+
let numberBackgroundPixels = 0;
|
|
1338
|
+
for (let i = -radius; i <= radius; i++) {
|
|
1339
|
+
for (let j = -radius; j <= radius; j++) {
|
|
1340
|
+
if (i !== 0 && j !== 0) {
|
|
1341
|
+
const n = (row + i) * width + (column + j);
|
|
1342
|
+
const foregroundColor = data[4 * n];
|
|
1343
|
+
const alphaColor = data[4 * n + 3];
|
|
1344
|
+
if (foregroundColor !== void 0 && !isForegroundId[foregroundColor] || alphaColor !== void 0 && alphaColor < alphaCutoff) {
|
|
1345
|
+
numberBackgroundPixels += 1;
|
|
1346
|
+
}
|
|
1347
|
+
}
|
|
1348
|
+
}
|
|
1349
|
+
}
|
|
1350
|
+
return numberBackgroundPixels > 0;
|
|
1351
|
+
};
|
|
1352
|
+
var toBinaryMask = async (segmentation, foreground = {
|
|
1353
|
+
r: 0,
|
|
1354
|
+
g: 0,
|
|
1355
|
+
b: 0,
|
|
1356
|
+
a: 0
|
|
1357
|
+
}, background = {
|
|
1358
|
+
r: 0,
|
|
1359
|
+
g: 0,
|
|
1360
|
+
b: 0,
|
|
1361
|
+
a: 255
|
|
1362
|
+
}, drawContour = false, foregroundThreshold = 0.5, foregroundMaskValues = Array.from(Array(256).keys())) => {
|
|
1363
|
+
const segmentations = !Array.isArray(segmentation) ? [segmentation] : segmentation;
|
|
1364
|
+
if (segmentations.length === 0) {
|
|
1365
|
+
return null;
|
|
1366
|
+
}
|
|
1367
|
+
const masks = await Promise.all(
|
|
1368
|
+
segmentations.map((segmentation2) => segmentation2.mask.toImageData())
|
|
1369
|
+
);
|
|
1370
|
+
const [imageData] = masks;
|
|
1371
|
+
if (!imageData) {
|
|
1372
|
+
return null;
|
|
1373
|
+
}
|
|
1374
|
+
const { width, height } = imageData;
|
|
1375
|
+
const bytes = new Uint8ClampedArray(width * height * 4);
|
|
1376
|
+
const alphaCutoff = Math.round(255 * foregroundThreshold);
|
|
1377
|
+
const isForegroundId = new Array(256).fill(false);
|
|
1378
|
+
foregroundMaskValues.forEach((id) => isForegroundId[id] = true);
|
|
1379
|
+
for (let i = 0; i < height; i++) {
|
|
1380
|
+
for (let j = 0; j < width; j++) {
|
|
1381
|
+
const n = i * width + j;
|
|
1382
|
+
bytes[4 * n + 0] = background.r;
|
|
1383
|
+
bytes[4 * n + 1] = background.g;
|
|
1384
|
+
bytes[4 * n + 2] = background.b;
|
|
1385
|
+
bytes[4 * n + 3] = background.a;
|
|
1386
|
+
for (const mask of masks) {
|
|
1387
|
+
const maskForegroundColor = mask.data[4 * n];
|
|
1388
|
+
const maskAlphaColor = mask.data[4 * n + 3];
|
|
1389
|
+
if (maskForegroundColor !== void 0 && isForegroundId[maskForegroundColor] && maskAlphaColor !== void 0 && maskAlphaColor >= alphaCutoff) {
|
|
1390
|
+
bytes[4 * n] = foreground.r;
|
|
1391
|
+
bytes[4 * n + 1] = foreground.g;
|
|
1392
|
+
bytes[4 * n + 2] = foreground.b;
|
|
1393
|
+
bytes[4 * n + 3] = foreground.a;
|
|
1394
|
+
if (drawContour && i - 1 >= 0 && i + 1 < height && j - 1 >= 0 && j + 1 < width && isSegmentationBoundary(
|
|
1395
|
+
mask.data,
|
|
1396
|
+
i,
|
|
1397
|
+
j,
|
|
1398
|
+
width,
|
|
1399
|
+
isForegroundId,
|
|
1400
|
+
alphaCutoff
|
|
1401
|
+
)) {
|
|
1402
|
+
drawStroke(bytes, i, j, width, 1);
|
|
1403
|
+
}
|
|
1404
|
+
}
|
|
1405
|
+
}
|
|
1406
|
+
}
|
|
1407
|
+
}
|
|
1408
|
+
return new ImageData(bytes, width, height);
|
|
1409
|
+
};
|
|
1410
|
+
var createInputImageConvertor = (draw) => async (input) => {
|
|
1411
|
+
return input instanceof ImageData || input instanceof ImageBitmap ? await draw(input) : input;
|
|
1412
|
+
};
|
|
1413
|
+
var ensure = (prop, message = "Processor is not opened, please call open() method first") => {
|
|
1414
|
+
if (!prop) {
|
|
1415
|
+
throw new Error(message);
|
|
1416
|
+
}
|
|
1417
|
+
return prop;
|
|
1418
|
+
};
|
|
1419
|
+
var hasRequestVideoFrameCallback = (input) => typeof input === "object" && input instanceof HTMLVideoElement && "requestVideoFrameCallback" in HTMLVideoElement.prototype;
|
|
1420
|
+
var getFrameRate = (input, frameRate) => {
|
|
1421
|
+
if (input instanceof HTMLVideoElement) {
|
|
1422
|
+
const quality = input.getVideoPlaybackQuality();
|
|
1423
|
+
return input.mozPresentedFrames || quality.totalVideoFrames - quality.droppedVideoFrames;
|
|
1424
|
+
}
|
|
1425
|
+
return frameRate;
|
|
1426
|
+
};
|
|
1427
|
+
var createFrameCallbackRequest = (callback, frameRate, {
|
|
1428
|
+
subscribeVisibilityChange = subscribeVisibilityChangeEvent
|
|
1429
|
+
} = {}) => {
|
|
1430
|
+
const props = { callbackId: 0, frameRate, started: false };
|
|
1431
|
+
const getCallbackLoopRunner = (input) => {
|
|
1432
|
+
if (!props.runner) {
|
|
1433
|
+
const fallbackRequestCallback = async (input2) => {
|
|
1434
|
+
if (props.runner) {
|
|
1435
|
+
props.runner.frameRate = getFrameRate(
|
|
1436
|
+
input2,
|
|
1437
|
+
props.frameRate
|
|
1438
|
+
);
|
|
1439
|
+
}
|
|
1440
|
+
await callback(input2);
|
|
1441
|
+
};
|
|
1442
|
+
props.runner = createAsyncCallbackLoop(
|
|
1443
|
+
fallbackRequestCallback,
|
|
1444
|
+
getFrameRate(input, props.frameRate)
|
|
1445
|
+
);
|
|
1446
|
+
}
|
|
1447
|
+
return props.runner;
|
|
1448
|
+
};
|
|
1449
|
+
const cancelFrameCallback = () => {
|
|
1450
|
+
if (hasRequestVideoFrameCallback(props.input) && props.callbackId) {
|
|
1451
|
+
props.input.cancelVideoFrameCallback(props.callbackId);
|
|
1452
|
+
props.callbackId = 0;
|
|
1453
|
+
}
|
|
1454
|
+
};
|
|
1455
|
+
return {
|
|
1456
|
+
start: async (input) => {
|
|
1457
|
+
props.input = input;
|
|
1458
|
+
if (hasRequestVideoFrameCallback(input)) {
|
|
1459
|
+
props.unsubscribe = subscribeVisibilityChange(async (hidden) => {
|
|
1460
|
+
if (props.started && props.input) {
|
|
1461
|
+
if (hidden && props.callbackId) {
|
|
1462
|
+
await getCallbackLoopRunner(props.input).start(
|
|
1463
|
+
props.input
|
|
1464
|
+
);
|
|
1465
|
+
} else {
|
|
1466
|
+
getCallbackLoopRunner(props.input).stop();
|
|
1467
|
+
}
|
|
1468
|
+
}
|
|
1469
|
+
});
|
|
1470
|
+
await new Promise((resolve) => {
|
|
1471
|
+
const wrap = async () => {
|
|
1472
|
+
await callback(input);
|
|
1473
|
+
if (!props.started) {
|
|
1474
|
+
props.started = true;
|
|
1475
|
+
}
|
|
1476
|
+
resolve();
|
|
1477
|
+
props.callbackId = input.requestVideoFrameCallback(wrap);
|
|
1478
|
+
};
|
|
1479
|
+
props.callbackId = input.requestVideoFrameCallback(wrap);
|
|
1480
|
+
});
|
|
1481
|
+
} else {
|
|
1482
|
+
await getCallbackLoopRunner(input).start(input);
|
|
1483
|
+
props.started = true;
|
|
1484
|
+
}
|
|
1485
|
+
},
|
|
1486
|
+
stop: () => {
|
|
1487
|
+
cancelFrameCallback();
|
|
1488
|
+
props.runner?.stop();
|
|
1489
|
+
props.unsubscribe?.();
|
|
1490
|
+
props.started = false;
|
|
1491
|
+
},
|
|
1492
|
+
get frameRate() {
|
|
1493
|
+
return getFrameRate(props.input, frameRate);
|
|
1494
|
+
},
|
|
1495
|
+
set frameRate(value) {
|
|
1496
|
+
props.frameRate = value;
|
|
1497
|
+
if (props.runner) {
|
|
1498
|
+
props.runner.frameRate = frameRate;
|
|
1499
|
+
}
|
|
1500
|
+
}
|
|
1501
|
+
};
|
|
1502
|
+
};
|
|
1503
|
+
|
|
1504
|
+
// src/video/canvasRenderUtils.ts
|
|
1505
|
+
import { Tensor as Tensor2, browser as browser2 } from "@tensorflow/tfjs-core/dist/base.js";
|
|
1506
|
+
var createCanvasRenderUtils = (processingWidth, processingHeight) => {
|
|
1507
|
+
const props = {};
|
|
1508
|
+
const getImage = (imageName) => {
|
|
1509
|
+
const image = props[imageName];
|
|
1510
|
+
if (!image) {
|
|
1511
|
+
const img = new Image();
|
|
1512
|
+
props[imageName] = img;
|
|
1513
|
+
return img;
|
|
1514
|
+
}
|
|
1515
|
+
return image;
|
|
1516
|
+
};
|
|
1517
|
+
const getCanvas = (canvasName) => {
|
|
1518
|
+
const canvas = props[canvasName];
|
|
1519
|
+
if (!canvas) {
|
|
1520
|
+
const canvas2 = createOffscreenCanvas(
|
|
1521
|
+
processingWidth,
|
|
1522
|
+
processingHeight
|
|
1523
|
+
);
|
|
1524
|
+
props[canvasName] = canvas2;
|
|
1525
|
+
return canvas2;
|
|
1526
|
+
}
|
|
1527
|
+
return canvas;
|
|
1528
|
+
};
|
|
1529
|
+
const renderImageDataToOffScreenCanvas = (image, canvasName) => {
|
|
1530
|
+
const canvas = getCanvas(canvasName);
|
|
1531
|
+
const context = getCanvasRenderingContext2D(canvas, {
|
|
1532
|
+
desynchronized: true
|
|
1533
|
+
});
|
|
1534
|
+
context.putImageData(image, 0, 0);
|
|
1535
|
+
return canvas;
|
|
1536
|
+
};
|
|
1537
|
+
const drawImage = async (ctx, image, sx, sy, sw, sh, dx, dy, dw, dh) => {
|
|
1538
|
+
if (image instanceof Tensor2) {
|
|
1539
|
+
const pixels = await browser2.toPixels(image);
|
|
1540
|
+
const { height, width } = getImageSize(image);
|
|
1541
|
+
image = new ImageData(pixels, width, height);
|
|
1542
|
+
}
|
|
1543
|
+
const source = image instanceof ImageData ? renderImageDataToOffScreenCanvas(image, "drawImageDataCanvas") : image;
|
|
1544
|
+
if (sw === void 0 || sh === void 0) {
|
|
1545
|
+
ctx.drawImage(source, sx, sy);
|
|
1546
|
+
} else if (dx === void 0 || dy === void 0 || dw === void 0 || dh === void 0) {
|
|
1547
|
+
ctx.drawImage(source, sx, sy, sw, sh);
|
|
1548
|
+
} else {
|
|
1549
|
+
ctx.drawImage(source, sx, sy, sw, sh, dx, dy, dw, dh);
|
|
1550
|
+
}
|
|
1551
|
+
};
|
|
1552
|
+
const renderImageToCanvas = async (image, canvas, dw = processingWidth, dh = processingHeight, options = {}) => {
|
|
1553
|
+
const { height, width } = getImageSize(image);
|
|
1554
|
+
const rect = fitDestinationSize(width, height, dw, dh);
|
|
1555
|
+
const ctx = getCanvasRenderingContext2D(canvas, {
|
|
1556
|
+
desynchronized: true,
|
|
1557
|
+
...options
|
|
1558
|
+
});
|
|
1559
|
+
await drawImage(ctx, image, rect.x, rect.y, rect.width, rect.height);
|
|
1560
|
+
};
|
|
1561
|
+
const renderImageToOffScreenCanvas = async (image, canvasName) => {
|
|
1562
|
+
const canvas = getCanvas(canvasName);
|
|
1563
|
+
await renderImageToCanvas(image, canvas);
|
|
1564
|
+
return canvas;
|
|
1565
|
+
};
|
|
1566
|
+
const drawWithCompositing = async (ctx, image, compositeOperation) => {
|
|
1567
|
+
ctx.globalCompositeOperation = compositeOperation;
|
|
1568
|
+
await drawImage(ctx, image, 0, 0);
|
|
1569
|
+
};
|
|
1570
|
+
const cpuBlur = async (canvas, image, blur) => {
|
|
1571
|
+
const ctx = getCanvasRenderingContext2D(canvas, { desynchronized: true });
|
|
1572
|
+
let sum2 = 0;
|
|
1573
|
+
const delta = 5;
|
|
1574
|
+
const alphaLeft = 1 / (2 * Math.PI * delta * delta);
|
|
1575
|
+
const step = blur < 3 ? 1 : 2;
|
|
1576
|
+
for (let y = -blur; y <= blur; y += step) {
|
|
1577
|
+
for (let x = -blur; x <= blur; x += step) {
|
|
1578
|
+
const weight = alphaLeft * Math.exp(-(x * x + y * y) / (2 * delta * delta));
|
|
1579
|
+
sum2 += weight;
|
|
1580
|
+
}
|
|
1581
|
+
}
|
|
1582
|
+
for (let y = -blur; y <= blur; y += step) {
|
|
1583
|
+
for (let x = -blur; x <= blur; x += step) {
|
|
1584
|
+
ctx.globalAlpha = alphaLeft * Math.exp(-(x * x + y * y) / (2 * delta * delta)) / sum2 * blur;
|
|
1585
|
+
await drawImage(ctx, image, x, y);
|
|
1586
|
+
}
|
|
1587
|
+
}
|
|
1588
|
+
ctx.globalAlpha = 1;
|
|
1589
|
+
};
|
|
1590
|
+
const drawAndBlurImageOnCanvas = async (image, blurAmount, canvas) => {
|
|
1591
|
+
const { height, width } = getImageSize(image);
|
|
1592
|
+
const ctx = getCanvasRenderingContext2D(canvas, { desynchronized: true });
|
|
1593
|
+
ctx.clearRect(0, 0, width, height);
|
|
1594
|
+
if (blurAmount <= 0) {
|
|
1595
|
+
return drawImage(ctx, image, 0, 0, width, height);
|
|
1596
|
+
}
|
|
1597
|
+
ctx.save();
|
|
1598
|
+
if ("filter" in ctx) {
|
|
1599
|
+
await drawImage(ctx, image, 0, 0, width, height);
|
|
1600
|
+
ctx.filter = `blur(${blurAmount}px)`;
|
|
1601
|
+
await drawImage(ctx, image, 0, 0, width, height);
|
|
1602
|
+
} else {
|
|
1603
|
+
await cpuBlur(canvas, image, blurAmount);
|
|
1604
|
+
}
|
|
1605
|
+
ctx.restore();
|
|
1606
|
+
};
|
|
1607
|
+
const drawAndBlurImageOnOffScreenCanvas = async (image, blurAmount, offscreenCanvasName) => {
|
|
1608
|
+
const canvas = getCanvas(offscreenCanvasName);
|
|
1609
|
+
if (blurAmount === 0) {
|
|
1610
|
+
await renderImageToCanvas(image, canvas);
|
|
1611
|
+
} else {
|
|
1612
|
+
await drawAndBlurImageOnCanvas(image, blurAmount, canvas);
|
|
1613
|
+
}
|
|
1614
|
+
return canvas;
|
|
1615
|
+
};
|
|
1616
|
+
const createPersonMask = async (segmentation, foregroundThreshold, edgeBlurAmount) => {
|
|
1617
|
+
const backgroundMaskImage = await toBinaryMask(
|
|
1618
|
+
segmentation,
|
|
1619
|
+
{ r: 0, g: 0, b: 0, a: 255 },
|
|
1620
|
+
{ r: 0, g: 0, b: 0, a: 0 },
|
|
1621
|
+
false,
|
|
1622
|
+
foregroundThreshold
|
|
1623
|
+
);
|
|
1624
|
+
if (!backgroundMaskImage) {
|
|
1625
|
+
return getCanvas("maskCanvas");
|
|
1626
|
+
}
|
|
1627
|
+
const backgroundMask = renderImageDataToOffScreenCanvas(
|
|
1628
|
+
backgroundMaskImage,
|
|
1629
|
+
"maskCanvas"
|
|
1630
|
+
);
|
|
1631
|
+
if (edgeBlurAmount === 0) {
|
|
1632
|
+
return backgroundMask;
|
|
1633
|
+
} else {
|
|
1634
|
+
return drawAndBlurImageOnOffScreenCanvas(
|
|
1635
|
+
backgroundMask,
|
|
1636
|
+
edgeBlurAmount,
|
|
1637
|
+
"blurredMaskCanvas"
|
|
1638
|
+
);
|
|
1639
|
+
}
|
|
1640
|
+
};
|
|
1641
|
+
const loadImageElement = async (url, imageName) => {
|
|
1642
|
+
const image = getImage(imageName);
|
|
1643
|
+
await loadImage(image, url);
|
|
1644
|
+
return image;
|
|
1645
|
+
};
|
|
1646
|
+
const loadAndDrawImageOnOffscreenCanvas = async (url, canvasName, imageName) => {
|
|
1647
|
+
const image = await loadImageElement(url, imageName);
|
|
1648
|
+
const canvas = getCanvas(canvasName);
|
|
1649
|
+
const context = getCanvasRenderingContext2D(canvas, {
|
|
1650
|
+
desynchronized: true
|
|
1651
|
+
});
|
|
1652
|
+
const imageSize = getImageSize(image);
|
|
1653
|
+
const rect = fitDestinationSize(
|
|
1654
|
+
imageSize.width,
|
|
1655
|
+
imageSize.height,
|
|
1656
|
+
processingWidth,
|
|
1657
|
+
processingHeight
|
|
1658
|
+
);
|
|
1659
|
+
await drawImage(
|
|
1660
|
+
context,
|
|
1661
|
+
image,
|
|
1662
|
+
rect.x,
|
|
1663
|
+
rect.y,
|
|
1664
|
+
rect.width,
|
|
1665
|
+
rect.height
|
|
1666
|
+
);
|
|
1667
|
+
return canvas;
|
|
1668
|
+
};
|
|
1669
|
+
const loadBackgroundImage = (url) => loadAndDrawImageOnOffscreenCanvas(
|
|
1670
|
+
url,
|
|
1671
|
+
"backgroundImageCanvas",
|
|
1672
|
+
"backgroundImage"
|
|
1673
|
+
);
|
|
1674
|
+
const drawBokehEffect = async (canvas, inputImage, backgroundImage, segmentations, foregroundThreshold = 0.5, backgroundBlurAmount = 3, edgeBlurAmount = 3, flipHorizontal = false) => {
|
|
1675
|
+
const blurredImage = await drawAndBlurImageOnOffScreenCanvas(
|
|
1676
|
+
backgroundImage,
|
|
1677
|
+
backgroundBlurAmount,
|
|
1678
|
+
"blurredCanvas"
|
|
1679
|
+
);
|
|
1680
|
+
const ctx = getCanvasRenderingContext2D(canvas, { desynchronized: true });
|
|
1681
|
+
if (Array.isArray(segmentations) && segmentations.length === 0) {
|
|
1682
|
+
return drawImage(ctx, blurredImage, 0, 0);
|
|
1683
|
+
}
|
|
1684
|
+
const personMask = await createPersonMask(
|
|
1685
|
+
segmentations,
|
|
1686
|
+
foregroundThreshold,
|
|
1687
|
+
edgeBlurAmount
|
|
1688
|
+
);
|
|
1689
|
+
ctx.save();
|
|
1690
|
+
if (flipHorizontal) {
|
|
1691
|
+
flipCanvasHorizontal(canvas);
|
|
1692
|
+
}
|
|
1693
|
+
const { height, width } = getImageSize(inputImage);
|
|
1694
|
+
await drawImage(ctx, inputImage, 0, 0, width, height);
|
|
1695
|
+
await drawWithCompositing(ctx, personMask, "destination-in");
|
|
1696
|
+
await drawWithCompositing(ctx, blurredImage, "destination-over");
|
|
1697
|
+
ctx.restore();
|
|
1698
|
+
};
|
|
1699
|
+
const drawBlurEffect = (canvas, inputImage, segmentations, foregroundThreshold = 0.5, backgroundBlurAmount = 3, edgeBlurAmount = 3, flipHorizontal = false) => drawBokehEffect(
|
|
1700
|
+
canvas,
|
|
1701
|
+
inputImage,
|
|
1702
|
+
inputImage,
|
|
1703
|
+
segmentations,
|
|
1704
|
+
foregroundThreshold,
|
|
1705
|
+
backgroundBlurAmount,
|
|
1706
|
+
edgeBlurAmount,
|
|
1707
|
+
flipHorizontal
|
|
1708
|
+
);
|
|
1709
|
+
const drawOverlayEffect = (canvas, inputImage, backgroundImage, segmentations, foregroundThreshold = 0.5, backgroundBlurAmount = 0, edgeBlurAmount = 3, flipHorizontal = false) => drawBokehEffect(
|
|
1710
|
+
canvas,
|
|
1711
|
+
inputImage,
|
|
1712
|
+
backgroundImage,
|
|
1713
|
+
segmentations,
|
|
1714
|
+
foregroundThreshold,
|
|
1715
|
+
backgroundBlurAmount,
|
|
1716
|
+
edgeBlurAmount,
|
|
1717
|
+
flipHorizontal
|
|
1718
|
+
);
|
|
1719
|
+
const evaluateInput = async (inputImage) => {
|
|
1720
|
+
const image = await renderImageToOffScreenCanvas(
|
|
1721
|
+
inputImage,
|
|
1722
|
+
"inputCanvas"
|
|
1723
|
+
);
|
|
1724
|
+
return image;
|
|
1725
|
+
};
|
|
1726
|
+
return {
|
|
1727
|
+
evaluateInput,
|
|
1728
|
+
renderImageToCanvas,
|
|
1729
|
+
drawBlurEffect,
|
|
1730
|
+
drawOverlayEffect,
|
|
1731
|
+
loadBackgroundImage,
|
|
1732
|
+
renderImageToOffScreenCanvas,
|
|
1733
|
+
renderImageDataToOffScreenCanvas,
|
|
1734
|
+
drawAndBlurImageOnOffScreenCanvas
|
|
1735
|
+
};
|
|
1736
|
+
};
|
|
1737
|
+
|
|
1738
|
+
// src/video/canvasTransform.ts
|
|
1739
|
+
var createAssertInRange = (from, to) => (value) => {
|
|
1740
|
+
if (value < from || value > to) {
|
|
1741
|
+
throw new Error(`Invalid value (${value}) to range [${from}, ${to}]`);
|
|
1742
|
+
}
|
|
1743
|
+
};
|
|
1744
|
+
var assertInRangeFrom0To1 = createAssertInRange(0, 1);
|
|
1745
|
+
var assertInRangeFrom0To20 = createAssertInRange(0, 20);
|
|
1746
|
+
var NOT_INITED_ERROR_MSG = "Please call init() method first!";
|
|
1747
|
+
var createTransform = (segmenter, {
|
|
1748
|
+
width = PROCESSING_WIDTH,
|
|
1749
|
+
height = PROCESSING_HEIGHT,
|
|
1750
|
+
foregroundThreshold = FOREGROUND_THRESHOLD,
|
|
1751
|
+
backgroundBlurAmount = BACKGROUND_BLUR_AMOUNT,
|
|
1752
|
+
edgeBlurAmount = EDGE_BLUR_AMOUNT,
|
|
1753
|
+
flipHorizontal = FLIP_HORIZONTAL,
|
|
1754
|
+
effects = "none",
|
|
1755
|
+
selfManageSegmenter,
|
|
1756
|
+
bgImageUrl
|
|
1757
|
+
} = {}) => {
|
|
1758
|
+
const props = {
|
|
1759
|
+
segmenter,
|
|
1760
|
+
width,
|
|
1761
|
+
height,
|
|
1762
|
+
foregroundThreshold,
|
|
1763
|
+
backgroundBlurAmount,
|
|
1764
|
+
edgeBlurAmount,
|
|
1765
|
+
flipHorizontal,
|
|
1766
|
+
utils: createCanvasRenderUtils(width, height),
|
|
1767
|
+
effects,
|
|
1768
|
+
backgroundImage: void 0,
|
|
1769
|
+
status: "created",
|
|
1770
|
+
backgroundImageUrl: bgImageUrl
|
|
1771
|
+
};
|
|
1772
|
+
const processInput = async (input) => {
|
|
1773
|
+
if (props.segmenter.status === "created" || props.segmenter.status === "closed") {
|
|
1774
|
+
await props.segmenter.open();
|
|
1775
|
+
}
|
|
1776
|
+
if (props.segmenter.status === "opening") {
|
|
1777
|
+
return [];
|
|
1778
|
+
}
|
|
1779
|
+
return await props.segmenter.process(input);
|
|
1780
|
+
};
|
|
1781
|
+
const loadBackgroundImage = async (url) => {
|
|
1782
|
+
if (props.backgroundImageUrl === url && props.backgroundImage) {
|
|
1783
|
+
return;
|
|
1784
|
+
}
|
|
1785
|
+
props.backgroundImageUrl = url;
|
|
1786
|
+
props.backgroundImage = await props.utils.loadBackgroundImage(url);
|
|
1787
|
+
};
|
|
1788
|
+
return {
|
|
1789
|
+
get status() {
|
|
1790
|
+
return props.status;
|
|
1791
|
+
},
|
|
1792
|
+
get width() {
|
|
1793
|
+
return props.width;
|
|
1794
|
+
},
|
|
1795
|
+
get height() {
|
|
1796
|
+
return props.height;
|
|
1797
|
+
},
|
|
1798
|
+
get foregroundThreshold() {
|
|
1799
|
+
return props.foregroundThreshold;
|
|
1800
|
+
},
|
|
1801
|
+
get backgroundBlurAmount() {
|
|
1802
|
+
return props.backgroundBlurAmount;
|
|
1803
|
+
},
|
|
1804
|
+
get edgeBlurAmount() {
|
|
1805
|
+
return props.edgeBlurAmount;
|
|
1806
|
+
},
|
|
1807
|
+
get flipHorizontal() {
|
|
1808
|
+
return props.flipHorizontal;
|
|
1809
|
+
},
|
|
1810
|
+
get effects() {
|
|
1811
|
+
return props.effects;
|
|
1812
|
+
},
|
|
1813
|
+
get backgroundImage() {
|
|
1814
|
+
return props.backgroundImage;
|
|
1815
|
+
},
|
|
1816
|
+
set foregroundThreshold(value) {
|
|
1817
|
+
assertInRangeFrom0To1(value);
|
|
1818
|
+
props.foregroundThreshold = value;
|
|
1819
|
+
},
|
|
1820
|
+
set backgroundBlurAmount(value) {
|
|
1821
|
+
assertInRangeFrom0To20(value);
|
|
1822
|
+
props.backgroundBlurAmount = value;
|
|
1823
|
+
},
|
|
1824
|
+
set edgeBlurAmount(value) {
|
|
1825
|
+
assertInRangeFrom0To20(value);
|
|
1826
|
+
props.edgeBlurAmount = value;
|
|
1827
|
+
},
|
|
1828
|
+
set flipHorizontal(value) {
|
|
1829
|
+
props.flipHorizontal = value;
|
|
1830
|
+
},
|
|
1831
|
+
set effects(value) {
|
|
1832
|
+
props.effects = value;
|
|
1833
|
+
},
|
|
1834
|
+
get backgroundImageUrl() {
|
|
1835
|
+
return props.backgroundImageUrl;
|
|
1836
|
+
},
|
|
1837
|
+
set backgroundImage(canvas) {
|
|
1838
|
+
props.backgroundImage = canvas;
|
|
1839
|
+
},
|
|
1840
|
+
loadBackgroundImage,
|
|
1841
|
+
get segmenter() {
|
|
1842
|
+
return props.segmenter;
|
|
1843
|
+
},
|
|
1844
|
+
set segmenter(value) {
|
|
1845
|
+
if (value !== props.segmenter) {
|
|
1846
|
+
props.segmenter = value;
|
|
1847
|
+
}
|
|
1848
|
+
},
|
|
1849
|
+
init: async () => {
|
|
1850
|
+
props.outputCanvas = createCanvas(width, height);
|
|
1851
|
+
if (props.backgroundImageUrl) {
|
|
1852
|
+
await loadBackgroundImage(props.backgroundImageUrl);
|
|
1853
|
+
}
|
|
1854
|
+
props.status = "opened";
|
|
1855
|
+
},
|
|
1856
|
+
transform: async (videoFrame, controller) => {
|
|
1857
|
+
if (!props.outputCanvas) {
|
|
1858
|
+
throw new Error(NOT_INITED_ERROR_MSG);
|
|
1859
|
+
}
|
|
1860
|
+
switch (props.effects) {
|
|
1861
|
+
case "blur": {
|
|
1862
|
+
const image = await props.utils.renderImageToOffScreenCanvas(
|
|
1863
|
+
videoFrame,
|
|
1864
|
+
"inputCanvas"
|
|
1865
|
+
);
|
|
1866
|
+
const segmentations = await processInput(image);
|
|
1867
|
+
if (props.status === "closed") {
|
|
1868
|
+
break;
|
|
1869
|
+
}
|
|
1870
|
+
if (props.outputCanvas.height !== height) {
|
|
1871
|
+
props.outputCanvas.width = width;
|
|
1872
|
+
props.outputCanvas.height = height;
|
|
1873
|
+
}
|
|
1874
|
+
await props.utils.drawBlurEffect(
|
|
1875
|
+
props.outputCanvas,
|
|
1876
|
+
image,
|
|
1877
|
+
segmentations,
|
|
1878
|
+
props.foregroundThreshold,
|
|
1879
|
+
props.backgroundBlurAmount,
|
|
1880
|
+
props.edgeBlurAmount,
|
|
1881
|
+
props.flipHorizontal
|
|
1882
|
+
);
|
|
1883
|
+
controller.enqueue(props.outputCanvas);
|
|
1884
|
+
break;
|
|
1885
|
+
}
|
|
1886
|
+
case "overlay": {
|
|
1887
|
+
if (!props.backgroundImage) {
|
|
1888
|
+
throw new Error(
|
|
1889
|
+
"Please call setBackgroundImage() method first"
|
|
1890
|
+
);
|
|
1891
|
+
}
|
|
1892
|
+
const image = await props.utils.renderImageToOffScreenCanvas(
|
|
1893
|
+
videoFrame,
|
|
1894
|
+
"inputCanvas"
|
|
1895
|
+
);
|
|
1896
|
+
const segmentations = await processInput(image);
|
|
1897
|
+
if (props.status === "closed") {
|
|
1898
|
+
break;
|
|
1899
|
+
}
|
|
1900
|
+
if (props.outputCanvas.height !== height) {
|
|
1901
|
+
props.outputCanvas.width = width;
|
|
1902
|
+
props.outputCanvas.height = height;
|
|
1903
|
+
}
|
|
1904
|
+
await props.utils.drawOverlayEffect(
|
|
1905
|
+
props.outputCanvas,
|
|
1906
|
+
image,
|
|
1907
|
+
props.backgroundImage,
|
|
1908
|
+
segmentations,
|
|
1909
|
+
props.foregroundThreshold,
|
|
1910
|
+
0,
|
|
1911
|
+
// No blur for overlay
|
|
1912
|
+
props.edgeBlurAmount,
|
|
1913
|
+
props.flipHorizontal
|
|
1914
|
+
);
|
|
1915
|
+
controller.enqueue(props.outputCanvas);
|
|
1916
|
+
break;
|
|
1917
|
+
}
|
|
1918
|
+
case "none": {
|
|
1919
|
+
controller.enqueue(videoFrame);
|
|
1920
|
+
break;
|
|
1921
|
+
}
|
|
1922
|
+
}
|
|
1923
|
+
props.status = "processing";
|
|
1924
|
+
},
|
|
1925
|
+
close: () => {
|
|
1926
|
+
if (!selfManageSegmenter) {
|
|
1927
|
+
segmenter.close();
|
|
1928
|
+
}
|
|
1929
|
+
props.outputCanvas = void 0;
|
|
1930
|
+
stopStreamTracks(props.outputStream);
|
|
1931
|
+
props.outputStream = void 0;
|
|
1932
|
+
props.status = "closed";
|
|
1933
|
+
},
|
|
1934
|
+
destroy: async () => {
|
|
1935
|
+
if (!selfManageSegmenter) {
|
|
1936
|
+
await segmenter.destroy();
|
|
1937
|
+
}
|
|
1938
|
+
props.status = "destroyed";
|
|
1939
|
+
}
|
|
1940
|
+
};
|
|
1941
|
+
};
|
|
1942
|
+
|
|
1943
|
+
// src/processor.ts
|
|
1944
|
+
var createMediaStreamTrackProcessor = (init) => {
|
|
1945
|
+
const processor = new MediaStreamTrackProcessor(init);
|
|
1946
|
+
return processor;
|
|
1947
|
+
};
|
|
1948
|
+
|
|
1949
|
+
// src/generator.ts
|
|
1950
|
+
var createMediaStreamTrackGenerator = (init) => {
|
|
1951
|
+
const generator = new MediaStreamTrackGenerator(init);
|
|
1952
|
+
return generator;
|
|
1953
|
+
};
|
|
1954
|
+
|
|
1955
|
+
// src/transformer.ts
|
|
1956
|
+
var createStreamTransformer = (transformer, writableStrategy, readableStrategy) => {
|
|
1957
|
+
const transformStream = new TransformStream(
|
|
1958
|
+
transformer,
|
|
1959
|
+
writableStrategy,
|
|
1960
|
+
readableStrategy
|
|
1961
|
+
);
|
|
1962
|
+
return transformStream;
|
|
1963
|
+
};
|
|
1964
|
+
|
|
1965
|
+
// src/video/transformer.ts
|
|
1966
|
+
var wrapTransformController = (videoFrame, controller) => {
|
|
1967
|
+
return {
|
|
1968
|
+
get desiredSize() {
|
|
1969
|
+
return controller.desiredSize;
|
|
1970
|
+
},
|
|
1971
|
+
enqueue: (frame) => {
|
|
1972
|
+
if (!frame) {
|
|
1973
|
+
return;
|
|
1974
|
+
}
|
|
1975
|
+
if (frame instanceof VideoFrame) {
|
|
1976
|
+
return controller.enqueue(frame);
|
|
1977
|
+
}
|
|
1978
|
+
if ("OffscreenCanvas" in window && frame instanceof OffscreenCanvas || frame instanceof HTMLCanvasElement || frame instanceof HTMLVideoElement || frame instanceof HTMLImageElement || frame instanceof ImageBitmap) {
|
|
1979
|
+
const timestamp = videoFrame.timestamp ?? 0;
|
|
1980
|
+
videoFrame.close();
|
|
1981
|
+
return controller.enqueue(
|
|
1982
|
+
new VideoFrame(frame, { timestamp, alpha: "discard" })
|
|
1983
|
+
);
|
|
1984
|
+
}
|
|
1985
|
+
throw new Error("Unexpected input frame");
|
|
1986
|
+
},
|
|
1987
|
+
error: (reason) => controller.error(reason),
|
|
1988
|
+
terminate: () => controller.terminate()
|
|
1989
|
+
};
|
|
1990
|
+
};
|
|
1991
|
+
var nullTransformController = (controller) => {
|
|
1992
|
+
return {
|
|
1993
|
+
get desiredSize() {
|
|
1994
|
+
return 0;
|
|
1995
|
+
},
|
|
1996
|
+
enqueue: (frame) => {
|
|
1997
|
+
if ("OffscreenCanvas" in window && frame instanceof OffscreenCanvas || frame instanceof HTMLCanvasElement || frame instanceof HTMLVideoElement || frame instanceof HTMLImageElement || frame instanceof ImageBitmap) {
|
|
1998
|
+
return controller.enqueue(frame);
|
|
1999
|
+
}
|
|
2000
|
+
throw new Error("Unexpected input frame");
|
|
2001
|
+
},
|
|
2002
|
+
error: (reason) => {
|
|
2003
|
+
throw new Error(reason);
|
|
2004
|
+
},
|
|
2005
|
+
terminate: () => controller.terminate()
|
|
2006
|
+
};
|
|
2007
|
+
};
|
|
2008
|
+
var adaptInputFrameTransformer = (transformer) => {
|
|
2009
|
+
const transform = (frame, controller) => {
|
|
2010
|
+
return transformer.transform?.(
|
|
2011
|
+
frame,
|
|
2012
|
+
wrapTransformController(frame, controller)
|
|
2013
|
+
);
|
|
2014
|
+
};
|
|
2015
|
+
return { ...transformer, transform };
|
|
2016
|
+
};
|
|
2017
|
+
|
|
2018
|
+
// src/video/videoStreamTrackProcessor.ts
|
|
2019
|
+
var createVideoTrackProcessor = () => (track, transformers, { signal } = {}) => {
|
|
2020
|
+
if (!transformers.length) {
|
|
2021
|
+
return Promise.resolve(track);
|
|
2022
|
+
}
|
|
2023
|
+
const processor = createMediaStreamTrackProcessor({ track });
|
|
2024
|
+
const trackGenerator = createMediaStreamTrackGenerator({ kind: "video" });
|
|
2025
|
+
let readable = processor.readable;
|
|
2026
|
+
transformers.forEach((transformer) => {
|
|
2027
|
+
readable = readable.pipeThrough(
|
|
2028
|
+
createStreamTransformer(
|
|
2029
|
+
adaptInputFrameTransformer(transformer)
|
|
2030
|
+
),
|
|
2031
|
+
{ signal }
|
|
2032
|
+
);
|
|
2033
|
+
});
|
|
2034
|
+
readable.pipeTo(trackGenerator.writable, {
|
|
2035
|
+
signal
|
|
2036
|
+
}).catch((error) => {
|
|
2037
|
+
if (signal && !(signal.aborted && (signal.reason === "close" /* Close */ || // AbortSignal['reason'] is only supported from Chromium v98 or Firefox v97
|
|
2038
|
+
!("reason" in AbortSignal.prototype)))) {
|
|
2039
|
+
throw error;
|
|
2040
|
+
}
|
|
2041
|
+
});
|
|
2042
|
+
return Promise.resolve(trackGenerator);
|
|
2043
|
+
};
|
|
2044
|
+
var createVideoTrackProcessorWithFallback = ({
|
|
2045
|
+
width = PROCESSING_WIDTH,
|
|
2046
|
+
height = PROCESSING_HEIGHT,
|
|
2047
|
+
frameRate = FRAME_RATE
|
|
2048
|
+
} = {}) => async (track, transformers, { signal } = {}) => {
|
|
2049
|
+
const [transformer] = transformers;
|
|
2050
|
+
if (!transformer?.transform) {
|
|
2051
|
+
return track;
|
|
2052
|
+
}
|
|
2053
|
+
if (transformers.length > 1) {
|
|
2054
|
+
throw new Error("Multi-transformer is NOT supported");
|
|
2055
|
+
}
|
|
2056
|
+
const outputCanvas = createCanvas(width, height);
|
|
2057
|
+
const ctx = getCanvasRenderingContext2D(outputCanvas, {
|
|
2058
|
+
desynchronized: true,
|
|
2059
|
+
alpha: false
|
|
2060
|
+
});
|
|
2061
|
+
const render = (input) => {
|
|
2062
|
+
if (!transformer.transform) {
|
|
2063
|
+
throw new Error("Transform is undefined");
|
|
2064
|
+
}
|
|
2065
|
+
return transformer.transform(
|
|
2066
|
+
input,
|
|
2067
|
+
nullTransformController({
|
|
2068
|
+
enqueue: (frame) => {
|
|
2069
|
+
if (!frame) {
|
|
2070
|
+
return;
|
|
2071
|
+
}
|
|
2072
|
+
const frameSize = getImageSize(frame);
|
|
2073
|
+
if (frameSize.height !== outputCanvas.height) {
|
|
2074
|
+
outputCanvas.height = frameSize.height;
|
|
2075
|
+
outputCanvas.width = frameSize.width;
|
|
2076
|
+
}
|
|
2077
|
+
ctx.drawImage(
|
|
2078
|
+
frame,
|
|
2079
|
+
0,
|
|
2080
|
+
0,
|
|
2081
|
+
frameSize.width,
|
|
2082
|
+
frameSize.height
|
|
2083
|
+
);
|
|
2084
|
+
},
|
|
2085
|
+
terminate: () => {
|
|
2086
|
+
stop();
|
|
2087
|
+
}
|
|
2088
|
+
})
|
|
2089
|
+
);
|
|
2090
|
+
};
|
|
2091
|
+
const runner = createFrameCallbackRequest(render, frameRate);
|
|
2092
|
+
const videoElement = toVideoElement(
|
|
2093
|
+
new MediaStream([track]),
|
|
2094
|
+
width,
|
|
2095
|
+
height
|
|
2096
|
+
);
|
|
2097
|
+
await playVideo(videoElement);
|
|
2098
|
+
await runner.start(videoElement);
|
|
2099
|
+
const stream = outputCanvas.captureStream(frameRate);
|
|
2100
|
+
const [trackGenerated] = stream.getVideoTracks();
|
|
2101
|
+
if (!trackGenerated) {
|
|
2102
|
+
throw new Error("Canvas captureStream returns no video track");
|
|
2103
|
+
}
|
|
2104
|
+
const stop = () => {
|
|
2105
|
+
stopStreamTracks(stream);
|
|
2106
|
+
runner.stop();
|
|
2107
|
+
};
|
|
2108
|
+
signal?.addEventListener("abort", stop);
|
|
2109
|
+
return trackGenerated;
|
|
2110
|
+
};
|
|
2111
|
+
|
|
2112
|
+
// src/video/segmenters/mediapipe.ts
|
|
2113
|
+
var createSegmenter = (basePath = "/", {
|
|
2114
|
+
modelType = "general",
|
|
2115
|
+
tfjsCoreLoaded = false,
|
|
2116
|
+
tfjsBackendLoaded = false,
|
|
2117
|
+
glueLoaded = false,
|
|
2118
|
+
processingWidth = PROCESSING_WIDTH,
|
|
2119
|
+
processingHeight = PROCESSING_HEIGHT,
|
|
2120
|
+
gluePath = "",
|
|
2121
|
+
selfieMode = false,
|
|
2122
|
+
prodMode = true
|
|
2123
|
+
} = {}) => {
|
|
2124
|
+
const props = {
|
|
2125
|
+
selfieMode,
|
|
2126
|
+
segmentation: [],
|
|
2127
|
+
renderUtils: createCanvasRenderUtils(processingWidth, processingHeight),
|
|
2128
|
+
status: "created",
|
|
2129
|
+
tfjsCoreLoaded,
|
|
2130
|
+
tfjsBackendLoaded,
|
|
2131
|
+
glueLoaded
|
|
2132
|
+
};
|
|
2133
|
+
const toInputImage = createInputImageConvertor(
|
|
2134
|
+
(input) => props.renderUtils.drawAndBlurImageOnOffScreenCanvas(
|
|
2135
|
+
input,
|
|
2136
|
+
0,
|
|
2137
|
+
"blurredCanvas"
|
|
2138
|
+
)
|
|
2139
|
+
);
|
|
2140
|
+
const segmentPerson = async (input) => {
|
|
2141
|
+
const segmenter = ensure(props.segmenter);
|
|
2142
|
+
props.status = "processing";
|
|
2143
|
+
const image = await toInputImage(input);
|
|
2144
|
+
await segmenter.send({ image });
|
|
2145
|
+
props.status = "idle";
|
|
2146
|
+
};
|
|
2147
|
+
return {
|
|
2148
|
+
get model() {
|
|
2149
|
+
return "mediapipeSelfie";
|
|
2150
|
+
},
|
|
2151
|
+
get width() {
|
|
2152
|
+
return processingWidth;
|
|
2153
|
+
},
|
|
2154
|
+
get height() {
|
|
2155
|
+
return processingHeight;
|
|
2156
|
+
},
|
|
2157
|
+
get status() {
|
|
2158
|
+
return props.status;
|
|
2159
|
+
},
|
|
2160
|
+
open: async () => {
|
|
2161
|
+
props.status = "opening";
|
|
2162
|
+
if (!props.tfjsCoreLoaded) {
|
|
2163
|
+
await loadTfjsCore(prodMode);
|
|
2164
|
+
props.tfjsCoreLoaded = true;
|
|
2165
|
+
}
|
|
2166
|
+
if (!props.tfjsBackendLoaded) {
|
|
2167
|
+
await loadTfjsBackendWebGl();
|
|
2168
|
+
props.tfjsBackendLoaded = true;
|
|
2169
|
+
}
|
|
2170
|
+
if (!props.glueLoaded) {
|
|
2171
|
+
await loadScript(gluePath, gluePath);
|
|
2172
|
+
props.glueLoaded = true;
|
|
2173
|
+
}
|
|
2174
|
+
props.segmenter = new SelfieSegmentation({
|
|
2175
|
+
locateFile: (path) => `${basePath.replace(/\/+$/, "")}/${path}`
|
|
2176
|
+
});
|
|
2177
|
+
const modelSelection = modelType === "landscape" ? 1 : 0;
|
|
2178
|
+
props.segmenter.setOptions({
|
|
2179
|
+
modelSelection,
|
|
2180
|
+
selfieMode: props.selfieMode
|
|
2181
|
+
});
|
|
2182
|
+
props.segmenter.onResults((results) => {
|
|
2183
|
+
props.segmentation = [
|
|
2184
|
+
toSegmentation(
|
|
2185
|
+
results.segmentationMask,
|
|
2186
|
+
"canvasimagesource",
|
|
2187
|
+
() => "person"
|
|
2188
|
+
)
|
|
2189
|
+
];
|
|
2190
|
+
});
|
|
2191
|
+
await props.segmenter.initialize();
|
|
2192
|
+
props.status = "opened";
|
|
2193
|
+
},
|
|
2194
|
+
process: async (input) => {
|
|
2195
|
+
await segmentPerson(input);
|
|
2196
|
+
return props.segmentation;
|
|
2197
|
+
},
|
|
2198
|
+
close: () => {
|
|
2199
|
+
props.segmenter?.reset();
|
|
2200
|
+
props.status = "closed";
|
|
2201
|
+
},
|
|
2202
|
+
destroy: async () => {
|
|
2203
|
+
props.status = "destroying";
|
|
2204
|
+
await props.segmenter?.close();
|
|
2205
|
+
props.status = "destroyed";
|
|
2206
|
+
}
|
|
2207
|
+
};
|
|
2208
|
+
};
|
|
2209
|
+
|
|
2210
|
+
// src/video/typeGuards.ts
|
|
2211
|
+
var isRenderEffects = (t) => {
|
|
2212
|
+
if (typeof t !== "string") {
|
|
2213
|
+
return false;
|
|
2214
|
+
}
|
|
2215
|
+
return RENDER_EFFECTS.some((effect) => effect === t);
|
|
2216
|
+
};
|
|
2217
|
+
var isSegmentationModel = (t) => {
|
|
2218
|
+
if (typeof t !== "string") {
|
|
2219
|
+
return false;
|
|
2220
|
+
}
|
|
2221
|
+
return SEG_MODELS.some((model) => model === t);
|
|
2222
|
+
};
|
|
2223
|
+
|
|
2224
|
+
// src/path.ts
|
|
2225
|
+
function toCoordinateString(p) {
|
|
2226
|
+
return p ? [p.x, p.y].join(",") : "";
|
|
2227
|
+
}
|
|
2228
|
+
function moveTo(p) {
|
|
2229
|
+
return p ? ["M", toCoordinateString(p)].join(" ") : "";
|
|
2230
|
+
}
|
|
2231
|
+
function lineTo(p) {
|
|
2232
|
+
return p ? ["L", toCoordinateString(p)].join(" ") : "";
|
|
2233
|
+
}
|
|
2234
|
+
function horizontalLineTo(x) {
|
|
2235
|
+
return `H ${x}`;
|
|
2236
|
+
}
|
|
2237
|
+
function verticalLineTo(y) {
|
|
2238
|
+
return `V ${y}`;
|
|
2239
|
+
}
|
|
2240
|
+
function cubicCurveTo({ scp, ecp, ep }) {
|
|
2241
|
+
return scp && ecp && ep ? ["C", ...[scp, ecp, ep].map(toCoordinateString)].join(" ") : "";
|
|
2242
|
+
}
|
|
2243
|
+
function closePath() {
|
|
2244
|
+
return "Z";
|
|
2245
|
+
}
|
|
2246
|
+
|
|
2247
|
+
// src/visual.ts
|
|
2248
|
+
var toPoint = (t) => t ? t : { x: 0, y: 0 };
|
|
2249
|
+
function calculateDistance(p1, p2) {
|
|
2250
|
+
return Math.sqrt((p2.x - p1.x) ** 2 + (p2.y - p1.y) ** 2);
|
|
2251
|
+
}
|
|
2252
|
+
function getBezierCurveControlPoints({
|
|
2253
|
+
p1,
|
|
2254
|
+
p2,
|
|
2255
|
+
p3,
|
|
2256
|
+
t
|
|
2257
|
+
}) {
|
|
2258
|
+
const d12 = calculateDistance(p1, p2);
|
|
2259
|
+
const d23 = calculateDistance(p2, p3);
|
|
2260
|
+
const widthOfT = p3.x - p1.x;
|
|
2261
|
+
const heightOfT = p3.y - p1.y;
|
|
2262
|
+
const scaleA = t * d12 / (d12 + d23);
|
|
2263
|
+
const scaleB = t - scaleA;
|
|
2264
|
+
const cp12 = {
|
|
2265
|
+
x: p2.x - scaleA * widthOfT,
|
|
2266
|
+
y: p2.y - scaleA * heightOfT
|
|
2267
|
+
};
|
|
2268
|
+
const cp23 = {
|
|
2269
|
+
x: p2.x + scaleB * widthOfT,
|
|
2270
|
+
y: p2.y + scaleB * heightOfT
|
|
2271
|
+
};
|
|
2272
|
+
return [cp12, cp23];
|
|
2273
|
+
}
|
|
2274
|
+
var line = (data) => {
|
|
2275
|
+
if (data.length < 2) {
|
|
2276
|
+
return "";
|
|
2277
|
+
}
|
|
2278
|
+
const [start, ...rest] = data;
|
|
2279
|
+
return [moveTo(start), ...rest.map(lineTo)].join(" ");
|
|
2280
|
+
};
|
|
2281
|
+
var curve = (data) => {
|
|
2282
|
+
if (data.length <= 2) {
|
|
2283
|
+
return line(data);
|
|
2284
|
+
}
|
|
2285
|
+
const [start, ...rest] = data;
|
|
2286
|
+
const knots = rest.slice(0, -1);
|
|
2287
|
+
const [end] = rest.slice(-1);
|
|
2288
|
+
const tension = 1 / 2;
|
|
2289
|
+
const cps = knots.map((current, idx, pts) => {
|
|
2290
|
+
const prev = pts[idx - 1] || start;
|
|
2291
|
+
const nxt = pts[idx + 1] || end;
|
|
2292
|
+
return getBezierCurveControlPoints({
|
|
2293
|
+
p1: toPoint(prev),
|
|
2294
|
+
p2: current,
|
|
2295
|
+
p3: toPoint(nxt),
|
|
2296
|
+
t: tension
|
|
2297
|
+
});
|
|
2298
|
+
});
|
|
2299
|
+
const curveTo = (ep, idx) => {
|
|
2300
|
+
const [scp] = cps[idx - 1]?.slice(-1) || [start];
|
|
2301
|
+
const [ecp] = cps[idx] || [end];
|
|
2302
|
+
return cubicCurveTo({
|
|
2303
|
+
scp: toPoint(scp),
|
|
2304
|
+
ecp: toPoint(ecp),
|
|
2305
|
+
ep
|
|
2306
|
+
});
|
|
2307
|
+
};
|
|
2308
|
+
return [moveTo(start), ...rest.map(curveTo)].join(" ");
|
|
2309
|
+
};
|
|
2310
|
+
var closedCurve = ({ x, y }) => (data) => {
|
|
2311
|
+
return [
|
|
2312
|
+
curve(data),
|
|
2313
|
+
verticalLineTo(y),
|
|
2314
|
+
horizontalLineTo(x),
|
|
2315
|
+
closePath()
|
|
2316
|
+
].join(" ");
|
|
2317
|
+
};
|
|
2318
|
+
|
|
2319
|
+
// src/benchUtils.ts
|
|
2320
|
+
var createBenchmark = (clock = performance, { calculationThresholdMS = 1e3 } = {}) => {
|
|
2321
|
+
const props = {
|
|
2322
|
+
beginTime: 0,
|
|
2323
|
+
endTime: 0,
|
|
2324
|
+
sumDelta: 0,
|
|
2325
|
+
count: 0,
|
|
2326
|
+
lastCalculateTime: 0,
|
|
2327
|
+
lastResult: 0
|
|
2328
|
+
};
|
|
2329
|
+
return {
|
|
2330
|
+
begin: () => {
|
|
2331
|
+
props.beginTime = clock.now();
|
|
2332
|
+
},
|
|
2333
|
+
end: () => {
|
|
2334
|
+
props.endTime = clock.now();
|
|
2335
|
+
props.sumDelta += props.endTime - props.beginTime;
|
|
2336
|
+
++props.count;
|
|
2337
|
+
},
|
|
2338
|
+
calculateFps: () => {
|
|
2339
|
+
const calculateTime = clock.now();
|
|
2340
|
+
if (calculateTime - props.lastCalculateTime >= calculationThresholdMS) {
|
|
2341
|
+
const result = props.count === 0 ? 0 : props.sumDelta / props.count;
|
|
2342
|
+
props.sumDelta = 0;
|
|
2343
|
+
props.count = 0;
|
|
2344
|
+
props.lastCalculateTime = calculateTime;
|
|
2345
|
+
props.lastResult = result;
|
|
2346
|
+
return result;
|
|
2347
|
+
}
|
|
2348
|
+
return props.lastResult;
|
|
2349
|
+
}
|
|
2350
|
+
};
|
|
2351
|
+
};
|
|
2352
|
+
var calculateFps = (time) => time === 0 ? time : 1e3 / time;
|
|
2353
|
+
|
|
2354
|
+
// src/index.ts
|
|
2355
|
+
var urls = {
|
|
2356
|
+
denoise: () => new URL("./worklets/denoise.worklet.js", import.meta.url)
|
|
2357
|
+
};
|
|
2358
|
+
export {
|
|
2359
|
+
AbortReason,
|
|
2360
|
+
BACKGROUND_BLUR_AMOUNT,
|
|
2361
|
+
CLIP_COUNT_THRESHOLD,
|
|
2362
|
+
CLIP_THRESHOLD,
|
|
2363
|
+
EDGE_BLUR_AMOUNT,
|
|
2364
|
+
FLIP_HORIZONTAL,
|
|
2365
|
+
FOREGROUND_THRESHOLD,
|
|
2366
|
+
FRAME_RATE,
|
|
2367
|
+
LOW_VOLUME_THRESHOLD,
|
|
2368
|
+
MONO_THRESHOLD,
|
|
2369
|
+
PROCESSING_HEIGHT,
|
|
2370
|
+
PROCESSING_WIDTH,
|
|
2371
|
+
RENDER_EFFECTS,
|
|
2372
|
+
SEG_MODELS,
|
|
2373
|
+
SILENT_THRESHOLD,
|
|
2374
|
+
VOICE_PROBABILITY_THRESHOLD,
|
|
2375
|
+
avg,
|
|
2376
|
+
calculateDistance,
|
|
2377
|
+
calculateFps,
|
|
2378
|
+
closedCurve,
|
|
2379
|
+
copyByteBufferToFloatBuffer,
|
|
2380
|
+
createAnalyzerGraphNode,
|
|
2381
|
+
createAnalyzerSubscribableGraphNode,
|
|
2382
|
+
createAsyncCallbackLoop,
|
|
2383
|
+
createAudioContext,
|
|
2384
|
+
createAudioDestinationGraphNode,
|
|
2385
|
+
createAudioGraph,
|
|
2386
|
+
createAudioGraphProxy,
|
|
2387
|
+
createAudioSignalDetector,
|
|
2388
|
+
createAudioStats,
|
|
2389
|
+
createBenchmark,
|
|
2390
|
+
createTransform as createCanvasTransform,
|
|
2391
|
+
createChannelMergerGraphNode,
|
|
2392
|
+
createChannelSplitterGraphNode,
|
|
2393
|
+
createDelayGraphNode,
|
|
2394
|
+
createDenoiseWorkletGraphNode,
|
|
2395
|
+
createFrameCallbackRequest,
|
|
2396
|
+
createGainGraphNode,
|
|
2397
|
+
createMediaElementSourceGraphNode,
|
|
2398
|
+
createSegmenter as createMediapipeSegmenter,
|
|
2399
|
+
createStreamDestinationGraphNode,
|
|
2400
|
+
createStreamSourceGraphNode,
|
|
2401
|
+
createVADetector,
|
|
2402
|
+
createVideoProcessor,
|
|
2403
|
+
createVideoTrackProcessor,
|
|
2404
|
+
createVideoTrackProcessorWithFallback,
|
|
2405
|
+
createVoiceDetectorFromProbability,
|
|
2406
|
+
createVoiceDetectorFromTimeData,
|
|
2407
|
+
curve,
|
|
2408
|
+
fitDestinationSize,
|
|
2409
|
+
fromByteToFloat,
|
|
2410
|
+
fromFloatToByte,
|
|
2411
|
+
getAudioStats,
|
|
2412
|
+
getBezierCurveControlPoints,
|
|
2413
|
+
isAnalyzerNodeInit,
|
|
2414
|
+
isAudioNode,
|
|
2415
|
+
isAudioNodeInit,
|
|
2416
|
+
isAudioParam,
|
|
2417
|
+
isClipping,
|
|
2418
|
+
isEqualSize,
|
|
2419
|
+
isLowVolume,
|
|
2420
|
+
isMono,
|
|
2421
|
+
isRenderEffects,
|
|
2422
|
+
isSegmentationModel,
|
|
2423
|
+
isSilent,
|
|
2424
|
+
isVoiceActivity,
|
|
2425
|
+
line,
|
|
2426
|
+
loadScript,
|
|
2427
|
+
loadTfjsBackendWebGl,
|
|
2428
|
+
loadTfjsCore,
|
|
2429
|
+
loadWasms,
|
|
2430
|
+
pow,
|
|
2431
|
+
processAverageVolume,
|
|
2432
|
+
resumeAudioOnInterruption,
|
|
2433
|
+
resumeAudioOnUnmute,
|
|
2434
|
+
rms,
|
|
2435
|
+
round,
|
|
2436
|
+
subscribeTimeoutAnalyzerNode,
|
|
2437
|
+
subscribeWorkletNode,
|
|
2438
|
+
sum,
|
|
2439
|
+
toDecibel,
|
|
2440
|
+
urls
|
|
2441
|
+
};
|