@twick/browser-render 0.15.7 → 0.15.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -10
- package/dist/index.js +505 -65
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +505 -65
- package/dist/index.mjs.map +1 -1
- package/package.json +4 -3
- package/public/audio-worker.js +1 -4
package/dist/index.mjs
CHANGED
|
@@ -1,6 +1,169 @@
|
|
|
1
1
|
// src/browser-renderer.ts
|
|
2
2
|
import { Renderer, Vector2 } from "@twick/core";
|
|
3
3
|
import defaultProject from "@twick/visualizer/dist/project.js";
|
|
4
|
+
import { hasAudio } from "@twick/media-utils";
|
|
5
|
+
|
|
6
|
+
// src/audio/video-audio-extractor.ts
|
|
7
|
+
var VideoElementAudioExtractor = class {
|
|
8
|
+
audioContext;
|
|
9
|
+
video;
|
|
10
|
+
destination = null;
|
|
11
|
+
mediaRecorder = null;
|
|
12
|
+
audioChunks = [];
|
|
13
|
+
constructor(videoSrc, sampleRate = 48e3) {
|
|
14
|
+
this.audioContext = new AudioContext({ sampleRate });
|
|
15
|
+
this.video = document.createElement("video");
|
|
16
|
+
this.video.crossOrigin = "anonymous";
|
|
17
|
+
this.video.src = videoSrc;
|
|
18
|
+
this.video.muted = true;
|
|
19
|
+
}
|
|
20
|
+
async initialize() {
|
|
21
|
+
return new Promise((resolve, reject) => {
|
|
22
|
+
this.video.addEventListener("loadedmetadata", () => resolve(), { once: true });
|
|
23
|
+
this.video.addEventListener("error", (e) => {
|
|
24
|
+
reject(new Error(`Failed to load video for audio extraction: ${e}`));
|
|
25
|
+
}, { once: true });
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Extract audio by playing the video and capturing audio output
|
|
30
|
+
*/
|
|
31
|
+
async extractAudio(startTime, duration, playbackRate = 1) {
|
|
32
|
+
try {
|
|
33
|
+
const source = this.audioContext.createMediaElementSource(this.video);
|
|
34
|
+
this.destination = this.audioContext.createMediaStreamDestination();
|
|
35
|
+
source.connect(this.destination);
|
|
36
|
+
} catch (err) {
|
|
37
|
+
throw new Error("Video has no audio track");
|
|
38
|
+
}
|
|
39
|
+
this.audioChunks = [];
|
|
40
|
+
let mimeType = "audio/webm";
|
|
41
|
+
if (!MediaRecorder.isTypeSupported(mimeType)) {
|
|
42
|
+
mimeType = "";
|
|
43
|
+
}
|
|
44
|
+
try {
|
|
45
|
+
this.mediaRecorder = new MediaRecorder(this.destination.stream, {
|
|
46
|
+
mimeType: mimeType || void 0
|
|
47
|
+
});
|
|
48
|
+
} catch (err) {
|
|
49
|
+
throw new Error(`Failed to create MediaRecorder: ${err}. Video may have no audio track.`);
|
|
50
|
+
}
|
|
51
|
+
this.mediaRecorder.ondataavailable = (event) => {
|
|
52
|
+
if (event.data && event.data.size > 0) {
|
|
53
|
+
this.audioChunks.push(event.data);
|
|
54
|
+
}
|
|
55
|
+
};
|
|
56
|
+
this.video.currentTime = startTime;
|
|
57
|
+
this.video.playbackRate = playbackRate;
|
|
58
|
+
await new Promise((resolve, reject) => {
|
|
59
|
+
const seekTimeout = setTimeout(() => {
|
|
60
|
+
reject(new Error("Video seek timeout"));
|
|
61
|
+
}, 5e3);
|
|
62
|
+
this.video.addEventListener("seeked", () => {
|
|
63
|
+
clearTimeout(seekTimeout);
|
|
64
|
+
resolve();
|
|
65
|
+
}, { once: true });
|
|
66
|
+
this.video.addEventListener("error", () => {
|
|
67
|
+
clearTimeout(seekTimeout);
|
|
68
|
+
reject(new Error("Video seek error"));
|
|
69
|
+
}, { once: true });
|
|
70
|
+
});
|
|
71
|
+
return new Promise((resolve, reject) => {
|
|
72
|
+
const recordingTimeout = setTimeout(() => {
|
|
73
|
+
this.video.pause();
|
|
74
|
+
if (this.mediaRecorder && this.mediaRecorder.state !== "inactive") {
|
|
75
|
+
this.mediaRecorder.stop();
|
|
76
|
+
}
|
|
77
|
+
reject(new Error("Audio extraction timeout - video may have no audio track"));
|
|
78
|
+
}, (duration / playbackRate + 10) * 1e3);
|
|
79
|
+
let hasData = false;
|
|
80
|
+
const dataCheckInterval = setInterval(() => {
|
|
81
|
+
if (this.audioChunks.length > 0 && this.audioChunks.some((chunk) => chunk.size > 0)) {
|
|
82
|
+
hasData = true;
|
|
83
|
+
}
|
|
84
|
+
}, 1e3);
|
|
85
|
+
this.mediaRecorder.onerror = (event) => {
|
|
86
|
+
clearInterval(dataCheckInterval);
|
|
87
|
+
clearTimeout(recordingTimeout);
|
|
88
|
+
this.video.pause();
|
|
89
|
+
reject(new Error(`MediaRecorder error: ${event}. Video may have no audio track.`));
|
|
90
|
+
};
|
|
91
|
+
try {
|
|
92
|
+
this.mediaRecorder.start(100);
|
|
93
|
+
this.video.play().catch((playErr) => {
|
|
94
|
+
clearInterval(dataCheckInterval);
|
|
95
|
+
clearTimeout(recordingTimeout);
|
|
96
|
+
reject(new Error(`Failed to play video: ${playErr}`));
|
|
97
|
+
});
|
|
98
|
+
} catch (startErr) {
|
|
99
|
+
clearInterval(dataCheckInterval);
|
|
100
|
+
clearTimeout(recordingTimeout);
|
|
101
|
+
reject(new Error(`Failed to start recording: ${startErr}`));
|
|
102
|
+
}
|
|
103
|
+
setTimeout(async () => {
|
|
104
|
+
clearInterval(dataCheckInterval);
|
|
105
|
+
clearTimeout(recordingTimeout);
|
|
106
|
+
this.video.pause();
|
|
107
|
+
if (this.mediaRecorder && this.mediaRecorder.state !== "inactive") {
|
|
108
|
+
this.mediaRecorder.stop();
|
|
109
|
+
}
|
|
110
|
+
const stopTimeout = setTimeout(() => {
|
|
111
|
+
if (this.audioChunks.length === 0 || !hasData) {
|
|
112
|
+
reject(new Error("No audio data captured - video has no audio track"));
|
|
113
|
+
}
|
|
114
|
+
}, 2e3);
|
|
115
|
+
await new Promise((res) => {
|
|
116
|
+
if (this.mediaRecorder) {
|
|
117
|
+
this.mediaRecorder.addEventListener("stop", () => {
|
|
118
|
+
clearTimeout(stopTimeout);
|
|
119
|
+
res();
|
|
120
|
+
}, { once: true });
|
|
121
|
+
} else {
|
|
122
|
+
clearTimeout(stopTimeout);
|
|
123
|
+
res();
|
|
124
|
+
}
|
|
125
|
+
});
|
|
126
|
+
try {
|
|
127
|
+
if (this.audioChunks.length === 0 || !this.audioChunks.some((chunk) => chunk.size > 0)) {
|
|
128
|
+
throw new Error("No audio data captured - video has no audio track");
|
|
129
|
+
}
|
|
130
|
+
const audioBlob = new Blob(this.audioChunks, { type: "audio/webm" });
|
|
131
|
+
if (audioBlob.size === 0) {
|
|
132
|
+
throw new Error("Audio blob is empty - video has no audio track");
|
|
133
|
+
}
|
|
134
|
+
const arrayBuffer = await audioBlob.arrayBuffer();
|
|
135
|
+
const audioBuffer = await this.audioContext.decodeAudioData(arrayBuffer);
|
|
136
|
+
if (audioBuffer.length === 0 || audioBuffer.duration === 0) {
|
|
137
|
+
throw new Error("Audio buffer is empty - video has no audio track");
|
|
138
|
+
}
|
|
139
|
+
resolve(audioBuffer);
|
|
140
|
+
} catch (err) {
|
|
141
|
+
reject(new Error(`Failed to decode recorded audio: ${err}`));
|
|
142
|
+
}
|
|
143
|
+
}, duration / playbackRate * 1e3);
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
async close() {
|
|
147
|
+
if (this.mediaRecorder && this.mediaRecorder.state !== "inactive") {
|
|
148
|
+
this.mediaRecorder.stop();
|
|
149
|
+
}
|
|
150
|
+
this.video.pause();
|
|
151
|
+
this.video.src = "";
|
|
152
|
+
if (this.audioContext.state !== "closed") {
|
|
153
|
+
await this.audioContext.close();
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
};
|
|
157
|
+
async function extractAudioFromVideo(videoSrc, startTime, duration, playbackRate = 1, sampleRate = 48e3) {
|
|
158
|
+
const extractor = new VideoElementAudioExtractor(videoSrc, sampleRate);
|
|
159
|
+
try {
|
|
160
|
+
await extractor.initialize();
|
|
161
|
+
const audioBuffer = await extractor.extractAudio(startTime, duration, playbackRate);
|
|
162
|
+
return audioBuffer;
|
|
163
|
+
} finally {
|
|
164
|
+
await extractor.close();
|
|
165
|
+
}
|
|
166
|
+
}
|
|
4
167
|
|
|
5
168
|
// src/audio/audio-processor.ts
|
|
6
169
|
function getAssetPlacement(frames) {
|
|
@@ -54,11 +217,26 @@ var BrowserAudioProcessor = class {
|
|
|
54
217
|
audioContext;
|
|
55
218
|
/**
|
|
56
219
|
* Fetch and decode audio from a media source
|
|
220
|
+
* Falls back to video element extraction if decodeAudioData fails
|
|
57
221
|
*/
|
|
58
222
|
async fetchAndDecodeAudio(src) {
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
223
|
+
try {
|
|
224
|
+
const response = await fetch(src);
|
|
225
|
+
const arrayBuffer = await response.arrayBuffer();
|
|
226
|
+
return await this.audioContext.decodeAudioData(arrayBuffer);
|
|
227
|
+
} catch (err) {
|
|
228
|
+
try {
|
|
229
|
+
return await extractAudioFromVideo(
|
|
230
|
+
src,
|
|
231
|
+
0,
|
|
232
|
+
999999,
|
|
233
|
+
1,
|
|
234
|
+
this.sampleRate
|
|
235
|
+
);
|
|
236
|
+
} catch (fallbackErr) {
|
|
237
|
+
throw new Error(`Failed to extract audio: ${err}. Fallback also failed: ${fallbackErr}`);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
62
240
|
}
|
|
63
241
|
/**
|
|
64
242
|
* Process audio asset with playback rate, volume, and timing
|
|
@@ -172,6 +350,99 @@ var BrowserAudioProcessor = class {
|
|
|
172
350
|
}
|
|
173
351
|
};
|
|
174
352
|
|
|
353
|
+
// src/audio/audio-video-muxer.ts
|
|
354
|
+
function getFFmpegBaseURL() {
|
|
355
|
+
if (typeof window !== "undefined") {
|
|
356
|
+
return `${window.location.origin}/ffmpeg`;
|
|
357
|
+
}
|
|
358
|
+
return "/ffmpeg";
|
|
359
|
+
}
|
|
360
|
+
async function muxAudioVideo(options) {
|
|
361
|
+
const muxStartTime = Date.now();
|
|
362
|
+
try {
|
|
363
|
+
console.log("Starting FFmpeg muxing...");
|
|
364
|
+
console.log(` Video blob size: ${options.videoBlob.size} bytes (${(options.videoBlob.size / 1024 / 1024).toFixed(2)} MB)`);
|
|
365
|
+
console.log(` Audio buffer size: ${options.audioBuffer.byteLength} bytes (${(options.audioBuffer.byteLength / 1024 / 1024).toFixed(2)} MB)`);
|
|
366
|
+
const { FFmpeg } = await import("@ffmpeg/ffmpeg");
|
|
367
|
+
const { fetchFile } = await import("@ffmpeg/util");
|
|
368
|
+
const ffmpeg = new FFmpeg();
|
|
369
|
+
const base = getFFmpegBaseURL();
|
|
370
|
+
const coreURL = `${base}/ffmpeg-core.js`;
|
|
371
|
+
const wasmURL = `${base}/ffmpeg-core.wasm`;
|
|
372
|
+
console.log(`Loading FFmpeg from ${base}`);
|
|
373
|
+
const loadStartTime = Date.now();
|
|
374
|
+
await ffmpeg.load({
|
|
375
|
+
coreURL,
|
|
376
|
+
wasmURL
|
|
377
|
+
});
|
|
378
|
+
const loadDuration = Date.now() - loadStartTime;
|
|
379
|
+
console.log(`FFmpeg loaded successfully in ${loadDuration}ms`);
|
|
380
|
+
console.log("Writing video and audio files...");
|
|
381
|
+
const writeStartTime = Date.now();
|
|
382
|
+
await ffmpeg.writeFile(
|
|
383
|
+
"video.mp4",
|
|
384
|
+
await fetchFile(options.videoBlob)
|
|
385
|
+
);
|
|
386
|
+
console.log(` Video file written: ${options.videoBlob.size} bytes`);
|
|
387
|
+
await ffmpeg.writeFile(
|
|
388
|
+
"audio.wav",
|
|
389
|
+
new Uint8Array(options.audioBuffer)
|
|
390
|
+
);
|
|
391
|
+
const writeDuration = Date.now() - writeStartTime;
|
|
392
|
+
console.log(` Audio file written: ${options.audioBuffer.byteLength} bytes`);
|
|
393
|
+
console.log(`Files written successfully in ${writeDuration}ms`);
|
|
394
|
+
console.log("Executing FFmpeg muxing command...");
|
|
395
|
+
const execStartTime = Date.now();
|
|
396
|
+
const ffmpegLogs = [];
|
|
397
|
+
ffmpeg.on("log", ({ message }) => {
|
|
398
|
+
ffmpegLogs.push(message);
|
|
399
|
+
console.log(` [FFmpeg] ${message}`);
|
|
400
|
+
});
|
|
401
|
+
await ffmpeg.exec([
|
|
402
|
+
"-i",
|
|
403
|
+
"video.mp4",
|
|
404
|
+
"-i",
|
|
405
|
+
"audio.wav",
|
|
406
|
+
"-c:v",
|
|
407
|
+
"copy",
|
|
408
|
+
"-c:a",
|
|
409
|
+
"aac",
|
|
410
|
+
"-b:a",
|
|
411
|
+
"192k",
|
|
412
|
+
"-shortest",
|
|
413
|
+
"output.mp4"
|
|
414
|
+
]);
|
|
415
|
+
const execDuration = Date.now() - execStartTime;
|
|
416
|
+
console.log(`FFmpeg muxing completed in ${execDuration}ms`);
|
|
417
|
+
const readStartTime = Date.now();
|
|
418
|
+
const data = await ffmpeg.readFile("output.mp4");
|
|
419
|
+
const readDuration = Date.now() - readStartTime;
|
|
420
|
+
console.log(`Output file read successfully in ${readDuration}ms`);
|
|
421
|
+
const uint8 = typeof data === "string" ? new TextEncoder().encode(data) : new Uint8Array(data);
|
|
422
|
+
const result = new Blob([uint8], { type: "video/mp4" });
|
|
423
|
+
const totalDuration = Date.now() - muxStartTime;
|
|
424
|
+
console.log(`Muxing successful: ${result.size} bytes (${(result.size / 1024 / 1024).toFixed(2)} MB) in ${totalDuration}ms`);
|
|
425
|
+
console.log(` Breakdown: load=${loadDuration}ms, write=${writeDuration}ms, exec=${execDuration}ms, read=${readDuration}ms`);
|
|
426
|
+
return result;
|
|
427
|
+
} catch (error) {
|
|
428
|
+
const totalDuration = Date.now() - muxStartTime;
|
|
429
|
+
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
430
|
+
const errorStack = error instanceof Error ? error.stack : void 0;
|
|
431
|
+
console.error("FFmpeg muxing failed:", errorMsg);
|
|
432
|
+
if (errorStack) {
|
|
433
|
+
console.error("Error stack:", errorStack);
|
|
434
|
+
}
|
|
435
|
+
console.error("Error details:", {
|
|
436
|
+
errorType: error instanceof Error ? error.constructor.name : typeof error,
|
|
437
|
+
errorMessage: errorMsg,
|
|
438
|
+
duration: `${totalDuration}ms`,
|
|
439
|
+
videoBlobSize: options.videoBlob.size,
|
|
440
|
+
audioBufferSize: options.audioBuffer.byteLength
|
|
441
|
+
});
|
|
442
|
+
throw error;
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
|
|
175
446
|
// src/browser-renderer.ts
|
|
176
447
|
var BrowserWasmExporter = class _BrowserWasmExporter {
|
|
177
448
|
constructor(settings) {
|
|
@@ -232,7 +503,6 @@ var BrowserWasmExporter = class _BrowserWasmExporter {
|
|
|
232
503
|
fps: this.fps
|
|
233
504
|
});
|
|
234
505
|
} catch (error) {
|
|
235
|
-
console.error("WASM loading error:", error);
|
|
236
506
|
throw error;
|
|
237
507
|
}
|
|
238
508
|
}
|
|
@@ -255,42 +525,99 @@ var BrowserWasmExporter = class _BrowserWasmExporter {
|
|
|
255
525
|
}
|
|
256
526
|
async generateAudio(assets, startFrame, endFrame) {
|
|
257
527
|
try {
|
|
258
|
-
console.log(
|
|
259
|
-
frames: assets.length,
|
|
260
|
-
startFrame,
|
|
261
|
-
endFrame
|
|
262
|
-
});
|
|
528
|
+
console.log(`Generating audio from ${assets.length} frames`);
|
|
263
529
|
const processor = new BrowserAudioProcessor();
|
|
264
530
|
const assetPlacements = getAssetPlacement(assets);
|
|
265
|
-
console.log(`\u{1F4CA} Found ${assetPlacements.length} audio assets to process`);
|
|
266
531
|
if (assetPlacements.length === 0) {
|
|
267
|
-
console.log("
|
|
532
|
+
console.log("No asset placements found");
|
|
268
533
|
return null;
|
|
269
534
|
}
|
|
535
|
+
console.log(`Processing ${assetPlacements.length} asset placements`);
|
|
270
536
|
const processedBuffers = [];
|
|
271
|
-
for (
|
|
537
|
+
for (let i = 0; i < assetPlacements.length; i++) {
|
|
538
|
+
const asset = assetPlacements[i];
|
|
539
|
+
console.log(`[${i + 1}/${assetPlacements.length}] Processing asset: ${asset.src} (type: ${asset.type}, volume: ${asset.volume}, playbackRate: ${asset.playbackRate})`);
|
|
272
540
|
if (asset.volume > 0 && asset.playbackRate > 0) {
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
asset
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
541
|
+
const startTime = Date.now();
|
|
542
|
+
try {
|
|
543
|
+
if (asset.type === "video") {
|
|
544
|
+
console.log(` \u2192 Checking if asset has audio: ${asset.src.substring(0, 50)}...`);
|
|
545
|
+
try {
|
|
546
|
+
const assetHasAudio = await hasAudio(asset.src);
|
|
547
|
+
if (!assetHasAudio) {
|
|
548
|
+
console.log(` \u23ED Skipping asset (no audio detected): ${asset.src.substring(0, 50)}...`);
|
|
549
|
+
continue;
|
|
550
|
+
}
|
|
551
|
+
console.log(` \u2713 Asset has audio, proceeding: ${asset.src.substring(0, 50)}...`);
|
|
552
|
+
} catch (audioCheckError) {
|
|
553
|
+
const errorMsg = audioCheckError instanceof Error ? audioCheckError.message : String(audioCheckError);
|
|
554
|
+
const errorStack = audioCheckError instanceof Error ? audioCheckError.stack : void 0;
|
|
555
|
+
console.warn(` \u26A0 Audio check failed, proceeding anyway: ${asset.src.substring(0, 50)}...`);
|
|
556
|
+
console.warn(` Error: ${errorMsg}`);
|
|
557
|
+
if (errorStack) {
|
|
558
|
+
console.warn(` Stack: ${errorStack}`);
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
console.log(` \u2192 Starting processAudioAsset for: ${asset.src}`);
|
|
563
|
+
const processPromise = processor.processAudioAsset(
|
|
564
|
+
asset,
|
|
565
|
+
this.settings.fps || 30,
|
|
566
|
+
endFrame - startFrame
|
|
567
|
+
);
|
|
568
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
569
|
+
setTimeout(() => {
|
|
570
|
+
reject(new Error(`Timeout processing audio asset after 20s - video may have no audio track`));
|
|
571
|
+
}, 2e4);
|
|
572
|
+
});
|
|
573
|
+
const buffer = await Promise.race([processPromise, timeoutPromise]);
|
|
574
|
+
const duration = Date.now() - startTime;
|
|
575
|
+
console.log(` \u2713 Successfully processed audio asset in ${duration}ms: ${asset.src.substring(0, 50)}...`);
|
|
576
|
+
processedBuffers.push(buffer);
|
|
577
|
+
} catch (error) {
|
|
578
|
+
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
579
|
+
const errorStack = error instanceof Error ? error.stack : void 0;
|
|
580
|
+
const duration = Date.now() - startTime;
|
|
581
|
+
console.warn(` \u2717 Failed to process audio asset after ${duration}ms: ${asset.src.substring(0, 50)}...`);
|
|
582
|
+
console.warn(` Error: ${errorMsg}`);
|
|
583
|
+
if (errorStack) {
|
|
584
|
+
console.warn(` Stack: ${errorStack}`);
|
|
585
|
+
}
|
|
586
|
+
console.warn(` Asset details: type=${asset.type}, volume=${asset.volume}, playbackRate=${asset.playbackRate}, startFrame=${asset.startInVideo}, endFrame=${asset.endInVideo}`);
|
|
587
|
+
}
|
|
588
|
+
} else {
|
|
589
|
+
console.log(` \u23ED Skipping asset: volume=${asset.volume}, playbackRate=${asset.playbackRate}`);
|
|
280
590
|
}
|
|
281
591
|
}
|
|
282
592
|
if (processedBuffers.length === 0) {
|
|
283
|
-
console.
|
|
593
|
+
console.warn("No audio buffers were successfully processed");
|
|
594
|
+
console.warn(` Total assets attempted: ${assetPlacements.length}`);
|
|
595
|
+
console.warn(` Assets with volume>0 and playbackRate>0: ${assetPlacements.filter((a) => a.volume > 0 && a.playbackRate > 0).length}`);
|
|
284
596
|
return null;
|
|
285
597
|
}
|
|
286
|
-
console.log(
|
|
598
|
+
console.log(`Mixing ${processedBuffers.length} audio buffers`);
|
|
599
|
+
const mixStartTime = Date.now();
|
|
287
600
|
const mixedBuffer = processor.mixAudioBuffers(processedBuffers);
|
|
601
|
+
const mixDuration = Date.now() - mixStartTime;
|
|
602
|
+
console.log(`Audio mixing completed in ${mixDuration}ms`);
|
|
603
|
+
const wavStartTime = Date.now();
|
|
288
604
|
const wavData = processor.audioBufferToWav(mixedBuffer);
|
|
605
|
+
const wavDuration = Date.now() - wavStartTime;
|
|
606
|
+
console.log(`WAV conversion completed in ${wavDuration}ms`);
|
|
607
|
+
console.log(`Audio generation complete: ${wavData.byteLength} bytes (${(wavData.byteLength / 1024 / 1024).toFixed(2)} MB)`);
|
|
289
608
|
await processor.close();
|
|
290
|
-
console.log(`\u2705 Audio processed: ${(wavData.byteLength / 1024 / 1024).toFixed(2)} MB`);
|
|
291
609
|
return wavData;
|
|
292
610
|
} catch (error) {
|
|
293
|
-
|
|
611
|
+
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
612
|
+
const errorStack = error instanceof Error ? error.stack : void 0;
|
|
613
|
+
console.error("Audio generation error:", errorMsg);
|
|
614
|
+
if (errorStack) {
|
|
615
|
+
console.error("Error stack:", errorStack);
|
|
616
|
+
}
|
|
617
|
+
console.error("Error details:", {
|
|
618
|
+
errorType: error instanceof Error ? error.constructor.name : typeof error,
|
|
619
|
+
errorMessage: errorMsg
|
|
620
|
+
});
|
|
294
621
|
return null;
|
|
295
622
|
}
|
|
296
623
|
}
|
|
@@ -335,12 +662,7 @@ var renderTwickVideoInBrowser = async (config) => {
|
|
|
335
662
|
const width = settings.width || variables.input.properties?.width || 1920;
|
|
336
663
|
const height = settings.height || variables.input.properties?.height || 1080;
|
|
337
664
|
const fps = settings.fps || variables.input.properties?.fps || 30;
|
|
338
|
-
|
|
339
|
-
if (!projectFile) {
|
|
340
|
-
project = defaultProject;
|
|
341
|
-
} else {
|
|
342
|
-
project = projectFile;
|
|
343
|
-
}
|
|
665
|
+
const project = !projectFile ? defaultProject : projectFile;
|
|
344
666
|
project.variables = variables;
|
|
345
667
|
const renderSettings = {
|
|
346
668
|
name: "browser-render",
|
|
@@ -368,6 +690,65 @@ var renderTwickVideoInBrowser = async (config) => {
|
|
|
368
690
|
renderer.playback.fps = renderSettings.fps;
|
|
369
691
|
renderer.playback.state = 1;
|
|
370
692
|
const totalFrames = await renderer.getNumberOfFrames(renderSettings);
|
|
693
|
+
if (totalFrames === 0 || !isFinite(totalFrames)) {
|
|
694
|
+
throw new Error(
|
|
695
|
+
"Cannot render: Video has zero duration. Please ensure your project has valid content with non-zero duration. Check that all video elements have valid sources and are properly loaded."
|
|
696
|
+
);
|
|
697
|
+
}
|
|
698
|
+
const videoElements = [];
|
|
699
|
+
if (variables.input.tracks) {
|
|
700
|
+
variables.input.tracks.forEach((track) => {
|
|
701
|
+
if (track.elements) {
|
|
702
|
+
track.elements.forEach((el) => {
|
|
703
|
+
if (el.type === "video") videoElements.push(el);
|
|
704
|
+
});
|
|
705
|
+
}
|
|
706
|
+
});
|
|
707
|
+
}
|
|
708
|
+
let hasAnyAudio = false;
|
|
709
|
+
console.log(`Found ${videoElements.length} video element(s) to check for audio`);
|
|
710
|
+
if (videoElements.length > 0) {
|
|
711
|
+
for (const videoEl of videoElements) {
|
|
712
|
+
const src = videoEl.props?.src;
|
|
713
|
+
if (!src || src === "undefined") continue;
|
|
714
|
+
const preloadVideo = document.createElement("video");
|
|
715
|
+
preloadVideo.crossOrigin = "anonymous";
|
|
716
|
+
preloadVideo.preload = "metadata";
|
|
717
|
+
preloadVideo.src = src;
|
|
718
|
+
await new Promise((resolve, reject) => {
|
|
719
|
+
const timeout = setTimeout(
|
|
720
|
+
() => reject(new Error(`Timeout loading video metadata: ${src.substring(0, 80)}`)),
|
|
721
|
+
3e4
|
|
722
|
+
);
|
|
723
|
+
preloadVideo.addEventListener("loadedmetadata", () => {
|
|
724
|
+
clearTimeout(timeout);
|
|
725
|
+
resolve();
|
|
726
|
+
}, { once: true });
|
|
727
|
+
preloadVideo.addEventListener("error", () => {
|
|
728
|
+
clearTimeout(timeout);
|
|
729
|
+
const err = preloadVideo.error;
|
|
730
|
+
reject(new Error(`Failed to load video: ${err?.message || "Unknown error"}`));
|
|
731
|
+
}, { once: true });
|
|
732
|
+
});
|
|
733
|
+
if (settings.includeAudio) {
|
|
734
|
+
try {
|
|
735
|
+
console.log(`Checking if video has audio: ${src.substring(0, 50)}...`);
|
|
736
|
+
const videoHasAudio = await hasAudio(src);
|
|
737
|
+
console.log(`Audio check result for ${src.substring(0, 50)}...: ${videoHasAudio ? "HAS AUDIO" : "NO AUDIO"}`);
|
|
738
|
+
if (videoHasAudio) {
|
|
739
|
+
hasAnyAudio = true;
|
|
740
|
+
console.log(`\u2713 Video has audio: ${src.substring(0, 50)}...`);
|
|
741
|
+
} else {
|
|
742
|
+
console.log(`\u2717 Video has no audio: ${src.substring(0, 50)}...`);
|
|
743
|
+
}
|
|
744
|
+
} catch (error) {
|
|
745
|
+
console.warn(`Failed to check audio for ${src.substring(0, 50)}...:`, error);
|
|
746
|
+
hasAnyAudio = true;
|
|
747
|
+
console.log(`\u26A0 Assuming video might have audio due to check error`);
|
|
748
|
+
}
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
}
|
|
371
752
|
await renderer.playback.recalculate();
|
|
372
753
|
await renderer.playback.reset();
|
|
373
754
|
await renderer.playback.seek(0);
|
|
@@ -384,40 +765,107 @@ var renderTwickVideoInBrowser = async (config) => {
|
|
|
384
765
|
mediaAssets.push(currentAssets);
|
|
385
766
|
const canvas = renderer.stage.finalBuffer;
|
|
386
767
|
await exporter.handleFrame(canvas, frame);
|
|
387
|
-
if (settings.onProgress)
|
|
388
|
-
settings.onProgress(frame / totalFrames);
|
|
389
|
-
}
|
|
768
|
+
if (settings.onProgress) settings.onProgress(frame / totalFrames);
|
|
390
769
|
}
|
|
391
770
|
await exporter.stop();
|
|
392
771
|
let audioData = null;
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
772
|
+
console.log(`Audio detection summary: hasAnyAudio=${hasAnyAudio}, includeAudio=${settings.includeAudio}, mediaAssets=${mediaAssets.length}`);
|
|
773
|
+
if (settings.includeAudio && mediaAssets.length > 0 && hasAnyAudio) {
|
|
774
|
+
console.log("Starting audio processing (audio detected in videos)");
|
|
775
|
+
if (settings.onProgress) {
|
|
776
|
+
settings.onProgress(0.98);
|
|
777
|
+
}
|
|
778
|
+
try {
|
|
779
|
+
console.log("Calling generateAudio...");
|
|
780
|
+
audioData = await exporter.generateAudio(mediaAssets, 0, totalFrames);
|
|
781
|
+
console.log("generateAudio completed");
|
|
782
|
+
if (audioData) {
|
|
783
|
+
console.log(`\u2713 Audio generation successful: ${audioData.byteLength} bytes`);
|
|
784
|
+
} else {
|
|
785
|
+
console.log("\u2717 No audio data generated");
|
|
786
|
+
}
|
|
787
|
+
if (settings.onProgress) {
|
|
788
|
+
settings.onProgress(0.99);
|
|
789
|
+
}
|
|
790
|
+
} catch (audioError) {
|
|
791
|
+
const errorMsg = audioError instanceof Error ? audioError.message : String(audioError);
|
|
792
|
+
const errorStack = audioError instanceof Error ? audioError.stack : void 0;
|
|
793
|
+
console.error("\u2717 Audio generation failed, continuing without audio");
|
|
794
|
+
console.error(` Error: ${errorMsg}`);
|
|
795
|
+
if (errorStack) {
|
|
796
|
+
console.error(` Stack: ${errorStack}`);
|
|
797
|
+
}
|
|
798
|
+
console.error(" Context:", {
|
|
799
|
+
hasAnyAudio,
|
|
800
|
+
includeAudio: settings.includeAudio,
|
|
801
|
+
mediaAssetsCount: mediaAssets.length,
|
|
802
|
+
totalFrames
|
|
803
|
+
});
|
|
804
|
+
audioData = null;
|
|
805
|
+
}
|
|
806
|
+
} else if (settings.includeAudio && mediaAssets.length > 0 && !hasAnyAudio) {
|
|
807
|
+
console.log("\u23ED Skipping audio processing: no audio detected in videos");
|
|
808
|
+
} else {
|
|
809
|
+
console.log(`\u23ED Skipping audio processing: includeAudio=${settings.includeAudio}, mediaAssets=${mediaAssets.length}, hasAnyAudio=${hasAnyAudio}`);
|
|
396
810
|
}
|
|
397
811
|
let finalBlob = exporter.getVideoBlob();
|
|
398
812
|
if (!finalBlob) {
|
|
399
813
|
throw new Error("Failed to create video blob");
|
|
400
814
|
}
|
|
815
|
+
if (finalBlob.size === 0) {
|
|
816
|
+
throw new Error("Video blob is empty. Rendering may have failed.");
|
|
817
|
+
}
|
|
401
818
|
if (audioData && settings.includeAudio) {
|
|
402
|
-
console.log(
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
819
|
+
console.log(`Attempting to mux audio (${audioData.byteLength} bytes) with video (${finalBlob.size} bytes)`);
|
|
820
|
+
try {
|
|
821
|
+
const muxedBlob = await muxAudioVideo({
|
|
822
|
+
videoBlob: finalBlob,
|
|
823
|
+
audioBuffer: audioData
|
|
824
|
+
});
|
|
825
|
+
if (!muxedBlob || muxedBlob.size === 0) {
|
|
826
|
+
throw new Error("Muxed video blob is empty");
|
|
827
|
+
}
|
|
828
|
+
if (muxedBlob.size === finalBlob.size) {
|
|
829
|
+
console.warn("Muxed blob size unchanged - muxing may have failed silently");
|
|
830
|
+
} else {
|
|
831
|
+
console.log(`Muxing successful: ${finalBlob.size} bytes -> ${muxedBlob.size} bytes`);
|
|
832
|
+
}
|
|
833
|
+
finalBlob = muxedBlob;
|
|
834
|
+
} catch (muxError) {
|
|
835
|
+
const errorMsg = muxError instanceof Error ? muxError.message : String(muxError);
|
|
836
|
+
const errorStack = muxError instanceof Error ? muxError.stack : void 0;
|
|
837
|
+
console.error("Audio muxing failed");
|
|
838
|
+
console.error(` Error: ${errorMsg}`);
|
|
839
|
+
if (errorStack) {
|
|
840
|
+
console.error(` Stack: ${errorStack}`);
|
|
841
|
+
}
|
|
842
|
+
console.error(" Context:", {
|
|
843
|
+
videoBlobSize: finalBlob.size,
|
|
844
|
+
audioDataSize: audioData?.byteLength || 0
|
|
845
|
+
});
|
|
846
|
+
if (settings.downloadAudioSeparately && audioData) {
|
|
847
|
+
const audioBlob = new Blob([audioData], { type: "audio/wav" });
|
|
848
|
+
const audioUrl = URL.createObjectURL(audioBlob);
|
|
849
|
+
const a = document.createElement("a");
|
|
850
|
+
a.href = audioUrl;
|
|
851
|
+
a.download = "audio.wav";
|
|
852
|
+
a.click();
|
|
853
|
+
URL.revokeObjectURL(audioUrl);
|
|
854
|
+
}
|
|
855
|
+
console.warn("Continuing with video without audio due to muxing failure");
|
|
856
|
+
finalBlob = exporter.getVideoBlob();
|
|
857
|
+
if (!finalBlob || finalBlob.size === 0) {
|
|
858
|
+
throw new Error("Video blob is invalid after muxing failure");
|
|
859
|
+
}
|
|
417
860
|
}
|
|
418
|
-
|
|
419
|
-
console.
|
|
420
|
-
|
|
861
|
+
} else if (settings.includeAudio && !audioData) {
|
|
862
|
+
console.warn("Audio processing was enabled but no audio data was generated");
|
|
863
|
+
}
|
|
864
|
+
if (!finalBlob || finalBlob.size === 0) {
|
|
865
|
+
throw new Error("Final video blob is empty or invalid");
|
|
866
|
+
}
|
|
867
|
+
if (settings.onProgress) {
|
|
868
|
+
settings.onProgress(1);
|
|
421
869
|
}
|
|
422
870
|
if (settings.onComplete) {
|
|
423
871
|
settings.onComplete(finalBlob);
|
|
@@ -461,17 +909,13 @@ var useBrowserRenderer = (options = {}) => {
|
|
|
461
909
|
}, []);
|
|
462
910
|
const download = useCallback((filename) => {
|
|
463
911
|
if (!videoBlob) {
|
|
464
|
-
|
|
465
|
-
setError(downloadError);
|
|
466
|
-
console.error(downloadError.message);
|
|
912
|
+
setError(new Error("No video available to download. Please render the video first."));
|
|
467
913
|
return;
|
|
468
914
|
}
|
|
469
915
|
try {
|
|
470
916
|
downloadVideoBlob(videoBlob, filename || options.downloadFilename || "video.mp4");
|
|
471
917
|
} catch (err) {
|
|
472
|
-
|
|
473
|
-
setError(downloadError);
|
|
474
|
-
console.error("Download error:", downloadError);
|
|
918
|
+
setError(err instanceof Error ? err : new Error("Failed to download video"));
|
|
475
919
|
}
|
|
476
920
|
}, [videoBlob, options.downloadFilename]);
|
|
477
921
|
const render = useCallback(async (variables) => {
|
|
@@ -501,9 +945,7 @@ var useBrowserRenderer = (options = {}) => {
|
|
|
501
945
|
try {
|
|
502
946
|
downloadVideoBlob(blob2, downloadFilename || "video.mp4");
|
|
503
947
|
} catch (downloadErr) {
|
|
504
|
-
|
|
505
|
-
setError(error2);
|
|
506
|
-
console.error("Auto-download error:", error2);
|
|
948
|
+
setError(downloadErr instanceof Error ? downloadErr : new Error("Failed to auto-download video"));
|
|
507
949
|
}
|
|
508
950
|
}
|
|
509
951
|
},
|
|
@@ -519,9 +961,7 @@ var useBrowserRenderer = (options = {}) => {
|
|
|
519
961
|
setProgress(1);
|
|
520
962
|
return blob;
|
|
521
963
|
} catch (err) {
|
|
522
|
-
|
|
523
|
-
setError(error2);
|
|
524
|
-
console.error("Render error:", error2);
|
|
964
|
+
setError(err instanceof Error ? err : new Error(String(err)));
|
|
525
965
|
return null;
|
|
526
966
|
} finally {
|
|
527
967
|
setIsRendering(false);
|