audio-video-sync 0.1.0

package/dist/index.esm.js

@@ -0,0 +1,475 @@
+import { FFmpeg } from '@ffmpeg/ffmpeg';
+import { fetchFile } from '@ffmpeg/util';
+
+/**
+ * 音频提取模块
+ * 使用 FFmpeg.wasm 从视频文件中提取音频 PCM 数据
+ */
+const DEFAULT_OPTIONS = {
+    sampleRate: 16000,
+    mono: true,
+    maxDuration: 60
+};
+/**
+ * 从视频文件提取音频 PCM 数据
+ *
+ * @param ffmpeg 已加载的 FFmpeg 实例
+ * @param videoFile 视频文件（File 或 Blob）
+ * @param options 提取选项
+ * @returns 音频 PCM 数据
+ */
+async function extractAudio(ffmpeg, videoFile, options = {}) {
+    const opts = { ...DEFAULT_OPTIONS, ...options };
+    const inputName = 'input_video';
+    const outputName = 'output_audio.pcm';
+    // 写入输入文件
+    await ffmpeg.writeFile(inputName, await fetchFile(videoFile));
+    // 构建 FFmpeg 命令
+    const args = [
+        '-i', inputName,
+        '-vn', // 不处理视频
+        '-ac', opts.mono ? '1' : '2', // 声道数
+        '-ar', opts.sampleRate.toString(), // 采样率
+        '-f', 's16le', // 16位小端 PCM
+        '-t', opts.maxDuration.toString(), // 最大时长
+        outputName
+    ];
+    await ffmpeg.exec(args);
+    // 读取输出文件
+    const data = await ffmpeg.readFile(outputName);
+    const pcmData = data instanceof Uint8Array ? data : new Uint8Array(data);
+    // 清理临时文件
+    await ffmpeg.deleteFile(inputName);
+    await ffmpeg.deleteFile(outputName);
+    // 将 16 位 PCM 转换为 Float32Array
+    const samples = pcm16ToFloat32(pcmData);
+    return {
+        samples,
+        sampleRate: opts.sampleRate,
+        duration: samples.length / opts.sampleRate,
+        channels: opts.mono ? 1 : 2
+    };
+}
+/**
+ * 将 16 位 PCM 数据转换为 Float32Array（归一化到 -1 到 1）
+ */
+function pcm16ToFloat32(pcmData) {
+    const numSamples = pcmData.length / 2;
+    const float32 = new Float32Array(numSamples);
+    const view = new DataView(pcmData.buffer, pcmData.byteOffset, pcmData.byteLength);
+    for (let i = 0; i < numSamples; i++) {
+        // 读取 16 位有符号整数（小端）
+        const int16 = view.getInt16(i * 2, true);
+        // 归一化到 -1 到 1
+        float32[i] = int16 / 32768;
+    }
+    return float32;
+}
+/**
+ * 从 AudioBuffer 获取 PCM 数据（用于 Web Audio API）
+ */
+function audioBufferToFloat32(audioBuffer) {
+    // 如果是多声道，混合为单声道
+    if (audioBuffer.numberOfChannels === 1) {
+        return audioBuffer.getChannelData(0);
+    }
+    const length = audioBuffer.length;
+    const mixed = new Float32Array(length);
+    const numChannels = audioBuffer.numberOfChannels;
+    for (let ch = 0; ch < numChannels; ch++) {
+        const channelData = audioBuffer.getChannelData(ch);
+        for (let i = 0; i < length; i++) {
+            mixed[i] += channelData[i] / numChannels;
+        }
+    }
+    return mixed;
+}
+/**
+ * 对音频数据进行降采样
+ */
+function downsample(samples, fromRate, toRate) {
+    if (fromRate === toRate) {
+        return samples;
+    }
+    const ratio = fromRate / toRate;
+    const newLength = Math.floor(samples.length / ratio);
+    const result = new Float32Array(newLength);
+    for (let i = 0; i < newLength; i++) {
+        const srcIndex = Math.floor(i * ratio);
+        result[i] = samples[srcIndex];
+    }
+    return result;
+}
+/**
+ * 对音频应用简单的预处理（去直流偏移、归一化）
+ */
+function preprocessAudio(samples) {
+    const result = new Float32Array(samples.length);
+    // 计算均值（直流偏移）
+    let sum = 0;
+    for (let i = 0; i < samples.length; i++) {
+        sum += samples[i];
+    }
+    const mean = sum / samples.length;
+    // 去除直流偏移并找最大值
+    let maxAbs = 0;
+    for (let i = 0; i < samples.length; i++) {
+        result[i] = samples[i] - mean;
+        maxAbs = Math.max(maxAbs, Math.abs(result[i]));
+    }
+    // 归一化
+    if (maxAbs > 0) {
+        for (let i = 0; i < result.length; i++) {
+            result[i] /= maxAbs;
+        }
+    }
+    return result;
+}
+
+/**
+ * FFT (Fast Fourier Transform) 实现
+ * 用于音频信号的频域分析和互相关计算
+ */
+/**
+ * 将数组填充到 2 的幂次长度
+ */
+function padToPowerOfTwo(arr, targetLength) {
+    const padded = new Float32Array(targetLength);
+    padded.set(arr);
+    return padded;
+}
+/**
+ * 计算下一个 2 的幂次
+ */
+function nextPowerOfTwo(n) {
+    return 1 << Math.ceil(Math.log2(n));
+}
+/**
+ * Cooley-Tukey FFT 算法
+ */
+function fft(input) {
+    const n = input.length;
+    if (n === 1) {
+        return [[input[0], 0]];
+    }
+    if (n & (n - 1)) {
+        throw new Error('FFT 输入长度必须是 2 的幂次');
+    }
+    // 分离奇偶项
+    const even = new Float32Array(n / 2);
+    const odd = new Float32Array(n / 2);
+    for (let i = 0; i < n / 2; i++) {
+        even[i] = input[2 * i];
+        odd[i] = input[2 * i + 1];
+    }
+    const evenFFT = fft(even);
+    const oddFFT = fft(odd);
+    const result = new Array(n);
+    for (let k = 0; k < n / 2; k++) {
+        const angle = -2 * Math.PI * k / n;
+        const twiddle = [Math.cos(angle), Math.sin(angle)];
+        // 复数乘法: twiddle * oddFFT[k]
+        const t = [
+            twiddle[0] * oddFFT[k][0] - twiddle[1] * oddFFT[k][1],
+            twiddle[0] * oddFFT[k][1] + twiddle[1] * oddFFT[k][0]
+        ];
+        result[k] = [evenFFT[k][0] + t[0], evenFFT[k][1] + t[1]];
+        result[k + n / 2] = [evenFFT[k][0] - t[0], evenFFT[k][1] - t[1]];
+    }
+    return result;
+}
+/**
+ * 逆 FFT
+ */
+function ifft(input) {
+    const n = input.length;
+    // 共轭
+    const conjugated = input.map(([re, im]) => [re, -im]);
+    // 转换为 Float32Array 进行 FFT
+    const realPart = new Float32Array(n);
+    for (let i = 0; i < n; i++) {
+        realPart[i] = conjugated[i][0];
+    }
+    // 对共轭进行 FFT（这里需要处理复数输入）
+    const result = fftComplex(conjugated);
+    // 共轭并除以 n
+    return result.map(([re, im]) => [re / n, -im / n]);
+}
+/**
+ * 复数输入的 FFT
+ */
+function fftComplex(input) {
+    const n = input.length;
+    if (n === 1) {
+        return [input[0]];
+    }
+    if (n & (n - 1)) {
+        throw new Error('FFT 输入长度必须是 2 的幂次');
+    }
+    const even = [];
+    const odd = [];
+    for (let i = 0; i < n / 2; i++) {
+        even.push(input[2 * i]);
+        odd.push(input[2 * i + 1]);
+    }
+    const evenFFT = fftComplex(even);
+    const oddFFT = fftComplex(odd);
+    const result = new Array(n);
+    for (let k = 0; k < n / 2; k++) {
+        const angle = -2 * Math.PI * k / n;
+        const twiddle = [Math.cos(angle), Math.sin(angle)];
+        const t = [
+            twiddle[0] * oddFFT[k][0] - twiddle[1] * oddFFT[k][1],
+            twiddle[0] * oddFFT[k][1] + twiddle[1] * oddFFT[k][0]
+        ];
+        result[k] = [evenFFT[k][0] + t[0], evenFFT[k][1] + t[1]];
+        result[k + n / 2] = [evenFFT[k][0] - t[0], evenFFT[k][1] - t[1]];
+    }
+    return result;
+}
+/**
+ * 计算两个信号的互相关
+ * 使用 FFT 加速: corr(a,b) = IFFT(FFT(a) * conj(FFT(b)))
+ *
+ * @param signalA 参考信号
+ * @param signalB 待对齐信号
+ * @returns 互相关结果数组
+ */
+function crossCorrelate(signalA, signalB) {
+    // 计算需要的 FFT 长度（两个信号长度之和的下一个 2 的幂次）
+    const n = nextPowerOfTwo(signalA.length + signalB.length - 1);
+    // 填充信号到相同长度
+    const paddedA = padToPowerOfTwo(signalA, n);
+    const paddedB = padToPowerOfTwo(signalB, n);
+    // 计算 FFT
+    const fftA = fft(paddedA);
+    const fftB = fft(paddedB);
+    // 计算 FFT(A) * conj(FFT(B))
+    const product = fftA.map((a, i) => {
+        const b = fftB[i];
+        // a * conj(b) = (a.re * b.re + a.im * b.im) + i(a.im * b.re - a.re * b.im)
+        return [
+            a[0] * b[0] + a[1] * b[1],
+            a[1] * b[0] - a[0] * b[1]
+        ];
+    });
+    // 计算 IFFT
+    const correlation = ifft(product);
+    // 提取实部
+    const result = new Float32Array(n);
+    for (let i = 0; i < n; i++) {
+        result[i] = correlation[i][0];
+    }
+    return result;
+}
+/**
+ * 从互相关结果中找到最大峰值位置
+ *
+ * @param correlation 互相关结果
+ * @param signalBLength 信号 B 的原始长度
+ * @returns 偏移量（正值表示 B 相对于 A 延迟，负值表示 B 领先）
+ */
+function findPeakOffset(correlation, signalBLength) {
+    let maxValue = -Infinity;
+    let maxIndex = 0;
+    for (let i = 0; i < correlation.length; i++) {
+        if (correlation[i] > maxValue) {
+            maxValue = correlation[i];
+            maxIndex = i;
+        }
+    }
+    // 将索引转换为偏移量
+    // 如果 maxIndex < signalBLength，则 B 相对于 A 延迟
+    // 如果 maxIndex >= signalBLength，则需要从另一端计算
+    const n = correlation.length;
+    if (maxIndex > n / 2) {
+        return maxIndex - n;
+    }
+    return maxIndex;
+}
+/**
+ * 计算互相关的置信度（归一化相关系数）
+ */
+function calculateConfidence(signalA, signalB, correlation, peakIndex) {
+    const maxCorr = correlation[peakIndex < 0 ? correlation.length + peakIndex : peakIndex];
+    // 计算信号的能量
+    let energyA = 0;
+    let energyB = 0;
+    for (let i = 0; i < signalA.length; i++) {
+        energyA += signalA[i] * signalA[i];
+    }
+    for (let i = 0; i < signalB.length; i++) {
+        energyB += signalB[i] * signalB[i];
+    }
+    // 归一化相关系数
+    const normFactor = Math.sqrt(energyA * energyB);
+    if (normFactor === 0)
+        return 0;
+    return Math.abs(maxCorr) / normFactor;
+}
+
+/**
+ * 多机位视频音频同步主模块
+ *
+ * 使用音频互相关算法自动对齐多个视频的时间轴
+ */
+/**
+ * 音频视频同步器
+ */
+class AudioVideoSync {
+    constructor(ffmpeg) {
+        this.loaded = false;
+        this.ffmpeg = ffmpeg || new FFmpeg();
+    }
+    /**
+     * 加载 FFmpeg（如果尚未加载）
+     */
+    async load() {
+        if (this.loaded)
+            return;
+        if (!this.ffmpeg.loaded) {
+            await this.ffmpeg.load();
+        }
+        this.loaded = true;
+    }
+    /**
+     * 同步多个视频
+     *
+     * @param videos 视频文件数组
+     * @param options 同步选项
+     * @returns 同步结果
+     */
+    async syncVideos(videos, options = {}) {
+        const { referenceIndex = 0, minConfidence = 0.3, onProgress, ...extractOptions } = options;
+        if (videos.length < 2) {
+            return {
+                referenceId: videos[0]?.id || '0',
+                results: [],
+                sampleRate: extractOptions.sampleRate || 16000,
+                success: false,
+                error: '至少需要 2 个视频进行同步'
+            };
+        }
+        if (referenceIndex < 0 || referenceIndex >= videos.length) {
+            return {
+                referenceId: '',
+                results: [],
+                sampleRate: extractOptions.sampleRate || 16000,
+                success: false,
+                error: '参考视频索引无效'
+            };
+        }
+        try {
+            await this.load();
+            // 提取所有视频的音频
+            const audioDataList = [];
+            for (let i = 0; i < videos.length; i++) {
+                onProgress?.('extracting', (i + 1) / videos.length);
+                const audio = await extractAudio(this.ffmpeg, videos[i].file, extractOptions);
+                audioDataList.push(audio);
+            }
+            // 预处理音频
+            const processedAudio = audioDataList.map(a => preprocessAudio(a.samples));
+            // 获取参考音频
+            const referenceAudio = processedAudio[referenceIndex];
+            const referenceVideo = videos[referenceIndex];
+            const sampleRate = audioDataList[referenceIndex].sampleRate;
+            // 计算每个视频相对于参考视频的偏移
+            const results = [];
+            for (let i = 0; i < videos.length; i++) {
+                onProgress?.('correlating', (i + 1) / videos.length);
+                const video = videos[i];
+                const videoId = video.id || i.toString();
+                if (i === referenceIndex) {
+                    // 参考视频偏移为 0
+                    results.push({
+                        id: videoId,
+                        offsetSeconds: 0,
+                        offsetSamples: 0,
+                        confidence: 1,
+                        correctedStartTime: video.originalStartTime || null
+                    });
+                    continue;
+                }
+                const targetAudio = processedAudio[i];
+                // 计算互相关
+                const correlation = crossCorrelate(referenceAudio, targetAudio);
+                // 找到峰值偏移
+                const offsetSamples = findPeakOffset(correlation, targetAudio.length);
+                const offsetSeconds = offsetSamples / sampleRate;
+                // 计算置信度
+                const confidence = calculateConfidence(referenceAudio, targetAudio, correlation, offsetSamples);
+                // 计算校正后的开始时间
+                let correctedStartTime = null;
+                if (referenceVideo.originalStartTime) {
+                    correctedStartTime = new Date(referenceVideo.originalStartTime.getTime() - offsetSeconds * 1000);
+                }
+                results.push({
+                    id: videoId,
+                    offsetSeconds,
+                    offsetSamples,
+                    confidence,
+                    correctedStartTime
+                });
+            }
+            // 检查是否所有同步都成功
+            const allSuccessful = results.every(r => r.confidence >= minConfidence);
+            return {
+                referenceId: referenceVideo.id || referenceIndex.toString(),
+                results,
+                sampleRate,
+                success: allSuccessful,
+                error: allSuccessful ? undefined : '部分视频同步置信度过低'
+            };
+        }
+        catch (error) {
+            return {
+                referenceId: videos[referenceIndex]?.id || referenceIndex.toString(),
+                results: [],
+                sampleRate: extractOptions.sampleRate || 16000,
+                success: false,
+                error: error instanceof Error ? error.message : '同步过程发生错误'
+            };
+        }
+    }
+    /**
+     * 计算两个视频之间的时间偏移
+     *
+     * @param referenceVideo 参考视频
+     * @param targetVideo 目标视频
+     * @param options 提取选项
+     * @returns 偏移结果
+     */
+    async calculateOffset(referenceVideo, targetVideo, options = {}) {
+        await this.load();
+        // 提取音频
+        const refAudio = await extractAudio(this.ffmpeg, referenceVideo, options);
+        const targetAudio = await extractAudio(this.ffmpeg, targetVideo, options);
+        // 预处理
+        const refProcessed = preprocessAudio(refAudio.samples);
+        const targetProcessed = preprocessAudio(targetAudio.samples);
+        // 计算互相关
+        const correlation = crossCorrelate(refProcessed, targetProcessed);
+        // 找到峰值
+        const offsetSamples = findPeakOffset(correlation, targetProcessed.length);
+        const offsetSeconds = offsetSamples / refAudio.sampleRate;
+        // 计算置信度
+        const confidence = calculateConfidence(refProcessed, targetProcessed, correlation, offsetSamples);
+        return { offsetSeconds, confidence };
+    }
+}
+/**
+ * 创建同步器实例的便捷函数
+ */
+function createSync(ffmpeg) {
+    return new AudioVideoSync(ffmpeg);
+}
+/**
+ * 快速同步多个视频（一次性使用）
+ */
+async function syncVideos(videos, options) {
+    const sync = new AudioVideoSync();
+    return sync.syncVideos(videos, options);
+}
+
+export { AudioVideoSync, audioBufferToFloat32, calculateConfidence, createSync, crossCorrelate, downsample, extractAudio, fft, findPeakOffset, ifft, nextPowerOfTwo, padToPowerOfTwo, preprocessAudio, syncVideos };