@vunk/plus 1.10.2 → 1.10.3

package/shared/StreamingInferenceService.mjs

@@ -1,204 +0,0 @@
-import { Deferred } from '@vunk/shared/promise';
-import { w as workerConfig } from './config.mjs';
-
-class StreamingInferenceService {
-  /**
-   * 构造函数，初始化Worker并注册消息回调。
-   * @param modelPath 模型路径
-   * @param onReady Worker初始化完成回调
-   */
-  constructor(modelPath, onReady) {
-    /** 后台推理Worker实例 */
-    this.worker = null;
-    /** 推理回调函数集合 */
-    this.callbacks = {};
-    /** Worker是否已初始化 */
-    this.isInitialized = false;
-    /** 待推理的chunk数据映射 */
-    this.pendingChunks = /* @__PURE__ */ new Map();
-    /** 已完成推理的chunk索引集合 */
-    this.completedChunks = /* @__PURE__ */ new Set();
-    /** 当前累计的总帧数 */
-    this.totalFrames = 0;
-    /** 是否处于推理处理中 */
-    this.isProcessing = false;
-    /** 全局chunk索引计数器，确保每个chunk有唯一索引 */
-    this.globalChunkIndex = 0;
-    this._def = new Deferred();
-    this.worker = new Worker(
-      /* @vite-ignore */
-      `${workerConfig.path}/streaming.inference.worker.js`,
-      { type: "module" }
-    );
-    this.worker.onmessage = (event) => {
-      const { type } = event.data;
-      switch (type) {
-        case "ready":
-          this.isInitialized = true;
-          onReady?.();
-          this._def.resolve();
-          break;
-        case "progress":
-          this.callbacks.onProgress?.(
-            event.data.payload.processed,
-            event.data.payload.total
-          );
-          break;
-        case "frame": {
-          const frameData = event.data.payload;
-          this.callbacks.onFrame?.(frameData.frame, frameData.frameIndex);
-          break;
-        }
-        case "chunk_complete": {
-          const chunkData = event.data.payload;
-          this.completedChunks.add(chunkData.chunkIndex);
-          this.callbacks.onChunkComplete?.(
-            chunkData.chunkIndex,
-            chunkData.timings
-          );
-          if (this.completedChunks.size === this.pendingChunks.size && this.pendingChunks.size > 0) {
-            this.callbacks.onAllComplete?.({});
-            this.lightCleanup();
-          }
-          break;
-        }
-        case "error":
-          this.callbacks.onError?.(event.data.payload);
-          this.cleanup();
-          break;
-      }
-    };
-    this.worker.onerror = (error) => {
-      this.callbacks.onError?.(`Worker error: ${error.message}`);
-      this.cleanup();
-    };
-    this.worker.postMessage({ type: "init", modelPath });
-  }
-  /**
-   * 判断推理服务是否已准备就绪。
-   * @returns 是否已初始化
-   */
-  isReady() {
-    return this.isInitialized;
-  }
-  when() {
-    return this._def.promise;
-  }
-  /**
-   * 启动流式推理流程，发送共享数据到worker。
-   * @param sharedData 所有chunk共享的数据（图像、配置等）
-   * @param callbacks 推理过程中的回调函数
-   * @param startImageIndex 起始图像帧索引，默认为0
-   */
-  async startStreaming(sharedData, callbacks, startImageIndex = 0) {
-    if (!this.worker || !this.isInitialized) {
-      throw new Error("\u63A8\u7406\u670D\u52A1\u5C1A\u672A\u521D\u59CB\u5316\u5B8C\u6210\u3002");
-    }
-    this.callbacks = callbacks;
-    this.pendingChunks.clear();
-    this.completedChunks.clear();
-    this.totalFrames = 0;
-    this.isProcessing = true;
-    const zipBuffer = await sharedData.zipBlob.arrayBuffer();
-    const message = {
-      type: "init_streaming",
-      dataset: sharedData.dataset,
-      zipBuffer,
-      blendingMaskBitmap: sharedData.blendingMaskBitmap,
-      startImageIndex
-    };
-    this.worker.postMessage(message, [
-      sharedData.blendingMaskBitmap,
-      zipBuffer
-    ]);
-  }
-  /**
-   * 添加一个chunk进行推理。
-   * @param chunkResult 特征提取完成的chunk
-   */
-  addChunk(chunkResult) {
-    if (!this.worker || !this.isProcessing) {
-      console.warn("\u65E0\u6CD5\u6DFB\u52A0chunk\uFF1A\u63A8\u7406\u670D\u52A1\u672A\u51C6\u5907\u5C31\u7EEA\u6216\u672A\u5728\u5904\u7406\u4E2D");
-      return;
-    }
-    const chunkFrames = chunkResult.dimensions[0];
-    const startFrame = this.totalFrames;
-    const endFrame = startFrame + chunkFrames;
-    this.totalFrames = endFrame;
-    const uniqueChunkIndex = this.globalChunkIndex++;
-    const chunkData = {
-      chunkIndex: uniqueChunkIndex,
-      // 使用全局唯一索引
-      audioFeatures: chunkResult.features,
-      // 直接使用，不复制
-      audioDimensions: chunkResult.dimensions,
-      startFrame,
-      endFrame
-    };
-    this.pendingChunks.set(uniqueChunkIndex, chunkData);
-    const message = {
-      type: "streaming_run",
-      chunkData,
-      dataset: {},
-      // 已在init_streaming中发送
-      zipBlob: new Blob(),
-      // 已在init_streaming中发送
-      blendingMaskBitmap: {},
-      // 已在init_streaming中发送
-      startImageIndex: 0
-      // 已在init_streaming中发送
-    };
-    this.worker.postMessage(message, [chunkData.audioFeatures.buffer]);
-  }
-  /**
-   * 通知所有chunk都已添加完成，发送总帧数到worker。
-   */
-  finishAddingChunks() {
-    if (!this.worker)
-      return;
-    this.worker.postMessage({
-      type: "finish_chunks",
-      totalFrames: this.totalFrames
-    });
-  }
-  /**
-   * 清理内部状态和资源。
-   */
-  cleanup() {
-    this.isProcessing = false;
-    this.callbacks = {};
-    this.pendingChunks.clear();
-    this.completedChunks.clear();
-    this.totalFrames = 0;
-    this.globalChunkIndex = 0;
-  }
-  /**
-   * 轻度清理，保持连续性，用于音频文件间的清理。
-   */
-  lightCleanup() {
-    this.completedChunks.clear();
-    this.pendingChunks.clear();
-  }
-  /**
-   * 停止推理并清理资源，通知worker停止。
-   */
-  stop() {
-    this.cleanup();
-    if (this.worker) {
-      this.worker.postMessage({ type: "stop" });
-    }
-  }
-  /**
-   * 彻底终止worker并清理所有资源。
-   */
-  terminate() {
-    if (this.worker) {
-      this.worker.terminate();
-      this.worker = null;
-      this.isInitialized = false;
-    }
-    this.cleanup();
-  }
-}
-
-export { StreamingInferenceService as S };

package/shared/audioToFrames/index.d.ts

@@ -1,19 +0,0 @@
-import type { ChunkFeatureResult } from './src/audio';
-import { workerConfig } from './src/config';
-import { StreamingInferenceService } from './src/inference';
-import { PredictiveFrameLoader } from './src/media';
-export type { ImageDataResponse } from './src/media';
-export { PredictiveFrameLoader, StreamingInferenceService, workerConfig, };
-export declare function blobToAudioBuffer(blob: Blob): Promise<AudioBuffer>;
-export declare function processStreaming(buffer: AudioBuffer, callbacks?: {
-    onChunkComplete?: (result: ChunkFeatureResult) => void;
-}): Promise<number[]>;
-export declare function getStremingStartData(options: {
-    sourceUrl: string;
-    datasetUrl: string;
-}): Promise<{
-    dataset: import("./src/media").ImageDataResponse;
-    zip: import("jszip");
-    zipBlob: Blob;
-    blendingMaskBitmap: ImageBitmap;
-}>;

package/shared/audioToFrames/index.mjs

@@ -1,44 +0,0 @@
-import { createAudioContext, StreamingFeatureExtractorService } from './src/audio/index.mjs';
-export { w as workerConfig } from '../config.mjs';
-export { S as StreamingInferenceService } from '../StreamingInferenceService.mjs';
-import { loadImageData } from './src/media/index.mjs';
-export { PredictiveFrameLoader } from './src/media/index.mjs';
-import '../constants.mjs';
-import '@vunk/shared/promise';
-
-var maskUrl = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGAAAABgCAAAAADH8yjkAAADsElEQVRoBbXBsaoYBAwF0KTP0vBQgppBNGNn6VTu5Cj55ODoFDoV545RHKIuUlLqq/5EPIfp2cPzh2f0f/j09PHpEz97ePEonzHd+/efff/hiZ+/sG/t8YHuPb2f3+YDP37+8vuXXz2nex//fPfLu7/5iy9f/fDqmxd078Pvb39++xfr169/fP2d0L399c1Pb/5gNQRc6N52ZQ2rIeBC97Yra1gNARe6t11Zw2oIuNC97coaVkPAhe5tV9awGgIudG+7sobVEHChe9uVNayGgAvd266sYTUEXOjedmUNqyHgQve2K2tYDQEXurddWcNqCLjQve3KGlZDwIXubVfWsBoCLnRvu7KG1RBwoXvblTWshoAL3duurGE1BFzo3nZlDash4EL3titrWA0BF7q3XVnDagi40L3tyhpWQ8CF7m1X1rAaAi50b7uyhtUQcKF725U1rIaAC93brqxhNQRc6N52ZQ2rIeBC97Yra1gNARe6t11Zw2oIuNC97coaVkPAhe5tV9awGgIudG+7sobVEHChe9uVNayGgAvd266sYTUEXOjedmUNqyHgQve2K2tYDQEXurddWcNqCLjQve3KGlZDwIXubVfWsBoCLnRvu7KG1RBwoXvblTWshoAL3duurGE1BFzo3nZlDash4EL3titrWA0BF7q3XVnDagi40L3tyhpWQ8CF7m1X1rAaAi50b7uyhtUQcKF725U1rIaAC93brqxhNQRc6N52ZQ2rIeBC97Yra1gNARe6t11Zw2oIuNC97coaVkPAhe5tV9awGgIudG+7sobVEHChe9uVNayGgAvd266sYTUEXOjedmUNqyHgQve2K2tYDQEXurddWcNqCLjQve3KGlZDwIXubVfWsBoCLnRvu7KG1RBwoXvblTWshoAL3duurGE1BFzo3nZlDash4EL3titrWA0BF7q3XVnDagi40L3tyhpWQ8CF7m1X1rAaAi50b7uyhtUQcKF725U1rIaAC93brqxhNQRc6N52ZQ2rIeBC97Yra1gNARe6t11Zw2oIuNC97coaVkPAhe5tV9awGgIudG+7sobVEHChe9uVNayGgAvd266sYTUEXOjedmUNqyHgQve2K2tYDQEXurddWcNqCLjQve3KGtavX//4+juhe/vrm5/e/MFffPnqh1ffvKB7H35/+/Pbv/jx85ffv/zqOd37+Oe7X979zc9f2Lf2+ED3nt7Pb/OBnz28eJTPmO79+8++//DE9Ozh+cMz+j98evr49Ok/l/tDxY3JIlEAAAAASUVORK5CYII=";
-
-async function blobToAudioBuffer(blob) {
-  const audioContext = await createAudioContext();
-  const buffer = await blob.arrayBuffer();
-  return audioContext.decodeAudioData(buffer);
-}
-async function processStreaming(buffer, callbacks) {
-  return new Promise((resolve, reject) => {
-    const service = new StreamingFeatureExtractorService();
-    service.processStreaming(buffer, {
-      onChunkComplete: callbacks?.onChunkComplete,
-      onComplete: (dimensions) => {
-        resolve(dimensions);
-      },
-      onError: (error) => {
-        reject(new Error(error));
-      }
-    });
-  });
-}
-async function getStremingStartData(options) {
-  const { imageData, zip, zipBlob } = await loadImageData(options.datasetUrl, options.sourceUrl);
-  const img = new Image();
-  img.src = maskUrl;
-  await img.decode();
-  const blendingMaskBitmap = await createImageBitmap(img);
-  return {
-    dataset: imageData,
-    zip,
-    zipBlob,
-    blendingMaskBitmap
-  };
-}
-
-export { blobToAudioBuffer, getStremingStartData, processStreaming };

package/shared/audioToFrames/src/assets/mask.d.ts

@@ -1,2 +0,0 @@
-declare const _default: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAGAAAABgCAAAAADH8yjkAAADsElEQVRoBbXBsaoYBAwF0KTP0vBQgppBNGNn6VTu5Cj55ODoFDoV545RHKIuUlLqq/5EPIfp2cPzh2f0f/j09PHpEz97ePEonzHd+/efff/hiZ+/sG/t8YHuPb2f3+YDP37+8vuXXz2nex//fPfLu7/5iy9f/fDqmxd078Pvb39++xfr169/fP2d0L399c1Pb/5gNQRc6N52ZQ2rIeBC97Yra1gNARe6t11Zw2oIuNC97coaVkPAhe5tV9awGgIudG+7sobVEHChe9uVNayGgAvd266sYTUEXOjedmUNqyHgQve2K2tYDQEXurddWcNqCLjQve3KGlZDwIXubVfWsBoCLnRvu7KG1RBwoXvblTWshoAL3duurGE1BFzo3nZlDash4EL3titrWA0BF7q3XVnDagi40L3tyhpWQ8CF7m1X1rAaAi50b7uyhtUQcKF725U1rIaAC93brqxhNQRc6N52ZQ2rIeBC97Yra1gNARe6t11Zw2oIuNC97coaVkPAhe5tV9awGgIudG+7sobVEHChe9uVNayGgAvd266sYTUEXOjedmUNqyHgQve2K2tYDQEXurddWcNqCLjQve3KGlZDwIXubVfWsBoCLnRvu7KG1RBwoXvblTWshoAL3duurGE1BFzo3nZlDash4EL3titrWA0BF7q3XVnDagi40L3tyhpWQ8CF7m1X1rAaAi50b7uyhtUQcKF725U1rIaAC93brqxhNQRc6N52ZQ2rIeBC97Yra1gNARe6t11Zw2oIuNC97coaVkPAhe5tV9awGgIudG+7sobVEHChe9uVNayGgAvd266sYTUEXOjedmUNqyHgQve2K2tYDQEXurddWcNqCLjQve3KGlZDwIXubVfWsBoCLnRvu7KG1RBwoXvblTWshoAL3duurGE1BFzo3nZlDash4EL3titrWA0BF7q3XVnDagi40L3tyhpWQ8CF7m1X1rAaAi50b7uyhtUQcKF725U1rIaAC93brqxhNQRc6N52ZQ2rIeBC97Yra1gNARe6t11Zw2oIuNC97coaVkPAhe5tV9awGgIudG+7sobVEHChe9uVNayGgAvd266sYTUEXOjedmUNqyHgQve2K2tYDQEXurddWcNqCLjQve3KGtavX//4+juhe/vrm5/e/MFffPnqh1ffvKB7H35/+/Pbv/jx85ffv/zqOd37+Oe7X979zc9f2Lf2+ED3nt7Pb/OBnz28eJTPmO79+8++//DE9Ozh+cMz+j98evr49Ok/l/tDxY3JIlEAAAAASUVORK5CYII=";
-export default _default;

package/shared/audioToFrames/src/audio/StreamingFeatureExtractorService.d.ts

@@ -1,125 +0,0 @@
-/**
- * @file StreamingFeatureExtractorService.ts
- *
- * 流式音频特征提取服务模块。
- * 1. 支持将长音频分割为5秒的chunk进行增量特征提取，显著降低首帧视频延迟。
- * 2. 每个chunk独立提取特征，处理完成后立即通过回调通知上层。
- * 3. 所有chunk处理完成后自动合并特征，输出完整特征序列。
- *
- * @author Zhonghan Li
- */
-/**
- * 创建AudioContext的兼容函数，处理Safari的限制。
- * 在Safari中，AudioContext需要在用户交互后才能创建。
- */
-export declare function createAudioContext(): Promise<AudioContext>;
-/**
- * 特征提取结果接口
- */
-export interface FeatureExtractionResult {
-    features: Float32Array;
-    dimensions: number[];
-}
-/**
- * 单个chunk的特征提取结果结构体。
- */
-export interface ChunkFeatureResult {
-    /** chunk索引（从0开始） */
-    chunkIndex: number;
-    /** 特征数据（Float32Array） */
-    features: Float32Array;
-    /** 特征维度信息 [帧数, 序列长度, mel频带数] */
-    dimensions: number[];
-    /** chunk起始时间（秒） */
-    startTimeSeconds: number;
-    /** chunk结束时间（秒） */
-    endTimeSeconds: number;
-}
-/**
- * 流式特征提取回调接口定义。
- */
-export interface StreamingCallbacks {
-    /**
-     * 单个chunk处理完成时回调。
-     * @param result 当前chunk的特征结果。
-     */
-    onChunkComplete?: (result: ChunkFeatureResult) => void;
-    /**
-     * 进度更新回调。
-     * @param completed 已完成的chunk数。
-     * @param total 总chunk数。
-     */
-    onProgress?: (completed: number, total: number) => void;
-    /**
-     * 全部chunk处理完成时回调。
-     * @param totalDimensions 合并后特征的维度信息。
-     */
-    onComplete?: (totalDimensions: number[]) => void;
-    /**
-     * 处理出错时回调。
-     * @param error 错误信息。
-     */
-    onError?: (error: string) => void;
-}
-/**
- * 流式音频特征提取服务类。
- * 支持将长音频分块并行处理，实时回调chunk结果，最终合并输出全部特征。
- */
-export declare class StreamingFeatureExtractorService {
-    private queue;
-    /** 回调函数集合 */
-    private callbacks;
-    /** 存储每个chunk的处理结果，用于最终合并 */
-    private chunkResults;
-    /** 音频总chunk数 */
-    private totalChunks;
-    /** 已完成的chunk数 */
-    private completedChunks;
-    /** 已处理的总帧数累加器 */
-    private totalFramesProcessed;
-    /** 当前是否有任务在运行 */
-    private isRunning;
-    /** 当前处理的音频缓冲区 */
-    private audioBuffer;
-    /** 音频上下文实例，用于创建AudioBuffer */
-    private audioContext;
-    /** 复用的Worker实例，以提高性能 */
-    private worker;
-    /**
-     * 创建AudioBuffer的兼容方法，处理Safari的限制。
-     * @param numberOfChannels 声道数
-     * @param length 缓冲区长度
-     * @param sampleRate 采样率
-     * @returns AudioBuffer实例
-     */
-    private createAudioBuffer;
-    /**
-     * 使用Worker处理AudioBuffer，提取特征
-     * @param audioBuffer 输入的AudioBuffer
-     * @returns Promise<FeatureExtractionResult>
-     */
-    private processWithWorker;
-    /**
-     * 启动流式音频特征提取。
-     * @param audioBuffer 输入的音频缓冲区。
-     * @param callbacks 处理过程中的回调函数集合。
-     */
-    processStreaming(audioBuffer: AudioBuffer, callbacks: StreamingCallbacks): Promise<void>;
-    /**
-     * 处理单个音频chunk，提取特征并回调。
-     * @param chunkIndex chunk的索引（从0开始）。
-     */
-    private processChunk;
-    /**
-     * 所有chunk处理完成后，合并特征并调用onComplete。
-     */
-    private finalize;
-    /**
-     * 清理内部状态和资源。
-     */
-    private cleanup;
-    /**
-     * 停止当前的流式处理。
-     */
-    stop(): void;
-}

package/shared/audioToFrames/src/audio/index.d.ts

@@ -1,5 +0,0 @@
-/**
- * @file 音频处理服务模块导出
- * @author Zhonghan Li
- */
-export * from './StreamingFeatureExtractorService';