@meframe/core 0.0.3 → 0.0.4

package/dist/workers/stages/compose/video-compose.worker.js

@@ -832,8 +832,11 @@ class VideoComposer {
     this.filterProcessor.clearCache();
   }
 }
-function resolveActiveLayers(layers, timestamp, _frame) {
+function resolveActiveLayers(layers, timestamp) {
   return layers.filter((layer) => {
+    if (layer.layerId.includes("base-video")) {
+      return true;
+    }
     if (layer.status !== "ready") {
       return false;
     }
@@ -892,11 +895,12 @@ class VideoComposeWorker {
   channel;
   composer = null;
   composeStream = null;
-  downstreamPorts = /* @__PURE__ */ new Map();
-  upstreamPorts = /* @__PURE__ */ new Map();
-  instructionRegistry = /* @__PURE__ */ new Map();
-  pendingReplay = /* @__PURE__ */ new Map();
-  streamState = /* @__PURE__ */ new Map();
+  clipId = null;
+  downstreamPort = null;
+  upstreamPort = null;
+  instructions = null;
+  streamState = null;
+  imageBitmap = null;
   constructor() {
     this.channel = new WorkerChannel(self, {
       name: "VideoComposeWorker",
@@ -910,7 +914,7 @@ class VideoComposeWorker {
     this.channel.registerHandler("flush", this.handleFlush.bind(this));
     this.channel.registerHandler("get_stats", this.handleGetStats.bind(this));
     this.channel.registerHandler("install_instructions", this.handleInstallInstructions.bind(this));
-    this.channel.registerHandler("sync_clip", this.handleSyncClip.bind(this));
+    this.channel.registerHandler("receive_image", this.handleReceiveImage.bind(this));
     this.channel.registerHandler("dispose_clip", this.handleDisposeClip.bind(this));
     this.channel.registerHandler(WorkerMessageType.Dispose, this.handleDispose.bind(this));
   }
@@ -918,9 +922,12 @@ class VideoComposeWorker {
    * Unified connect handler used by stream pipeline
    */
   async handleConnect(payload) {
-    const { port, direction, clipId = "default" } = payload;
+    const { port, direction, clipId } = payload;
+    if (clipId && !this.clipId) {
+      this.clipId = clipId;
+    }
     if (direction === "upstream") {
-      this.upstreamPorts.set(clipId, port);
+      this.upstreamPort = port;
       const channel = new WorkerChannel(port, {
         name: "VideoCompose-Decode",
         timeout: 3e4
@@ -928,7 +935,7 @@ class VideoComposeWorker {
       channel.receiveStream(this.handleReceiveStream.bind(this));
     }
     if (direction === "downstream") {
-      this.downstreamPorts.set(clipId, port);
+      this.downstreamPort = port;
     }
     return { success: true };
   }
@@ -967,14 +974,12 @@ class VideoComposeWorker {
     };
   }
   async handleReceiveStream(stream, metadata) {
-    const { clipId = "default" } = metadata || {};
     if (!this.composer) {
       console.error("[VideoComposeWorker] Composer not configured");
       return;
     }
-    const instruction = this.instructionRegistry.get(clipId);
-    if (!instruction) {
-      console.warn("[VideoComposeWorker] No instructions for clip", clipId);
+    if (!this.instructions) {
+      console.warn("[VideoComposeWorker] No instructions installed");
       return;
     }
     const filteredStream = stream.pipeThrough(
@@ -984,12 +989,7 @@ class VideoComposeWorker {
             const frame = wrappedFrame.frame || wrappedFrame;
             const gopSerial = wrappedFrame.gopSerial;
             const isKeyframe = wrappedFrame.isKeyframe;
-            const timestamp = frame.timestamp ?? 0;
-            if (this.shouldSkipFrame(clipId, timestamp)) {
-              frame.close();
-              return;
-            }
-            const request = this.buildComposeRequest(clipId, instruction, frame, timestamp);
+            const request = this.buildComposeRequest(this.instructions, frame);
             if (!request) {
               frame.close();
               return;
@@ -1008,7 +1008,7 @@ class VideoComposeWorker {
     const { composeStream, cacheStream } = this.composer.createStreams();
     this.channel.sendStream(cacheStream, metadata);
     filteredStream.pipeTo(composeStream).catch((error) => {
-      console.error("[VideoComposeWorker] compose stream error", clipId, error);
+      console.error("[VideoComposeWorker] compose stream error", this.clipId, error);
     });
   }
   // private handleGetStream(): ReadableStream<VideoFrame> | undefined {
@@ -1047,61 +1047,96 @@ class VideoComposeWorker {
       this.composer = null;
     }
     this.composeStream = null;
-    this.downstreamPorts.get("default")?.close();
-    this.upstreamPorts.get("default")?.close();
-    this.downstreamPorts.clear();
-    this.upstreamPorts.clear();
+    this.downstreamPort?.close();
+    this.upstreamPort?.close();
+    this.downstreamPort = null;
+    this.upstreamPort = null;
+    this.imageBitmap?.close();
+    this.imageBitmap = null;
+    this.instructions = null;
+    this.streamState = null;
     this.channel.state = WorkerState.Disposed;
     return { success: true };
   }
-  async handleInstallInstructions(data) {
-    const { clipId, revision } = data;
-    const current = this.instructionRegistry.get(clipId);
-    if (current && current.revision > revision) {
+  async handleInstallInstructions(payload) {
+    const { clipId, revision } = payload;
+    if (!this.clipId) {
+      this.clipId = clipId;
+    }
+    if (this.instructions && this.instructions.revision > revision) {
       return { success: false };
     }
-    this.instructionRegistry.set(clipId, data);
+    this.instructions = payload;
     return { success: true };
   }
-  async handleSyncClip(payload) {
-    const { clipId, revision, range } = payload;
-    const current = this.instructionRegistry.get(clipId);
-    if (!current || current.revision > revision) {
-      return { success: false };
+  /**
+   * Receive image data from ResourceLoader
+   * Note: ImageBitmap is required because VideoFrame constructor in Worker context
+   * only accepts ImageBitmap/OffscreenCanvas, not HTMLImageElement or Blob
+   */
+  async handleReceiveImage(payload) {
+    const { clipId, imageBitmap } = payload;
+    if (!this.clipId) {
+      this.clipId = clipId;
+    }
+    if (this.imageBitmap) {
+      this.imageBitmap.close();
+    }
+    this.imageBitmap = imageBitmap;
+    if (this.instructions) {
+      await this.startImageFrameStream();
     }
-    this.pendingReplay.set(clipId, { ...range, revision });
-    this.channel.notify("sync_ack", { clipId, revision });
     return { success: true };
   }
-  async handleDisposeClip(payload) {
-    const { clipId } = payload;
-    this.instructionRegistry.delete(clipId);
-    this.pendingReplay.delete(clipId);
-    this.downstreamPorts.get(clipId)?.close();
-    this.upstreamPorts.get(clipId)?.close();
-    this.downstreamPorts.delete(clipId);
-    this.upstreamPorts.delete(clipId);
+  async handleDisposeClip() {
+    this.instructions = null;
+    this.streamState = null;
+    this.downstreamPort?.close();
+    this.upstreamPort?.close();
+    this.downstreamPort = null;
+    this.upstreamPort = null;
+    this.imageBitmap?.close();
+    this.imageBitmap = null;
     return { success: true };
   }
-  /**
-   * Check if frame should be skipped (outside dirty range)
-   * Returns true if frame is NOT in the dirty range and should use cached version
-   */
-  shouldSkipFrame(clipId, timestamp) {
-    const dirtyRange = this.pendingReplay.get(clipId);
-    if (!dirtyRange) {
-      return false;
-    }
-    if (timestamp >= dirtyRange.startUs && timestamp <= dirtyRange.endUs) {
-      return false;
+  async startImageFrameStream() {
+    if (!this.instructions || !this.composer) {
+      return;
     }
-    if (timestamp > dirtyRange.endUs) {
-      this.pendingReplay.delete(clipId);
+    const timeline = this.instructions.baseConfig.timeline;
+    if (!timeline) {
+      return;
     }
-    return true;
+    const { composeStream, cacheStream } = this.composer.createStreams();
+    const { clipDurationUs, compositionFps } = timeline;
+    let currentTimeUs = 0;
+    const readableStream = new ReadableStream({
+      pull: (controller) => {
+        if (currentTimeUs >= clipDurationUs) {
+          controller.close();
+          return;
+        }
+        const videoFrame = new VideoFrame(this.imageBitmap, {
+          timestamp: currentTimeUs,
+          duration: frameDurationFromFps(compositionFps)
+        });
+        const request = this.buildComposeRequest(this.instructions, videoFrame);
+        if (request) {
+          controller.enqueue(request);
+        }
+        currentTimeUs += frameDurationFromFps(compositionFps);
+      }
+    });
+    this.channel.sendStream(cacheStream, {
+      streamType: "video",
+      clipId: this.clipId
+    });
+    readableStream.pipeTo(composeStream).catch((error) => {
+      console.error("[VideoComposeWorker] image frame stream error", this.clipId, error);
+    });
   }
-  buildComposeRequest(clipId, instruction, frame, _timestamp) {
-    const normalizedTime = this.computeTimelineTimestamp(clipId, frame, instruction.baseConfig);
+  buildComposeRequest(instruction, frame) {
+    const normalizedTime = this.computeTimelineTimestamp(frame, instruction.baseConfig);
     const clipStartUs = instruction.baseConfig.timeline?.clipStartUs ?? 0;
     const clipDurationUs = instruction.baseConfig.timeline?.clipDurationUs ?? Infinity;
     const clipEndUs = clipStartUs + clipDurationUs;
@@ -1142,43 +1177,40 @@ class VideoComposeWorker {
       direction: entry.params.payload?.direction
     };
   }
-  computeTimelineTimestamp(clipId, frame, config) {
-    const key = clipId;
-    let state = this.streamState.get(key);
-    if (!state) {
-      state = {
+  computeTimelineTimestamp(frame, config) {
+    if (!this.streamState) {
+      this.streamState = {
         baseTimestamp: null,
         lastSourceTimestamp: null,
         nextFrameIndex: 0
       };
-      this.streamState.set(key, state);
     }
     const timeline = config.timeline;
     if (!timeline) {
       const ts = frame.timestamp ?? 0;
-      state.lastSourceTimestamp = frame.timestamp ?? null;
+      this.streamState.lastSourceTimestamp = frame.timestamp ?? null;
       return ts;
     }
     const { clipStartUs, compositionFps } = timeline;
     const sourceTimestamp = frame.timestamp ?? null;
-    if (sourceTimestamp !== null && state.lastSourceTimestamp !== null && sourceTimestamp < state.lastSourceTimestamp) {
-      state.baseTimestamp = null;
-      state.nextFrameIndex = 0;
-    }
-    if (state.baseTimestamp === null) {
-      state.baseTimestamp = sourceTimestamp ?? 0;
-      state.nextFrameIndex = 0;
-      if (state.baseTimestamp > 1e3) {
+    if (sourceTimestamp !== null && this.streamState.lastSourceTimestamp !== null && sourceTimestamp < this.streamState.lastSourceTimestamp) {
+      this.streamState.baseTimestamp = null;
+      this.streamState.nextFrameIndex = 0;
+    }
+    if (this.streamState.baseTimestamp === null) {
+      this.streamState.baseTimestamp = sourceTimestamp ?? 0;
+      this.streamState.nextFrameIndex = 0;
+      if (this.streamState.baseTimestamp > 1e3) {
         console.warn(
-          `[VideoComposeWorker] First frame timestamp is ${state.baseTimestamp}us, expected ~0. Check MP4Demuxer normalization.`
+          `[VideoComposeWorker] First frame timestamp is ${this.streamState.baseTimestamp}us, expected ~0. Check MP4Demuxer normalization.`
         );
       }
     }
     const frameDuration = frameDurationFromFps(compositionFps);
-    let frameIndex = state.nextFrameIndex;
+    let frameIndex = this.streamState.nextFrameIndex;
     if (sourceTimestamp !== null) {
       const approxIndex = frameIndexFromTimestamp(
-        state.baseTimestamp,
+        this.streamState.baseTimestamp,
         sourceTimestamp,
         compositionFps,
         "nearest"
@@ -1192,8 +1224,8 @@ class VideoComposeWorker {
       compositionFps,
       "nearest"
     );
-    state.nextFrameIndex = frameIndex + 1;
-    state.lastSourceTimestamp = sourceTimestamp;
+    this.streamState.nextFrameIndex = frameIndex + 1;
+    this.streamState.lastSourceTimestamp = sourceTimestamp;
     return timelineTime;
   }
 }