@meframe/core 0.0.10 → 0.0.12

package/dist/workers/stages/compose/video-compose.worker.js

@@ -12,30 +12,6 @@ function frameDurationFromFps(fps) {
   const duration = MICROSECONDS_PER_SECOND / normalized;
   return Math.max(Math.round(duration), 1);
 }
-function frameIndexFromTimestamp(baseTimestampUs, timestampUs, fps, strategy = "nearest") {
-  const frameDurationUs = frameDurationFromFps(fps);
-  if (frameDurationUs <= 0) {
-    return 0;
-  }
-  const delta = timestampUs - baseTimestampUs;
-  const rawIndex = delta / frameDurationUs;
-  if (!Number.isFinite(rawIndex)) {
-    return 0;
-  }
-  switch (strategy) {
-    case "floor":
-      return Math.floor(rawIndex);
-    case "ceil":
-      return Math.ceil(rawIndex);
-    default:
-      return Math.round(rawIndex);
-  }
-}
-function quantizeTimestampToFrame(timestampUs, baseTimestampUs, fps, strategy = "nearest") {
-  const frameDurationUs = frameDurationFromFps(fps);
-  const index = frameIndexFromTimestamp(baseTimestampUs, timestampUs, fps, strategy);
-  return baseTimestampUs + index * frameDurationUs;
-}
 class LayerRenderer {
   ctx;
   width;
@@ -63,7 +39,8 @@ class LayerRenderer {
         this.ctx.globalCompositeOperation = layer.blendMode;
       }
       if (layer.transform) {
-        this.applyTransform(layer.transform);
+        const layerDimensions = this.getLayerDimensions(layer);
+        this.applyTransform(layer.transform, layerDimensions);
       }
       switch (layer.type) {
         case "video":
@@ -83,9 +60,60 @@ class LayerRenderer {
       this.ctx.restore();
     }
   }
-  applyTransform(transform) {
-    const centerX = this.width * (transform.anchorX ?? 0.5);
-    const centerY = this.height * (transform.anchorY ?? 0.5);
+  parseDimension(value, canvasSize) {
+    if (value === void 0) return void 0;
+    if (typeof value === "number") return value;
+    const strValue = value;
+    if (strValue.includes("%")) {
+      const numValue = parseFloat(strValue);
+      return isNaN(numValue) ? void 0 : numValue / 100 * canvasSize;
+    }
+    const parsed = parseFloat(strValue);
+    return isNaN(parsed) ? void 0 : parsed;
+  }
+  getLayerDimensions(layer) {
+    if (layer.type === "image") {
+      const imageLayer = layer;
+      if (imageLayer.source) {
+        const imgWidth = imageLayer.source.width;
+        const imgHeight = imageLayer.source.height;
+        const isAttachment = !!imageLayer.attachmentId;
+        if (isAttachment) {
+          const targetWidthRaw = imageLayer.targetWidth;
+          const targetHeightRaw = imageLayer.targetHeight;
+          const targetWidth = this.parseDimension(targetWidthRaw, this.width);
+          const targetHeight = this.parseDimension(targetHeightRaw, this.height);
+          if (targetWidth && targetHeight) {
+            return { width: targetWidth, height: targetHeight };
+          } else if (targetWidth) {
+            return {
+              width: targetWidth,
+              height: imgHeight / imgWidth * targetWidth
+            };
+          } else if (targetHeight) {
+            return {
+              width: imgWidth / imgHeight * targetHeight,
+              height: targetHeight
+            };
+          }
+        }
+        return { width: imgWidth, height: imgHeight };
+      }
+    } else if (layer.type === "video") {
+      const videoLayer = layer;
+      const videoFrame = videoLayer.videoFrame;
+      return {
+        width: videoFrame.displayWidth || videoFrame.codedWidth,
+        height: videoFrame.displayHeight || videoFrame.codedHeight
+      };
+    }
+    return { width: this.width, height: this.height };
+  }
+  applyTransform(transform, layerDimensions) {
+    const anchorX = transform.anchorX ?? 0.5;
+    const anchorY = transform.anchorY ?? 0.5;
+    const centerX = layerDimensions.width * anchorX;
+    const centerY = layerDimensions.height * anchorY;
     this.ctx.translate(transform.x + centerX, transform.y + centerY);
     if (transform.rotation) {
       this.ctx.rotate(transform.rotation);
@@ -138,6 +166,33 @@ class LayerRenderer {
       if (!source) {
         return;
       }
+      const isAttachment = !!layer.attachmentId;
+      const imgWidth = source.width;
+      const imgHeight = source.height;
+      let renderWidth;
+      let renderHeight;
+      if (isAttachment) {
+        const targetWidthRaw = layer.targetWidth;
+        const targetHeightRaw = layer.targetHeight;
+        const targetWidth = this.parseDimension(targetWidthRaw, this.width);
+        const targetHeight = this.parseDimension(targetHeightRaw, this.height);
+        if (targetWidth && targetHeight) {
+          renderWidth = targetWidth;
+          renderHeight = targetHeight;
+        } else if (targetWidth) {
+          renderWidth = targetWidth;
+          renderHeight = imgHeight / imgWidth * targetWidth;
+        } else if (targetHeight) {
+          renderHeight = targetHeight;
+          renderWidth = imgWidth / imgHeight * targetHeight;
+        } else {
+          renderWidth = imgWidth;
+          renderHeight = imgHeight;
+        }
+      } else {
+        renderWidth = this.width;
+        renderHeight = this.height;
+      }
       if (crop) {
         this.ctx.drawImage(
           source,
@@ -147,11 +202,11 @@ class LayerRenderer {
           crop.height,
           0,
           0,
-          this.width,
-          this.height
+          renderWidth,
+          renderHeight
         );
       } else {
-        this.ctx.drawImage(source, 0, 0, this.width, this.height);
+        this.ctx.drawImage(source, 0, 0, renderWidth, renderHeight);
       }
     }
   }
@@ -834,9 +889,87 @@ class VideoComposer {
     this.filterProcessor.clearCache();
   }
 }
+function interpolateKeyframes(keyframes, timeUs) {
+  const defaultTransform = {
+    x: 0,
+    y: 0,
+    scaleX: 1,
+    scaleY: 1,
+    rotation: 0,
+    anchorX: 0.5,
+    anchorY: 0.5
+  };
+  if (keyframes.length === 0) {
+    return { transform: defaultTransform };
+  }
+  const firstFrame = keyframes[0];
+  const lastFrame = keyframes[keyframes.length - 1];
+  if (timeUs <= firstFrame.time) {
+    return {
+      transform: { ...defaultTransform, ...firstFrame.transform },
+      opacity: firstFrame.opacity
+    };
+  }
+  if (timeUs >= lastFrame.time) {
+    return {
+      transform: { ...defaultTransform, ...lastFrame.transform },
+      opacity: lastFrame.opacity
+    };
+  }
+  let prevFrame = firstFrame;
+  let nextFrame = lastFrame;
+  for (let i = 0; i < keyframes.length - 1; i++) {
+    const currentFrame = keyframes[i];
+    const followingFrame = keyframes[i + 1];
+    if (timeUs >= currentFrame.time && timeUs < followingFrame.time) {
+      prevFrame = currentFrame;
+      nextFrame = followingFrame;
+      break;
+    }
+  }
+  const duration = nextFrame.time - prevFrame.time;
+  const elapsed = timeUs - prevFrame.time;
+  const progress = elapsed / duration;
+  const easedProgress = applyEasing(progress, prevFrame.easing ?? "linear");
+  const prevTransform = prevFrame.transform ?? { x: 0, y: 0 };
+  const nextTransform = nextFrame.transform ?? { x: 0, y: 0 };
+  const transform = interpolateTransform(prevTransform, nextTransform, easedProgress);
+  const opacity = prevFrame.opacity !== void 0 && nextFrame.opacity !== void 0 ? lerp(prevFrame.opacity, nextFrame.opacity, easedProgress) : void 0;
+  return { transform, opacity };
+}
+function interpolateTransform(from, to, t) {
+  return {
+    x: lerp(from.x ?? 0, to.x ?? 0, t),
+    y: lerp(from.y ?? 0, to.y ?? 0, t),
+    scaleX: lerp(from.scaleX ?? 1, to.scaleX ?? 1, t),
+    scaleY: lerp(from.scaleY ?? 1, to.scaleY ?? 1, t),
+    rotation: lerp(from.rotation ?? 0, to.rotation ?? 0, t),
+    anchorX: lerp(from.anchorX ?? 0.5, to.anchorX ?? 0.5, t),
+    anchorY: lerp(from.anchorY ?? 0.5, to.anchorY ?? 0.5, t)
+  };
+}
+function lerp(a, b, t) {
+  return a + (b - a) * t;
+}
+function applyEasing(t, easing) {
+  switch (easing) {
+    case "linear":
+      return t;
+    case "ease-in":
+      return t * t * t;
+    case "ease-out": {
+      const oneMinusT = 1 - t;
+      return 1 - oneMinusT * oneMinusT * oneMinusT;
+    }
+    case "ease-in-out":
+      return t < 0.5 ? 4 * t * t * t : 1 - Math.pow(-2 * t + 2, 3) / 2;
+    default:
+      return t;
+  }
+}
 function resolveActiveLayers(layers, timestamp) {
   return layers.filter((layer) => {
-    if (layer.layerId.includes("base-video")) {
+    if (!layer.payload.attachmentId) {
       return true;
     }
     if (layer.status !== "ready") {
@@ -847,7 +980,7 @@ function resolveActiveLayers(layers, timestamp) {
     );
   });
 }
-function materializeLayer(layer, frame) {
+function materializeLayer(layer, frame, imageMap, globalTimeUs) {
   const baseLayer = {
     id: layer.layerId,
     type: layer.type,
@@ -877,22 +1010,74 @@ function materializeLayer(layer, frame) {
       lineHeight: payload.lineHeight,
       textAlign: payload.align,
       verticalAlign: "bottom"
-      // Subtitles positioned at bottom
     };
   }
   if (layer.type === "image") {
     const payload = layer.payload;
-    const source = payload.bitmapHandle ?? null;
-    if (source) {
-      return {
-        ...baseLayer,
-        type: "image",
-        source
+    const resourceId = payload.resourceId;
+    const source = imageMap.get(resourceId) ?? null;
+    const imageLayer = {
+      ...baseLayer,
+      type: "image",
+      source,
+      attachmentId: payload.attachmentId
+    };
+    if (payload.targetWidth !== void 0) {
+      imageLayer.targetWidth = payload.targetWidth;
+    }
+    if (payload.targetHeight !== void 0) {
+      imageLayer.targetHeight = payload.targetHeight;
+    }
+    if (payload.animation && globalTimeUs !== void 0) {
+      const animState = computeAnimationState(payload.animation, globalTimeUs);
+      if (!animState.visible) {
+        return null;
+      }
+      imageLayer.transform = {
+        x: animState.transform.x ?? 0,
+        y: animState.transform.y ?? 0,
+        scaleX: animState.transform.scaleX ?? 1,
+        scaleY: animState.transform.scaleY ?? 1,
+        rotation: animState.transform.rotation ?? 0,
+        anchorX: animState.transform.anchorX ?? 0.5,
+        anchorY: animState.transform.anchorY ?? 0.5
       };
+      if (animState.opacity !== void 0) {
+        imageLayer.opacity = animState.opacity;
+      }
     }
+    return imageLayer;
   }
   return baseLayer;
 }
+function computeAnimationState(animation, globalTimeUs) {
+  const { position, keyframes, overlayClipStartUs } = animation;
+  const relativeTimeUs = globalTimeUs - overlayClipStartUs;
+  if (relativeTimeUs < 0 || relativeTimeUs > keyframes[keyframes.length - 1].time) {
+    return {
+      transform: { x: 0, y: 0, scaleX: 1, scaleY: 1, rotation: 0, anchorX: 0.5, anchorY: 0.5 },
+      opacity: 0,
+      visible: false
+    };
+  }
+  const animState = interpolateKeyframes(keyframes, relativeTimeUs);
+  const rotationDeg = animState.transform?.rotation ?? 0;
+  const rotationRad = rotationDeg * Math.PI / 180;
+  const finalTransform = {
+    x: position.x + (animState.transform?.x ?? 0),
+    y: position.y + (animState.transform?.y ?? 0),
+    scaleX: animState.transform?.scaleX ?? 1,
+    scaleY: animState.transform?.scaleY ?? 1,
+    rotation: rotationRad,
+    anchorX: animState.transform?.anchorX ?? 0.5,
+    anchorY: animState.transform?.anchorY ?? 0.5
+  };
+  return {
+    transform: finalTransform,
+    opacity: animState.opacity,
+    visible: true
+  };
+}
 class VideoComposeWorker {
   channel;
   composer = null;
@@ -902,7 +1087,7 @@ class VideoComposeWorker {
   upstreamPort = null;
   instructions = null;
   streamState = null;
-  imageBitmap = null;
+  imageMap = /* @__PURE__ */ new Map();
   constructor() {
     this.channel = new WorkerChannel(self, {
       name: "VideoComposeWorker",
@@ -947,7 +1132,6 @@ class VideoComposeWorker {
    */
   async handleConfigure(payload) {
     const { config, initial } = payload;
-    console.log("[VideoComposeWorker] handleConfigure", config, initial);
     const hasValidDimensions = config.width > 0 && config.height > 0;
     const hasValidFps = config.fps > 0;
     if (!hasValidDimensions || !hasValidFps) {
@@ -1067,8 +1251,8 @@ class VideoComposeWorker {
     this.upstreamPort?.close();
     this.downstreamPort = null;
     this.upstreamPort = null;
-    this.imageBitmap?.close();
-    this.imageBitmap = null;
+    this.imageMap.forEach((bitmap) => bitmap.close());
+    this.imageMap.clear();
     this.instructions = null;
     this.streamState = null;
     this.channel.state = WorkerState.Disposed;
@@ -1091,16 +1275,21 @@ class VideoComposeWorker {
    * only accepts ImageBitmap/OffscreenCanvas, not HTMLImageElement or Blob
    */
   async handleReceiveImage(payload) {
-    const { sessionId, imageBitmap } = payload;
+    const { resourceId, sessionId, imageBitmap } = payload;
     if (!this.sessionId) {
       this.sessionId = sessionId;
     }
-    if (this.imageBitmap) {
-      this.imageBitmap.close();
+    const existing = this.imageMap.get(resourceId);
+    if (existing) {
+      existing.close();
     }
-    this.imageBitmap = imageBitmap;
+    this.imageMap.set(resourceId, imageBitmap);
     if (this.instructions) {
-      await this.startImageFrameStream();
+      const mainLayer = this.instructions.layers.find((l) => !l.payload.attachmentId);
+      const mainResourceId = mainLayer?.payload.resourceId;
+      if (resourceId === mainResourceId) {
+        await this.startImageFrameStream();
+      }
     }
     return { success: true };
   }
@@ -1111,8 +1300,8 @@ class VideoComposeWorker {
     this.upstreamPort?.close();
     this.downstreamPort = null;
     this.upstreamPort = null;
-    this.imageBitmap?.close();
-    this.imageBitmap = null;
+    this.imageMap.forEach((bitmap) => bitmap.close());
+    this.imageMap.clear();
     return { success: true };
   }
   async startImageFrameStream() {
@@ -1123,6 +1312,14 @@ class VideoComposeWorker {
     if (!timeline) {
       return;
     }
+    const mainLayer = this.instructions.layers.find((l) => !l.payload.attachmentId);
+    if (!mainLayer) return;
+    const mainResourceId = mainLayer.payload.resourceId;
+    const imageBitmap = this.imageMap.get(mainResourceId);
+    if (!imageBitmap) {
+      console.warn("[VideoComposeWorker] Main track ImageBitmap not found:", mainResourceId);
+      return;
+    }
     const { composeStream, cacheStream, encodeStream } = this.composer.createStreams();
     const { clipDurationUs, compositionFps } = timeline;
     let currentTimeUs = 0;
@@ -1132,7 +1329,7 @@ class VideoComposeWorker {
           controller.close();
           return;
         }
-        const videoFrame = new VideoFrame(this.imageBitmap, {
+        const videoFrame = new VideoFrame(imageBitmap, {
           timestamp: currentTimeUs,
           duration: frameDurationFromFps(compositionFps)
         });
@@ -1164,21 +1361,24 @@ class VideoComposeWorker {
     }
   }
   buildComposeRequest(instruction, frame) {
-    const normalizedTime = this.computeTimelineTimestamp(frame, instruction.baseConfig);
-    const clipStartUs = instruction.baseConfig.timeline?.clipStartUs ?? 0;
+    const clipRelativeTime = this.computeTimelineTimestamp(frame, instruction.baseConfig);
     const clipDurationUs = instruction.baseConfig.timeline?.clipDurationUs ?? Infinity;
-    const clipEndUs = clipStartUs + clipDurationUs;
-    if (normalizedTime < clipStartUs || normalizedTime >= clipEndUs) {
+    if (clipRelativeTime < 0 || clipRelativeTime >= clipDurationUs) {
       return null;
     }
-    const clipRelativeTime = normalizedTime - clipStartUs;
     const activeLayers = resolveActiveLayers(instruction.layers, clipRelativeTime);
     if (!activeLayers.length) {
       return null;
     }
-    const layers = activeLayers.map((layer) => materializeLayer(layer, frame));
+    const clipStartUs = instruction.baseConfig.timeline?.clipStartUs ?? 0;
+    const globalTimeUs = clipStartUs + clipRelativeTime;
+    const layers = activeLayers.map((layer) => materializeLayer(layer, frame, this.imageMap, globalTimeUs)).filter((layer) => layer !== null);
+    if (!layers.length) {
+      return null;
+    }
     return {
-      timeUs: normalizedTime,
+      timeUs: clipRelativeTime,
+      globalTimeUs,
       layers,
       transition: VideoComposeWorker.buildTransition(
         instruction.transitions,
@@ -1208,7 +1408,6 @@ class VideoComposeWorker {
   computeTimelineTimestamp(frame, config) {
     if (!this.streamState) {
       this.streamState = {
-        baseTimestamp: null,
         lastSourceTimestamp: null,
         nextFrameIndex: 0
       };
@@ -1219,42 +1418,21 @@ class VideoComposeWorker {
       this.streamState.lastSourceTimestamp = frame.timestamp ?? null;
       return ts;
     }
-    const { clipStartUs, compositionFps } = timeline;
+    const { compositionFps } = timeline;
     const sourceTimestamp = frame.timestamp ?? null;
     if (sourceTimestamp !== null && this.streamState.lastSourceTimestamp !== null && sourceTimestamp < this.streamState.lastSourceTimestamp) {
-      this.streamState.baseTimestamp = null;
       this.streamState.nextFrameIndex = 0;
     }
-    if (this.streamState.baseTimestamp === null) {
-      this.streamState.baseTimestamp = sourceTimestamp ?? 0;
-      this.streamState.nextFrameIndex = 0;
-      if (this.streamState.baseTimestamp > 1e3) {
-        console.warn(
-          `[VideoComposeWorker] First frame timestamp is ${this.streamState.baseTimestamp}us, expected ~0. Check MP4Demuxer normalization.`
-        );
-      }
-    }
     const frameDuration = frameDurationFromFps(compositionFps);
     let frameIndex = this.streamState.nextFrameIndex;
     if (sourceTimestamp !== null) {
-      const approxIndex = frameIndexFromTimestamp(
-        this.streamState.baseTimestamp,
-        sourceTimestamp,
-        compositionFps,
-        "nearest"
-      );
+      const approxIndex = Math.round(sourceTimestamp / frameDuration);
       frameIndex = Math.max(frameIndex, approxIndex);
     }
-    const rawTimeline = clipStartUs + frameIndex * frameDuration;
-    const timelineTime = quantizeTimestampToFrame(
-      rawTimeline,
-      clipStartUs,
-      compositionFps,
-      "nearest"
-    );
+    const relativeTimeUs = frameIndex * frameDuration;
     this.streamState.nextFrameIndex = frameIndex + 1;
     this.streamState.lastSourceTimestamp = sourceTimestamp;
-    return timelineTime;
+    return relativeTimeUs;
   }
 }
 const worker = new VideoComposeWorker();