@meframe/core 0.0.28 → 0.0.30-beta

package/dist/orchestrator/Orchestrator.js

@@ -1,18 +1,16 @@
 import { EventBus } from "../event/EventBus.js";
 import { WorkerPool } from "../worker/WorkerPool.js";
 import { applyPatch } from "../model/patch.js";
-import { ResourceLoader, ResourceConflictError } from "../stages/load/ResourceLoader.js";
+import { ResourceLoader } from "../stages/load/ResourceLoader.js";
 import { CacheManager } from "../cache/CacheManager.js";
 import { ConfigLoader } from "../config/ConfigLoader.js";
-import { hasResourceId } from "../model/types.js";
+import { isVideoClip, hasResourceId } from "../model/types.js";
 import { MeframeEvent } from "../event/events.js";
 import { CompositionPlanner } from "./CompositionPlanner.js";
-import { VideoClipSession } from "./VideoClipSession.js";
-import { ClipSessionManager } from "./ClipSessionManager.js";
 import { GlobalAudioSession } from "./GlobalAudioSession.js";
 import { MuxManager } from "../stages/mux/MuxManager.js";
-import { VideoChunkDecoder } from "../stages/decode/VideoChunkDecoder.js";
-import { quantizeTimestampToFrame } from "../utils/time-utils.js";
+import { OnDemandVideoSession } from "./OnDemandVideoSession.js";
+import { ExportScheduler } from "./ExportScheduler.js";
 class Orchestrator {
   workers;
   eventBus;
@@ -22,13 +20,11 @@ class Orchestrator {
   planner;
   audioSession;
   muxManager;
-  activeClips = /* @__PURE__ */ new Set();
+  exportScheduler;
   isInitialized = false;
   config = ConfigLoader.getInstance().getConfig();
-  clipSessionManager;
-  currentClipId = null;
   ensureCacheDebounceTimer = null;
-  ensureCacheDebounceDelay = 150;
+  currentClipId = null;
   events;
   constructor(config) {
     this.eventBus = config.eventBus || new EventBus();
@@ -64,13 +60,6 @@ class Orchestrator {
       },
       this.eventBus
     );
-    this.clipSessionManager = new ClipSessionManager({
-      maxConcurrent: 2,
-      factory: {
-        createSession: (clipId) => this.createSession(clipId)
-      },
-      cacheManager: this.cacheManager
-    });
     this.audioSession = new GlobalAudioSession({
       cacheManager: this.cacheManager,
       workers: this.workers,
@@ -84,26 +73,48 @@ class Orchestrator {
       this.audioSession,
       this.config.encode.audio
     );
+    this.exportScheduler = new ExportScheduler({
+      workerPool: this.workers,
+      planner: this.planner,
+      cacheManager: this.cacheManager,
+      resourceLoader: this.resourceLoader,
+      muxManager: this.muxManager,
+      audioSession: this.audioSession,
+      workerConfigsProvider: () => this.buildWorkerConfigs(),
+      eventBus: this.eventBus
+    });
+    this.setupResourceFirstFrameHandler();
+    this.setupPreloadHandlers();
   }
-  get workerStatus() {
-    const status = this.workers.status;
-    const result = {};
-    const workerTypes = [
-      "videoDemux",
-      "audioDemux",
-      "videoDecode",
-      "audioDecode",
-      "videoCompose",
-      "audioCompose",
-      "videoEncode"
-    ];
-    for (const type of workerTypes) {
-      result[type] = status[type] || {
-        state: "idle",
-        taskCount: 0
-      };
-    }
-    return result;
+  setupResourceFirstFrameHandler() {
+    this.eventBus.on(MeframeEvent.ResourceFirstFrameReady, async (payload) => {
+      const { resourceId, clipId, index, chunks } = payload;
+      if (!this.compositionModel) return;
+      const clip = this.compositionModel.findClip(clipId);
+      if (!clip || !clip.trackId) return;
+      if (clip.startUs === 0) {
+        const fps = this.compositionModel.fps ?? 30;
+        await OnDemandVideoSession.decodeAndCacheFirstFrame(
+          resourceId,
+          chunks,
+          index,
+          clip,
+          this.cacheManager,
+          fps
+        );
+      }
+    });
+  }
+  setupPreloadHandlers() {
+    this.eventBus.on(MeframeEvent.PlaybackPlay, () => {
+      this.resourceLoader.setPreloadingEnabled(false);
+    });
+    this.eventBus.on(MeframeEvent.PlaybackPause, () => {
+      this.resourceLoader.setPreloadingEnabled(true);
+    });
+    this.eventBus.on(MeframeEvent.PlaybackStop, () => {
+      this.resourceLoader.setPreloadingEnabled(true);
+    });
   }
   async initialize() {
     if (this.isInitialized) return;
@@ -130,31 +141,25 @@ class Orchestrator {
     }
     this.compositionModel = model;
     this.planner.setModel(model);
-    this.currentClipId = null;
     this.eventBus.emit(MeframeEvent.ModelSet, model);
     this.eventBus.emit(MeframeEvent.CompositionUpdated, {
       trackCount: model.tracks.length,
       clipCount: model.tracks.reduce((acc, track) => acc + track.clips.length, 0),
       durationUs: model.durationUs
     });
-    await this.audioSession.activateAllAudioClips();
   }
   async applyPatch(patch) {
     if (!this.compositionModel) {
       throw new Error("No composition model set");
     }
     const affectedClipIds = applyPatch(this.compositionModel, patch);
-    const clipUpdates = this.planner.applyPatch(patch, affectedClipIds);
+    this.planner.applyPatch(patch, affectedClipIds);
     this.eventBus.emit(MeframeEvent.PatchApplied, {
       operations: patch.operations.length,
       affectedClips: Array.from(affectedClipIds)
     });
-    for (const update of clipUpdates) {
-      if (update.type === "remove") {
-        this.activeClips.delete(update.clipId);
-      }
-      this.cacheManager.invalidateClip(update.clipId);
-      await this.clipSessionManager.handlePlannerUpdate(update.clipId, update);
+    for (const clipId of affectedClipIds) {
+      this.cacheManager.invalidateClip(clipId);
     }
     const reactivatedAudioClips = [];
     const reactivatedVideoClips = [];
@@ -185,62 +190,10 @@ class Orchestrator {
     if (state !== "ready") {
       return;
     }
-    const resource = this.compositionModel.getResource(resourceId);
-    if (!resource) {
-      return;
-    }
-    if (resource.type === "video" || resource.type === "audio") {
-      return;
-    }
-    const clipIds = this.compositionModel.getClipIdsByResourceId(resourceId);
-    for (const clipId of clipIds) {
-      if (!this.clipSessionManager.isClipActive(clipId)) {
-        continue;
-      }
-      const clip = this.compositionModel.findClip(clipId);
-      if (!clip) {
-        continue;
-      }
-      const instructions = this.planner.getInstructions(clipId);
-      if (!instructions) {
-        continue;
-      }
-      const session = this.clipSessionManager.getSession(clipId);
-      const composeWorker = session?.composeWorker;
-      if (composeWorker) {
-        composeWorker.send("install_instructions", instructions);
-      }
-    }
-  }
-  async restartWorker(type, clipId) {
-    const clipLocalTypes = [
-      "videoDemux",
-      "audioDemux",
-      "videoDecode",
-      "audioDecode",
-      "videoCompose",
-      "videoEncode"
-    ];
-    if (clipLocalTypes.includes(type) && !clipId) {
-      throw new Error(`clipId required for restarting ${type} worker`);
-    }
-    this.workers.terminate(type, clipId);
-    const worker = await this.workers.getOrCreate(type, clipId);
-    this.eventBus.emit(MeframeEvent.WorkerRestarted, {
-      type,
-      workerId: worker.getWorkerId(),
-      reason: "Manual restart"
-    });
-    if (clipId) {
-      const session = this.clipSessionManager.getSession(clipId);
-      if (session) {
-        await session.activate();
-      }
-    }
   }
-  async renderFrame(timeUs, options) {
+  async getFrame(timeUs, options) {
     const signal = options?.signal;
-    const immediate = options?.immediate ?? true;
+    const preheat = options?.preheat ?? false;
     if (!this.compositionModel) {
       throw new Error("No composition model set");
     }
@@ -248,113 +201,89 @@ class Orchestrator {
     if (!clip) {
       return null;
     }
-    if (this.currentClipId !== clip.id) {
-      this.currentClipId = clip.id;
-      void this.ensureClipCache(timeUs, immediate);
-    }
     let relativeTimeUs = options?.relativeTimeUs ?? timeUs - clip.startUs;
-    relativeTimeUs = quantizeTimestampToFrame(relativeTimeUs, 0, this.compositionModel.fps);
     relativeTimeUs = Math.min(relativeTimeUs, clip.durationUs - 1);
-    const cachedFrame = this.cacheManager.getFrame(relativeTimeUs, clip.id);
-    if (cachedFrame) {
-      this.eventBus.emit(MeframeEvent.CacheHit, {
+    if (!preheat) {
+      this.cacheManager.setWindow(timeUs);
+      const cachedFrame = this.cacheManager.getFrame(relativeTimeUs, clip.id);
+      if (cachedFrame) {
+        this.preheatNextClip(clip);
+        this.eventBus.emit(MeframeEvent.CacheHit, {
+          timeUs,
+          level: "L1",
+          key: `${clip.id}-${relativeTimeUs}`
+        });
+        return cachedFrame;
+      }
+      this.eventBus.emit(MeframeEvent.CacheMiss, {
         timeUs,
         level: "L1",
         key: `${clip.id}-${relativeTimeUs}`
       });
-      return cachedFrame;
     }
-    this.eventBus.emit(MeframeEvent.CacheMiss, {
-      timeUs,
-      level: "L1",
-      key: `${clip.id}-${relativeTimeUs}`
-    });
     if (signal?.aborted) {
       throw new DOMException("Render aborted", "AbortError");
     }
-    const l2Frame = await this.decodeFromL2(relativeTimeUs, clip);
-    if (l2Frame) {
-      void this.ensureAudioFromL2(clip.id);
-      return l2Frame;
-    }
-    return null;
-  }
-  async ensureAudioFromL2(clipId) {
-    try {
-      if (this.cacheManager.hasClipPCM(clipId)) {
-        return;
-      }
-      const hasAudio = await this.cacheManager.hasClipInL2(clipId, "audio");
-      if (!hasAudio) return;
-      await this.audioSession.ensureClipAudioFromL2(clipId);
-    } catch (error) {
-      console.warn("[Orchestrator] ensureAudioFromL2IfNeeded error:", error);
-    }
+    const resourceFrame = await this.decodeFromResource(clip, relativeTimeUs, timeUs, options);
+    return resourceFrame;
   }
-  // TODO: move to @ClipSessionManager
-  async decodeFromL2(timeUs, clip) {
-    const { id, trackId, startUs } = clip;
-    const [chunkStream, metadata] = await Promise.all([
-      this.cacheManager.l2Cache.createReadStream(id, "video"),
-      this.cacheManager.l2Cache.getClipMetadata(id, "video")
-    ]);
-    if (!chunkStream || !metadata?.codec) {
-      return null;
-    }
-    const decoder = new VideoChunkDecoder(`l2-temp-${id}`, {
-      codec: metadata.codec,
-      width: metadata.codedWidth,
-      height: metadata.codedHeight,
-      description: metadata.description,
-      hardwareAcceleration: metadata.hardwareAcceleration || "no-preference",
-      thread: "main"
-    });
-    try {
-      const decodeStream = chunkStream.pipeThrough(decoder.createStream());
-      let targetFrame = null;
-      await this.cacheManager.receiveComposedFrames(decodeStream, {
-        clipId: id,
-        trackId: trackId || this.compositionModel?.mainTrackId || "main",
-        fps: this.compositionModel?.fps ?? 30,
-        clipStartUs: startUs,
-        onFrame: (info) => {
-          if (info.timeUs === timeUs) {
-            targetFrame = this.cacheManager.getFrame(timeUs, id);
-          }
-        }
+  async preheatNextClip(clip) {
+    if (clip.id === this.currentClipId) return;
+    this.currentClipId = clip.id;
+    const nextClip = this.compositionModel?.getClipsAtTime(
+      clip.startUs + clip.durationUs,
+      this.compositionModel?.mainTrackId
+    )[0];
+    if (nextClip && isVideoClip(nextClip)) {
+      this.resourceLoader.fetch(nextClip.resourceId, {
+        priority: "normal",
+        clipId: nextClip.id,
+        trackId: nextClip.trackId
       });
-      return targetFrame;
-    } finally {
-      await decoder.close();
     }
   }
   /**
-   * Ensure clips are cached using 2-Clip strategy
-   * Debounced to avoid excessive session activation during fast seek
-   * @param timeUs - Target time for cache window
-   * @param immediate - Skip debounce if true (used for initial load)
+   * Compose frame from OPFS resource (on-demand decoding)
+   * This is the new path for long clips with window caching
    */
-  async ensureClipCache(timeUs, immediate = false) {
-    const executeCache = async () => {
-      if (!this.compositionModel) return;
-      const clipIds = this.compositionModel.getClipsToCacheAtTime(timeUs);
-      if (clipIds.size === 0) return;
-      await this.clipSessionManager.ensureClips(clipIds);
+  async decodeFromResource(clip, relativeTimeUs, globalTimeUs, options) {
+    if (!hasResourceId(clip)) return null;
+    const resourceId = clip.resourceId;
+    const resource = this.compositionModel?.getResource(resourceId);
+    const isReady = resource?.state === "ready";
+    const fetchOptions = {
+      priority: "high",
+      clipId: clip.id,
+      trackId: clip.trackId
     };
-    if (this.ensureCacheDebounceTimer !== null) {
-      clearTimeout(this.ensureCacheDebounceTimer);
-      this.ensureCacheDebounceTimer = null;
-    }
-    if (immediate) {
-      return executeCache();
+    if (options?.immediate && !isReady) {
+      this.resourceLoader.fetch(resourceId, fetchOptions);
+      return null;
     }
-    return new Promise((resolve) => {
-      this.ensureCacheDebounceTimer = setTimeout(async () => {
-        this.ensureCacheDebounceTimer = null;
-        await executeCache();
-        resolve();
-      }, this.ensureCacheDebounceDelay);
+    await this.resourceLoader.fetch(resourceId, fetchOptions);
+    const session = await OnDemandVideoSession.create({
+      clipId: clip.id,
+      resourceId,
+      targetTimeUs: relativeTimeUs,
+      globalTimeUs,
+      resourceCache: this.cacheManager.resourceCache,
+      mp4IndexCache: this.cacheManager.mp4IndexCache,
+      cacheManager: this.cacheManager,
+      compositionModel: this.compositionModel,
+      fps: this.compositionModel?.fps ?? 30
     });
+    try {
+      const DECODE_WINDOW_SIZE = 3e6;
+      const windowStart = relativeTimeUs;
+      const windowEnd = Math.min(clip.durationUs, relativeTimeUs + DECODE_WINDOW_SIZE);
+      await session.decodeWindow(windowStart, windowEnd);
+      return this.cacheManager.getFrame(relativeTimeUs, clip.id);
+    } catch (error) {
+      console.error("[Orchestrator] Error composing from resource:", error);
+      return null;
+    } finally {
+      await session.dispose();
+    }
   }
   /**
    * Wait for clip cache to be ready for playback
@@ -375,115 +304,7 @@ class Orchestrator {
     return this.cacheManager.waitForClipReady(currentClip.id, {
       minFrameCount: options?.minFrameCount ?? 5,
       timeoutMs: options?.timeoutMs ?? 5e3
-      // Don't pass startTimeUs - count all frames in the clip
-    });
-  }
-  // TODO: move to @ClipSessionManager
-  async renderClipForL2(clipId) {
-    const sessionId = `${clipId}#l2`;
-    let session = null;
-    return new Promise((resolve, reject) => {
-      this.createSession(sessionId, {
-        forL2Only: true,
-        clipId,
-        onL2Complete: () => {
-          resolve(true);
-        },
-        onL2Error: (error) => {
-          console.error("[Orchestrator] L2 rendering failed for", clipId, error);
-          reject(error);
-        }
-      }).then((s) => {
-        session = s;
-        return session.activate();
-      }).catch(async (error) => {
-        if (session) {
-          await session.dispose();
-        }
-        if (error instanceof ResourceConflictError) {
-          resolve(false);
-        } else {
-          reject(error);
-        }
-      });
-    });
-  }
-  // TODO: move to @ClipSessionManager
-  async createSession(sessionId, options) {
-    const clipId = options?.clipId ?? sessionId;
-    const clip = this.compositionModel?.findClip(clipId);
-    if (!clip) {
-      throw new Error(`Clip ${clipId} not found`);
-    }
-    const session = await VideoClipSession.create({
-      clipId,
-      sessionId,
-      forL2Only: options?.forL2Only ?? false,
-      planner: this.planner,
-      workerPool: this.workers,
-      cacheManager: this.cacheManager,
-      compositionModel: this.compositionModel,
-      workerConfigs: this.buildWorkerConfigs(),
-      resourceLoader: this.resourceLoader,
-      callbacks: {
-        onComposedStreamReady: async (stream, fps) => {
-          await this.cacheManager.receiveComposedFrames(stream, {
-            clipId,
-            trackId: this.compositionModel.mainTrackId,
-            fps,
-            clipStartUs: clip.startUs,
-            onFrame: () => {
-            }
-          });
-        },
-        onEncodedStreamReady: async (stream, track) => {
-          await this.cacheManager.receiveEncodedChunks(stream, clipId, track, {
-            onComplete: () => {
-              session.dispose();
-              if (track === "video" && options?.onL2Complete) {
-                options.onL2Complete();
-              }
-            },
-            onError: (error) => {
-              console.error(`[Orchestrator] L2 encode stream error for ${clipId} ${track}:`, error);
-              session.dispose();
-              if (options?.onL2Error) {
-                options.onL2Error(error);
-              }
-            }
-          });
-        },
-        onAudioStreamReady: (stream, metadata) => {
-          if (options?.forL2Only) {
-            stream.cancel();
-          } else {
-            this.audioSession.handleAudioStream(stream, metadata);
-          }
-        },
-        onPipelineReady: async (attachmentResourceIds) => {
-          const clip2 = this.compositionModel?.findClip(clipId);
-          if (clip2 && hasResourceId(clip2)) {
-            await this.resourceLoader.fetch(clip2.resourceId, {
-              priority: options?.forL2Only ? "low" : "high",
-              sessionId,
-              trackId: clip2.trackId,
-              isMainTrack: true
-            });
-          }
-          if (attachmentResourceIds && attachmentResourceIds.length > 0) {
-            for (const resourceId of attachmentResourceIds) {
-              await this.resourceLoader.fetch(resourceId, {
-                priority: "normal",
-                sessionId,
-                isMainTrack: false
-              });
-            }
-          }
-        }
-      }
     });
-    this.activeClips.add(sessionId);
-    return session;
   }
   async dispose() {
     if (this.ensureCacheDebounceTimer !== null) {
@@ -491,10 +312,7 @@ class Orchestrator {
       this.ensureCacheDebounceTimer = null;
     }
     this.resourceLoader.dispose();
-    await this.clipSessionManager.dispose();
     await this.cacheManager.clear();
-    this.currentClipId = null;
-    this.activeClips.clear();
     this.workers.terminateAll();
     this.compositionModel = null;
     this.eventBus.dispose();
@@ -504,6 +322,7 @@ class Orchestrator {
     const defaultCanvasWidth = config.global?.defaultCanvasWidth ?? 720;
     const defaultCanvasHeight = config.global?.defaultCanvasHeight ?? 1280;
     const defaultFps = config.global?.defaultFps ?? 30;
+    const targetFps = this.compositionModel?.fps ?? defaultFps;
     return {
       videoDemux: {
         highWaterMark: config.demux?.backpressure?.highWaterMark ?? 10
@@ -516,7 +335,7 @@ class Orchestrator {
       videoCompose: {
         width: config.compose?.canvas?.width ?? defaultCanvasWidth,
         height: config.compose?.canvas?.height ?? defaultCanvasHeight,
-        fps: config.global?.defaultFps ?? defaultFps,
+        fps: targetFps,
         backgroundColor: config.compose?.canvas?.backgroundColor ?? "#000000",
         enableSmoothing: config.compose?.visual?.enableSmoothing ?? true,
         enableHardwareAcceleration: config.compose?.visual?.enableHardwareAcceleration ?? true,
@@ -531,7 +350,7 @@ class Orchestrator {
         width: config.compose?.canvas?.width || defaultCanvasWidth,
         height: config.compose?.canvas?.height || defaultCanvasHeight,
         bitrate: config.encode?.video?.bitrateKbps ? config.encode.video.bitrateKbps * 1e3 : 12e6,
-        framerate: config.encode?.video?.framerate || defaultFps,
+        framerate: targetFps,
         latencyMode: "quality",
         bitrateMode: "variable",
         hardwareAcceleration: "no-preference",
@@ -540,7 +359,121 @@ class Orchestrator {
     };
   }
   async export(model, options) {
-    return this.muxManager.export(model, options);
+    return this.exportScheduler.execute(model, options);
+  }
+  /**
+   * Get render state for real-time composition
+   * Returns layers ready for VideoComposer
+   */
+  async getRenderState(timeUs, options) {
+    if (!this.compositionModel) {
+      return null;
+    }
+    const frame = await this.getFrame(timeUs, options);
+    if (options?.immediate && !frame) {
+      return null;
+    }
+    const clip = this.compositionModel.getClipsAtTime(timeUs, this.compositionModel.mainTrackId)[0];
+    if (!clip) {
+      return null;
+    }
+    const relativeTimeUs = timeUs - clip.startUs;
+    const instructions = this.planner.getInstructions(clip.id);
+    if (!instructions) {
+      return null;
+    }
+    const layers = [];
+    const activeLayers = instructions.layers.filter((layer) => {
+      if (!layer.payload.attachmentId) {
+        return true;
+      }
+      if (layer.status !== "ready") {
+        return false;
+      }
+      return layer.activeRanges.some(
+        (range) => relativeTimeUs >= range.startUs && relativeTimeUs < range.endUs
+      );
+    });
+    for (const layerPlan of activeLayers) {
+      const layer = this.materializeLayer(layerPlan, clip, relativeTimeUs, timeUs);
+      if (layer) {
+        layers.push(layer);
+      }
+    }
+    return { layers };
+  }
+  /**
+   * Materialize a serialized layer plan into concrete Layer
+   */
+  materializeLayer(layerPlan, clip, clipRelativeTimeUs, globalTimeUs) {
+    const baseLayer = {
+      id: layerPlan.layerId,
+      type: layerPlan.type,
+      zIndex: layerPlan.zIndex ?? 0,
+      visible: true,
+      opacity: layerPlan.opacity ?? 1
+    };
+    if (layerPlan.type === "video" && !layerPlan.payload.attachmentId) {
+      const rcFrame = this.cacheManager.getFrame(clipRelativeTimeUs, clip.id);
+      if (!rcFrame) {
+        console.warn("[Orchestrator] Video frame not found in L1:", clip.id, clipRelativeTimeUs);
+        return null;
+      }
+      return {
+        ...baseLayer,
+        type: "video",
+        rcFrame
+      };
+    }
+    if (layerPlan.type === "text") {
+      const payload = layerPlan.payload;
+      return {
+        ...baseLayer,
+        type: "text",
+        text: payload.text,
+        localeCode: payload.localeCode,
+        fontConfig: payload.fontConfig,
+        animation: payload.animation,
+        wordTimings: payload.wordTimings,
+        letterCase: payload.letterCase
+      };
+    }
+    if (layerPlan.type === "image") {
+      const payload = layerPlan.payload;
+      const resourceId = payload.resourceId;
+      const source = this.cacheManager.imageBitmapCache.get(resourceId);
+      const imageLayer = {
+        ...baseLayer,
+        type: "image",
+        source,
+        attachmentId: payload.attachmentId
+      };
+      if (payload.targetWidth !== void 0) {
+        imageLayer.targetWidth = payload.targetWidth;
+      }
+      if (payload.targetHeight !== void 0) {
+        imageLayer.targetHeight = payload.targetHeight;
+      }
+      if (payload.animation) {
+        const { position, keyframes, overlayClipStartUs } = payload.animation;
+        const relativeTimeUs = globalTimeUs - overlayClipStartUs;
+        if (relativeTimeUs < 0 || relativeTimeUs > keyframes[keyframes.length - 1].time) {
+          return null;
+        }
+        const rotationRad = 0;
+        imageLayer.transform = {
+          x: position.x,
+          y: position.y,
+          scaleX: 1,
+          scaleY: 1,
+          rotation: rotationRad,
+          anchorX: 0.5,
+          anchorY: 0.5
+        };
+      }
+      return imageLayer;
+    }
+    return baseLayer;
   }
 }
 export {