@meframe/core 0.0.31 → 0.0.33

package/dist/stages/load/ResourceLoader.js

@@ -4,12 +4,7 @@ import { StreamFactory } from "./StreamFactory.js";
 import { MeframeEvent } from "../../event/events.js";
 import { createImageBitmapFromBlob } from "../../utils/image-utils.js";
 import { MP4IndexParser } from "../demux/MP4IndexParser.js";
-class ResourceConflictError extends Error {
-  constructor(message) {
-    super(message);
-    this.name = "ResourceConflictError";
-  }
-}
+import { MP3FrameParser } from "../demux/MP3FrameParser.js";
 class ResourceLoader {
   cacheManager;
   workerPool;
@@ -19,19 +14,16 @@ class ResourceLoader {
   eventBus;
   onStateChange;
   blobCache = /* @__PURE__ */ new Map();
-  pendingTransfers = /* @__PURE__ */ new Map();
   parsingIndexes = /* @__PURE__ */ new Set();
   // Track in-progress index parsing
   // Preloading state
   isPreloadingEnabled = true;
   preloadQueue = [];
-  activePreloads = /* @__PURE__ */ new Set();
-  // TODO: make this configurable
-  IDLE_PRELOAD_CONCURRENCY = 2;
   constructor(options) {
-    const maxConcurrent = options.config?.maxConcurrent ?? 4;
+    const config = options.config || {};
+    const maxConcurrent = config.maxConcurrent ?? 2;
     this.taskManager = new TaskManager(maxConcurrent);
-    this.streamFactory = new StreamFactory(options.onProgress, options.config);
+    this.streamFactory = new StreamFactory(options.onProgress, config);
     this.eventBus = options.eventBus;
     this.onStateChange = options.onStateChange;
     this.cacheManager = options.cacheManager;
@@ -41,7 +33,7 @@ class ResourceLoader {
     this.model = model;
     const mainTrack = model.tracks.find((track) => track.id === (model.mainTrackId || "main"));
     if (mainTrack?.clips?.[0] && hasResourceId(mainTrack.clips[0])) {
-      await this.fetch(mainTrack.clips[0].resourceId, {
+      await this.load(mainTrack.clips[0].resourceId, {
         priority: "high",
         clipId: mainTrack.clips[0].id,
         trackId: mainTrack.id
@@ -53,8 +45,6 @@ class ResourceLoader {
     this.isPreloadingEnabled = enabled;
     if (enabled) {
       this.startPreloading();
-    } else {
-      this.preloadQueue = [];
     }
   }
   startPreloading() {
@@ -63,7 +53,6 @@ class ResourceLoader {
       (track) => track.id === (this.model?.mainTrackId || "main")
     );
     if (!mainTrack) return;
-    const newQueue = [];
     for (const clip of mainTrack.clips) {
       if (!hasResourceId(clip)) continue;
       const resource = this.model.getResource(clip.resourceId);
@@ -71,56 +60,27 @@ class ResourceLoader {
       if (!resource || resource.state === "ready" || resource.state === "loading" || resource.state === "error") {
         continue;
       }
-      if (this.activePreloads.has(resource.id) || this.taskManager.hasActiveTask(resource.id)) {
-        continue;
-      }
-      newQueue.push(resource.id);
+      this.load(resource.id, { priority: "low" });
     }
-    this.preloadQueue = newQueue;
-    this.processPreloadQueue();
   }
   processPreloadQueue() {
     if (!this.isPreloadingEnabled || this.preloadQueue.length === 0) return;
-    while (this.activePreloads.size < this.IDLE_PRELOAD_CONCURRENCY && this.preloadQueue.length > 0) {
+    while (this.preloadQueue.length > 0) {
       const resourceId = this.preloadQueue.shift();
       if (!resourceId) break;
-      this.activePreloads.add(resourceId);
-      this.fetch(resourceId, { priority: "low" }).finally(() => {
-        this.activePreloads.delete(resourceId);
+      this.load(resourceId, { priority: "low" }).finally(() => {
         this.processPreloadQueue();
       });
     }
   }
-  enqueueLoad(resource, priority = "normal", sessionId, clipId, trackId, isMainTrack = false) {
-    if (this.taskManager.hasActiveTask(resource.id)) {
-      if (priority === "high") ;
-      else if (isMainTrack) {
-        throw new ResourceConflictError(
-          `Resource ${resource.id} is being loaded by another session. Preview channel has priority.`
-        );
-      } else {
-        if (this.blobCache.has(resource.id)) {
-          void this.transferCachedImage(resource.id, sessionId);
-          return;
-        } else {
-          this.registerPendingTransfer(resource.id, sessionId);
-          console.debug(
-            `[ResourceLoader] Attachment resource ${resource.id} loading, registered for pending transfer`
-          );
-          return;
-        }
-      }
+  enqueueLoad(resource, priority = "normal", sessionId, clipId, trackId) {
+    const existingTask = this.taskManager.getActiveTask(resource.id);
+    if (existingTask) {
+      return existingTask;
     }
-    this.taskManager.enqueue(resource, priority, sessionId, clipId, trackId);
+    const task = this.taskManager.enqueue(resource, priority, sessionId, clipId, trackId);
     this.processQueue();
-  }
-  registerPendingTransfer(resourceId, sessionId) {
-    if (!sessionId) return;
-    const pending = this.pendingTransfers.get(resourceId) || [];
-    if (!pending.includes(sessionId)) {
-      pending.push(sessionId);
-      this.pendingTransfers.set(resourceId, pending);
-    }
+    return task;
   }
   processQueue() {
     while (this.taskManager.canProcess) {
@@ -129,6 +89,103 @@ class ResourceLoader {
       this.startLoad(task);
     }
   }
+  /**
+   * Check if resource is cached and ready (without loading)
+   */
+  async isResourceCached(resourceId, type) {
+    switch (type) {
+      case "video": {
+        const hasOPFS = await this.cacheManager.hasResourceInCache(resourceId);
+        const hasIndex = this.cacheManager.mp4IndexCache.has(resourceId);
+        return hasOPFS && hasIndex;
+      }
+      case "audio":
+        return this.cacheManager.audioSampleCache.has(resourceId);
+      case "image":
+        return this.blobCache.has(resourceId);
+      case "json":
+      case "text":
+        return this.blobCache.has(resourceId);
+      default:
+        return false;
+    }
+  }
+  /**
+   * Transfer video stream to worker
+   */
+  async transferVideoToWorker(resourceId, sessionId) {
+    const stream = await this.createOPFSReadStream(resourceId);
+    const demuxWorker = await this.workerPool.get("videoDemux", sessionId);
+    if (demuxWorker) {
+      await demuxWorker.sendStream(stream, {
+        sessionId,
+        resourceId
+      });
+    } else {
+      stream.cancel();
+    }
+  }
+  /**
+   * Transfer image to worker
+   */
+  async transferImageToWorker(resourceId, sessionId, imageBitmap) {
+    const composeWorker = await this.workerPool.get("videoCompose", sessionId);
+    await composeWorker?.send?.(
+      "receive_image",
+      { resourceId, sessionId, imageBitmap },
+      { transfer: [imageBitmap] }
+    );
+  }
+  /**
+   * Load video resource (download + cache or read from cache)
+   */
+  async loadVideoResource(task) {
+    const cached = await this.cacheManager.hasResourceInCache(task.resourceId);
+    if (cached) {
+      await this.ensureIndexParsed(task.resourceId);
+      if (task.sessionId) {
+        await this.transferVideoToWorker(task.resourceId, task.sessionId);
+      }
+    } else {
+      await this.loadWithOPFSCache(task);
+    }
+  }
+  /**
+   * Load audio resource (download + parse or reuse cache)
+   */
+  async loadAudioResource(task) {
+    if (!this.cacheManager.audioSampleCache.has(task.resourceId)) {
+      await this.loadAndParseAudioFile(task);
+    }
+  }
+  /**
+   * Load image resource (download + cache or read from cache) and transfer to worker
+   */
+  async loadImageResource(task) {
+    let blob = this.blobCache.get(task.resourceId);
+    if (!blob) {
+      if (task.controller) {
+        blob = await this.fetchBlob(task.resource.uri, task.controller.signal);
+        this.blobCache.set(task.resourceId, blob);
+      } else {
+        return null;
+      }
+    }
+    const imageBitmap = await createImageBitmapFromBlob(blob);
+    if (task.sessionId) {
+      await this.transferImageToWorker(task.resourceId, task.sessionId, imageBitmap);
+      return null;
+    }
+    return imageBitmap;
+  }
+  /**
+   * Load text resource (json/text)
+   */
+  async loadTextResource(task) {
+    if (task.controller) {
+      await this.fetchBlob(task.resource.uri, task.controller.signal);
+    }
+  }
   /**
    * Start loading a resource
    * Handles state management (loading → ready/error) for all resource types
@@ -139,29 +196,24 @@ class ResourceLoader {
     try {
       this.updateResourceState(task.resourceId, "loading");
       task.controller = new AbortController();
-      if (task.resource.type === "image") {
-        await this.loadImageBitmap(task);
-      } else if (task.resource.type === "video") {
-        const cached = await this.cacheManager.hasResourceInCache(task.resourceId);
-        if (cached) {
-          await this.ensureIndexParsed(task.resourceId);
-          if (task.sessionId) {
-            const stream = await this.createOPFSReadStream(task.resourceId);
-            task.stream = stream;
-            await this.transferToDemuxWorker(task);
+      switch (task.resource.type) {
+        case "image": {
+          const image = await this.loadImageResource(task);
+          if (image) {
+            image.close();
           }
-        } else {
-          await this.loadWithOPFSCache(task);
-        }
-      } else if (task.resource.type === "audio") {
-        const stream = await this.streamFactory.createRegularStream(task);
-        if (!stream) {
-          throw new Error(`Failed to create stream for ${task.resourceId}`);
+          break;
         }
-        task.stream = stream;
-        await this.transferToDemuxWorker(task);
-      } else if (task.resource.type === "json" || task.resource.type === "text") {
-        await this.loadTextResource(task);
+        case "video":
+          await this.loadVideoResource(task);
+          break;
+        case "audio":
+          await this.loadAudioResource(task);
+          break;
+        case "json":
+        case "text":
+          await this.loadTextResource(task);
+          break;
       }
       this.updateResourceState(task.resourceId, "ready");
     } catch (error) {
@@ -258,6 +310,44 @@ class ResourceLoader {
   async writeToOPFS(resourceId, stream) {
     await this.cacheManager.resourceCache.writeResource(resourceId, stream);
   }
+  /**
+   * Load and parse audio file (MP3/WAV) in main thread
+   * Extract EncodedAudioChunk and cache to AudioSampleCache
+   * Aligned with video audio track extraction (unified architecture)
+   */
+  async loadAndParseAudioFile(task) {
+    const { resourceId } = task;
+    try {
+      const blob = await this.fetchBlob(task.resource.uri, task.controller.signal);
+      const arrayBuffer = await blob.arrayBuffer();
+      const uint8Array = new Uint8Array(arrayBuffer);
+      const parser = new MP3FrameParser();
+      const { frames, config } = parser.push(uint8Array);
+      const remainingFrames = parser.flush();
+      const allFrames = [...frames, ...remainingFrames];
+      if (!config) {
+        throw new Error(`Failed to parse audio config for ${resourceId}`);
+      }
+      const audioChunks = allFrames.map((frame) => {
+        return new EncodedAudioChunk({
+          type: "key",
+          // MP3 frames are all key frames
+          timestamp: frame.timestampUs,
+          duration: frame.durationUs,
+          data: frame.data
+        });
+      });
+      const audioConfig = {
+        codec: "mp3",
+        sampleRate: config.sampleRate,
+        numberOfChannels: config.channels
+      };
+      this.cacheManager.audioSampleCache.set(resourceId, audioChunks, audioConfig);
+    } catch (error) {
+      console.error(`[ResourceLoader] Failed to parse audio file ${resourceId}:`, error);
+      throw error;
+    }
+  }
   /**
    * Parse moov from stream and cache index + audio samples + decode first frame
    */
@@ -296,30 +386,21 @@ class ResourceLoader {
       throw error;
     }
   }
-  /**
-   * Load text-based resources (json, text)
-   * Just download the content - state management is handled by startLoad()
-   */
-  async loadTextResource(task) {
-    await this.fetchBlob(task.resource.uri, task.controller.signal);
-  }
-  /**
-   * Load image resource: fetch blob → create ImageBitmap → cache in CacheManager
-   * Note: Images don't need streaming (typically < 5MB)
-   *
-   * Strategy for window cache architecture:
-   * - Store ImageBitmap in CacheManager.imageBitmapCache (main thread accessible)
-   * - OnDemandComposeSession retrieves from cache for composition
-   * - No longer transfer to Worker (composition is in main thread)
-   */
-  async loadImageBitmap(task) {
-    let blob = this.blobCache.get(task.resourceId);
-    if (!blob) {
-      blob = await this.fetchBlob(task.resource.uri, task.controller.signal);
-      this.blobCache.set(task.resourceId, blob);
+  async loadImage(resource) {
+    const task = {
+      resourceId: resource.id,
+      resource,
+      bytesLoaded: 0,
+      totalBytes: 0,
+      startTime: Date.now(),
+      priority: "normal",
+      controller: new AbortController()
+    };
+    const imageBitmap = await this.loadImageResource(task);
+    if (!imageBitmap) {
+      throw new Error(`Failed to load image ${resource.id}`);
     }
-    const imageBitmap = await createImageBitmapFromBlob(blob);
-    this.cacheManager.imageBitmapCache.set(task.resourceId, imageBitmap);
+    return imageBitmap;
   }
   /**
    * Fetch resource as blob (for images, json, etc.)
@@ -331,57 +412,14 @@ class ResourceLoader {
     }
     return response.blob();
   }
-  /**
-   * Transfer ImageBitmap to VideoComposeWorker
-   * Legacy: Not used in window cache architecture (images accessed via CacheManager)
-   */
-  // private async transferImageToWorker(task: LoadTask, imageBitmap: ImageBitmap): Promise<void> {
-  //   if (!this.orchestrator) return;
-  //   if (!task.sessionId) {
-  //     throw new Error(
-  //       `[ResourceLoader] sessionId required for resource ${task.resourceId}. ` +
-  //         `In Clip-based architecture, use fetch(resourceId, { sessionId })`
-  //     );
-  //   }
-  //   const composeWorker = await this.orchestrator.workers.get('videoCompose', task.sessionId);
-  //   await composeWorker?.send?.(
-  //     'receive_image',
-  //     {
-  //       resourceId: task.resourceId,
-  //       sessionId: task.sessionId,
-  //       imageBitmap,
-  //     },
-  //     { transfer: [imageBitmap] }
-  //   );
-  // }
-  /**
-   * Transfer cached image to a session
-   * Creates new ImageBitmap from cached Blob and transfers to worker
-   */
-  async transferCachedImage(resourceId, sessionId) {
-    const blob = this.blobCache.get(resourceId);
-    if (!blob || !sessionId) return;
-    const imageBitmap = await createImageBitmapFromBlob(blob);
-    const composeWorker = await this.workerPool.get("videoCompose", sessionId);
-    await composeWorker?.send?.(
-      "receive_image",
-      {
-        resourceId,
-        sessionId,
-        imageBitmap
-      },
-      { transfer: [imageBitmap] }
-    );
-  }
   /**
    * Transfer stream to demux worker (for audio files)
    */
   async transferToDemuxWorker(task) {
     if (!task.stream) return;
     if (!task.sessionId) {
-      throw new Error(
-        `[ResourceLoader] sessionId required for resource ${task.resourceId}. In Clip-based architecture, use fetch(resourceId, { sessionId })`
-      );
+      task.stream.cancel();
+      return;
     }
     const workerType = task.resource.type === "video" ? "videoDemux" : "audioDemux";
     const demuxWorker = await this.workerPool.get(workerType, task.sessionId);
@@ -410,7 +448,19 @@ class ResourceLoader {
     }
     this.onStateChange?.(resourceId, state);
   }
-  async fetch(resourceId, options) {
+  /**
+   * Fetch a resource and wait for loading + parsing to complete
+   *
+   * Returns a Promise that resolves when:
+   * - Resource is fully loaded, parsed, and cached (state='ready')
+   * - Or rejects if loading/parsing fails
+   *
+   * Promise lifecycle:
+   * 1. enqueueLoad() creates LoadTask with promise/resolve/reject (or reuses existing)
+   * 2. processQueue() → startLoad() executes async in background
+   * 3. startLoad() completes → finally → completeTask() → task.resolve()/reject()
+   */
+  async load(resourceId, options) {
     if (!resourceId) {
       return;
     }
@@ -419,54 +469,72 @@ class ResourceLoader {
       console.warn(`Resource ${resourceId} not found in model`);
       return;
     }
-    const transferResourceToWorker = async (rId, sId, type) => {
-      if (type === "video") {
-        const stream = await this.createOPFSReadStream(rId);
-        const task = {
-          resourceId: rId,
-          sessionId: sId,
-          resource: this.model?.resources.get(rId),
-          stream,
-          bytesLoaded: 0,
-          totalBytes: 0,
-          startTime: Date.now(),
-          priority: "normal"
-        };
-        await this.transferToDemuxWorker(task);
-      } else if (type === "image") {
-        await this.transferCachedImage(rId, sId);
-      }
-    };
     if (resource.state === "ready") {
-      if (options?.sessionId) {
-        await transferResourceToWorker(resourceId, options.sessionId, resource.type);
+      if (!options?.sessionId) {
+        return;
+      }
+      const isCached = await this.isResourceCached(resourceId, resource.type);
+      const hasActiveTaskForSession = this.taskManager.hasActiveTaskForSession(
+        resourceId,
+        options.sessionId
+      );
+      if (isCached && !hasActiveTaskForSession) {
+        switch (resource.type) {
+          case "video":
+            await this.transferVideoToWorker(resourceId, options.sessionId);
+            break;
+          case "image": {
+            const blob = this.blobCache.get(resourceId);
+            if (blob) {
+              const imageBitmap = await createImageBitmapFromBlob(blob);
+              await this.transferImageToWorker(resourceId, options.sessionId, imageBitmap);
+            }
+            break;
+          }
+        }
+        return;
       }
-      return;
     }
-    let taskPromise = this.taskManager.getTaskPromise(resourceId);
-    let isCoveredByTask = false;
-    if (taskPromise) {
-      const activeTask = this.taskManager.activeTasks.get(resourceId) || this.taskManager.taskQueue.find((t) => t.resourceId === resourceId);
-      if (activeTask && activeTask.sessionId === options?.sessionId) {
-        isCoveredByTask = true;
+    if (resource.state === "loading") {
+      const existingTask2 = this.taskManager.getActiveTask(resourceId);
+      if (existingTask2) {
+        if (!options?.sessionId || existingTask2.sessionId === options.sessionId) {
+          return existingTask2.promise;
+        }
       }
-    } else {
-      this.enqueueLoad(
-        resource,
-        options?.priority || "normal",
-        options?.sessionId,
-        options?.clipId,
-        options?.trackId,
-        options?.isMainTrack ?? false
-      );
-      taskPromise = this.taskManager.getTaskPromise(resourceId);
-      isCoveredByTask = true;
     }
-    await taskPromise;
-    const updatedResource = this.model?.resources.get(resourceId);
-    if (!isCoveredByTask && updatedResource?.state === "ready" && options?.sessionId) {
-      await transferResourceToWorker(resourceId, options.sessionId, resource.type);
+    const existingTask = this.taskManager.getActiveTask(resourceId);
+    if (existingTask) {
+      if (options?.sessionId && !existingTask.sessionId) {
+        await existingTask.promise;
+        const isCached = await this.isResourceCached(resourceId, resource.type);
+        if (isCached) {
+          switch (resource.type) {
+            case "video":
+              await this.transferVideoToWorker(resourceId, options.sessionId);
+              break;
+            case "image": {
+              const blob = this.blobCache.get(resourceId);
+              if (blob) {
+                const imageBitmap = await createImageBitmapFromBlob(blob);
+                await this.transferImageToWorker(resourceId, options.sessionId, imageBitmap);
+              }
+              break;
+            }
+          }
+        }
+        return;
+      }
+      return existingTask.promise;
     }
+    const task = this.enqueueLoad(
+      resource,
+      options?.priority || "normal",
+      options?.sessionId,
+      options?.clipId,
+      options?.trackId
+    );
+    return task.promise;
   }
   cancel(resourceId) {
     this.taskManager.cancelTask(resourceId);
@@ -499,7 +567,7 @@ class ResourceLoader {
         options?.trackId
       );
     } else {
-      await this.fetch(resourceId, options);
+      await this.load(resourceId, options);
     }
   }
   get activeTasks() {
@@ -511,11 +579,9 @@ class ResourceLoader {
   dispose() {
     this.taskManager.clear();
     this.blobCache.clear();
-    this.pendingTransfers.clear();
   }
 }
 export {
-  ResourceConflictError,
   ResourceLoader
 };
 //# sourceMappingURL=ResourceLoader.js.map