@effing/ffs 0.1.2 → 0.3.0

package/dist/server.js

@@ -1,13 +1,13 @@
 #!/usr/bin/env node
 
 // src/server.ts
-import express4 from "express";
+import express5 from "express";
 import bodyParser from "body-parser";
 
 // src/handlers/shared.ts
 import "express";
 
-// src/cache.ts
+// src/storage.ts
 import {
   S3Client,
   PutObjectCommand,
@@ -22,11 +22,14 @@ import { pipeline } from "stream/promises";
 import path from "path";
 import os from "os";
 import crypto from "crypto";
-var S3CacheStorage = class {
+var DEFAULT_SOURCE_TTL_MS = 60 * 60 * 1e3;
+var DEFAULT_JOB_METADATA_TTL_MS = 8 * 60 * 60 * 1e3;
+var S3TransientStore = class {
   client;
   bucket;
   prefix;
-  ttlMs;
+  sourceTtlMs;
+  jobMetadataTtlMs;
   constructor(options) {
     this.client = new S3Client({
       endpoint: options.endpoint,
@@ -39,22 +42,23 @@ var S3CacheStorage = class {
     });
     this.bucket = options.bucket;
     this.prefix = options.prefix ?? "";
-    this.ttlMs = options.ttlMs ?? 60 * 60 * 1e3;
+    this.sourceTtlMs = options.sourceTtlMs ?? DEFAULT_SOURCE_TTL_MS;
+    this.jobMetadataTtlMs = options.jobMetadataTtlMs ?? DEFAULT_JOB_METADATA_TTL_MS;
   }
-  getExpires() {
-    return new Date(Date.now() + this.ttlMs);
+  getExpires(ttlMs) {
+    return new Date(Date.now() + ttlMs);
   }
   getFullKey(key) {
     return `${this.prefix}${key}`;
   }
-  async put(key, stream) {
+  async put(key, stream, ttlMs) {
     const upload = new Upload({
       client: this.client,
       params: {
         Bucket: this.bucket,
         Key: this.getFullKey(key),
         Body: stream,
-        Expires: this.getExpires()
+        Expires: this.getExpires(ttlMs ?? this.sourceTtlMs)
       }
     });
     await upload.done();
@@ -115,14 +119,14 @@ var S3CacheStorage = class {
       throw err;
     }
   }
-  async putJson(key, data) {
+  async putJson(key, data, ttlMs) {
     await this.client.send(
       new PutObjectCommand({
         Bucket: this.bucket,
         Key: this.getFullKey(key),
         Body: JSON.stringify(data),
         ContentType: "application/json",
-        Expires: this.getExpires()
+        Expires: this.getExpires(ttlMs ?? this.jobMetadataTtlMs)
       })
     );
   }
@@ -148,20 +152,25 @@ var S3CacheStorage = class {
   close() {
   }
 };
-var LocalCacheStorage = class {
+var LocalTransientStore = class {
   baseDir;
   initialized = false;
   cleanupInterval;
-  ttlMs;
-  constructor(baseDir, ttlMs = 60 * 60 * 1e3) {
-    this.baseDir = baseDir ?? path.join(os.tmpdir(), "ffs-cache");
-    this.ttlMs = ttlMs;
+  sourceTtlMs;
+  jobMetadataTtlMs;
+  /** For cleanup, use the longer of the two TTLs */
+  maxTtlMs;
+  constructor(options) {
+    this.baseDir = options?.baseDir ?? path.join(os.tmpdir(), "ffs-transient");
+    this.sourceTtlMs = options?.sourceTtlMs ?? DEFAULT_SOURCE_TTL_MS;
+    this.jobMetadataTtlMs = options?.jobMetadataTtlMs ?? DEFAULT_JOB_METADATA_TTL_MS;
+    this.maxTtlMs = Math.max(this.sourceTtlMs, this.jobMetadataTtlMs);
     this.cleanupInterval = setInterval(() => {
       this.cleanupExpired().catch(console.error);
     }, 3e5);
   }
   /**
-   * Remove files older than TTL
+   * Remove files older than max TTL
    */
   async cleanupExpired() {
     if (!this.initialized) return;
@@ -186,7 +195,7 @@ var LocalCacheStorage = class {
       } else if (entry.isFile()) {
         try {
           const stat = await fs.stat(fullPath);
-          if (now - stat.mtimeMs > this.ttlMs) {
+          if (now - stat.mtimeMs > this.maxTtlMs) {
             await fs.rm(fullPath, { force: true });
           }
         } catch {
@@ -205,7 +214,7 @@ var LocalCacheStorage = class {
     const rand = crypto.randomBytes(8).toString("hex");
     return `${finalPath}.tmp-${process.pid}-${rand}`;
   }
-  async put(key, stream) {
+  async put(key, stream, _ttlMs) {
     const fp = this.filePath(key);
     await this.ensureDir(fp);
     const tmpPath = this.tmpPathFor(fp);
@@ -241,7 +250,7 @@ var LocalCacheStorage = class {
   async delete(key) {
     await fs.rm(this.filePath(key), { force: true });
   }
-  async putJson(key, data) {
+  async putJson(key, data, _ttlMs) {
     const fp = this.filePath(key);
     await this.ensureDir(fp);
     const tmpPath = this.tmpPathFor(fp);
@@ -269,48 +278,208 @@ var LocalCacheStorage = class {
     }
   }
 };
-function createCacheStorage() {
-  const ttlMs = process.env.FFS_CACHE_TTL_MS ? parseInt(process.env.FFS_CACHE_TTL_MS, 10) : 60 * 60 * 1e3;
-  if (process.env.FFS_CACHE_BUCKET) {
-    return new S3CacheStorage({
-      endpoint: process.env.FFS_CACHE_ENDPOINT,
-      region: process.env.FFS_CACHE_REGION ?? "auto",
-      bucket: process.env.FFS_CACHE_BUCKET,
-      prefix: process.env.FFS_CACHE_PREFIX,
-      accessKeyId: process.env.FFS_CACHE_ACCESS_KEY,
-      secretAccessKey: process.env.FFS_CACHE_SECRET_KEY,
-      ttlMs
+function createTransientStore() {
+  const sourceTtlMs = process.env.FFS_SOURCE_CACHE_TTL_MS ? parseInt(process.env.FFS_SOURCE_CACHE_TTL_MS, 10) : DEFAULT_SOURCE_TTL_MS;
+  const jobMetadataTtlMs = process.env.FFS_JOB_METADATA_TTL_MS ? parseInt(process.env.FFS_JOB_METADATA_TTL_MS, 10) : DEFAULT_JOB_METADATA_TTL_MS;
+  if (process.env.FFS_TRANSIENT_STORE_BUCKET) {
+    return new S3TransientStore({
+      endpoint: process.env.FFS_TRANSIENT_STORE_ENDPOINT,
+      region: process.env.FFS_TRANSIENT_STORE_REGION ?? "auto",
+      bucket: process.env.FFS_TRANSIENT_STORE_BUCKET,
+      prefix: process.env.FFS_TRANSIENT_STORE_PREFIX,
+      accessKeyId: process.env.FFS_TRANSIENT_STORE_ACCESS_KEY,
+      secretAccessKey: process.env.FFS_TRANSIENT_STORE_SECRET_KEY,
+      sourceTtlMs,
+      jobMetadataTtlMs
     });
   }
-  return new LocalCacheStorage(process.env.FFS_CACHE_LOCAL_DIR, ttlMs);
+  return new LocalTransientStore({
+    baseDir: process.env.FFS_TRANSIENT_STORE_LOCAL_DIR,
+    sourceTtlMs,
+    jobMetadataTtlMs
+  });
 }
 function hashUrl(url) {
   return crypto.createHash("sha256").update(url).digest("hex").slice(0, 16);
 }
-function sourceCacheKey(url) {
+function sourceStoreKey(url) {
   return `sources/${hashUrl(url)}`;
 }
-function warmupJobCacheKey(jobId) {
+function warmupJobStoreKey(jobId) {
   return `jobs/warmup/${jobId}.json`;
 }
-function renderJobCacheKey(jobId) {
+function renderJobStoreKey(jobId) {
   return `jobs/render/${jobId}.json`;
 }
-var cacheKeys = {
-  source: sourceCacheKey,
-  warmupJob: warmupJobCacheKey,
-  renderJob: renderJobCacheKey
+function warmupAndRenderJobStoreKey(jobId) {
+  return `jobs/warmup-and-render/${jobId}.json`;
+}
+var storeKeys = {
+  source: sourceStoreKey,
+  warmupJob: warmupJobStoreKey,
+  renderJob: renderJobStoreKey,
+  warmupAndRenderJob: warmupAndRenderJobStoreKey
+};
+
+// src/proxy.ts
+import http from "http";
+import { Readable } from "stream";
+
+// src/fetch.ts
+import { fetch, Agent } from "undici";
+async function ffsFetch(url, options) {
+  const {
+    method,
+    body,
+    headers,
+    headersTimeout = 3e5,
+    // 5 minutes
+    bodyTimeout = 3e5
+    // 5 minutes
+  } = options ?? {};
+  const agent = new Agent({ headersTimeout, bodyTimeout });
+  return fetch(url, {
+    method,
+    body,
+    headers: { "User-Agent": "FFS (+https://effing.dev/ffs)", ...headers },
+    dispatcher: agent
+  });
+}
+
+// src/proxy.ts
+var HttpProxy = class {
+  server = null;
+  _port = null;
+  startPromise = null;
+  get port() {
+    return this._port;
+  }
+  /**
+   * Transform a URL to go through the proxy.
+   * @throws Error if proxy not started
+   */
+  transformUrl(url) {
+    if (this._port === null) throw new Error("Proxy not started");
+    return `http://127.0.0.1:${this._port}/${url}`;
+  }
+  /**
+   * Start the proxy server. Safe to call multiple times.
+   */
+  async start() {
+    if (this._port !== null) return;
+    if (this.startPromise) {
+      await this.startPromise;
+      return;
+    }
+    this.startPromise = this.doStart();
+    await this.startPromise;
+  }
+  async doStart() {
+    this.server = http.createServer(async (req, res) => {
+      try {
+        const originalUrl = this.parseProxyPath(req.url || "");
+        if (!originalUrl) {
+          res.writeHead(400, { "Content-Type": "text/plain" });
+          res.end("Bad Request: invalid proxy path");
+          return;
+        }
+        const response = await ffsFetch(originalUrl, {
+          method: req.method,
+          headers: this.filterHeaders(req.headers),
+          bodyTimeout: 0
+          // No timeout for streaming
+        });
+        const headers = {};
+        response.headers.forEach((value, key) => {
+          headers[key] = value;
+        });
+        res.writeHead(response.status, headers);
+        if (response.body) {
+          const nodeStream = Readable.fromWeb(response.body);
+          nodeStream.pipe(res);
+          nodeStream.on("error", (err) => {
+            console.error("Proxy stream error:", err);
+            res.destroy();
+          });
+        } else {
+          res.end();
+        }
+      } catch (err) {
+        console.error("Proxy request error:", err);
+        if (!res.headersSent) {
+          res.writeHead(502, { "Content-Type": "text/plain" });
+          res.end("Bad Gateway");
+        } else {
+          res.destroy();
+        }
+      }
+    });
+    await new Promise((resolve) => {
+      this.server.listen(0, "127.0.0.1", () => {
+        this._port = this.server.address().port;
+        resolve();
+      });
+    });
+  }
+  /**
+   * Parse the proxy path to extract the original URL.
+   * Path format: /{originalUrl}
+   */
+  parseProxyPath(path3) {
+    if (!path3.startsWith("/http://") && !path3.startsWith("/https://")) {
+      return null;
+    }
+    return path3.slice(1);
+  }
+  /**
+   * Filter headers to forward to the upstream server.
+   * Removes hop-by-hop headers that shouldn't be forwarded.
+   */
+  filterHeaders(headers) {
+    const skip = /* @__PURE__ */ new Set([
+      "host",
+      "connection",
+      "keep-alive",
+      "transfer-encoding",
+      "te",
+      "trailer",
+      "upgrade",
+      "proxy-authorization",
+      "proxy-authenticate"
+    ]);
+    const result = {};
+    for (const [key, value] of Object.entries(headers)) {
+      if (!skip.has(key.toLowerCase()) && typeof value === "string") {
+        result[key] = value;
+      }
+    }
+    return result;
+  }
+  /**
+   * Close the proxy server and reset state.
+   */
+  close() {
+    this.server?.close();
+    this.server = null;
+    this._port = null;
+    this.startPromise = null;
+  }
 };
 
 // src/handlers/shared.ts
 import { effieDataSchema } from "@effing/effie";
-function createServerContext() {
+async function createServerContext() {
   const port2 = process.env.FFS_PORT || 2e3;
+  const httpProxy = new HttpProxy();
+  await httpProxy.start();
   return {
-    cacheStorage: createCacheStorage(),
+    transientStore: createTransientStore(),
+    httpProxy,
     baseUrl: process.env.FFS_BASE_URL || `http://localhost:${port2}`,
     skipValidation: !!process.env.FFS_SKIP_VALIDATION && process.env.FFS_SKIP_VALIDATION !== "false",
-    cacheConcurrency: parseInt(process.env.FFS_CACHE_CONCURRENCY || "4", 10)
+    warmupConcurrency: parseInt(process.env.FFS_WARMUP_CONCURRENCY || "4", 10),
+    warmupBackendBaseUrl: process.env.FFS_WARMUP_BACKEND_BASE_URL,
+    renderBackendBaseUrl: process.env.FFS_RENDER_BACKEND_BASE_URL
   };
 }
 function parseEffieData(body, skipValidation) {
@@ -357,215 +526,21 @@ data: ${JSON.stringify(data)}
 
 // src/handlers/caching.ts
 import "express";
-import { Readable, Transform } from "stream";
-import { randomUUID } from "crypto";
-
-// src/fetch.ts
-import { fetch, Agent } from "undici";
-async function ffsFetch(url, options) {
-  const {
-    method,
-    body,
-    headers,
-    headersTimeout = 3e5,
-    // 5 minutes
-    bodyTimeout = 3e5
-    // 5 minutes
-  } = options ?? {};
-  const agent = new Agent({ headersTimeout, bodyTimeout });
-  return fetch(url, {
-    method,
-    body,
-    headers: { "User-Agent": "FFS (+https://effing.dev/ffs)", ...headers },
-    dispatcher: agent
-  });
-}
+import { Readable as Readable3, Transform } from "stream";
+import { randomUUID as randomUUID3 } from "crypto";
+import {
+  extractEffieSources,
+  extractEffieSourcesWithTypes as extractEffieSourcesWithTypes2
+} from "@effing/effie";
 
-// src/handlers/caching.ts
-import { extractEffieSources } from "@effing/effie";
-var inFlightFetches = /* @__PURE__ */ new Map();
-async function createWarmupJob(req, res, ctx2) {
-  try {
-    const parseResult = parseEffieData(req.body, ctx2.skipValidation);
-    if ("error" in parseResult) {
-      res.status(400).json(parseResult);
-      return;
-    }
-    const sources = extractEffieSources(parseResult.effie);
-    const jobId = randomUUID();
-    await ctx2.cacheStorage.putJson(cacheKeys.warmupJob(jobId), { sources });
-    res.json({
-      id: jobId,
-      url: `${ctx2.baseUrl}/warmup/${jobId}`
-    });
-  } catch (error) {
-    console.error("Error creating warmup job:", error);
-    res.status(500).json({ error: "Failed to create warmup job" });
-  }
-}
-async function streamWarmupJob(req, res, ctx2) {
-  try {
-    setupCORSHeaders(res);
-    const jobId = req.params.id;
-    const jobCacheKey = cacheKeys.warmupJob(jobId);
-    const job = await ctx2.cacheStorage.getJson(jobCacheKey);
-    ctx2.cacheStorage.delete(jobCacheKey);
-    if (!job) {
-      res.status(404).json({ error: "Job not found" });
-      return;
-    }
-    setupSSEResponse(res);
-    const sendEvent = createSSEEventSender(res);
-    try {
-      await warmupSources(job.sources, sendEvent, ctx2);
-      sendEvent("complete", { status: "ready" });
-    } catch (error) {
-      sendEvent("error", { message: String(error) });
-    } finally {
-      res.end();
-    }
-  } catch (error) {
-    console.error("Error in warmup streaming:", error);
-    if (!res.headersSent) {
-      res.status(500).json({ error: "Warmup streaming failed" });
-    } else {
-      res.end();
-    }
-  }
-}
-async function purgeCache(req, res, ctx2) {
-  try {
-    const parseResult = parseEffieData(req.body, ctx2.skipValidation);
-    if ("error" in parseResult) {
-      res.status(400).json(parseResult);
-      return;
-    }
-    const sources = extractEffieSources(parseResult.effie);
-    let purged = 0;
-    for (const url of sources) {
-      const ck = cacheKeys.source(url);
-      if (await ctx2.cacheStorage.exists(ck)) {
-        await ctx2.cacheStorage.delete(ck);
-        purged++;
-      }
-    }
-    res.json({ purged, total: sources.length });
-  } catch (error) {
-    console.error("Error purging cache:", error);
-    res.status(500).json({ error: "Failed to purge cache" });
-  }
-}
-async function warmupSources(sources, sendEvent, ctx2) {
-  const total = sources.length;
-  const sourceCacheKeys = sources.map(cacheKeys.source);
-  sendEvent("start", { total });
-  const existsMap = await ctx2.cacheStorage.existsMany(sourceCacheKeys);
-  let cached = 0;
-  let failed = 0;
-  for (let i = 0; i < sources.length; i++) {
-    if (existsMap.get(sourceCacheKeys[i])) {
-      cached++;
-      sendEvent("progress", {
-        url: sources[i],
-        status: "hit",
-        cached,
-        failed,
-        total
-      });
-    }
-  }
-  const uncached = sources.filter((_, i) => !existsMap.get(sourceCacheKeys[i]));
-  if (uncached.length === 0) {
-    sendEvent("summary", { cached, failed, total });
-    return;
-  }
-  const keepalive = setInterval(() => {
-    sendEvent("keepalive", { cached, failed, total });
-  }, 25e3);
-  const queue = [...uncached];
-  const workers = Array.from(
-    { length: Math.min(ctx2.cacheConcurrency, queue.length) },
-    async () => {
-      while (queue.length > 0) {
-        const url = queue.shift();
-        const cacheKey = cacheKeys.source(url);
-        const startTime = Date.now();
-        try {
-          let fetchPromise = inFlightFetches.get(cacheKey);
-          if (!fetchPromise) {
-            fetchPromise = fetchAndCache(url, cacheKey, sendEvent, ctx2);
-            inFlightFetches.set(cacheKey, fetchPromise);
-          }
-          await fetchPromise;
-          inFlightFetches.delete(cacheKey);
-          cached++;
-          sendEvent("progress", {
-            url,
-            status: "cached",
-            cached,
-            failed,
-            total,
-            ms: Date.now() - startTime
-          });
-        } catch (error) {
-          inFlightFetches.delete(cacheKey);
-          failed++;
-          sendEvent("progress", {
-            url,
-            status: "error",
-            error: String(error),
-            cached,
-            failed,
-            total,
-            ms: Date.now() - startTime
-          });
-        }
-      }
-    }
-  );
-  await Promise.all(workers);
-  clearInterval(keepalive);
-  sendEvent("summary", { cached, failed, total });
-}
-async function fetchAndCache(url, cacheKey, sendEvent, ctx2) {
-  const response = await ffsFetch(url, {
-    headersTimeout: 10 * 60 * 1e3,
-    // 10 minutes
-    bodyTimeout: 20 * 60 * 1e3
-    // 20 minutes
-  });
-  if (!response.ok) {
-    throw new Error(`${response.status} ${response.statusText}`);
-  }
-  sendEvent("downloading", { url, status: "started", bytesReceived: 0 });
-  const sourceStream = Readable.fromWeb(
-    response.body
-  );
-  let totalBytes = 0;
-  let lastEventTime = Date.now();
-  const PROGRESS_INTERVAL = 1e4;
-  const progressStream = new Transform({
-    transform(chunk, _encoding, callback) {
-      totalBytes += chunk.length;
-      const now = Date.now();
-      if (now - lastEventTime >= PROGRESS_INTERVAL) {
-        sendEvent("downloading", {
-          url,
-          status: "downloading",
-          bytesReceived: totalBytes
-        });
-        lastEventTime = now;
-      }
-      callback(null, chunk);
-    }
-  });
-  const trackedStream = sourceStream.pipe(progressStream);
-  await ctx2.cacheStorage.put(cacheKey, trackedStream);
-}
+// src/handlers/orchestrating.ts
+import "express";
+import { randomUUID as randomUUID2 } from "crypto";
+import { extractEffieSourcesWithTypes, effieDataSchema as effieDataSchema3 } from "@effing/effie";
 
 // src/handlers/rendering.ts
 import "express";
-import { randomUUID as randomUUID2 } from "crypto";
+import { randomUUID } from "crypto";
 
 // src/render.ts
 import { Readable as Readable2 } from "stream";
@@ -815,14 +790,14 @@ var FFmpegRunner = class {
   constructor(command) {
     this.command = command;
   }
-  async run(sourceResolver, imageTransformer) {
+  async run(sourceFetcher, imageTransformer, referenceResolver, urlTransformer) {
     const tempDir = await fs2.mkdtemp(path2.join(os2.tmpdir(), "ffs-"));
     const fileMapping = /* @__PURE__ */ new Map();
-    const sourceCache = /* @__PURE__ */ new Map();
-    const fetchAndSaveSource = async (input, inputName) => {
-      const stream = await sourceResolver({
+    const fetchCache = /* @__PURE__ */ new Map();
+    const fetchAndSaveSource = async (input, sourceUrl, inputName) => {
+      const stream = await sourceFetcher({
         type: input.type,
-        src: input.source
+        src: sourceUrl
       });
       if (input.type === "animation") {
         const extractionDir = path2.join(tempDir, inputName);
@@ -864,17 +839,27 @@ var FFmpegRunner = class {
       this.command.inputs.map(async (input) => {
         if (input.type === "color") return;
         const inputName = `ffmpeg_input_${input.index.toString().padStart(3, "0")}`;
+        const sourceUrl = referenceResolver ? referenceResolver(input.source) : input.source;
+        if ((input.type === "video" || input.type === "audio") && (sourceUrl.startsWith("http://") || sourceUrl.startsWith("https://"))) {
+          const finalUrl = urlTransformer ? urlTransformer(sourceUrl) : sourceUrl;
+          fileMapping.set(input.index, finalUrl);
+          return;
+        }
         const shouldCache = input.source.startsWith("#");
         if (shouldCache) {
-          let fetchPromise = sourceCache.get(input.source);
+          let fetchPromise = fetchCache.get(input.source);
           if (!fetchPromise) {
-            fetchPromise = fetchAndSaveSource(input, inputName);
-            sourceCache.set(input.source, fetchPromise);
+            fetchPromise = fetchAndSaveSource(input, sourceUrl, inputName);
+            fetchCache.set(input.source, fetchPromise);
           }
           const filePath = await fetchPromise;
           fileMapping.set(input.index, filePath);
         } else {
-          const filePath = await fetchAndSaveSource(input, inputName);
+          const filePath = await fetchAndSaveSource(
+            input,
+            sourceUrl,
+            inputName
+          );
           fileMapping.set(input.index, filePath);
         }
       })
@@ -969,20 +954,15 @@ var EffieRenderer = class {
   effieData;
   ffmpegRunner;
   allowLocalFiles;
-  cacheStorage;
+  transientStore;
+  httpProxy;
   constructor(effieData, options) {
     this.effieData = effieData;
     this.allowLocalFiles = options?.allowLocalFiles ?? false;
-    this.cacheStorage = options?.cacheStorage;
+    this.transientStore = options?.transientStore;
+    this.httpProxy = options?.httpProxy;
   }
   async fetchSource(src) {
-    if (src.startsWith("#")) {
-      const sourceName = src.slice(1);
-      if (!(sourceName in this.effieData.sources)) {
-        throw new Error(`Named source "${sourceName}" not found`);
-      }
-      src = this.effieData.sources[sourceName];
-    }
     if (src.startsWith("data:")) {
       const commaIndex = src.indexOf(",");
       if (commaIndex === -1) {
@@ -1002,9 +982,9 @@ var EffieRenderer = class {
       }
       return createReadStream2(fileURLToPath(src));
     }
-    if (this.cacheStorage) {
-      const cachedStream = await this.cacheStorage.getStream(
-        cacheKeys.source(src)
+    if (this.transientStore) {
+      const cachedStream = await this.transientStore.getStream(
+        storeKeys.source(src)
       );
       if (cachedStream) {
         return cachedStream;
@@ -1276,7 +1256,13 @@ var EffieRenderer = class {
         segmentBgInputIndices.push(null);
       }
     }
-    for (const segment of this.effieData.segments) {
+    const globalBgSegmentIndices = [];
+    for (let i = 0; i < this.effieData.segments.length; i++) {
+      if (segmentBgInputIndices[i] === null) {
+        globalBgSegmentIndices.push(i);
+      }
+    }
+    for (const segment of this.effieData.segments) {
       for (const layer of segment.layers) {
         inputs.push(this.buildLayerInput(layer, segment.duration, inputIndex));
         inputIndex++;
@@ -1312,6 +1298,26 @@ var EffieRenderer = class {
     const filterParts = [];
     const videoSegmentLabels = [];
     const audioSegmentLabels = [];
+    const globalBgFifoLabels = /* @__PURE__ */ new Map();
+    const bgFilter = `fps=${this.effieData.fps},scale=${frameWidth}x${frameHeight}:force_original_aspect_ratio=increase,crop=${frameWidth}:${frameHeight}`;
+    if (globalBgSegmentIndices.length === 1) {
+      const fifoLabel = `bg_fifo_0`;
+      filterParts.push(`[${globalBgInputIdx}:v]${bgFilter},fifo[${fifoLabel}]`);
+      globalBgFifoLabels.set(globalBgSegmentIndices[0], fifoLabel);
+    } else if (globalBgSegmentIndices.length > 1) {
+      const splitCount = globalBgSegmentIndices.length;
+      const splitOutputLabels = globalBgSegmentIndices.map(
+        (_, i) => `bg_split_${i}`
+      );
+      filterParts.push(
+        `[${globalBgInputIdx}:v]${bgFilter},split=${splitCount}${splitOutputLabels.map((l) => `[${l}]`).join("")}`
+      );
+      for (let i = 0; i < splitCount; i++) {
+        const fifoLabel = `bg_fifo_${i}`;
+        filterParts.push(`[${splitOutputLabels[i]}]fifo[${fifoLabel}]`);
+        globalBgFifoLabels.set(globalBgSegmentIndices[i], fifoLabel);
+      }
+    }
     for (let segIdx = 0; segIdx < this.effieData.segments.length; segIdx++) {
       const segment = this.effieData.segments[segIdx];
       const bgLabel = `bg_seg${segIdx}`;
@@ -1322,9 +1328,12 @@ var EffieRenderer = class {
           `[${segBgInputIdx}:v]fps=${this.effieData.fps},scale=${frameWidth}x${frameHeight},trim=start=${segBgSeek}:duration=${segment.duration},setpts=PTS-STARTPTS[${bgLabel}]`
         );
       } else {
-        filterParts.push(
-          `[${globalBgInputIdx}:v]fps=${this.effieData.fps},scale=${frameWidth}x${frameHeight},trim=start=${backgroundSeek + currentTime}:duration=${segment.duration},setpts=PTS-STARTPTS[${bgLabel}]`
-        );
+        const fifoLabel = globalBgFifoLabels.get(segIdx);
+        if (fifoLabel) {
+          filterParts.push(
+            `[${fifoLabel}]trim=start=${backgroundSeek + currentTime}:duration=${segment.duration},setpts=PTS-STARTPTS[${bgLabel}]`
+          );
+        }
       }
       const vidLabel = `vid_seg${segIdx}`;
       filterParts.push(
@@ -1408,6 +1417,20 @@ var EffieRenderer = class {
       }
     };
   }
+  /**
+   * Resolves a source reference to its actual URL.
+   * If the source is a #reference, returns the resolved URL.
+   * Otherwise, returns the source as-is.
+   */
+  resolveReference(src) {
+    if (src.startsWith("#")) {
+      const sourceName = src.slice(1);
+      if (sourceName in this.effieData.sources) {
+        return this.effieData.sources[sourceName];
+      }
+    }
+    return src;
+  }
   /**
    * Renders the effie data to a video stream.
    * @param scaleFactor - Scale factor for output dimensions
@@ -1415,9 +1438,12 @@ var EffieRenderer = class {
   async render(scaleFactor = 1) {
     const ffmpegCommand = this.buildFFmpegCommand("-", scaleFactor);
     this.ffmpegRunner = new FFmpegRunner(ffmpegCommand);
+    const urlTransformer = this.httpProxy ? (url) => this.httpProxy.transformUrl(url) : void 0;
     return this.ffmpegRunner.run(
       async ({ src }) => this.fetchSource(src),
-      this.createImageTransformer(scaleFactor)
+      this.createImageTransformer(scaleFactor),
+      (src) => this.resolveReference(src),
+      urlTransformer
     );
   }
   close() {
@@ -1476,14 +1502,18 @@ async function createRenderJob(req, res, ctx2) {
       }
       effie = data;
     }
-    const jobId = randomUUID2();
+    const jobId = randomUUID();
     const job = {
       effie,
       scale,
       upload,
       createdAt: Date.now()
     };
-    await ctx2.cacheStorage.putJson(cacheKeys.renderJob(jobId), job);
+    await ctx2.transientStore.putJson(
+      storeKeys.renderJob(jobId),
+      job,
+      ctx2.transientStore.jobMetadataTtlMs
+    );
     res.json({
       id: jobId,
       url: `${ctx2.baseUrl}/render/${jobId}`
@@ -1497,9 +1527,13 @@ async function streamRenderJob(req, res, ctx2) {
   try {
     setupCORSHeaders(res);
     const jobId = req.params.id;
-    const jobCacheKey = cacheKeys.renderJob(jobId);
-    const job = await ctx2.cacheStorage.getJson(jobCacheKey);
-    ctx2.cacheStorage.delete(jobCacheKey);
+    if (ctx2.renderBackendBaseUrl) {
+      await proxyRenderFromBackend(res, jobId, ctx2);
+      return;
+    }
+    const jobCacheKey = storeKeys.renderJob(jobId);
+    const job = await ctx2.transientStore.getJson(jobCacheKey);
+    ctx2.transientStore.delete(jobCacheKey);
     if (!job) {
       res.status(404).json({ error: "Job not found or expired" });
       return;
@@ -1520,7 +1554,8 @@ async function streamRenderJob(req, res, ctx2) {
 }
 async function streamRenderDirect(res, job, ctx2) {
   const renderer = new EffieRenderer(job.effie, {
-    cacheStorage: ctx2.cacheStorage
+    transientStore: ctx2.transientStore,
+    httpProxy: ctx2.httpProxy
   });
   const videoStream = await renderer.render(job.scale);
   res.on("close", () => {
@@ -1582,7 +1617,10 @@ async function renderAndUploadInternal(effie, scale, upload, sendEvent, ctx2) {
     timings.uploadCoverTime = Date.now() - uploadCoverStartTime;
   }
   const renderStartTime = Date.now();
-  const renderer = new EffieRenderer(effie, { cacheStorage: ctx2.cacheStorage });
+  const renderer = new EffieRenderer(effie, {
+    transientStore: ctx2.transientStore,
+    httpProxy: ctx2.httpProxy
+  });
   const videoStream = await renderer.render(scale);
   const chunks = [];
   for await (const chunk of videoStream) {
@@ -1608,11 +1646,534 @@ async function renderAndUploadInternal(effie, scale, upload, sendEvent, ctx2) {
   timings.uploadTime = Date.now() - uploadStartTime;
   return timings;
 }
+async function proxyRenderFromBackend(res, jobId, ctx2) {
+  const backendUrl = `${ctx2.renderBackendBaseUrl}/render/${jobId}`;
+  const response = await ffsFetch(backendUrl);
+  if (!response.ok) {
+    res.status(response.status).json({ error: "Backend render failed" });
+    return;
+  }
+  const contentType = response.headers.get("content-type") || "";
+  if (contentType.includes("text/event-stream")) {
+    setupSSEResponse(res);
+    const sendEvent = createSSEEventSender(res);
+    const reader = response.body?.getReader();
+    if (!reader) {
+      sendEvent("error", { message: "No response body from backend" });
+      res.end();
+      return;
+    }
+    const decoder = new TextDecoder();
+    let buffer = "";
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        if (res.destroyed) {
+          reader.cancel();
+          break;
+        }
+        buffer += decoder.decode(value, { stream: true });
+        const lines = buffer.split("\n");
+        buffer = lines.pop() || "";
+        let currentEvent = "";
+        let currentData = "";
+        for (const line of lines) {
+          if (line.startsWith("event: ")) {
+            currentEvent = line.slice(7);
+          } else if (line.startsWith("data: ")) {
+            currentData = line.slice(6);
+          } else if (line === "" && currentEvent && currentData) {
+            try {
+              const data = JSON.parse(currentData);
+              sendEvent(currentEvent, data);
+            } catch {
+            }
+            currentEvent = "";
+            currentData = "";
+          }
+        }
+      }
+    } finally {
+      reader.releaseLock();
+      res.end();
+    }
+  } else {
+    await proxyBinaryStream(response, res);
+  }
+}
+
+// src/handlers/orchestrating.ts
+async function createWarmupAndRenderJob(req, res, ctx2) {
+  try {
+    const options = req.body;
+    let rawEffieData;
+    if (typeof options.effie === "string") {
+      const response = await ffsFetch(options.effie);
+      if (!response.ok) {
+        throw new Error(
+          `Failed to fetch Effie data: ${response.status} ${response.statusText}`
+        );
+      }
+      rawEffieData = await response.json();
+    } else {
+      rawEffieData = options.effie;
+    }
+    let effie;
+    if (!ctx2.skipValidation) {
+      const result = effieDataSchema3.safeParse(rawEffieData);
+      if (!result.success) {
+        res.status(400).json({
+          error: "Invalid effie data",
+          issues: result.error.issues.map((issue) => ({
+            path: issue.path.join("."),
+            message: issue.message
+          }))
+        });
+        return;
+      }
+      effie = result.data;
+    } else {
+      const data = rawEffieData;
+      if (!data?.segments) {
+        res.status(400).json({ error: "Invalid effie data: missing segments" });
+        return;
+      }
+      effie = data;
+    }
+    const sources = extractEffieSourcesWithTypes(effie);
+    const scale = options.scale ?? 1;
+    const upload = options.upload;
+    const jobId = randomUUID2();
+    const warmupJobId = randomUUID2();
+    const renderJobId = randomUUID2();
+    const job = {
+      effie,
+      sources,
+      scale,
+      upload,
+      warmupJobId,
+      renderJobId,
+      createdAt: Date.now()
+    };
+    await ctx2.transientStore.putJson(
+      storeKeys.warmupAndRenderJob(jobId),
+      job,
+      ctx2.transientStore.jobMetadataTtlMs
+    );
+    await ctx2.transientStore.putJson(
+      storeKeys.warmupJob(warmupJobId),
+      { sources },
+      ctx2.transientStore.jobMetadataTtlMs
+    );
+    await ctx2.transientStore.putJson(
+      storeKeys.renderJob(renderJobId),
+      {
+        effie,
+        scale,
+        upload,
+        createdAt: Date.now()
+      },
+      ctx2.transientStore.jobMetadataTtlMs
+    );
+    res.json({
+      id: jobId,
+      url: `${ctx2.baseUrl}/warmup-and-render/${jobId}`
+    });
+  } catch (error) {
+    console.error("Error creating warmup-and-render job:", error);
+    res.status(500).json({ error: "Failed to create warmup-and-render job" });
+  }
+}
+async function streamWarmupAndRenderJob(req, res, ctx2) {
+  try {
+    setupCORSHeaders(res);
+    const jobId = req.params.id;
+    const jobCacheKey = storeKeys.warmupAndRenderJob(jobId);
+    const job = await ctx2.transientStore.getJson(jobCacheKey);
+    ctx2.transientStore.delete(jobCacheKey);
+    if (!job) {
+      res.status(404).json({ error: "Job not found" });
+      return;
+    }
+    setupSSEResponse(res);
+    const sendEvent = createSSEEventSender(res);
+    let keepalivePhase = "warmup";
+    const keepalive = setInterval(() => {
+      sendEvent("keepalive", { phase: keepalivePhase });
+    }, 25e3);
+    try {
+      if (ctx2.warmupBackendBaseUrl) {
+        await proxyRemoteSSE(
+          `${ctx2.warmupBackendBaseUrl}/warmup/${job.warmupJobId}`,
+          sendEvent,
+          "warmup:",
+          res
+        );
+      } else {
+        const warmupSender = prefixEventSender(sendEvent, "warmup:");
+        await warmupSources(job.sources, warmupSender, ctx2);
+        warmupSender("complete", { status: "ready" });
+      }
+      keepalivePhase = "render";
+      if (ctx2.renderBackendBaseUrl) {
+        await proxyRemoteSSE(
+          `${ctx2.renderBackendBaseUrl}/render/${job.renderJobId}`,
+          sendEvent,
+          "render:",
+          res
+        );
+      } else {
+        const renderSender = prefixEventSender(sendEvent, "render:");
+        if (job.upload) {
+          renderSender("started", { status: "rendering" });
+          const timings = await renderAndUploadInternal(
+            job.effie,
+            job.scale,
+            job.upload,
+            renderSender,
+            ctx2
+          );
+          renderSender("complete", { status: "uploaded", timings });
+        } else {
+          const videoUrl = `${ctx2.baseUrl}/render/${job.renderJobId}`;
+          sendEvent("complete", { status: "ready", videoUrl });
+        }
+      }
+      if (job.upload && !ctx2.renderBackendBaseUrl) {
+        sendEvent("complete", { status: "done" });
+      }
+    } catch (error) {
+      sendEvent("error", {
+        phase: keepalivePhase,
+        message: String(error)
+      });
+    } finally {
+      clearInterval(keepalive);
+      res.end();
+    }
+  } catch (error) {
+    console.error("Error in warmup-and-render streaming:", error);
+    if (!res.headersSent) {
+      res.status(500).json({ error: "Warmup-and-render streaming failed" });
+    } else {
+      res.end();
+    }
+  }
+}
+function prefixEventSender(sendEvent, prefix) {
+  return (event, data) => {
+    sendEvent(`${prefix}${event}`, data);
+  };
+}
+async function proxyRemoteSSE(url, sendEvent, prefix, res) {
+  const response = await ffsFetch(url, {
+    headers: {
+      Accept: "text/event-stream"
+    }
+  });
+  if (!response.ok) {
+    throw new Error(`Remote backend error: ${response.status}`);
+  }
+  const reader = response.body?.getReader();
+  if (!reader) {
+    throw new Error("No response body from remote backend");
+  }
+  const decoder = new TextDecoder();
+  let buffer = "";
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      if (res.destroyed) {
+        reader.cancel();
+        break;
+      }
+      buffer += decoder.decode(value, { stream: true });
+      const lines = buffer.split("\n");
+      buffer = lines.pop() || "";
+      let currentEvent = "";
+      let currentData = "";
+      for (const line of lines) {
+        if (line.startsWith("event: ")) {
+          currentEvent = line.slice(7);
+        } else if (line.startsWith("data: ")) {
+          currentData = line.slice(6);
+        } else if (line === "" && currentEvent && currentData) {
+          try {
+            const data = JSON.parse(currentData);
+            sendEvent(`${prefix}${currentEvent}`, data);
+          } catch {
+          }
+          currentEvent = "";
+          currentData = "";
+        }
+      }
+    }
+  } finally {
+    reader.releaseLock();
+  }
+}
+async function proxyBinaryStream(response, res) {
+  const contentType = response.headers.get("content-type");
+  if (contentType) res.set("Content-Type", contentType);
+  const contentLength = response.headers.get("content-length");
+  if (contentLength) res.set("Content-Length", contentLength);
+  const reader = response.body?.getReader();
+  if (!reader) {
+    throw new Error("No response body");
+  }
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      if (res.destroyed) {
+        reader.cancel();
+        break;
+      }
+      res.write(value);
+    }
+  } finally {
+    reader.releaseLock();
+    res.end();
+  }
+}
+
+// src/handlers/caching.ts
+function shouldSkipWarmup(source) {
+  return source.type === "video" || source.type === "audio";
+}
+var inFlightFetches = /* @__PURE__ */ new Map();
+async function createWarmupJob(req, res, ctx2) {
+  try {
+    const parseResult = parseEffieData(req.body, ctx2.skipValidation);
+    if ("error" in parseResult) {
+      res.status(400).json(parseResult);
+      return;
+    }
+    const sources = extractEffieSourcesWithTypes2(parseResult.effie);
+    const jobId = randomUUID3();
+    await ctx2.transientStore.putJson(
+      storeKeys.warmupJob(jobId),
+      { sources },
+      ctx2.transientStore.jobMetadataTtlMs
+    );
+    res.json({
+      id: jobId,
+      url: `${ctx2.baseUrl}/warmup/${jobId}`
+    });
+  } catch (error) {
+    console.error("Error creating warmup job:", error);
+    res.status(500).json({ error: "Failed to create warmup job" });
+  }
+}
+async function streamWarmupJob(req, res, ctx2) {
+  try {
+    setupCORSHeaders(res);
+    const jobId = req.params.id;
+    if (ctx2.warmupBackendBaseUrl) {
+      setupSSEResponse(res);
+      const sendEvent2 = createSSEEventSender(res);
+      try {
+        await proxyRemoteSSE(
+          `${ctx2.warmupBackendBaseUrl}/warmup/${jobId}`,
+          sendEvent2,
+          "",
+          res
+        );
+      } finally {
+        res.end();
+      }
+      return;
+    }
+    const jobCacheKey = storeKeys.warmupJob(jobId);
+    const job = await ctx2.transientStore.getJson(jobCacheKey);
+    ctx2.transientStore.delete(jobCacheKey);
+    if (!job) {
+      res.status(404).json({ error: "Job not found" });
+      return;
+    }
+    setupSSEResponse(res);
+    const sendEvent = createSSEEventSender(res);
+    try {
+      await warmupSources(job.sources, sendEvent, ctx2);
+      sendEvent("complete", { status: "ready" });
+    } catch (error) {
+      sendEvent("error", { message: String(error) });
+    } finally {
+      res.end();
+    }
+  } catch (error) {
+    console.error("Error in warmup streaming:", error);
+    if (!res.headersSent) {
+      res.status(500).json({ error: "Warmup streaming failed" });
+    } else {
+      res.end();
+    }
+  }
+}
+async function purgeCache(req, res, ctx2) {
+  try {
+    const parseResult = parseEffieData(req.body, ctx2.skipValidation);
+    if ("error" in parseResult) {
+      res.status(400).json(parseResult);
+      return;
+    }
+    const sources = extractEffieSources(parseResult.effie);
+    let purged = 0;
+    for (const url of sources) {
+      const ck = storeKeys.source(url);
+      if (await ctx2.transientStore.exists(ck)) {
+        await ctx2.transientStore.delete(ck);
+        purged++;
+      }
+    }
+    res.json({ purged, total: sources.length });
+  } catch (error) {
+    console.error("Error purging cache:", error);
+    res.status(500).json({ error: "Failed to purge cache" });
+  }
+}
+async function warmupSources(sources, sendEvent, ctx2) {
+  const total = sources.length;
+  sendEvent("start", { total });
+  let cached = 0;
+  let failed = 0;
+  let skipped = 0;
+  const sourcesToCache = [];
+  for (const source of sources) {
+    if (shouldSkipWarmup(source)) {
+      skipped++;
+      sendEvent("progress", {
+        url: source.url,
+        status: "skipped",
+        reason: "http-video-audio-passthrough",
+        cached,
+        failed,
+        skipped,
+        total
+      });
+    } else {
+      sourcesToCache.push(source);
+    }
+  }
+  const sourceCacheKeys = sourcesToCache.map((s) => storeKeys.source(s.url));
+  const existsMap = await ctx2.transientStore.existsMany(sourceCacheKeys);
+  for (let i = 0; i < sourcesToCache.length; i++) {
+    if (existsMap.get(sourceCacheKeys[i])) {
+      cached++;
+      sendEvent("progress", {
+        url: sourcesToCache[i].url,
+        status: "hit",
+        cached,
+        failed,
+        skipped,
+        total
+      });
+    }
+  }
+  const uncached = sourcesToCache.filter(
+    (_, i) => !existsMap.get(sourceCacheKeys[i])
+  );
+  if (uncached.length === 0) {
+    sendEvent("summary", { cached, failed, skipped, total });
+    return;
+  }
+  const keepalive = setInterval(() => {
+    sendEvent("keepalive", { cached, failed, skipped, total });
+  }, 25e3);
+  const queue = [...uncached];
+  const workers = Array.from(
+    { length: Math.min(ctx2.warmupConcurrency, queue.length) },
+    async () => {
+      while (queue.length > 0) {
+        const source = queue.shift();
+        const cacheKey = storeKeys.source(source.url);
+        const startTime = Date.now();
+        try {
+          let fetchPromise = inFlightFetches.get(cacheKey);
+          if (!fetchPromise) {
+            fetchPromise = fetchAndCache(source.url, cacheKey, sendEvent, ctx2);
+            inFlightFetches.set(cacheKey, fetchPromise);
+          }
+          await fetchPromise;
+          inFlightFetches.delete(cacheKey);
+          cached++;
+          sendEvent("progress", {
+            url: source.url,
+            status: "cached",
+            cached,
+            failed,
+            skipped,
+            total,
+            ms: Date.now() - startTime
+          });
+        } catch (error) {
+          inFlightFetches.delete(cacheKey);
+          failed++;
+          sendEvent("progress", {
+            url: source.url,
+            status: "error",
+            error: String(error),
+            cached,
+            failed,
+            skipped,
+            total,
+            ms: Date.now() - startTime
+          });
+        }
+      }
+    }
+  );
+  await Promise.all(workers);
+  clearInterval(keepalive);
+  sendEvent("summary", { cached, failed, skipped, total });
+}
+async function fetchAndCache(url, cacheKey, sendEvent, ctx2) {
+  const response = await ffsFetch(url, {
+    headersTimeout: 10 * 60 * 1e3,
+    // 10 minutes
+    bodyTimeout: 20 * 60 * 1e3
+    // 20 minutes
+  });
+  if (!response.ok) {
+    throw new Error(`${response.status} ${response.statusText}`);
+  }
+  sendEvent("downloading", { url, status: "started", bytesReceived: 0 });
+  const sourceStream = Readable3.fromWeb(
+    response.body
+  );
+  let totalBytes = 0;
+  let lastEventTime = Date.now();
+  const PROGRESS_INTERVAL = 1e4;
+  const progressStream = new Transform({
+    transform(chunk, _encoding, callback) {
+      totalBytes += chunk.length;
+      const now = Date.now();
+      if (now - lastEventTime >= PROGRESS_INTERVAL) {
+        sendEvent("downloading", {
+          url,
+          status: "downloading",
+          bytesReceived: totalBytes
+        });
+        lastEventTime = now;
+      }
+      callback(null, chunk);
+    }
+  });
+  const trackedStream = sourceStream.pipe(progressStream);
+  await ctx2.transientStore.put(
+    cacheKey,
+    trackedStream,
+    ctx2.transientStore.sourceTtlMs
+  );
+}
 
 // src/server.ts
-var app = express4();
+var app = express5();
 app.use(bodyParser.json({ limit: "50mb" }));
-var ctx = createServerContext();
+var ctx = await createServerContext();
+console.log(`FFS HTTP proxy listening on port ${ctx.httpProxy.port}`);
 function validateAuth(req, res) {
   const apiKey = process.env.FFS_API_KEY;
   if (!apiKey) return true;
@@ -1635,15 +2196,24 @@ app.post("/render", (req, res) => {
   if (!validateAuth(req, res)) return;
   createRenderJob(req, res, ctx);
 });
+app.post("/warmup-and-render", (req, res) => {
+  if (!validateAuth(req, res)) return;
+  createWarmupAndRenderJob(req, res, ctx);
+});
 app.get("/warmup/:id", (req, res) => streamWarmupJob(req, res, ctx));
 app.get("/render/:id", (req, res) => streamRenderJob(req, res, ctx));
+app.get(
+  "/warmup-and-render/:id",
+  (req, res) => streamWarmupAndRenderJob(req, res, ctx)
+);
 var port = process.env.FFS_PORT || 2e3;
 var server = app.listen(port, () => {
   console.log(`FFS server listening on port ${port}`);
 });
 function shutdown() {
   console.log("Shutting down FFS server...");
-  ctx.cacheStorage.close();
+  ctx.httpProxy.close();
+  ctx.transientStore.close();
   server.close(() => {
     console.log("FFS server stopped");
     process.exit(0);