@effing/ffs 0.2.0 → 0.3.0

package/dist/server.js

@@ -1,13 +1,13 @@
 #!/usr/bin/env node
 
 // src/server.ts
-import express4 from "express";
+import express5 from "express";
 import bodyParser from "body-parser";
 
 // src/handlers/shared.ts
 import "express";
 
-// src/cache.ts
+// src/storage.ts
 import {
   S3Client,
   PutObjectCommand,
@@ -22,11 +22,14 @@ import { pipeline } from "stream/promises";
 import path from "path";
 import os from "os";
 import crypto from "crypto";
-var S3CacheStorage = class {
+var DEFAULT_SOURCE_TTL_MS = 60 * 60 * 1e3;
+var DEFAULT_JOB_METADATA_TTL_MS = 8 * 60 * 60 * 1e3;
+var S3TransientStore = class {
   client;
   bucket;
   prefix;
-  ttlMs;
+  sourceTtlMs;
+  jobMetadataTtlMs;
   constructor(options) {
     this.client = new S3Client({
       endpoint: options.endpoint,
@@ -39,22 +42,23 @@ var S3CacheStorage = class {
     });
     this.bucket = options.bucket;
     this.prefix = options.prefix ?? "";
-    this.ttlMs = options.ttlMs ?? 60 * 60 * 1e3;
+    this.sourceTtlMs = options.sourceTtlMs ?? DEFAULT_SOURCE_TTL_MS;
+    this.jobMetadataTtlMs = options.jobMetadataTtlMs ?? DEFAULT_JOB_METADATA_TTL_MS;
   }
-  getExpires() {
-    return new Date(Date.now() + this.ttlMs);
+  getExpires(ttlMs) {
+    return new Date(Date.now() + ttlMs);
   }
   getFullKey(key) {
     return `${this.prefix}${key}`;
   }
-  async put(key, stream) {
+  async put(key, stream, ttlMs) {
     const upload = new Upload({
       client: this.client,
       params: {
         Bucket: this.bucket,
         Key: this.getFullKey(key),
         Body: stream,
-        Expires: this.getExpires()
+        Expires: this.getExpires(ttlMs ?? this.sourceTtlMs)
       }
     });
     await upload.done();
@@ -115,14 +119,14 @@ var S3CacheStorage = class {
       throw err;
     }
   }
-  async putJson(key, data) {
+  async putJson(key, data, ttlMs) {
     await this.client.send(
       new PutObjectCommand({
         Bucket: this.bucket,
         Key: this.getFullKey(key),
         Body: JSON.stringify(data),
         ContentType: "application/json",
-        Expires: this.getExpires()
+        Expires: this.getExpires(ttlMs ?? this.jobMetadataTtlMs)
       })
     );
   }
@@ -148,20 +152,25 @@ var S3CacheStorage = class {
   close() {
   }
 };
-var LocalCacheStorage = class {
+var LocalTransientStore = class {
   baseDir;
   initialized = false;
   cleanupInterval;
-  ttlMs;
-  constructor(baseDir, ttlMs = 60 * 60 * 1e3) {
-    this.baseDir = baseDir ?? path.join(os.tmpdir(), "ffs-cache");
-    this.ttlMs = ttlMs;
+  sourceTtlMs;
+  jobMetadataTtlMs;
+  /** For cleanup, use the longer of the two TTLs */
+  maxTtlMs;
+  constructor(options) {
+    this.baseDir = options?.baseDir ?? path.join(os.tmpdir(), "ffs-transient");
+    this.sourceTtlMs = options?.sourceTtlMs ?? DEFAULT_SOURCE_TTL_MS;
+    this.jobMetadataTtlMs = options?.jobMetadataTtlMs ?? DEFAULT_JOB_METADATA_TTL_MS;
+    this.maxTtlMs = Math.max(this.sourceTtlMs, this.jobMetadataTtlMs);
     this.cleanupInterval = setInterval(() => {
       this.cleanupExpired().catch(console.error);
     }, 3e5);
   }
   /**
-   * Remove files older than TTL
+   * Remove files older than max TTL
    */
   async cleanupExpired() {
     if (!this.initialized) return;
@@ -186,7 +195,7 @@ var LocalCacheStorage = class {
       } else if (entry.isFile()) {
         try {
           const stat = await fs.stat(fullPath);
-          if (now - stat.mtimeMs > this.ttlMs) {
+          if (now - stat.mtimeMs > this.maxTtlMs) {
             await fs.rm(fullPath, { force: true });
           }
         } catch {
@@ -205,7 +214,7 @@ var LocalCacheStorage = class {
     const rand = crypto.randomBytes(8).toString("hex");
     return `${finalPath}.tmp-${process.pid}-${rand}`;
   }
-  async put(key, stream) {
+  async put(key, stream, _ttlMs) {
     const fp = this.filePath(key);
     await this.ensureDir(fp);
     const tmpPath = this.tmpPathFor(fp);
@@ -241,7 +250,7 @@ var LocalCacheStorage = class {
   async delete(key) {
     await fs.rm(this.filePath(key), { force: true });
   }
-  async putJson(key, data) {
+  async putJson(key, data, _ttlMs) {
     const fp = this.filePath(key);
     await this.ensureDir(fp);
     const tmpPath = this.tmpPathFor(fp);
@@ -269,37 +278,47 @@ var LocalCacheStorage = class {
     }
   }
 };
-function createCacheStorage() {
-  const ttlMs = process.env.FFS_CACHE_TTL_MS ? parseInt(process.env.FFS_CACHE_TTL_MS, 10) : 60 * 60 * 1e3;
-  if (process.env.FFS_CACHE_BUCKET) {
-    return new S3CacheStorage({
-      endpoint: process.env.FFS_CACHE_ENDPOINT,
-      region: process.env.FFS_CACHE_REGION ?? "auto",
-      bucket: process.env.FFS_CACHE_BUCKET,
-      prefix: process.env.FFS_CACHE_PREFIX,
-      accessKeyId: process.env.FFS_CACHE_ACCESS_KEY,
-      secretAccessKey: process.env.FFS_CACHE_SECRET_KEY,
-      ttlMs
+function createTransientStore() {
+  const sourceTtlMs = process.env.FFS_SOURCE_CACHE_TTL_MS ? parseInt(process.env.FFS_SOURCE_CACHE_TTL_MS, 10) : DEFAULT_SOURCE_TTL_MS;
+  const jobMetadataTtlMs = process.env.FFS_JOB_METADATA_TTL_MS ? parseInt(process.env.FFS_JOB_METADATA_TTL_MS, 10) : DEFAULT_JOB_METADATA_TTL_MS;
+  if (process.env.FFS_TRANSIENT_STORE_BUCKET) {
+    return new S3TransientStore({
+      endpoint: process.env.FFS_TRANSIENT_STORE_ENDPOINT,
+      region: process.env.FFS_TRANSIENT_STORE_REGION ?? "auto",
+      bucket: process.env.FFS_TRANSIENT_STORE_BUCKET,
+      prefix: process.env.FFS_TRANSIENT_STORE_PREFIX,
+      accessKeyId: process.env.FFS_TRANSIENT_STORE_ACCESS_KEY,
+      secretAccessKey: process.env.FFS_TRANSIENT_STORE_SECRET_KEY,
+      sourceTtlMs,
+      jobMetadataTtlMs
     });
   }
-  return new LocalCacheStorage(process.env.FFS_CACHE_LOCAL_DIR, ttlMs);
+  return new LocalTransientStore({
+    baseDir: process.env.FFS_TRANSIENT_STORE_LOCAL_DIR,
+    sourceTtlMs,
+    jobMetadataTtlMs
+  });
 }
 function hashUrl(url) {
   return crypto.createHash("sha256").update(url).digest("hex").slice(0, 16);
 }
-function sourceCacheKey(url) {
+function sourceStoreKey(url) {
   return `sources/${hashUrl(url)}`;
 }
-function warmupJobCacheKey(jobId) {
+function warmupJobStoreKey(jobId) {
   return `jobs/warmup/${jobId}.json`;
 }
-function renderJobCacheKey(jobId) {
+function renderJobStoreKey(jobId) {
   return `jobs/render/${jobId}.json`;
 }
-var cacheKeys = {
-  source: sourceCacheKey,
-  warmupJob: warmupJobCacheKey,
-  renderJob: renderJobCacheKey
+function warmupAndRenderJobStoreKey(jobId) {
+  return `jobs/warmup-and-render/${jobId}.json`;
+}
+var storeKeys = {
+  source: sourceStoreKey,
+  warmupJob: warmupJobStoreKey,
+  renderJob: renderJobStoreKey,
+  warmupAndRenderJob: warmupAndRenderJobStoreKey
 };
 
 // src/proxy.ts
@@ -454,11 +473,13 @@ async function createServerContext() {
   const httpProxy = new HttpProxy();
   await httpProxy.start();
   return {
-    cacheStorage: createCacheStorage(),
+    transientStore: createTransientStore(),
     httpProxy,
     baseUrl: process.env.FFS_BASE_URL || `http://localhost:${port2}`,
     skipValidation: !!process.env.FFS_SKIP_VALIDATION && process.env.FFS_SKIP_VALIDATION !== "false",
-    cacheConcurrency: parseInt(process.env.FFS_CACHE_CONCURRENCY || "4", 10)
+    warmupConcurrency: parseInt(process.env.FFS_WARMUP_CONCURRENCY || "4", 10),
+    warmupBackendBaseUrl: process.env.FFS_WARMUP_BACKEND_BASE_URL,
+    renderBackendBaseUrl: process.env.FFS_RENDER_BACKEND_BASE_URL
   };
 }
 function parseEffieData(body, skipValidation) {
@@ -505,224 +526,24 @@ data: ${JSON.stringify(data)}
 
 // src/handlers/caching.ts
 import "express";
-import { Readable as Readable2, Transform } from "stream";
-import { randomUUID } from "crypto";
+import { Readable as Readable3, Transform } from "stream";
+import { randomUUID as randomUUID3 } from "crypto";
 import {
   extractEffieSources,
-  extractEffieSourcesWithTypes
+  extractEffieSourcesWithTypes as extractEffieSourcesWithTypes2
 } from "@effing/effie";
-function shouldSkipWarmup(source) {
-  return source.type === "video" || source.type === "audio";
-}
-var inFlightFetches = /* @__PURE__ */ new Map();
-async function createWarmupJob(req, res, ctx2) {
-  try {
-    const parseResult = parseEffieData(req.body, ctx2.skipValidation);
-    if ("error" in parseResult) {
-      res.status(400).json(parseResult);
-      return;
-    }
-    const sources = extractEffieSourcesWithTypes(parseResult.effie);
-    const jobId = randomUUID();
-    await ctx2.cacheStorage.putJson(cacheKeys.warmupJob(jobId), { sources });
-    res.json({
-      id: jobId,
-      url: `${ctx2.baseUrl}/warmup/${jobId}`
-    });
-  } catch (error) {
-    console.error("Error creating warmup job:", error);
-    res.status(500).json({ error: "Failed to create warmup job" });
-  }
-}
-async function streamWarmupJob(req, res, ctx2) {
-  try {
-    setupCORSHeaders(res);
-    const jobId = req.params.id;
-    const jobCacheKey = cacheKeys.warmupJob(jobId);
-    const job = await ctx2.cacheStorage.getJson(jobCacheKey);
-    ctx2.cacheStorage.delete(jobCacheKey);
-    if (!job) {
-      res.status(404).json({ error: "Job not found" });
-      return;
-    }
-    setupSSEResponse(res);
-    const sendEvent = createSSEEventSender(res);
-    try {
-      await warmupSources(job.sources, sendEvent, ctx2);
-      sendEvent("complete", { status: "ready" });
-    } catch (error) {
-      sendEvent("error", { message: String(error) });
-    } finally {
-      res.end();
-    }
-  } catch (error) {
-    console.error("Error in warmup streaming:", error);
-    if (!res.headersSent) {
-      res.status(500).json({ error: "Warmup streaming failed" });
-    } else {
-      res.end();
-    }
-  }
-}
-async function purgeCache(req, res, ctx2) {
-  try {
-    const parseResult = parseEffieData(req.body, ctx2.skipValidation);
-    if ("error" in parseResult) {
-      res.status(400).json(parseResult);
-      return;
-    }
-    const sources = extractEffieSources(parseResult.effie);
-    let purged = 0;
-    for (const url of sources) {
-      const ck = cacheKeys.source(url);
-      if (await ctx2.cacheStorage.exists(ck)) {
-        await ctx2.cacheStorage.delete(ck);
-        purged++;
-      }
-    }
-    res.json({ purged, total: sources.length });
-  } catch (error) {
-    console.error("Error purging cache:", error);
-    res.status(500).json({ error: "Failed to purge cache" });
-  }
-}
-async function warmupSources(sources, sendEvent, ctx2) {
-  const total = sources.length;
-  sendEvent("start", { total });
-  let cached = 0;
-  let failed = 0;
-  let skipped = 0;
-  const sourcesToCache = [];
-  for (const source of sources) {
-    if (shouldSkipWarmup(source)) {
-      skipped++;
-      sendEvent("progress", {
-        url: source.url,
-        status: "skipped",
-        reason: "http-video-audio-passthrough",
-        cached,
-        failed,
-        skipped,
-        total
-      });
-    } else {
-      sourcesToCache.push(source);
-    }
-  }
-  const sourceCacheKeys = sourcesToCache.map((s) => cacheKeys.source(s.url));
-  const existsMap = await ctx2.cacheStorage.existsMany(sourceCacheKeys);
-  for (let i = 0; i < sourcesToCache.length; i++) {
-    if (existsMap.get(sourceCacheKeys[i])) {
-      cached++;
-      sendEvent("progress", {
-        url: sourcesToCache[i].url,
-        status: "hit",
-        cached,
-        failed,
-        skipped,
-        total
-      });
-    }
-  }
-  const uncached = sourcesToCache.filter(
-    (_, i) => !existsMap.get(sourceCacheKeys[i])
-  );
-  if (uncached.length === 0) {
-    sendEvent("summary", { cached, failed, skipped, total });
-    return;
-  }
-  const keepalive = setInterval(() => {
-    sendEvent("keepalive", { cached, failed, skipped, total });
-  }, 25e3);
-  const queue = [...uncached];
-  const workers = Array.from(
-    { length: Math.min(ctx2.cacheConcurrency, queue.length) },
-    async () => {
-      while (queue.length > 0) {
-        const source = queue.shift();
-        const cacheKey = cacheKeys.source(source.url);
-        const startTime = Date.now();
-        try {
-          let fetchPromise = inFlightFetches.get(cacheKey);
-          if (!fetchPromise) {
-            fetchPromise = fetchAndCache(source.url, cacheKey, sendEvent, ctx2);
-            inFlightFetches.set(cacheKey, fetchPromise);
-          }
-          await fetchPromise;
-          inFlightFetches.delete(cacheKey);
-          cached++;
-          sendEvent("progress", {
-            url: source.url,
-            status: "cached",
-            cached,
-            failed,
-            skipped,
-            total,
-            ms: Date.now() - startTime
-          });
-        } catch (error) {
-          inFlightFetches.delete(cacheKey);
-          failed++;
-          sendEvent("progress", {
-            url: source.url,
-            status: "error",
-            error: String(error),
-            cached,
-            failed,
-            skipped,
-            total,
-            ms: Date.now() - startTime
-          });
-        }
-      }
-    }
-  );
-  await Promise.all(workers);
-  clearInterval(keepalive);
-  sendEvent("summary", { cached, failed, skipped, total });
-}
-async function fetchAndCache(url, cacheKey, sendEvent, ctx2) {
-  const response = await ffsFetch(url, {
-    headersTimeout: 10 * 60 * 1e3,
-    // 10 minutes
-    bodyTimeout: 20 * 60 * 1e3
-    // 20 minutes
-  });
-  if (!response.ok) {
-    throw new Error(`${response.status} ${response.statusText}`);
-  }
-  sendEvent("downloading", { url, status: "started", bytesReceived: 0 });
-  const sourceStream = Readable2.fromWeb(
-    response.body
-  );
-  let totalBytes = 0;
-  let lastEventTime = Date.now();
-  const PROGRESS_INTERVAL = 1e4;
-  const progressStream = new Transform({
-    transform(chunk, _encoding, callback) {
-      totalBytes += chunk.length;
-      const now = Date.now();
-      if (now - lastEventTime >= PROGRESS_INTERVAL) {
-        sendEvent("downloading", {
-          url,
-          status: "downloading",
-          bytesReceived: totalBytes
-        });
-        lastEventTime = now;
-      }
-      callback(null, chunk);
-    }
-  });
-  const trackedStream = sourceStream.pipe(progressStream);
-  await ctx2.cacheStorage.put(cacheKey, trackedStream);
-}
 
-// src/handlers/rendering.ts
+// src/handlers/orchestrating.ts
 import "express";
 import { randomUUID as randomUUID2 } from "crypto";
+import { extractEffieSourcesWithTypes, effieDataSchema as effieDataSchema3 } from "@effing/effie";
+
+// src/handlers/rendering.ts
+import "express";
+import { randomUUID } from "crypto";
 
 // src/render.ts
-import { Readable as Readable3 } from "stream";
+import { Readable as Readable2 } from "stream";
 import { createReadStream as createReadStream2 } from "fs";
 
 // src/motion.ts
@@ -1133,12 +954,12 @@ var EffieRenderer = class {
   effieData;
   ffmpegRunner;
   allowLocalFiles;
-  cacheStorage;
+  transientStore;
   httpProxy;
   constructor(effieData, options) {
     this.effieData = effieData;
     this.allowLocalFiles = options?.allowLocalFiles ?? false;
-    this.cacheStorage = options?.cacheStorage;
+    this.transientStore = options?.transientStore;
     this.httpProxy = options?.httpProxy;
   }
   async fetchSource(src) {
@@ -1151,7 +972,7 @@ var EffieRenderer = class {
       const isBase64 = meta.endsWith(";base64");
       const data = src.slice(commaIndex + 1);
       const buffer = isBase64 ? Buffer.from(data, "base64") : Buffer.from(decodeURIComponent(data));
-      return Readable3.from(buffer);
+      return Readable2.from(buffer);
     }
     if (src.startsWith("file:")) {
       if (!this.allowLocalFiles) {
@@ -1161,9 +982,9 @@ var EffieRenderer = class {
       }
       return createReadStream2(fileURLToPath(src));
     }
-    if (this.cacheStorage) {
-      const cachedStream = await this.cacheStorage.getStream(
-        cacheKeys.source(src)
+    if (this.transientStore) {
+      const cachedStream = await this.transientStore.getStream(
+        storeKeys.source(src)
       );
       if (cachedStream) {
         return cachedStream;
@@ -1183,7 +1004,7 @@ var EffieRenderer = class {
     if (!response.body) {
       throw new Error(`No body for ${src}`);
     }
-    return Readable3.fromWeb(response.body);
+    return Readable2.fromWeb(response.body);
   }
   buildAudioFilter({
     duration,
@@ -1681,14 +1502,18 @@ async function createRenderJob(req, res, ctx2) {
       }
       effie = data;
     }
-    const jobId = randomUUID2();
+    const jobId = randomUUID();
     const job = {
       effie,
       scale,
       upload,
       createdAt: Date.now()
     };
-    await ctx2.cacheStorage.putJson(cacheKeys.renderJob(jobId), job);
+    await ctx2.transientStore.putJson(
+      storeKeys.renderJob(jobId),
+      job,
+      ctx2.transientStore.jobMetadataTtlMs
+    );
     res.json({
       id: jobId,
       url: `${ctx2.baseUrl}/render/${jobId}`
@@ -1702,9 +1527,13 @@ async function streamRenderJob(req, res, ctx2) {
   try {
     setupCORSHeaders(res);
     const jobId = req.params.id;
-    const jobCacheKey = cacheKeys.renderJob(jobId);
-    const job = await ctx2.cacheStorage.getJson(jobCacheKey);
-    ctx2.cacheStorage.delete(jobCacheKey);
+    if (ctx2.renderBackendBaseUrl) {
+      await proxyRenderFromBackend(res, jobId, ctx2);
+      return;
+    }
+    const jobCacheKey = storeKeys.renderJob(jobId);
+    const job = await ctx2.transientStore.getJson(jobCacheKey);
+    ctx2.transientStore.delete(jobCacheKey);
     if (!job) {
       res.status(404).json({ error: "Job not found or expired" });
       return;
@@ -1725,7 +1554,7 @@ async function streamRenderJob(req, res, ctx2) {
 }
 async function streamRenderDirect(res, job, ctx2) {
   const renderer = new EffieRenderer(job.effie, {
-    cacheStorage: ctx2.cacheStorage,
+    transientStore: ctx2.transientStore,
     httpProxy: ctx2.httpProxy
   });
   const videoStream = await renderer.render(job.scale);
@@ -1789,7 +1618,7 @@ async function renderAndUploadInternal(effie, scale, upload, sendEvent, ctx2) {
   }
   const renderStartTime = Date.now();
   const renderer = new EffieRenderer(effie, {
-    cacheStorage: ctx2.cacheStorage,
+    transientStore: ctx2.transientStore,
     httpProxy: ctx2.httpProxy
   });
   const videoStream = await renderer.render(scale);
@@ -1817,36 +1646,566 @@ async function renderAndUploadInternal(effie, scale, upload, sendEvent, ctx2) {
   timings.uploadTime = Date.now() - uploadStartTime;
   return timings;
 }
-
-// src/server.ts
-var app = express4();
-app.use(bodyParser.json({ limit: "50mb" }));
-var ctx = await createServerContext();
-console.log(`FFS HTTP proxy listening on port ${ctx.httpProxy.port}`);
-function validateAuth(req, res) {
-  const apiKey = process.env.FFS_API_KEY;
-  if (!apiKey) return true;
-  const authHeader = req.headers.authorization;
-  if (!authHeader || authHeader !== `Bearer ${apiKey}`) {
-    res.status(401).json({ error: "Unauthorized" });
-    return false;
+async function proxyRenderFromBackend(res, jobId, ctx2) {
+  const backendUrl = `${ctx2.renderBackendBaseUrl}/render/${jobId}`;
+  const response = await ffsFetch(backendUrl);
+  if (!response.ok) {
+    res.status(response.status).json({ error: "Backend render failed" });
+    return;
+  }
+  const contentType = response.headers.get("content-type") || "";
+  if (contentType.includes("text/event-stream")) {
+    setupSSEResponse(res);
+    const sendEvent = createSSEEventSender(res);
+    const reader = response.body?.getReader();
+    if (!reader) {
+      sendEvent("error", { message: "No response body from backend" });
+      res.end();
+      return;
+    }
+    const decoder = new TextDecoder();
+    let buffer = "";
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        if (res.destroyed) {
+          reader.cancel();
+          break;
+        }
+        buffer += decoder.decode(value, { stream: true });
+        const lines = buffer.split("\n");
+        buffer = lines.pop() || "";
+        let currentEvent = "";
+        let currentData = "";
+        for (const line of lines) {
+          if (line.startsWith("event: ")) {
+            currentEvent = line.slice(7);
+          } else if (line.startsWith("data: ")) {
+            currentData = line.slice(6);
+          } else if (line === "" && currentEvent && currentData) {
+            try {
+              const data = JSON.parse(currentData);
+              sendEvent(currentEvent, data);
+            } catch {
+            }
+            currentEvent = "";
+            currentData = "";
+          }
+        }
+      }
+    } finally {
+      reader.releaseLock();
+      res.end();
+    }
+  } else {
+    await proxyBinaryStream(response, res);
   }
-  return true;
 }
-app.post("/warmup", (req, res) => {
-  if (!validateAuth(req, res)) return;
-  createWarmupJob(req, res, ctx);
-});
-app.post("/purge", (req, res) => {
-  if (!validateAuth(req, res)) return;
-  purgeCache(req, res, ctx);
-});
-app.post("/render", (req, res) => {
-  if (!validateAuth(req, res)) return;
-  createRenderJob(req, res, ctx);
+
+// src/handlers/orchestrating.ts
+async function createWarmupAndRenderJob(req, res, ctx2) {
+  try {
+    const options = req.body;
+    let rawEffieData;
+    if (typeof options.effie === "string") {
+      const response = await ffsFetch(options.effie);
+      if (!response.ok) {
+        throw new Error(
+          `Failed to fetch Effie data: ${response.status} ${response.statusText}`
+        );
+      }
+      rawEffieData = await response.json();
+    } else {
+      rawEffieData = options.effie;
+    }
+    let effie;
+    if (!ctx2.skipValidation) {
+      const result = effieDataSchema3.safeParse(rawEffieData);
+      if (!result.success) {
+        res.status(400).json({
+          error: "Invalid effie data",
+          issues: result.error.issues.map((issue) => ({
+            path: issue.path.join("."),
+            message: issue.message
+          }))
+        });
+        return;
+      }
+      effie = result.data;
+    } else {
+      const data = rawEffieData;
+      if (!data?.segments) {
+        res.status(400).json({ error: "Invalid effie data: missing segments" });
+        return;
+      }
+      effie = data;
+    }
+    const sources = extractEffieSourcesWithTypes(effie);
+    const scale = options.scale ?? 1;
+    const upload = options.upload;
+    const jobId = randomUUID2();
+    const warmupJobId = randomUUID2();
+    const renderJobId = randomUUID2();
+    const job = {
+      effie,
+      sources,
+      scale,
+      upload,
+      warmupJobId,
+      renderJobId,
+      createdAt: Date.now()
+    };
+    await ctx2.transientStore.putJson(
+      storeKeys.warmupAndRenderJob(jobId),
+      job,
+      ctx2.transientStore.jobMetadataTtlMs
+    );
+    await ctx2.transientStore.putJson(
+      storeKeys.warmupJob(warmupJobId),
+      { sources },
+      ctx2.transientStore.jobMetadataTtlMs
+    );
+    await ctx2.transientStore.putJson(
+      storeKeys.renderJob(renderJobId),
+      {
+        effie,
+        scale,
+        upload,
+        createdAt: Date.now()
+      },
+      ctx2.transientStore.jobMetadataTtlMs
+    );
+    res.json({
+      id: jobId,
+      url: `${ctx2.baseUrl}/warmup-and-render/${jobId}`
+    });
+  } catch (error) {
+    console.error("Error creating warmup-and-render job:", error);
+    res.status(500).json({ error: "Failed to create warmup-and-render job" });
+  }
+}
+async function streamWarmupAndRenderJob(req, res, ctx2) {
+  try {
+    setupCORSHeaders(res);
+    const jobId = req.params.id;
+    const jobCacheKey = storeKeys.warmupAndRenderJob(jobId);
+    const job = await ctx2.transientStore.getJson(jobCacheKey);
+    ctx2.transientStore.delete(jobCacheKey);
+    if (!job) {
+      res.status(404).json({ error: "Job not found" });
+      return;
+    }
+    setupSSEResponse(res);
+    const sendEvent = createSSEEventSender(res);
+    let keepalivePhase = "warmup";
+    const keepalive = setInterval(() => {
+      sendEvent("keepalive", { phase: keepalivePhase });
+    }, 25e3);
+    try {
+      if (ctx2.warmupBackendBaseUrl) {
+        await proxyRemoteSSE(
+          `${ctx2.warmupBackendBaseUrl}/warmup/${job.warmupJobId}`,
+          sendEvent,
+          "warmup:",
+          res
+        );
+      } else {
+        const warmupSender = prefixEventSender(sendEvent, "warmup:");
+        await warmupSources(job.sources, warmupSender, ctx2);
+        warmupSender("complete", { status: "ready" });
+      }
+      keepalivePhase = "render";
+      if (ctx2.renderBackendBaseUrl) {
+        await proxyRemoteSSE(
+          `${ctx2.renderBackendBaseUrl}/render/${job.renderJobId}`,
+          sendEvent,
+          "render:",
+          res
+        );
+      } else {
+        const renderSender = prefixEventSender(sendEvent, "render:");
+        if (job.upload) {
+          renderSender("started", { status: "rendering" });
+          const timings = await renderAndUploadInternal(
+            job.effie,
+            job.scale,
+            job.upload,
+            renderSender,
+            ctx2
+          );
+          renderSender("complete", { status: "uploaded", timings });
+        } else {
+          const videoUrl = `${ctx2.baseUrl}/render/${job.renderJobId}`;
+          sendEvent("complete", { status: "ready", videoUrl });
+        }
+      }
+      if (job.upload && !ctx2.renderBackendBaseUrl) {
+        sendEvent("complete", { status: "done" });
+      }
+    } catch (error) {
+      sendEvent("error", {
+        phase: keepalivePhase,
+        message: String(error)
+      });
+    } finally {
+      clearInterval(keepalive);
+      res.end();
+    }
+  } catch (error) {
+    console.error("Error in warmup-and-render streaming:", error);
+    if (!res.headersSent) {
+      res.status(500).json({ error: "Warmup-and-render streaming failed" });
+    } else {
+      res.end();
+    }
+  }
+}
+function prefixEventSender(sendEvent, prefix) {
+  return (event, data) => {
+    sendEvent(`${prefix}${event}`, data);
+  };
+}
+async function proxyRemoteSSE(url, sendEvent, prefix, res) {
+  const response = await ffsFetch(url, {
+    headers: {
+      Accept: "text/event-stream"
+    }
+  });
+  if (!response.ok) {
+    throw new Error(`Remote backend error: ${response.status}`);
+  }
+  const reader = response.body?.getReader();
+  if (!reader) {
+    throw new Error("No response body from remote backend");
+  }
+  const decoder = new TextDecoder();
+  let buffer = "";
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      if (res.destroyed) {
+        reader.cancel();
+        break;
+      }
+      buffer += decoder.decode(value, { stream: true });
+      const lines = buffer.split("\n");
+      buffer = lines.pop() || "";
+      let currentEvent = "";
+      let currentData = "";
+      for (const line of lines) {
+        if (line.startsWith("event: ")) {
+          currentEvent = line.slice(7);
+        } else if (line.startsWith("data: ")) {
+          currentData = line.slice(6);
+        } else if (line === "" && currentEvent && currentData) {
+          try {
+            const data = JSON.parse(currentData);
+            sendEvent(`${prefix}${currentEvent}`, data);
+          } catch {
+          }
+          currentEvent = "";
+          currentData = "";
+        }
+      }
+    }
+  } finally {
+    reader.releaseLock();
+  }
+}
+async function proxyBinaryStream(response, res) {
+  const contentType = response.headers.get("content-type");
+  if (contentType) res.set("Content-Type", contentType);
+  const contentLength = response.headers.get("content-length");
+  if (contentLength) res.set("Content-Length", contentLength);
+  const reader = response.body?.getReader();
+  if (!reader) {
+    throw new Error("No response body");
+  }
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      if (res.destroyed) {
+        reader.cancel();
+        break;
+      }
+      res.write(value);
+    }
+  } finally {
+    reader.releaseLock();
+    res.end();
+  }
+}
+
+// src/handlers/caching.ts
+function shouldSkipWarmup(source) {
+  return source.type === "video" || source.type === "audio";
+}
+var inFlightFetches = /* @__PURE__ */ new Map();
+async function createWarmupJob(req, res, ctx2) {
+  try {
+    const parseResult = parseEffieData(req.body, ctx2.skipValidation);
+    if ("error" in parseResult) {
+      res.status(400).json(parseResult);
+      return;
+    }
+    const sources = extractEffieSourcesWithTypes2(parseResult.effie);
+    const jobId = randomUUID3();
+    await ctx2.transientStore.putJson(
+      storeKeys.warmupJob(jobId),
+      { sources },
+      ctx2.transientStore.jobMetadataTtlMs
+    );
+    res.json({
+      id: jobId,
+      url: `${ctx2.baseUrl}/warmup/${jobId}`
+    });
+  } catch (error) {
+    console.error("Error creating warmup job:", error);
+    res.status(500).json({ error: "Failed to create warmup job" });
+  }
+}
+async function streamWarmupJob(req, res, ctx2) {
+  try {
+    setupCORSHeaders(res);
+    const jobId = req.params.id;
+    if (ctx2.warmupBackendBaseUrl) {
+      setupSSEResponse(res);
+      const sendEvent2 = createSSEEventSender(res);
+      try {
+        await proxyRemoteSSE(
+          `${ctx2.warmupBackendBaseUrl}/warmup/${jobId}`,
+          sendEvent2,
+          "",
+          res
+        );
+      } finally {
+        res.end();
+      }
+      return;
+    }
+    const jobCacheKey = storeKeys.warmupJob(jobId);
+    const job = await ctx2.transientStore.getJson(jobCacheKey);
+    ctx2.transientStore.delete(jobCacheKey);
+    if (!job) {
+      res.status(404).json({ error: "Job not found" });
+      return;
+    }
+    setupSSEResponse(res);
+    const sendEvent = createSSEEventSender(res);
+    try {
+      await warmupSources(job.sources, sendEvent, ctx2);
+      sendEvent("complete", { status: "ready" });
+    } catch (error) {
+      sendEvent("error", { message: String(error) });
+    } finally {
+      res.end();
+    }
+  } catch (error) {
+    console.error("Error in warmup streaming:", error);
+    if (!res.headersSent) {
+      res.status(500).json({ error: "Warmup streaming failed" });
+    } else {
+      res.end();
+    }
+  }
+}
+async function purgeCache(req, res, ctx2) {
+  try {
+    const parseResult = parseEffieData(req.body, ctx2.skipValidation);
+    if ("error" in parseResult) {
+      res.status(400).json(parseResult);
+      return;
+    }
+    const sources = extractEffieSources(parseResult.effie);
+    let purged = 0;
+    for (const url of sources) {
+      const ck = storeKeys.source(url);
+      if (await ctx2.transientStore.exists(ck)) {
+        await ctx2.transientStore.delete(ck);
+        purged++;
+      }
+    }
+    res.json({ purged, total: sources.length });
+  } catch (error) {
+    console.error("Error purging cache:", error);
+    res.status(500).json({ error: "Failed to purge cache" });
+  }
+}
+async function warmupSources(sources, sendEvent, ctx2) {
+  const total = sources.length;
+  sendEvent("start", { total });
+  let cached = 0;
+  let failed = 0;
+  let skipped = 0;
+  const sourcesToCache = [];
+  for (const source of sources) {
+    if (shouldSkipWarmup(source)) {
+      skipped++;
+      sendEvent("progress", {
+        url: source.url,
+        status: "skipped",
+        reason: "http-video-audio-passthrough",
+        cached,
+        failed,
+        skipped,
+        total
+      });
+    } else {
+      sourcesToCache.push(source);
+    }
+  }
+  const sourceCacheKeys = sourcesToCache.map((s) => storeKeys.source(s.url));
+  const existsMap = await ctx2.transientStore.existsMany(sourceCacheKeys);
+  for (let i = 0; i < sourcesToCache.length; i++) {
+    if (existsMap.get(sourceCacheKeys[i])) {
+      cached++;
+      sendEvent("progress", {
+        url: sourcesToCache[i].url,
+        status: "hit",
+        cached,
+        failed,
+        skipped,
+        total
+      });
+    }
+  }
+  const uncached = sourcesToCache.filter(
+    (_, i) => !existsMap.get(sourceCacheKeys[i])
+  );
+  if (uncached.length === 0) {
+    sendEvent("summary", { cached, failed, skipped, total });
+    return;
+  }
+  const keepalive = setInterval(() => {
+    sendEvent("keepalive", { cached, failed, skipped, total });
+  }, 25e3);
+  const queue = [...uncached];
+  const workers = Array.from(
+    { length: Math.min(ctx2.warmupConcurrency, queue.length) },
+    async () => {
+      while (queue.length > 0) {
+        const source = queue.shift();
+        const cacheKey = storeKeys.source(source.url);
+        const startTime = Date.now();
+        try {
+          let fetchPromise = inFlightFetches.get(cacheKey);
+          if (!fetchPromise) {
+            fetchPromise = fetchAndCache(source.url, cacheKey, sendEvent, ctx2);
+            inFlightFetches.set(cacheKey, fetchPromise);
+          }
+          await fetchPromise;
+          inFlightFetches.delete(cacheKey);
+          cached++;
+          sendEvent("progress", {
+            url: source.url,
+            status: "cached",
+            cached,
+            failed,
+            skipped,
+            total,
+            ms: Date.now() - startTime
+          });
+        } catch (error) {
+          inFlightFetches.delete(cacheKey);
+          failed++;
+          sendEvent("progress", {
+            url: source.url,
+            status: "error",
+            error: String(error),
+            cached,
+            failed,
+            skipped,
+            total,
+            ms: Date.now() - startTime
+          });
+        }
+      }
+    }
+  );
+  await Promise.all(workers);
+  clearInterval(keepalive);
+  sendEvent("summary", { cached, failed, skipped, total });
+}
+async function fetchAndCache(url, cacheKey, sendEvent, ctx2) {
+  const response = await ffsFetch(url, {
+    headersTimeout: 10 * 60 * 1e3,
+    // 10 minutes
+    bodyTimeout: 20 * 60 * 1e3
+    // 20 minutes
+  });
+  if (!response.ok) {
+    throw new Error(`${response.status} ${response.statusText}`);
+  }
+  sendEvent("downloading", { url, status: "started", bytesReceived: 0 });
+  const sourceStream = Readable3.fromWeb(
+    response.body
+  );
+  let totalBytes = 0;
+  let lastEventTime = Date.now();
+  const PROGRESS_INTERVAL = 1e4;
+  const progressStream = new Transform({
+    transform(chunk, _encoding, callback) {
+      totalBytes += chunk.length;
+      const now = Date.now();
+      if (now - lastEventTime >= PROGRESS_INTERVAL) {
+        sendEvent("downloading", {
+          url,
+          status: "downloading",
+          bytesReceived: totalBytes
+        });
+        lastEventTime = now;
+      }
+      callback(null, chunk);
+    }
+  });
+  const trackedStream = sourceStream.pipe(progressStream);
+  await ctx2.transientStore.put(
+    cacheKey,
+    trackedStream,
+    ctx2.transientStore.sourceTtlMs
+  );
+}
+
+// src/server.ts
+var app = express5();
+app.use(bodyParser.json({ limit: "50mb" }));
+var ctx = await createServerContext();
+console.log(`FFS HTTP proxy listening on port ${ctx.httpProxy.port}`);
+function validateAuth(req, res) {
+  const apiKey = process.env.FFS_API_KEY;
+  if (!apiKey) return true;
+  const authHeader = req.headers.authorization;
+  if (!authHeader || authHeader !== `Bearer ${apiKey}`) {
+    res.status(401).json({ error: "Unauthorized" });
+    return false;
+  }
+  return true;
+}
+app.post("/warmup", (req, res) => {
+  if (!validateAuth(req, res)) return;
+  createWarmupJob(req, res, ctx);
+});
+app.post("/purge", (req, res) => {
+  if (!validateAuth(req, res)) return;
+  purgeCache(req, res, ctx);
+});
+app.post("/render", (req, res) => {
+  if (!validateAuth(req, res)) return;
+  createRenderJob(req, res, ctx);
+});
+app.post("/warmup-and-render", (req, res) => {
+  if (!validateAuth(req, res)) return;
+  createWarmupAndRenderJob(req, res, ctx);
 });
 app.get("/warmup/:id", (req, res) => streamWarmupJob(req, res, ctx));
 app.get("/render/:id", (req, res) => streamRenderJob(req, res, ctx));
+app.get(
+  "/warmup-and-render/:id",
+  (req, res) => streamWarmupAndRenderJob(req, res, ctx)
+);
 var port = process.env.FFS_PORT || 2e3;
 var server = app.listen(port, () => {
   console.log(`FFS server listening on port ${port}`);
@@ -1854,7 +2213,7 @@ var server = app.listen(port, () => {
 function shutdown() {
   console.log("Shutting down FFS server...");
   ctx.httpProxy.close();
-  ctx.cacheStorage.close();
+  ctx.transientStore.close();
   server.close(() => {
     console.log("FFS server stopped");
     process.exit(0);