@effing/ffs 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +140 -16
- package/dist/{chunk-6YHSYHDY.js → chunk-7FMPCMLO.js} +559 -215
- package/dist/chunk-7FMPCMLO.js.map +1 -0
- package/dist/{chunk-A7BAW24L.js → chunk-J64HSZNQ.js} +65 -46
- package/dist/chunk-J64HSZNQ.js.map +1 -0
- package/dist/handlers/index.d.ts +38 -4
- package/dist/handlers/index.js +10 -2
- package/dist/index.d.ts +5 -5
- package/dist/index.js +1 -1
- package/dist/{proxy-BI8OMQl0.d.ts → proxy-qTA69nOV.d.ts} +11 -7
- package/dist/server.js +660 -293
- package/dist/server.js.map +1 -1
- package/package.json +2 -2
- package/dist/chunk-6YHSYHDY.js.map +0 -1
- package/dist/chunk-A7BAW24L.js.map +0 -1
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import {
|
|
2
2
|
EffieRenderer,
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
} from "./chunk-
|
|
3
|
+
createTransientStore,
|
|
4
|
+
ffsFetch,
|
|
5
|
+
storeKeys
|
|
6
|
+
} from "./chunk-J64HSZNQ.js";
|
|
7
7
|
|
|
8
8
|
// src/handlers/shared.ts
|
|
9
9
|
import "express";
|
|
@@ -133,15 +133,19 @@ var HttpProxy = class {
|
|
|
133
133
|
// src/handlers/shared.ts
|
|
134
134
|
import { effieDataSchema } from "@effing/effie";
|
|
135
135
|
async function createServerContext() {
|
|
136
|
-
const port = process.env.FFS_PORT || 2e3;
|
|
136
|
+
const port = process.env.FFS_PORT || process.env.PORT || 2e3;
|
|
137
137
|
const httpProxy = new HttpProxy();
|
|
138
138
|
await httpProxy.start();
|
|
139
139
|
return {
|
|
140
|
-
|
|
140
|
+
transientStore: createTransientStore(),
|
|
141
141
|
httpProxy,
|
|
142
142
|
baseUrl: process.env.FFS_BASE_URL || `http://localhost:${port}`,
|
|
143
143
|
skipValidation: !!process.env.FFS_SKIP_VALIDATION && process.env.FFS_SKIP_VALIDATION !== "false",
|
|
144
|
-
|
|
144
|
+
warmupConcurrency: parseInt(process.env.FFS_WARMUP_CONCURRENCY || "4", 10),
|
|
145
|
+
warmupBackendBaseUrl: process.env.FFS_WARMUP_BACKEND_BASE_URL,
|
|
146
|
+
renderBackendBaseUrl: process.env.FFS_RENDER_BACKEND_BASE_URL,
|
|
147
|
+
warmupBackendApiKey: process.env.FFS_WARMUP_BACKEND_API_KEY,
|
|
148
|
+
renderBackendApiKey: process.env.FFS_RENDER_BACKEND_API_KEY
|
|
145
149
|
};
|
|
146
150
|
}
|
|
147
151
|
function parseEffieData(body, skipValidation) {
|
|
@@ -189,11 +193,511 @@ data: ${JSON.stringify(data)}
|
|
|
189
193
|
// src/handlers/caching.ts
|
|
190
194
|
import "express";
|
|
191
195
|
import { Readable as Readable2, Transform } from "stream";
|
|
192
|
-
import { randomUUID } from "crypto";
|
|
196
|
+
import { randomUUID as randomUUID3 } from "crypto";
|
|
193
197
|
import {
|
|
194
198
|
extractEffieSources,
|
|
195
|
-
extractEffieSourcesWithTypes
|
|
199
|
+
extractEffieSourcesWithTypes as extractEffieSourcesWithTypes2
|
|
196
200
|
} from "@effing/effie";
|
|
201
|
+
|
|
202
|
+
// src/handlers/orchestrating.ts
|
|
203
|
+
import "express";
|
|
204
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
205
|
+
import { extractEffieSourcesWithTypes, effieDataSchema as effieDataSchema3 } from "@effing/effie";
|
|
206
|
+
|
|
207
|
+
// src/handlers/rendering.ts
|
|
208
|
+
import "express";
|
|
209
|
+
import { randomUUID } from "crypto";
|
|
210
|
+
import { effieDataSchema as effieDataSchema2 } from "@effing/effie";
|
|
211
|
+
async function createRenderJob(req, res, ctx) {
|
|
212
|
+
try {
|
|
213
|
+
const isWrapped = "effie" in req.body;
|
|
214
|
+
let rawEffieData;
|
|
215
|
+
let scale;
|
|
216
|
+
let upload;
|
|
217
|
+
if (isWrapped) {
|
|
218
|
+
const options = req.body;
|
|
219
|
+
if (typeof options.effie === "string") {
|
|
220
|
+
const response = await ffsFetch(options.effie);
|
|
221
|
+
if (!response.ok) {
|
|
222
|
+
throw new Error(
|
|
223
|
+
`Failed to fetch Effie data: ${response.status} ${response.statusText}`
|
|
224
|
+
);
|
|
225
|
+
}
|
|
226
|
+
rawEffieData = await response.json();
|
|
227
|
+
} else {
|
|
228
|
+
rawEffieData = options.effie;
|
|
229
|
+
}
|
|
230
|
+
scale = options.scale ?? 1;
|
|
231
|
+
upload = options.upload;
|
|
232
|
+
} else {
|
|
233
|
+
rawEffieData = req.body;
|
|
234
|
+
scale = parseFloat(req.query.scale?.toString() || "1");
|
|
235
|
+
}
|
|
236
|
+
let effie;
|
|
237
|
+
if (!ctx.skipValidation) {
|
|
238
|
+
const result = effieDataSchema2.safeParse(rawEffieData);
|
|
239
|
+
if (!result.success) {
|
|
240
|
+
res.status(400).json({
|
|
241
|
+
error: "Invalid effie data",
|
|
242
|
+
issues: result.error.issues.map((issue) => ({
|
|
243
|
+
path: issue.path.join("."),
|
|
244
|
+
message: issue.message
|
|
245
|
+
}))
|
|
246
|
+
});
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
249
|
+
effie = result.data;
|
|
250
|
+
} else {
|
|
251
|
+
const data = rawEffieData;
|
|
252
|
+
if (!data?.segments) {
|
|
253
|
+
res.status(400).json({ error: "Invalid effie data: missing segments" });
|
|
254
|
+
return;
|
|
255
|
+
}
|
|
256
|
+
effie = data;
|
|
257
|
+
}
|
|
258
|
+
const jobId = randomUUID();
|
|
259
|
+
const job = {
|
|
260
|
+
effie,
|
|
261
|
+
scale,
|
|
262
|
+
upload,
|
|
263
|
+
createdAt: Date.now()
|
|
264
|
+
};
|
|
265
|
+
await ctx.transientStore.putJson(
|
|
266
|
+
storeKeys.renderJob(jobId),
|
|
267
|
+
job,
|
|
268
|
+
ctx.transientStore.jobMetadataTtlMs
|
|
269
|
+
);
|
|
270
|
+
res.json({
|
|
271
|
+
id: jobId,
|
|
272
|
+
url: `${ctx.baseUrl}/render/${jobId}`
|
|
273
|
+
});
|
|
274
|
+
} catch (error) {
|
|
275
|
+
console.error("Error creating render job:", error);
|
|
276
|
+
res.status(500).json({ error: "Failed to create render job" });
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
async function streamRenderJob(req, res, ctx) {
|
|
280
|
+
try {
|
|
281
|
+
setupCORSHeaders(res);
|
|
282
|
+
const jobId = req.params.id;
|
|
283
|
+
if (ctx.renderBackendBaseUrl) {
|
|
284
|
+
await proxyRenderFromBackend(res, jobId, ctx);
|
|
285
|
+
return;
|
|
286
|
+
}
|
|
287
|
+
const jobCacheKey = storeKeys.renderJob(jobId);
|
|
288
|
+
const job = await ctx.transientStore.getJson(jobCacheKey);
|
|
289
|
+
ctx.transientStore.delete(jobCacheKey);
|
|
290
|
+
if (!job) {
|
|
291
|
+
res.status(404).json({ error: "Job not found or expired" });
|
|
292
|
+
return;
|
|
293
|
+
}
|
|
294
|
+
if (job.upload) {
|
|
295
|
+
await streamRenderWithUpload(res, job, ctx);
|
|
296
|
+
} else {
|
|
297
|
+
await streamRenderDirect(res, job, ctx);
|
|
298
|
+
}
|
|
299
|
+
} catch (error) {
|
|
300
|
+
console.error("Error in render:", error);
|
|
301
|
+
if (!res.headersSent) {
|
|
302
|
+
res.status(500).json({ error: "Rendering failed" });
|
|
303
|
+
} else {
|
|
304
|
+
res.end();
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
async function streamRenderDirect(res, job, ctx) {
|
|
309
|
+
const renderer = new EffieRenderer(job.effie, {
|
|
310
|
+
transientStore: ctx.transientStore,
|
|
311
|
+
httpProxy: ctx.httpProxy
|
|
312
|
+
});
|
|
313
|
+
const videoStream = await renderer.render(job.scale);
|
|
314
|
+
res.on("close", () => {
|
|
315
|
+
videoStream.destroy();
|
|
316
|
+
renderer.close();
|
|
317
|
+
});
|
|
318
|
+
res.set("Content-Type", "video/mp4");
|
|
319
|
+
videoStream.pipe(res);
|
|
320
|
+
}
|
|
321
|
+
async function streamRenderWithUpload(res, job, ctx) {
|
|
322
|
+
setupSSEResponse(res);
|
|
323
|
+
const sendEvent = createSSEEventSender(res);
|
|
324
|
+
const keepalive = setInterval(() => {
|
|
325
|
+
sendEvent("keepalive", { status: "rendering" });
|
|
326
|
+
}, 25e3);
|
|
327
|
+
try {
|
|
328
|
+
sendEvent("started", { status: "rendering" });
|
|
329
|
+
const timings = await renderAndUploadInternal(
|
|
330
|
+
job.effie,
|
|
331
|
+
job.scale,
|
|
332
|
+
job.upload,
|
|
333
|
+
sendEvent,
|
|
334
|
+
ctx
|
|
335
|
+
);
|
|
336
|
+
sendEvent("complete", { status: "uploaded", timings });
|
|
337
|
+
} catch (error) {
|
|
338
|
+
sendEvent("error", { message: String(error) });
|
|
339
|
+
} finally {
|
|
340
|
+
clearInterval(keepalive);
|
|
341
|
+
res.end();
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
async function renderAndUploadInternal(effie, scale, upload, sendEvent, ctx) {
|
|
345
|
+
const timings = {};
|
|
346
|
+
if (upload.coverUrl) {
|
|
347
|
+
const fetchCoverStartTime = Date.now();
|
|
348
|
+
const coverFetchResponse = await ffsFetch(effie.cover);
|
|
349
|
+
if (!coverFetchResponse.ok) {
|
|
350
|
+
throw new Error(
|
|
351
|
+
`Failed to fetch cover image: ${coverFetchResponse.status} ${coverFetchResponse.statusText}`
|
|
352
|
+
);
|
|
353
|
+
}
|
|
354
|
+
const coverBuffer = Buffer.from(await coverFetchResponse.arrayBuffer());
|
|
355
|
+
timings.fetchCoverTime = Date.now() - fetchCoverStartTime;
|
|
356
|
+
const uploadCoverStartTime = Date.now();
|
|
357
|
+
const uploadCoverResponse = await ffsFetch(upload.coverUrl, {
|
|
358
|
+
method: "PUT",
|
|
359
|
+
body: coverBuffer,
|
|
360
|
+
headers: {
|
|
361
|
+
"Content-Type": "image/png",
|
|
362
|
+
"Content-Length": coverBuffer.length.toString()
|
|
363
|
+
}
|
|
364
|
+
});
|
|
365
|
+
if (!uploadCoverResponse.ok) {
|
|
366
|
+
throw new Error(
|
|
367
|
+
`Failed to upload cover: ${uploadCoverResponse.status} ${uploadCoverResponse.statusText}`
|
|
368
|
+
);
|
|
369
|
+
}
|
|
370
|
+
timings.uploadCoverTime = Date.now() - uploadCoverStartTime;
|
|
371
|
+
}
|
|
372
|
+
const renderStartTime = Date.now();
|
|
373
|
+
const renderer = new EffieRenderer(effie, {
|
|
374
|
+
transientStore: ctx.transientStore,
|
|
375
|
+
httpProxy: ctx.httpProxy
|
|
376
|
+
});
|
|
377
|
+
const videoStream = await renderer.render(scale);
|
|
378
|
+
const chunks = [];
|
|
379
|
+
for await (const chunk of videoStream) {
|
|
380
|
+
chunks.push(Buffer.from(chunk));
|
|
381
|
+
}
|
|
382
|
+
const videoBuffer = Buffer.concat(chunks);
|
|
383
|
+
timings.renderTime = Date.now() - renderStartTime;
|
|
384
|
+
sendEvent("keepalive", { status: "uploading" });
|
|
385
|
+
const uploadStartTime = Date.now();
|
|
386
|
+
const uploadResponse = await ffsFetch(upload.videoUrl, {
|
|
387
|
+
method: "PUT",
|
|
388
|
+
body: videoBuffer,
|
|
389
|
+
headers: {
|
|
390
|
+
"Content-Type": "video/mp4",
|
|
391
|
+
"Content-Length": videoBuffer.length.toString()
|
|
392
|
+
}
|
|
393
|
+
});
|
|
394
|
+
if (!uploadResponse.ok) {
|
|
395
|
+
throw new Error(
|
|
396
|
+
`Failed to upload rendered video: ${uploadResponse.status} ${uploadResponse.statusText}`
|
|
397
|
+
);
|
|
398
|
+
}
|
|
399
|
+
timings.uploadTime = Date.now() - uploadStartTime;
|
|
400
|
+
return timings;
|
|
401
|
+
}
|
|
402
|
+
async function proxyRenderFromBackend(res, jobId, ctx) {
|
|
403
|
+
const backendUrl = `${ctx.renderBackendBaseUrl}/render/${jobId}`;
|
|
404
|
+
const response = await ffsFetch(backendUrl, {
|
|
405
|
+
headers: ctx.renderBackendApiKey ? { Authorization: `Bearer ${ctx.renderBackendApiKey}` } : void 0
|
|
406
|
+
});
|
|
407
|
+
if (!response.ok) {
|
|
408
|
+
res.status(response.status).json({ error: "Backend render failed" });
|
|
409
|
+
return;
|
|
410
|
+
}
|
|
411
|
+
const contentType = response.headers.get("content-type") || "";
|
|
412
|
+
if (contentType.includes("text/event-stream")) {
|
|
413
|
+
setupSSEResponse(res);
|
|
414
|
+
const sendEvent = createSSEEventSender(res);
|
|
415
|
+
const reader = response.body?.getReader();
|
|
416
|
+
if (!reader) {
|
|
417
|
+
sendEvent("error", { message: "No response body from backend" });
|
|
418
|
+
res.end();
|
|
419
|
+
return;
|
|
420
|
+
}
|
|
421
|
+
const decoder = new TextDecoder();
|
|
422
|
+
let buffer = "";
|
|
423
|
+
try {
|
|
424
|
+
while (true) {
|
|
425
|
+
const { done, value } = await reader.read();
|
|
426
|
+
if (done) break;
|
|
427
|
+
if (res.destroyed) {
|
|
428
|
+
reader.cancel();
|
|
429
|
+
break;
|
|
430
|
+
}
|
|
431
|
+
buffer += decoder.decode(value, { stream: true });
|
|
432
|
+
const lines = buffer.split("\n");
|
|
433
|
+
buffer = lines.pop() || "";
|
|
434
|
+
let currentEvent = "";
|
|
435
|
+
let currentData = "";
|
|
436
|
+
for (const line of lines) {
|
|
437
|
+
if (line.startsWith("event: ")) {
|
|
438
|
+
currentEvent = line.slice(7);
|
|
439
|
+
} else if (line.startsWith("data: ")) {
|
|
440
|
+
currentData = line.slice(6);
|
|
441
|
+
} else if (line === "" && currentEvent && currentData) {
|
|
442
|
+
try {
|
|
443
|
+
const data = JSON.parse(currentData);
|
|
444
|
+
sendEvent(currentEvent, data);
|
|
445
|
+
} catch {
|
|
446
|
+
}
|
|
447
|
+
currentEvent = "";
|
|
448
|
+
currentData = "";
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
} finally {
|
|
453
|
+
reader.releaseLock();
|
|
454
|
+
res.end();
|
|
455
|
+
}
|
|
456
|
+
} else {
|
|
457
|
+
await proxyBinaryStream(response, res);
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
// src/handlers/orchestrating.ts
|
|
462
|
+
async function createWarmupAndRenderJob(req, res, ctx) {
|
|
463
|
+
try {
|
|
464
|
+
const options = req.body;
|
|
465
|
+
let rawEffieData;
|
|
466
|
+
if (typeof options.effie === "string") {
|
|
467
|
+
const response = await ffsFetch(options.effie);
|
|
468
|
+
if (!response.ok) {
|
|
469
|
+
throw new Error(
|
|
470
|
+
`Failed to fetch Effie data: ${response.status} ${response.statusText}`
|
|
471
|
+
);
|
|
472
|
+
}
|
|
473
|
+
rawEffieData = await response.json();
|
|
474
|
+
} else {
|
|
475
|
+
rawEffieData = options.effie;
|
|
476
|
+
}
|
|
477
|
+
let effie;
|
|
478
|
+
if (!ctx.skipValidation) {
|
|
479
|
+
const result = effieDataSchema3.safeParse(rawEffieData);
|
|
480
|
+
if (!result.success) {
|
|
481
|
+
res.status(400).json({
|
|
482
|
+
error: "Invalid effie data",
|
|
483
|
+
issues: result.error.issues.map((issue) => ({
|
|
484
|
+
path: issue.path.join("."),
|
|
485
|
+
message: issue.message
|
|
486
|
+
}))
|
|
487
|
+
});
|
|
488
|
+
return;
|
|
489
|
+
}
|
|
490
|
+
effie = result.data;
|
|
491
|
+
} else {
|
|
492
|
+
const data = rawEffieData;
|
|
493
|
+
if (!data?.segments) {
|
|
494
|
+
res.status(400).json({ error: "Invalid effie data: missing segments" });
|
|
495
|
+
return;
|
|
496
|
+
}
|
|
497
|
+
effie = data;
|
|
498
|
+
}
|
|
499
|
+
const sources = extractEffieSourcesWithTypes(effie);
|
|
500
|
+
const scale = options.scale ?? 1;
|
|
501
|
+
const upload = options.upload;
|
|
502
|
+
const jobId = randomUUID2();
|
|
503
|
+
const warmupJobId = randomUUID2();
|
|
504
|
+
const renderJobId = randomUUID2();
|
|
505
|
+
const job = {
|
|
506
|
+
effie,
|
|
507
|
+
sources,
|
|
508
|
+
scale,
|
|
509
|
+
upload,
|
|
510
|
+
warmupJobId,
|
|
511
|
+
renderJobId,
|
|
512
|
+
createdAt: Date.now()
|
|
513
|
+
};
|
|
514
|
+
await ctx.transientStore.putJson(
|
|
515
|
+
storeKeys.warmupAndRenderJob(jobId),
|
|
516
|
+
job,
|
|
517
|
+
ctx.transientStore.jobMetadataTtlMs
|
|
518
|
+
);
|
|
519
|
+
await ctx.transientStore.putJson(
|
|
520
|
+
storeKeys.warmupJob(warmupJobId),
|
|
521
|
+
{ sources },
|
|
522
|
+
ctx.transientStore.jobMetadataTtlMs
|
|
523
|
+
);
|
|
524
|
+
await ctx.transientStore.putJson(
|
|
525
|
+
storeKeys.renderJob(renderJobId),
|
|
526
|
+
{
|
|
527
|
+
effie,
|
|
528
|
+
scale,
|
|
529
|
+
upload,
|
|
530
|
+
createdAt: Date.now()
|
|
531
|
+
},
|
|
532
|
+
ctx.transientStore.jobMetadataTtlMs
|
|
533
|
+
);
|
|
534
|
+
res.json({
|
|
535
|
+
id: jobId,
|
|
536
|
+
url: `${ctx.baseUrl}/warmup-and-render/${jobId}`
|
|
537
|
+
});
|
|
538
|
+
} catch (error) {
|
|
539
|
+
console.error("Error creating warmup-and-render job:", error);
|
|
540
|
+
res.status(500).json({ error: "Failed to create warmup-and-render job" });
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
async function streamWarmupAndRenderJob(req, res, ctx) {
|
|
544
|
+
try {
|
|
545
|
+
setupCORSHeaders(res);
|
|
546
|
+
const jobId = req.params.id;
|
|
547
|
+
const jobCacheKey = storeKeys.warmupAndRenderJob(jobId);
|
|
548
|
+
const job = await ctx.transientStore.getJson(jobCacheKey);
|
|
549
|
+
ctx.transientStore.delete(jobCacheKey);
|
|
550
|
+
if (!job) {
|
|
551
|
+
res.status(404).json({ error: "Job not found" });
|
|
552
|
+
return;
|
|
553
|
+
}
|
|
554
|
+
setupSSEResponse(res);
|
|
555
|
+
const sendEvent = createSSEEventSender(res);
|
|
556
|
+
let keepalivePhase = "warmup";
|
|
557
|
+
const keepalive = setInterval(() => {
|
|
558
|
+
sendEvent("keepalive", { phase: keepalivePhase });
|
|
559
|
+
}, 25e3);
|
|
560
|
+
try {
|
|
561
|
+
if (ctx.warmupBackendBaseUrl) {
|
|
562
|
+
await proxyRemoteSSE(
|
|
563
|
+
`${ctx.warmupBackendBaseUrl}/warmup/${job.warmupJobId}`,
|
|
564
|
+
sendEvent,
|
|
565
|
+
"warmup:",
|
|
566
|
+
res,
|
|
567
|
+
ctx.warmupBackendApiKey ? { Authorization: `Bearer ${ctx.warmupBackendApiKey}` } : void 0
|
|
568
|
+
);
|
|
569
|
+
} else {
|
|
570
|
+
const warmupSender = prefixEventSender(sendEvent, "warmup:");
|
|
571
|
+
await warmupSources(job.sources, warmupSender, ctx);
|
|
572
|
+
warmupSender("complete", { status: "ready" });
|
|
573
|
+
}
|
|
574
|
+
keepalivePhase = "render";
|
|
575
|
+
if (ctx.renderBackendBaseUrl) {
|
|
576
|
+
await proxyRemoteSSE(
|
|
577
|
+
`${ctx.renderBackendBaseUrl}/render/${job.renderJobId}`,
|
|
578
|
+
sendEvent,
|
|
579
|
+
"render:",
|
|
580
|
+
res,
|
|
581
|
+
ctx.renderBackendApiKey ? { Authorization: `Bearer ${ctx.renderBackendApiKey}` } : void 0
|
|
582
|
+
);
|
|
583
|
+
} else {
|
|
584
|
+
const renderSender = prefixEventSender(sendEvent, "render:");
|
|
585
|
+
if (job.upload) {
|
|
586
|
+
renderSender("started", { status: "rendering" });
|
|
587
|
+
const timings = await renderAndUploadInternal(
|
|
588
|
+
job.effie,
|
|
589
|
+
job.scale,
|
|
590
|
+
job.upload,
|
|
591
|
+
renderSender,
|
|
592
|
+
ctx
|
|
593
|
+
);
|
|
594
|
+
renderSender("complete", { status: "uploaded", timings });
|
|
595
|
+
} else {
|
|
596
|
+
const videoUrl = `${ctx.baseUrl}/render/${job.renderJobId}`;
|
|
597
|
+
sendEvent("complete", { status: "ready", videoUrl });
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
if (job.upload && !ctx.renderBackendBaseUrl) {
|
|
601
|
+
sendEvent("complete", { status: "done" });
|
|
602
|
+
}
|
|
603
|
+
} catch (error) {
|
|
604
|
+
sendEvent("error", {
|
|
605
|
+
phase: keepalivePhase,
|
|
606
|
+
message: String(error)
|
|
607
|
+
});
|
|
608
|
+
} finally {
|
|
609
|
+
clearInterval(keepalive);
|
|
610
|
+
res.end();
|
|
611
|
+
}
|
|
612
|
+
} catch (error) {
|
|
613
|
+
console.error("Error in warmup-and-render streaming:", error);
|
|
614
|
+
if (!res.headersSent) {
|
|
615
|
+
res.status(500).json({ error: "Warmup-and-render streaming failed" });
|
|
616
|
+
} else {
|
|
617
|
+
res.end();
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
function prefixEventSender(sendEvent, prefix) {
|
|
622
|
+
return (event, data) => {
|
|
623
|
+
sendEvent(`${prefix}${event}`, data);
|
|
624
|
+
};
|
|
625
|
+
}
|
|
626
|
+
async function proxyRemoteSSE(url, sendEvent, prefix, res, headers) {
|
|
627
|
+
const response = await ffsFetch(url, {
|
|
628
|
+
headers: {
|
|
629
|
+
Accept: "text/event-stream",
|
|
630
|
+
...headers
|
|
631
|
+
}
|
|
632
|
+
});
|
|
633
|
+
if (!response.ok) {
|
|
634
|
+
throw new Error(`Remote backend error: ${response.status}`);
|
|
635
|
+
}
|
|
636
|
+
const reader = response.body?.getReader();
|
|
637
|
+
if (!reader) {
|
|
638
|
+
throw new Error("No response body from remote backend");
|
|
639
|
+
}
|
|
640
|
+
const decoder = new TextDecoder();
|
|
641
|
+
let buffer = "";
|
|
642
|
+
try {
|
|
643
|
+
while (true) {
|
|
644
|
+
const { done, value } = await reader.read();
|
|
645
|
+
if (done) break;
|
|
646
|
+
if (res.destroyed) {
|
|
647
|
+
reader.cancel();
|
|
648
|
+
break;
|
|
649
|
+
}
|
|
650
|
+
buffer += decoder.decode(value, { stream: true });
|
|
651
|
+
const lines = buffer.split("\n");
|
|
652
|
+
buffer = lines.pop() || "";
|
|
653
|
+
let currentEvent = "";
|
|
654
|
+
let currentData = "";
|
|
655
|
+
for (const line of lines) {
|
|
656
|
+
if (line.startsWith("event: ")) {
|
|
657
|
+
currentEvent = line.slice(7);
|
|
658
|
+
} else if (line.startsWith("data: ")) {
|
|
659
|
+
currentData = line.slice(6);
|
|
660
|
+
} else if (line === "" && currentEvent && currentData) {
|
|
661
|
+
try {
|
|
662
|
+
const data = JSON.parse(currentData);
|
|
663
|
+
sendEvent(`${prefix}${currentEvent}`, data);
|
|
664
|
+
} catch {
|
|
665
|
+
}
|
|
666
|
+
currentEvent = "";
|
|
667
|
+
currentData = "";
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
} finally {
|
|
672
|
+
reader.releaseLock();
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
async function proxyBinaryStream(response, res) {
|
|
676
|
+
const contentType = response.headers.get("content-type");
|
|
677
|
+
if (contentType) res.set("Content-Type", contentType);
|
|
678
|
+
const contentLength = response.headers.get("content-length");
|
|
679
|
+
if (contentLength) res.set("Content-Length", contentLength);
|
|
680
|
+
const reader = response.body?.getReader();
|
|
681
|
+
if (!reader) {
|
|
682
|
+
throw new Error("No response body");
|
|
683
|
+
}
|
|
684
|
+
try {
|
|
685
|
+
while (true) {
|
|
686
|
+
const { done, value } = await reader.read();
|
|
687
|
+
if (done) break;
|
|
688
|
+
if (res.destroyed) {
|
|
689
|
+
reader.cancel();
|
|
690
|
+
break;
|
|
691
|
+
}
|
|
692
|
+
res.write(value);
|
|
693
|
+
}
|
|
694
|
+
} finally {
|
|
695
|
+
reader.releaseLock();
|
|
696
|
+
res.end();
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
// src/handlers/caching.ts
|
|
197
701
|
function shouldSkipWarmup(source) {
|
|
198
702
|
return source.type === "video" || source.type === "audio";
|
|
199
703
|
}
|
|
@@ -205,9 +709,13 @@ async function createWarmupJob(req, res, ctx) {
|
|
|
205
709
|
res.status(400).json(parseResult);
|
|
206
710
|
return;
|
|
207
711
|
}
|
|
208
|
-
const sources =
|
|
209
|
-
const jobId =
|
|
210
|
-
await ctx.
|
|
712
|
+
const sources = extractEffieSourcesWithTypes2(parseResult.effie);
|
|
713
|
+
const jobId = randomUUID3();
|
|
714
|
+
await ctx.transientStore.putJson(
|
|
715
|
+
storeKeys.warmupJob(jobId),
|
|
716
|
+
{ sources },
|
|
717
|
+
ctx.transientStore.jobMetadataTtlMs
|
|
718
|
+
);
|
|
211
719
|
res.json({
|
|
212
720
|
id: jobId,
|
|
213
721
|
url: `${ctx.baseUrl}/warmup/${jobId}`
|
|
@@ -221,9 +729,25 @@ async function streamWarmupJob(req, res, ctx) {
|
|
|
221
729
|
try {
|
|
222
730
|
setupCORSHeaders(res);
|
|
223
731
|
const jobId = req.params.id;
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
732
|
+
if (ctx.warmupBackendBaseUrl) {
|
|
733
|
+
setupSSEResponse(res);
|
|
734
|
+
const sendEvent2 = createSSEEventSender(res);
|
|
735
|
+
try {
|
|
736
|
+
await proxyRemoteSSE(
|
|
737
|
+
`${ctx.warmupBackendBaseUrl}/warmup/${jobId}`,
|
|
738
|
+
sendEvent2,
|
|
739
|
+
"",
|
|
740
|
+
res,
|
|
741
|
+
ctx.warmupBackendApiKey ? { Authorization: `Bearer ${ctx.warmupBackendApiKey}` } : void 0
|
|
742
|
+
);
|
|
743
|
+
} finally {
|
|
744
|
+
res.end();
|
|
745
|
+
}
|
|
746
|
+
return;
|
|
747
|
+
}
|
|
748
|
+
const jobCacheKey = storeKeys.warmupJob(jobId);
|
|
749
|
+
const job = await ctx.transientStore.getJson(jobCacheKey);
|
|
750
|
+
ctx.transientStore.delete(jobCacheKey);
|
|
227
751
|
if (!job) {
|
|
228
752
|
res.status(404).json({ error: "Job not found" });
|
|
229
753
|
return;
|
|
@@ -257,9 +781,9 @@ async function purgeCache(req, res, ctx) {
|
|
|
257
781
|
const sources = extractEffieSources(parseResult.effie);
|
|
258
782
|
let purged = 0;
|
|
259
783
|
for (const url of sources) {
|
|
260
|
-
const ck =
|
|
261
|
-
if (await ctx.
|
|
262
|
-
await ctx.
|
|
784
|
+
const ck = storeKeys.source(url);
|
|
785
|
+
if (await ctx.transientStore.exists(ck)) {
|
|
786
|
+
await ctx.transientStore.delete(ck);
|
|
263
787
|
purged++;
|
|
264
788
|
}
|
|
265
789
|
}
|
|
@@ -292,8 +816,8 @@ async function warmupSources(sources, sendEvent, ctx) {
|
|
|
292
816
|
sourcesToCache.push(source);
|
|
293
817
|
}
|
|
294
818
|
}
|
|
295
|
-
const sourceCacheKeys = sourcesToCache.map((s) =>
|
|
296
|
-
const existsMap = await ctx.
|
|
819
|
+
const sourceCacheKeys = sourcesToCache.map((s) => storeKeys.source(s.url));
|
|
820
|
+
const existsMap = await ctx.transientStore.existsMany(sourceCacheKeys);
|
|
297
821
|
for (let i = 0; i < sourcesToCache.length; i++) {
|
|
298
822
|
if (existsMap.get(sourceCacheKeys[i])) {
|
|
299
823
|
cached++;
|
|
@@ -319,11 +843,11 @@ async function warmupSources(sources, sendEvent, ctx) {
|
|
|
319
843
|
}, 25e3);
|
|
320
844
|
const queue = [...uncached];
|
|
321
845
|
const workers = Array.from(
|
|
322
|
-
{ length: Math.min(ctx.
|
|
846
|
+
{ length: Math.min(ctx.warmupConcurrency, queue.length) },
|
|
323
847
|
async () => {
|
|
324
848
|
while (queue.length > 0) {
|
|
325
849
|
const source = queue.shift();
|
|
326
|
-
const cacheKey =
|
|
850
|
+
const cacheKey = storeKeys.source(source.url);
|
|
327
851
|
const startTime = Date.now();
|
|
328
852
|
try {
|
|
329
853
|
let fetchPromise = inFlightFetches.get(cacheKey);
|
|
@@ -397,203 +921,23 @@ async function fetchAndCache(url, cacheKey, sendEvent, ctx) {
|
|
|
397
921
|
}
|
|
398
922
|
});
|
|
399
923
|
const trackedStream = sourceStream.pipe(progressStream);
|
|
400
|
-
await ctx.
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
import { randomUUID as randomUUID2 } from "crypto";
|
|
406
|
-
import { effieDataSchema as effieDataSchema2 } from "@effing/effie";
|
|
407
|
-
async function createRenderJob(req, res, ctx) {
|
|
408
|
-
try {
|
|
409
|
-
const isWrapped = "effie" in req.body;
|
|
410
|
-
let rawEffieData;
|
|
411
|
-
let scale;
|
|
412
|
-
let upload;
|
|
413
|
-
if (isWrapped) {
|
|
414
|
-
const options = req.body;
|
|
415
|
-
if (typeof options.effie === "string") {
|
|
416
|
-
const response = await ffsFetch(options.effie);
|
|
417
|
-
if (!response.ok) {
|
|
418
|
-
throw new Error(
|
|
419
|
-
`Failed to fetch Effie data: ${response.status} ${response.statusText}`
|
|
420
|
-
);
|
|
421
|
-
}
|
|
422
|
-
rawEffieData = await response.json();
|
|
423
|
-
} else {
|
|
424
|
-
rawEffieData = options.effie;
|
|
425
|
-
}
|
|
426
|
-
scale = options.scale ?? 1;
|
|
427
|
-
upload = options.upload;
|
|
428
|
-
} else {
|
|
429
|
-
rawEffieData = req.body;
|
|
430
|
-
scale = parseFloat(req.query.scale?.toString() || "1");
|
|
431
|
-
}
|
|
432
|
-
let effie;
|
|
433
|
-
if (!ctx.skipValidation) {
|
|
434
|
-
const result = effieDataSchema2.safeParse(rawEffieData);
|
|
435
|
-
if (!result.success) {
|
|
436
|
-
res.status(400).json({
|
|
437
|
-
error: "Invalid effie data",
|
|
438
|
-
issues: result.error.issues.map((issue) => ({
|
|
439
|
-
path: issue.path.join("."),
|
|
440
|
-
message: issue.message
|
|
441
|
-
}))
|
|
442
|
-
});
|
|
443
|
-
return;
|
|
444
|
-
}
|
|
445
|
-
effie = result.data;
|
|
446
|
-
} else {
|
|
447
|
-
const data = rawEffieData;
|
|
448
|
-
if (!data?.segments) {
|
|
449
|
-
res.status(400).json({ error: "Invalid effie data: missing segments" });
|
|
450
|
-
return;
|
|
451
|
-
}
|
|
452
|
-
effie = data;
|
|
453
|
-
}
|
|
454
|
-
const jobId = randomUUID2();
|
|
455
|
-
const job = {
|
|
456
|
-
effie,
|
|
457
|
-
scale,
|
|
458
|
-
upload,
|
|
459
|
-
createdAt: Date.now()
|
|
460
|
-
};
|
|
461
|
-
await ctx.cacheStorage.putJson(cacheKeys.renderJob(jobId), job);
|
|
462
|
-
res.json({
|
|
463
|
-
id: jobId,
|
|
464
|
-
url: `${ctx.baseUrl}/render/${jobId}`
|
|
465
|
-
});
|
|
466
|
-
} catch (error) {
|
|
467
|
-
console.error("Error creating render job:", error);
|
|
468
|
-
res.status(500).json({ error: "Failed to create render job" });
|
|
469
|
-
}
|
|
470
|
-
}
|
|
471
|
-
async function streamRenderJob(req, res, ctx) {
|
|
472
|
-
try {
|
|
473
|
-
setupCORSHeaders(res);
|
|
474
|
-
const jobId = req.params.id;
|
|
475
|
-
const jobCacheKey = cacheKeys.renderJob(jobId);
|
|
476
|
-
const job = await ctx.cacheStorage.getJson(jobCacheKey);
|
|
477
|
-
ctx.cacheStorage.delete(jobCacheKey);
|
|
478
|
-
if (!job) {
|
|
479
|
-
res.status(404).json({ error: "Job not found or expired" });
|
|
480
|
-
return;
|
|
481
|
-
}
|
|
482
|
-
if (job.upload) {
|
|
483
|
-
await streamRenderWithUpload(res, job, ctx);
|
|
484
|
-
} else {
|
|
485
|
-
await streamRenderDirect(res, job, ctx);
|
|
486
|
-
}
|
|
487
|
-
} catch (error) {
|
|
488
|
-
console.error("Error in render:", error);
|
|
489
|
-
if (!res.headersSent) {
|
|
490
|
-
res.status(500).json({ error: "Rendering failed" });
|
|
491
|
-
} else {
|
|
492
|
-
res.end();
|
|
493
|
-
}
|
|
494
|
-
}
|
|
495
|
-
}
|
|
496
|
-
async function streamRenderDirect(res, job, ctx) {
|
|
497
|
-
const renderer = new EffieRenderer(job.effie, {
|
|
498
|
-
cacheStorage: ctx.cacheStorage,
|
|
499
|
-
httpProxy: ctx.httpProxy
|
|
500
|
-
});
|
|
501
|
-
const videoStream = await renderer.render(job.scale);
|
|
502
|
-
res.on("close", () => {
|
|
503
|
-
videoStream.destroy();
|
|
504
|
-
renderer.close();
|
|
505
|
-
});
|
|
506
|
-
res.set("Content-Type", "video/mp4");
|
|
507
|
-
videoStream.pipe(res);
|
|
508
|
-
}
|
|
509
|
-
async function streamRenderWithUpload(res, job, ctx) {
|
|
510
|
-
setupSSEResponse(res);
|
|
511
|
-
const sendEvent = createSSEEventSender(res);
|
|
512
|
-
const keepalive = setInterval(() => {
|
|
513
|
-
sendEvent("keepalive", { status: "rendering" });
|
|
514
|
-
}, 25e3);
|
|
515
|
-
try {
|
|
516
|
-
sendEvent("started", { status: "rendering" });
|
|
517
|
-
const timings = await renderAndUploadInternal(
|
|
518
|
-
job.effie,
|
|
519
|
-
job.scale,
|
|
520
|
-
job.upload,
|
|
521
|
-
sendEvent,
|
|
522
|
-
ctx
|
|
523
|
-
);
|
|
524
|
-
sendEvent("complete", { status: "uploaded", timings });
|
|
525
|
-
} catch (error) {
|
|
526
|
-
sendEvent("error", { message: String(error) });
|
|
527
|
-
} finally {
|
|
528
|
-
clearInterval(keepalive);
|
|
529
|
-
res.end();
|
|
530
|
-
}
|
|
531
|
-
}
|
|
532
|
-
async function renderAndUploadInternal(effie, scale, upload, sendEvent, ctx) {
|
|
533
|
-
const timings = {};
|
|
534
|
-
if (upload.coverUrl) {
|
|
535
|
-
const fetchCoverStartTime = Date.now();
|
|
536
|
-
const coverFetchResponse = await ffsFetch(effie.cover);
|
|
537
|
-
if (!coverFetchResponse.ok) {
|
|
538
|
-
throw new Error(
|
|
539
|
-
`Failed to fetch cover image: ${coverFetchResponse.status} ${coverFetchResponse.statusText}`
|
|
540
|
-
);
|
|
541
|
-
}
|
|
542
|
-
const coverBuffer = Buffer.from(await coverFetchResponse.arrayBuffer());
|
|
543
|
-
timings.fetchCoverTime = Date.now() - fetchCoverStartTime;
|
|
544
|
-
const uploadCoverStartTime = Date.now();
|
|
545
|
-
const uploadCoverResponse = await ffsFetch(upload.coverUrl, {
|
|
546
|
-
method: "PUT",
|
|
547
|
-
body: coverBuffer,
|
|
548
|
-
headers: {
|
|
549
|
-
"Content-Type": "image/png",
|
|
550
|
-
"Content-Length": coverBuffer.length.toString()
|
|
551
|
-
}
|
|
552
|
-
});
|
|
553
|
-
if (!uploadCoverResponse.ok) {
|
|
554
|
-
throw new Error(
|
|
555
|
-
`Failed to upload cover: ${uploadCoverResponse.status} ${uploadCoverResponse.statusText}`
|
|
556
|
-
);
|
|
557
|
-
}
|
|
558
|
-
timings.uploadCoverTime = Date.now() - uploadCoverStartTime;
|
|
559
|
-
}
|
|
560
|
-
const renderStartTime = Date.now();
|
|
561
|
-
const renderer = new EffieRenderer(effie, {
|
|
562
|
-
cacheStorage: ctx.cacheStorage,
|
|
563
|
-
httpProxy: ctx.httpProxy
|
|
564
|
-
});
|
|
565
|
-
const videoStream = await renderer.render(scale);
|
|
566
|
-
const chunks = [];
|
|
567
|
-
for await (const chunk of videoStream) {
|
|
568
|
-
chunks.push(Buffer.from(chunk));
|
|
569
|
-
}
|
|
570
|
-
const videoBuffer = Buffer.concat(chunks);
|
|
571
|
-
timings.renderTime = Date.now() - renderStartTime;
|
|
572
|
-
sendEvent("keepalive", { status: "uploading" });
|
|
573
|
-
const uploadStartTime = Date.now();
|
|
574
|
-
const uploadResponse = await ffsFetch(upload.videoUrl, {
|
|
575
|
-
method: "PUT",
|
|
576
|
-
body: videoBuffer,
|
|
577
|
-
headers: {
|
|
578
|
-
"Content-Type": "video/mp4",
|
|
579
|
-
"Content-Length": videoBuffer.length.toString()
|
|
580
|
-
}
|
|
581
|
-
});
|
|
582
|
-
if (!uploadResponse.ok) {
|
|
583
|
-
throw new Error(
|
|
584
|
-
`Failed to upload rendered video: ${uploadResponse.status} ${uploadResponse.statusText}`
|
|
585
|
-
);
|
|
586
|
-
}
|
|
587
|
-
timings.uploadTime = Date.now() - uploadStartTime;
|
|
588
|
-
return timings;
|
|
924
|
+
await ctx.transientStore.put(
|
|
925
|
+
cacheKey,
|
|
926
|
+
trackedStream,
|
|
927
|
+
ctx.transientStore.sourceTtlMs
|
|
928
|
+
);
|
|
589
929
|
}
|
|
590
930
|
|
|
591
931
|
export {
|
|
592
932
|
createServerContext,
|
|
933
|
+
createRenderJob,
|
|
934
|
+
streamRenderJob,
|
|
935
|
+
createWarmupAndRenderJob,
|
|
936
|
+
streamWarmupAndRenderJob,
|
|
937
|
+
proxyRemoteSSE,
|
|
938
|
+
proxyBinaryStream,
|
|
593
939
|
createWarmupJob,
|
|
594
940
|
streamWarmupJob,
|
|
595
|
-
purgeCache
|
|
596
|
-
createRenderJob,
|
|
597
|
-
streamRenderJob
|
|
941
|
+
purgeCache
|
|
598
942
|
};
|
|
599
|
-
//# sourceMappingURL=chunk-
|
|
943
|
+
//# sourceMappingURL=chunk-7FMPCMLO.js.map
|