@remotion/lambda 4.0.164 → 4.0.165

package/dist/functions/index.js

@@ -10,14 +10,15 @@ const leak_detection_1 = require("./helpers/leak-detection");
 const lifecycle_1 = require("./helpers/lifecycle");
 const print_cloudwatch_helper_1 = require("./helpers/print-cloudwatch-helper");
 const streamify_response_1 = require("./helpers/streamify-response");
-const streaming_payloads_1 = require("./helpers/streaming-payloads");
 const info_1 = require("./info");
 const launch_1 = require("./launch");
 const progress_1 = require("./progress");
 const renderer_2 = require("./renderer");
 const start_1 = require("./start");
 const still_1 = require("./still");
-const innerHandler = async (params, responseStream, context) => {
+const stream_writer_1 = require("./streaming/stream-writer");
+const streaming_1 = require("./streaming/streaming");
+const innerHandler = async ({ params, responseWriter, context, }) => {
     (0, leak_detection_1.setCurrentRequestId)(context.awsRequestId);
     process.env.__RESERVED_IS_INSIDE_REMOTION_LAMBDA = 'true';
     const timeoutInMilliseconds = context.getRemainingTimeInMillis();
@@ -38,19 +39,36 @@ const innerHandler = async (params, responseStream, context) => {
             inputProps: JSON.stringify(params.inputProps),
             isWarm,
         }, params.logLevel);
-        const renderIdDetermined = {
-            type: 'render-id-determined',
-            renderId,
-        };
-        (0, streaming_payloads_1.sendProgressEvent)(responseStream, renderIdDetermined);
-        const response = await (0, still_1.stillHandler)({
-            expectedBucketOwner: currentUserId,
-            params,
-            renderId,
-        });
-        responseStream.write(JSON.stringify(response), () => {
-            responseStream.end();
+        await new Promise((resolve, reject) => {
+            const onStream = (payload) => {
+                const message = (0, streaming_1.makeStreamPayload)({
+                    message: payload,
+                });
+                responseWriter.write(message).catch((err) => {
+                    reject(err);
+                });
+            };
+            if (params.streamed) {
+                onStream({
+                    type: 'render-id-determined',
+                    payload: { renderId },
+                });
+            }
+            (0, still_1.stillHandler)({
+                expectedBucketOwner: currentUserId,
+                params,
+                renderId,
+                onStream,
+                timeoutInMilliseconds,
+            })
+                .then((r) => {
+                resolve(r);
+            })
+                .catch((err) => {
+                reject(err);
+            });
         });
+        await responseWriter.end();
         return;
     }
     if (params.type === constants_1.LambdaRoutines.start) {
@@ -60,10 +78,10 @@ const innerHandler = async (params, responseStream, context) => {
         }, params.logLevel);
         const response = await (0, start_1.startHandler)(params, {
             expectedBucketOwner: currentUserId,
+            timeoutInMilliseconds,
         });
-        responseStream.write(JSON.stringify(response), () => {
-            responseStream.end();
-        });
+        await responseWriter.write(Buffer.from(JSON.stringify(response)));
+        await responseWriter.end();
         return;
     }
     if (params.type === constants_1.LambdaRoutines.launch) {
@@ -76,9 +94,8 @@ const innerHandler = async (params, responseStream, context) => {
             expectedBucketOwner: currentUserId,
             getRemainingTimeInMillis: context.getRemainingTimeInMillis,
         });
-        responseStream.write(JSON.stringify(response), () => {
-            responseStream.end();
-        });
+        await responseWriter.write(Buffer.from(JSON.stringify(response)));
+        await responseWriter.end();
         return;
     }
     if (params.type === constants_1.LambdaRoutines.status) {
@@ -91,9 +108,8 @@ const innerHandler = async (params, responseStream, context) => {
             timeoutInMilliseconds,
             retriesRemaining: 2,
         });
-        responseStream.write(JSON.stringify(response), () => {
-            responseStream.end();
-        });
+        await responseWriter.write(Buffer.from(JSON.stringify(response)));
+        await responseWriter.end();
         return;
     }
     if (params.type === constants_1.LambdaRoutines.renderer) {
@@ -104,13 +120,26 @@ const innerHandler = async (params, responseStream, context) => {
             resolvedProps: JSON.stringify(params.resolvedProps),
             isWarm,
         }, params.logLevel);
-        const response = await (0, renderer_2.rendererHandler)(params, {
-            expectedBucketOwner: currentUserId,
-            isWarm,
-        }, context);
-        responseStream.write(JSON.stringify(response), () => {
-            responseStream.end();
+        await new Promise((resolve, reject) => {
+            (0, renderer_2.rendererHandler)(params, {
+                expectedBucketOwner: currentUserId,
+                isWarm,
+            }, (payload) => {
+                const message = (0, streaming_1.makeStreamPayload)({
+                    message: payload,
+                });
+                responseWriter.write(message).catch((err) => {
+                    reject(err);
+                });
+            }, context)
+                .then((res) => {
+                resolve(res);
+            })
+                .catch((err) => {
+                reject(err);
+            });
         });
+        await responseWriter.end();
         return;
     }
     if (params.type === constants_1.LambdaRoutines.info) {
@@ -118,9 +147,8 @@ const innerHandler = async (params, responseStream, context) => {
             isWarm,
         }, params.logLevel);
         const response = await (0, info_1.infoHandler)(params);
-        responseStream.write(JSON.stringify(response), () => {
-            responseStream.end();
-        });
+        await responseWriter.write(Buffer.from(JSON.stringify(response)));
+        await responseWriter.end();
         return;
     }
     if (params.type === constants_1.LambdaRoutines.compositions) {
@@ -130,16 +158,20 @@ const innerHandler = async (params, responseStream, context) => {
         const response = await (0, compositions_1.compositionsHandler)(params, {
             expectedBucketOwner: currentUserId,
         });
-        responseStream.write(JSON.stringify(response), () => {
-            responseStream.end();
-        });
+        await responseWriter.write(Buffer.from(JSON.stringify(response)));
+        await responseWriter.end();
         return;
     }
     throw new Error(constants_1.COMMAND_NOT_FOUND);
 };
 const routine = async (params, responseStream, context) => {
+    const responseWriter = (0, stream_writer_1.streamWriter)(responseStream);
     try {
-        await innerHandler(params, responseStream, context);
+        await innerHandler({
+            params,
+            responseWriter,
+            context,
+        });
     }
     catch (err) {
         const res = {
@@ -147,8 +179,8 @@ const routine = async (params, responseStream, context) => {
             message: err.message,
             stack: err.stack,
         };
-        responseStream.write(JSON.stringify(res));
-        responseStream.end();
+        await responseWriter.write(Buffer.from(JSON.stringify(res)));
+        await responseWriter.end();
     }
 };
 exports.handler = (0, streamify_response_1.streamifyResponse)(routine);

package/dist/functions/launch.js

@@ -1,11 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.launchHandler = void 0;
-/* eslint-disable @typescript-eslint/no-use-before-define */
-const client_lambda_1 = require("@aws-sdk/client-lambda");
 const renderer_1 = require("@remotion/renderer");
+const fs_1 = require("fs");
+const path_1 = require("path");
 const version_1 = require("remotion/version");
-const aws_clients_1 = require("../shared/aws-clients");
 const compress_props_1 = require("../shared/compress-props");
 const constants_1 = require("../shared/constants");
 const docs_url_1 = require("../shared/docs-url");
@@ -20,40 +19,12 @@ const expected_out_name_1 = require("./helpers/expected-out-name");
 const find_output_file_in_bucket_1 = require("./helpers/find-output-file-in-bucket");
 const get_browser_instance_1 = require("./helpers/get-browser-instance");
 const get_current_region_1 = require("./helpers/get-current-region");
-const io_1 = require("./helpers/io");
 const merge_chunks_1 = require("./helpers/merge-chunks");
-const timer_1 = require("./helpers/timer");
+const overall_render_progress_1 = require("./helpers/overall-render-progress");
+const stream_renderer_1 = require("./helpers/stream-renderer");
 const validate_composition_1 = require("./helpers/validate-composition");
 const write_lambda_error_1 = require("./helpers/write-lambda-error");
-const callFunctionWithRetry = async ({ payload, retries, functionName, }) => {
-    try {
-        const result = await (0, aws_clients_1.getLambdaClient)((0, get_current_region_1.getCurrentRegionInFunction)()).send(new client_lambda_1.InvokeCommand({
-            FunctionName: process.env.AWS_LAMBDA_FUNCTION_NAME,
-            Payload: JSON.stringify(payload),
-            InvocationType: 'Event',
-        }));
-        if (result.FunctionError) {
-            throw new Error(`Lambda function ${functionName} returned an error: ${result.FunctionError} ${result.LogResult}`);
-        }
-    }
-    catch (err) {
-        if (err.name === 'ResourceConflictException') {
-            if (retries > 10) {
-                throw err;
-            }
-            await new Promise((resolve) => {
-                setTimeout(resolve, 1000);
-            });
-            return callFunctionWithRetry({
-                payload,
-                retries: retries + 1,
-                functionName,
-            });
-        }
-        throw err;
-    }
-};
-const innerLaunchHandler = async ({ functionName, params, options, }) => {
+const innerLaunchHandler = async ({ functionName, params, options, overallProgress, }) => {
     var _a, _b, _c, _d, _e;
     if (params.type !== constants_1.LambdaRoutines.launch) {
         throw new Error('Expected launch type');
@@ -75,6 +46,7 @@ const innerLaunchHandler = async ({ functionName, params, options, }) => {
     renderer_1.RenderInternals.Log.info(logOptions, 'Waiting for browser to be ready:', serializedInputPropsWithCustomSchema);
     const { instance } = await browserInstance;
     renderer_1.RenderInternals.Log.info(logOptions, 'Validating composition, input props:', serializedInputPropsWithCustomSchema);
+    const startTime = Date.now();
     const comp = await (0, validate_composition_1.validateComposition)({
         serveUrl: params.serveUrl,
         composition: params.composition,
@@ -131,8 +103,8 @@ const innerLaunchHandler = async ({ functionName, params, options, }) => {
     if (chunks.length > constants_1.MAX_FUNCTIONS_PER_RENDER) {
         throw new Error(`Too many functions: This render would cause ${chunks.length} functions to spawn. We limit this amount to ${constants_1.MAX_FUNCTIONS_PER_RENDER} functions as more would result in diminishing returns. Values set: frameCount = ${frameCount}, framesPerLambda=${framesPerLambda}. See ${docs_url_1.DOCS_URL}/docs/lambda/concurrency#too-many-functions for help.`);
     }
+    overallProgress.setExpectedChunks(chunks.length);
     const sortedChunks = chunks.slice().sort((a, b) => a[0] - b[0]);
-    const reqSend = (0, timer_1.timer)('sending off requests', params.logLevel);
     const serializedResolved = (0, compress_props_1.serializeOrThrow)(comp.props, 'resolved-props');
     const needsToUpload = (0, compress_props_1.getNeedsToUpload)('video-or-audio', [
         serializedResolved.length,
@@ -148,6 +120,9 @@ const innerLaunchHandler = async ({ functionName, params, options, }) => {
         needsToUpload,
     });
     const fps = comp.fps / params.everyNthFrame;
+    // If for 150 functions, we stream every frame, we DDos ourselves.
+    // Throttling a bit, allowing more progress if there is lower concurrency.
+    const progressEveryNthFrame = Math.ceil(chunks.length / 15);
     const lambdaPayloads = chunks.map((chunkPayload) => {
         var _a;
         const payload = {
@@ -195,6 +170,7 @@ const innerLaunchHandler = async ({ functionName, params, options, }) => {
             preferLossless: params.preferLossless,
             compositionStart: realFrameRange[0],
             framesPerLambda,
+            progressEveryNthFrame,
         };
         return payload;
     });
@@ -235,40 +211,36 @@ const innerLaunchHandler = async ({ functionName, params, options, }) => {
     const { key, renderBucketName, customCredentials } = (0, expected_out_name_1.getExpectedOutName)(renderMetadata, params.bucketName, typeof params.outName === 'string' || typeof params.outName === 'undefined'
         ? null
         : (_e = (_d = params.outName) === null || _d === void 0 ? void 0 : _d.s3OutputProvider) !== null && _e !== void 0 ? _e : null);
-    const output = await (0, find_output_file_in_bucket_1.findOutputFileInBucket)({
-        bucketName: params.bucketName,
-        customCredentials,
-        region: (0, get_current_region_1.getCurrentRegionInFunction)(),
-        renderMetadata,
-    });
-    if (output) {
-        if (params.overwrite) {
-            console.info('Deleting', { bucketName: renderBucketName, key }, 'because it already existed and will be overwritten');
-            await (0, io_1.lambdaDeleteFile)({
-                bucketName: renderBucketName,
-                customCredentials,
-                key,
-                region: (0, get_current_region_1.getCurrentRegionInFunction)(),
-            });
-        }
-        else {
-            throw new TypeError(`Output file "${key}" in bucket "${renderBucketName}" in region "${(0, get_current_region_1.getCurrentRegionInFunction)()}" already exists. Delete it before re-rendering, or use the overwrite option to delete it before render."`);
+    if (!params.overwrite) {
+        const output = await (0, find_output_file_in_bucket_1.findOutputFileInBucket)({
+            bucketName: params.bucketName,
+            customCredentials,
+            renderMetadata,
+            region: (0, get_current_region_1.getCurrentRegionInFunction)(),
+        });
+        if (output) {
+            throw new TypeError(`Output file "${key}" in bucket "${renderBucketName}" in region "${(0, get_current_region_1.getCurrentRegionInFunction)()}" already exists. Delete it before re-rendering, or set the 'overwrite' option in renderMediaOnLambda() to overwrite it."`);
         }
     }
-    await (0, io_1.lambdaWriteFile)({
-        bucketName: params.bucketName,
-        key: (0, constants_1.renderMetadataKey)(params.renderId),
-        body: JSON.stringify(renderMetadata),
-        region: (0, get_current_region_1.getCurrentRegionInFunction)(),
-        privacy: 'private',
-        expectedBucketOwner: options.expectedBucketOwner,
-        downloadBehavior: null,
-        customCredentials: null,
-    });
+    overallProgress.setRenderMetadata(renderMetadata);
+    const outdir = (0, path_1.join)(renderer_1.RenderInternals.tmpDir(constants_1.CONCAT_FOLDER_TOKEN), 'bucket');
+    if ((0, fs_1.existsSync)(outdir)) {
+        (0, fs_1.rmSync)(outdir, {
+            recursive: true,
+        });
+    }
+    (0, fs_1.mkdirSync)(outdir);
+    const files = [];
     await Promise.all(lambdaPayloads.map(async (payload) => {
-        await callFunctionWithRetry({ payload, retries: 0, functionName });
+        await (0, stream_renderer_1.streamRendererFunctionWithRetry)({
+            files,
+            functionName,
+            outdir,
+            overallProgress,
+            payload,
+            logLevel: params.logLevel,
+        });
     }));
-    reqSend.end();
     const postRenderData = await (0, merge_chunks_1.mergeChunksAndFinishRender)({
         bucketName: params.bucketName,
         renderId: params.renderId,
@@ -293,6 +265,10 @@ const innerLaunchHandler = async ({ functionName, params, options, }) => {
         binariesDirectory: null,
         preferLossless: params.preferLossless,
         compositionStart: realFrameRange[0],
+        outdir,
+        files: files.sort(),
+        overallProgress,
+        startTime,
     });
     return postRenderData;
 };
@@ -308,6 +284,7 @@ const launchHandler = async (params, options) => {
     };
     const onTimeout = async () => {
         var _a;
+        renderer_1.RenderInternals.Log.error({ indent: false, logLevel: params.logLevel }, 'Function is about to time out. Can not finish render.');
         if (!params.webhook) {
             return;
         }
@@ -335,34 +312,39 @@ const launchHandler = async (params, options) => {
             }
             renderer_1.RenderInternals.Log.error({ indent: false, logLevel: params.logLevel }, 'Failed to invoke webhook:');
             renderer_1.RenderInternals.Log.error({ indent: false, logLevel: params.logLevel }, err);
-            await (0, write_lambda_error_1.writeLambdaError)({
-                bucketName: params.bucketName,
-                errorInfo: {
-                    type: 'webhook',
-                    message: err.message,
-                    name: err.name,
-                    stack: err.stack,
-                    tmpDir: null,
-                    frame: 0,
-                    chunk: 0,
-                    isFatal: false,
-                    attempt: 1,
-                    willRetry: false,
-                    totalAttempts: 1,
-                },
-                renderId: params.renderId,
-                expectedBucketOwner: options.expectedBucketOwner,
+            overallProgress.addErrorWithoutUpload({
+                type: 'webhook',
+                message: err.message,
+                name: err.name,
+                stack: err.stack,
+                tmpDir: null,
+                frame: 0,
+                chunk: 0,
+                isFatal: false,
+                attempt: 1,
+                willRetry: false,
+                totalAttempts: 1,
             });
+            overallProgress.upload();
         }
     };
     let webhookInvoked = false;
     const webhookDueToTimeout = setTimeout(onTimeout, Math.max(options.getRemainingTimeInMillis() - 1000, 1000));
     renderer_1.RenderInternals.Log.info(logOptions, `Function has ${Math.max(options.getRemainingTimeInMillis() - 1000, 1000)} before it times out`);
+    const overallProgress = (0, overall_render_progress_1.makeOverallRenderProgress)({
+        renderId: params.renderId,
+        bucketName: params.bucketName,
+        expectedBucketOwner: options.expectedBucketOwner,
+        region: (0, get_current_region_1.getCurrentRegionInFunction)(),
+        timeoutTimestamp: options.getRemainingTimeInMillis() + Date.now(),
+        logLevel: params.logLevel,
+    });
     try {
         const postRenderData = await innerLaunchHandler({
             functionName,
             params,
             options,
+            overallProgress,
         });
         clearTimeout(webhookDueToTimeout);
         if (!params.webhook || webhookInvoked) {
@@ -394,24 +376,20 @@ const launchHandler = async (params, options) => {
             if (process.env.NODE_ENV === 'test') {
                 throw err;
             }
-            await (0, write_lambda_error_1.writeLambdaError)({
-                bucketName: params.bucketName,
-                errorInfo: {
-                    type: 'webhook',
-                    message: err.message,
-                    name: err.name,
-                    stack: err.stack,
-                    tmpDir: null,
-                    frame: 0,
-                    chunk: 0,
-                    isFatal: false,
-                    attempt: 1,
-                    willRetry: false,
-                    totalAttempts: 1,
-                },
-                renderId: params.renderId,
-                expectedBucketOwner: options.expectedBucketOwner,
+            overallProgress.addErrorWithoutUpload({
+                type: 'webhook',
+                message: err.message,
+                name: err.name,
+                stack: err.stack,
+                tmpDir: null,
+                frame: 0,
+                chunk: 0,
+                isFatal: false,
+                attempt: 1,
+                willRetry: false,
+                totalAttempts: 1,
             });
+            overallProgress.upload();
             renderer_1.RenderInternals.Log.error({ indent: false, logLevel: params.logLevel }, 'Failed to invoke webhook:');
             renderer_1.RenderInternals.Log.error({ indent: false, logLevel: params.logLevel }, err);
         }
@@ -424,24 +402,20 @@ const launchHandler = async (params, options) => {
             throw err;
         }
         renderer_1.RenderInternals.Log.error({ indent: false, logLevel: params.logLevel }, 'Error occurred', err);
-        await (0, write_lambda_error_1.writeLambdaError)({
-            bucketName: params.bucketName,
-            errorInfo: {
-                chunk: null,
-                frame: null,
-                name: err.name,
-                stack: err.stack,
-                type: 'stitcher',
-                isFatal: true,
-                tmpDir: (0, write_lambda_error_1.getTmpDirStateIfENoSp)(err.stack),
-                attempt: 1,
-                totalAttempts: 1,
-                willRetry: false,
-                message: err.message,
-            },
-            expectedBucketOwner: options.expectedBucketOwner,
-            renderId: params.renderId,
+        overallProgress.addErrorWithoutUpload({
+            chunk: null,
+            frame: null,
+            name: err.name,
+            stack: err.stack,
+            type: 'stitcher',
+            isFatal: true,
+            tmpDir: (0, write_lambda_error_1.getTmpDirStateIfENoSp)(err.stack),
+            attempt: 1,
+            totalAttempts: 1,
+            willRetry: false,
+            message: err.message,
         });
+        await overallProgress.upload();
         renderer_1.RenderInternals.Log.error({ indent: false, logLevel: params.logLevel }, 'Wrote error to S3');
         clearTimeout(webhookDueToTimeout);
         if (params.webhook && !webhookInvoked) {
@@ -469,24 +443,20 @@ const launchHandler = async (params, options) => {
                 if (process.env.NODE_ENV === 'test') {
                     throw error;
                 }
-                await (0, write_lambda_error_1.writeLambdaError)({
-                    bucketName: params.bucketName,
-                    errorInfo: {
-                        type: 'webhook',
-                        message: err.message,
-                        name: err.name,
-                        stack: err.stack,
-                        tmpDir: null,
-                        frame: 0,
-                        chunk: 0,
-                        isFatal: false,
-                        attempt: 1,
-                        willRetry: false,
-                        totalAttempts: 1,
-                    },
-                    renderId: params.renderId,
-                    expectedBucketOwner: options.expectedBucketOwner,
+                overallProgress.addErrorWithoutUpload({
+                    type: 'webhook',
+                    message: err.message,
+                    name: err.name,
+                    stack: err.stack,
+                    tmpDir: null,
+                    frame: 0,
+                    chunk: 0,
+                    isFatal: false,
+                    attempt: 1,
+                    willRetry: false,
+                    totalAttempts: 1,
                 });
+                overallProgress.upload();
                 renderer_1.RenderInternals.Log.error({ indent: false, logLevel: params.logLevel }, 'Failed to invoke webhook:');
                 renderer_1.RenderInternals.Log.error({ indent: false, logLevel: params.logLevel }, error);
             }

package/dist/functions/renderer.d.ts

@@ -1,11 +1,11 @@
 import type { LambdaPayload } from '../shared/constants';
 import type { RequestContext } from './helpers/request-context';
+import type { OnStream } from './streaming/streaming';
 type Options = {
     expectedBucketOwner: string;
     isWarm: boolean;
 };
-export declare const ENABLE_SLOW_LEAK_DETECTION = false;
-export declare const rendererHandler: (params: LambdaPayload, options: Options, requestContext: RequestContext) => Promise<{
+export declare const rendererHandler: (params: LambdaPayload, options: Options, onStream: OnStream, requestContext: RequestContext) => Promise<{
     type: 'success';
 }>;
 export {};