@remotion/lambda 4.0.163 → 4.0.165

package/dist/functions/launch.js

@@ -1,15 +1,13 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.launchHandler = void 0;
-/* eslint-disable @typescript-eslint/no-use-before-define */
-const client_lambda_1 = require("@aws-sdk/client-lambda");
 const renderer_1 = require("@remotion/renderer");
+const fs_1 = require("fs");
+const path_1 = require("path");
 const version_1 = require("remotion/version");
-const aws_clients_1 = require("../shared/aws-clients");
 const compress_props_1 = require("../shared/compress-props");
 const constants_1 = require("../shared/constants");
 const docs_url_1 = require("../shared/docs-url");
-const get_aws_urls_1 = require("../shared/get-aws-urls");
 const invoke_webhook_1 = require("../shared/invoke-webhook");
 const validate_1 = require("../shared/validate");
 const validate_frames_per_lambda_1 = require("../shared/validate-frames-per-lambda");
@@ -21,40 +19,12 @@ const expected_out_name_1 = require("./helpers/expected-out-name");
 const find_output_file_in_bucket_1 = require("./helpers/find-output-file-in-bucket");
 const get_browser_instance_1 = require("./helpers/get-browser-instance");
 const get_current_region_1 = require("./helpers/get-current-region");
-const io_1 = require("./helpers/io");
 const merge_chunks_1 = require("./helpers/merge-chunks");
-const timer_1 = require("./helpers/timer");
+const overall_render_progress_1 = require("./helpers/overall-render-progress");
+const stream_renderer_1 = require("./helpers/stream-renderer");
 const validate_composition_1 = require("./helpers/validate-composition");
 const write_lambda_error_1 = require("./helpers/write-lambda-error");
-const callFunctionWithRetry = async ({ payload, retries, functionName, }) => {
-    try {
-        const result = await (0, aws_clients_1.getLambdaClient)((0, get_current_region_1.getCurrentRegionInFunction)()).send(new client_lambda_1.InvokeCommand({
-            FunctionName: process.env.AWS_LAMBDA_FUNCTION_NAME,
-            Payload: JSON.stringify(payload),
-            InvocationType: 'Event',
-        }));
-        if (result.FunctionError) {
-            throw new Error(`Lambda function ${functionName} returned an error: ${result.FunctionError} ${result.LogResult}`);
-        }
-    }
-    catch (err) {
-        if (err.name === 'ResourceConflictException') {
-            if (retries > 10) {
-                throw err;
-            }
-            await new Promise((resolve) => {
-                setTimeout(resolve, 1000);
-            });
-            return callFunctionWithRetry({
-                payload,
-                retries: retries + 1,
-                functionName,
-            });
-        }
-        throw err;
-    }
-};
-const innerLaunchHandler = async ({ functionName, params, options, onAllChunksAvailable, }) => {
+const innerLaunchHandler = async ({ functionName, params, options, overallProgress, }) => {
     var _a, _b, _c, _d, _e;
     if (params.type !== constants_1.LambdaRoutines.launch) {
         throw new Error('Expected launch type');
@@ -76,6 +46,7 @@ const innerLaunchHandler = async ({ functionName, params, options, onAllChunksAv
     renderer_1.RenderInternals.Log.info(logOptions, 'Waiting for browser to be ready:', serializedInputPropsWithCustomSchema);
     const { instance } = await browserInstance;
     renderer_1.RenderInternals.Log.info(logOptions, 'Validating composition, input props:', serializedInputPropsWithCustomSchema);
+    const startTime = Date.now();
     const comp = await (0, validate_composition_1.validateComposition)({
         serveUrl: params.serveUrl,
         composition: params.composition,
@@ -132,8 +103,8 @@ const innerLaunchHandler = async ({ functionName, params, options, onAllChunksAv
     if (chunks.length > constants_1.MAX_FUNCTIONS_PER_RENDER) {
         throw new Error(`Too many functions: This render would cause ${chunks.length} functions to spawn. We limit this amount to ${constants_1.MAX_FUNCTIONS_PER_RENDER} functions as more would result in diminishing returns. Values set: frameCount = ${frameCount}, framesPerLambda=${framesPerLambda}. See ${docs_url_1.DOCS_URL}/docs/lambda/concurrency#too-many-functions for help.`);
     }
+    overallProgress.setExpectedChunks(chunks.length);
     const sortedChunks = chunks.slice().sort((a, b) => a[0] - b[0]);
-    const reqSend = (0, timer_1.timer)('sending off requests', params.logLevel);
     const serializedResolved = (0, compress_props_1.serializeOrThrow)(comp.props, 'resolved-props');
     const needsToUpload = (0, compress_props_1.getNeedsToUpload)('video-or-audio', [
         serializedResolved.length,
@@ -149,6 +120,9 @@ const innerLaunchHandler = async ({ functionName, params, options, onAllChunksAv
         needsToUpload,
     });
     const fps = comp.fps / params.everyNthFrame;
+    // If for 150 functions, we stream every frame, we DDos ourselves.
+    // Throttling a bit, allowing more progress if there is lower concurrency.
+    const progressEveryNthFrame = Math.ceil(chunks.length / 15);
     const lambdaPayloads = chunks.map((chunkPayload) => {
         var _a;
         const payload = {
@@ -196,6 +170,7 @@ const innerLaunchHandler = async ({ functionName, params, options, onAllChunksAv
             preferLossless: params.preferLossless,
             compositionStart: realFrameRange[0],
             framesPerLambda,
+            progressEveryNthFrame,
         };
         return payload;
     });
@@ -236,40 +211,36 @@ const innerLaunchHandler = async ({ functionName, params, options, onAllChunksAv
     const { key, renderBucketName, customCredentials } = (0, expected_out_name_1.getExpectedOutName)(renderMetadata, params.bucketName, typeof params.outName === 'string' || typeof params.outName === 'undefined'
         ? null
         : (_e = (_d = params.outName) === null || _d === void 0 ? void 0 : _d.s3OutputProvider) !== null && _e !== void 0 ? _e : null);
-    const output = await (0, find_output_file_in_bucket_1.findOutputFileInBucket)({
-        bucketName: params.bucketName,
-        customCredentials,
-        region: (0, get_current_region_1.getCurrentRegionInFunction)(),
-        renderMetadata,
-    });
-    if (output) {
-        if (params.overwrite) {
-            console.info('Deleting', { bucketName: renderBucketName, key }, 'because it already existed and will be overwritten');
-            await (0, io_1.lambdaDeleteFile)({
-                bucketName: renderBucketName,
-                customCredentials,
-                key,
-                region: (0, get_current_region_1.getCurrentRegionInFunction)(),
-            });
-        }
-        else {
-            throw new TypeError(`Output file "${key}" in bucket "${renderBucketName}" in region "${(0, get_current_region_1.getCurrentRegionInFunction)()}" already exists. Delete it before re-rendering, or use the overwrite option to delete it before render."`);
+    if (!params.overwrite) {
+        const output = await (0, find_output_file_in_bucket_1.findOutputFileInBucket)({
+            bucketName: params.bucketName,
+            customCredentials,
+            renderMetadata,
+            region: (0, get_current_region_1.getCurrentRegionInFunction)(),
+        });
+        if (output) {
+            throw new TypeError(`Output file "${key}" in bucket "${renderBucketName}" in region "${(0, get_current_region_1.getCurrentRegionInFunction)()}" already exists. Delete it before re-rendering, or set the 'overwrite' option in renderMediaOnLambda() to overwrite it."`);
         }
     }
-    await (0, io_1.lambdaWriteFile)({
-        bucketName: params.bucketName,
-        key: (0, constants_1.renderMetadataKey)(params.renderId),
-        body: JSON.stringify(renderMetadata),
-        region: (0, get_current_region_1.getCurrentRegionInFunction)(),
-        privacy: 'private',
-        expectedBucketOwner: options.expectedBucketOwner,
-        downloadBehavior: null,
-        customCredentials: null,
-    });
+    overallProgress.setRenderMetadata(renderMetadata);
+    const outdir = (0, path_1.join)(renderer_1.RenderInternals.tmpDir(constants_1.CONCAT_FOLDER_TOKEN), 'bucket');
+    if ((0, fs_1.existsSync)(outdir)) {
+        (0, fs_1.rmSync)(outdir, {
+            recursive: true,
+        });
+    }
+    (0, fs_1.mkdirSync)(outdir);
+    const files = [];
     await Promise.all(lambdaPayloads.map(async (payload) => {
-        await callFunctionWithRetry({ payload, retries: 0, functionName });
+        await (0, stream_renderer_1.streamRendererFunctionWithRetry)({
+            files,
+            functionName,
+            outdir,
+            overallProgress,
+            payload,
+            logLevel: params.logLevel,
+        });
     }));
-    reqSend.end();
     const postRenderData = await (0, merge_chunks_1.mergeChunksAndFinishRender)({
         bucketName: params.bucketName,
         renderId: params.renderId,
@@ -288,13 +259,16 @@ const innerLaunchHandler = async ({ functionName, params, options, onAllChunksAv
         inputProps: params.inputProps,
         serializedResolvedProps,
         renderMetadata,
-        onAllChunks: onAllChunksAvailable,
         audioBitrate: params.audioBitrate,
         logLevel: params.logLevel,
         framesPerLambda,
         binariesDirectory: null,
         preferLossless: params.preferLossless,
         compositionStart: realFrameRange[0],
+        outdir,
+        files: files.sort(),
+        overallProgress,
+        startTime,
     });
     return postRenderData;
 };
@@ -303,7 +277,6 @@ const launchHandler = async (params, options) => {
     if (params.type !== constants_1.LambdaRoutines.launch) {
         throw new Error('Expected launch type');
     }
-    let allChunksAvailable = null;
     const functionName = (_a = params.rendererFunctionName) !== null && _a !== void 0 ? _a : process.env.AWS_LAMBDA_FUNCTION_NAME;
     const logOptions = {
         indent: false,
@@ -311,62 +284,7 @@ const launchHandler = async (params, options) => {
     };
     const onTimeout = async () => {
         var _a;
-        if (allChunksAvailable) {
-            renderer_1.RenderInternals.Log.info(logOptions, 'All chunks are available, but the function is about to time out.');
-            renderer_1.RenderInternals.Log.info(logOptions, 'Spawning another function to merge chunks.');
-            try {
-                await callFunctionWithRetry({
-                    functionName,
-                    payload: {
-                        type: constants_1.LambdaRoutines.merge,
-                        renderId: params.renderId,
-                        bucketName: params.bucketName,
-                        outName: params.outName,
-                        serializedResolvedProps: allChunksAvailable.serializedResolvedProps,
-                        inputProps: allChunksAvailable.inputProps,
-                        logLevel: params.logLevel,
-                        framesPerLambda: allChunksAvailable.framesPerLambda,
-                        preferLossless: params.preferLossless,
-                        compositionStart: allChunksAvailable.compositionStart,
-                    },
-                    retries: 2,
-                });
-                renderer_1.RenderInternals.Log.info(logOptions, `New function successfully invoked. See the CloudWatch logs for it:`);
-                renderer_1.RenderInternals.Log.info(logOptions, (0, get_aws_urls_1.getCloudwatchMethodUrl)({
-                    functionName: process.env.AWS_LAMBDA_FUNCTION_NAME,
-                    method: constants_1.LambdaRoutines.merge,
-                    region: (0, get_current_region_1.getCurrentRegionInFunction)(),
-                    rendererFunctionName: params.rendererFunctionName,
-                    renderId: params.renderId,
-                }));
-                renderer_1.RenderInternals.Log.info(logOptions, 'This function will now time out.');
-            }
-            catch (err) {
-                if (process.env.NODE_ENV === 'test') {
-                    throw err;
-                }
-                renderer_1.RenderInternals.Log.error({ indent: false, logLevel: params.logLevel }, 'Failed to invoke additional function to merge videos:');
-                renderer_1.RenderInternals.Log.error({ indent: false, logLevel: params.logLevel }, err);
-                await (0, write_lambda_error_1.writeLambdaError)({
-                    bucketName: params.bucketName,
-                    errorInfo: {
-                        type: 'stitcher',
-                        message: err.message,
-                        name: err.name,
-                        stack: err.stack,
-                        tmpDir: null,
-                        frame: 0,
-                        chunk: 0,
-                        isFatal: false,
-                        attempt: 1,
-                        willRetry: false,
-                        totalAttempts: 1,
-                    },
-                    renderId: params.renderId,
-                    expectedBucketOwner: options.expectedBucketOwner,
-                });
-            }
-        }
+        renderer_1.RenderInternals.Log.error({ indent: false, logLevel: params.logLevel }, 'Function is about to time out. Can not finish render.');
         if (!params.webhook) {
             return;
         }
@@ -394,42 +312,39 @@ const launchHandler = async (params, options) => {
             }
             renderer_1.RenderInternals.Log.error({ indent: false, logLevel: params.logLevel }, 'Failed to invoke webhook:');
             renderer_1.RenderInternals.Log.error({ indent: false, logLevel: params.logLevel }, err);
-            await (0, write_lambda_error_1.writeLambdaError)({
-                bucketName: params.bucketName,
-                errorInfo: {
-                    type: 'webhook',
-                    message: err.message,
-                    name: err.name,
-                    stack: err.stack,
-                    tmpDir: null,
-                    frame: 0,
-                    chunk: 0,
-                    isFatal: false,
-                    attempt: 1,
-                    willRetry: false,
-                    totalAttempts: 1,
-                },
-                renderId: params.renderId,
-                expectedBucketOwner: options.expectedBucketOwner,
+            overallProgress.addErrorWithoutUpload({
+                type: 'webhook',
+                message: err.message,
+                name: err.name,
+                stack: err.stack,
+                tmpDir: null,
+                frame: 0,
+                chunk: 0,
+                isFatal: false,
+                attempt: 1,
+                willRetry: false,
+                totalAttempts: 1,
             });
+            overallProgress.upload();
         }
     };
     let webhookInvoked = false;
     const webhookDueToTimeout = setTimeout(onTimeout, Math.max(options.getRemainingTimeInMillis() - 1000, 1000));
     renderer_1.RenderInternals.Log.info(logOptions, `Function has ${Math.max(options.getRemainingTimeInMillis() - 1000, 1000)} before it times out`);
+    const overallProgress = (0, overall_render_progress_1.makeOverallRenderProgress)({
+        renderId: params.renderId,
+        bucketName: params.bucketName,
+        expectedBucketOwner: options.expectedBucketOwner,
+        region: (0, get_current_region_1.getCurrentRegionInFunction)(),
+        timeoutTimestamp: options.getRemainingTimeInMillis() + Date.now(),
+        logLevel: params.logLevel,
+    });
     try {
         const postRenderData = await innerLaunchHandler({
             functionName,
             params,
             options,
-            onAllChunksAvailable: ({ inputProps, serializedResolvedProps, framesPerLambda, compositionStart, }) => {
-                allChunksAvailable = {
-                    inputProps,
-                    serializedResolvedProps,
-                    framesPerLambda,
-                    compositionStart,
-                };
-            },
+            overallProgress,
         });
         clearTimeout(webhookDueToTimeout);
         if (!params.webhook || webhookInvoked) {
@@ -461,24 +376,20 @@ const launchHandler = async (params, options) => {
             if (process.env.NODE_ENV === 'test') {
                 throw err;
             }
-            await (0, write_lambda_error_1.writeLambdaError)({
-                bucketName: params.bucketName,
-                errorInfo: {
-                    type: 'webhook',
-                    message: err.message,
-                    name: err.name,
-                    stack: err.stack,
-                    tmpDir: null,
-                    frame: 0,
-                    chunk: 0,
-                    isFatal: false,
-                    attempt: 1,
-                    willRetry: false,
-                    totalAttempts: 1,
-                },
-                renderId: params.renderId,
-                expectedBucketOwner: options.expectedBucketOwner,
+            overallProgress.addErrorWithoutUpload({
+                type: 'webhook',
+                message: err.message,
+                name: err.name,
+                stack: err.stack,
+                tmpDir: null,
+                frame: 0,
+                chunk: 0,
+                isFatal: false,
+                attempt: 1,
+                willRetry: false,
+                totalAttempts: 1,
             });
+            overallProgress.upload();
             renderer_1.RenderInternals.Log.error({ indent: false, logLevel: params.logLevel }, 'Failed to invoke webhook:');
             renderer_1.RenderInternals.Log.error({ indent: false, logLevel: params.logLevel }, err);
         }
@@ -491,24 +402,20 @@ const launchHandler = async (params, options) => {
             throw err;
         }
         renderer_1.RenderInternals.Log.error({ indent: false, logLevel: params.logLevel }, 'Error occurred', err);
-        await (0, write_lambda_error_1.writeLambdaError)({
-            bucketName: params.bucketName,
-            errorInfo: {
-                chunk: null,
-                frame: null,
-                name: err.name,
-                stack: err.stack,
-                type: 'stitcher',
-                isFatal: true,
-                tmpDir: (0, write_lambda_error_1.getTmpDirStateIfENoSp)(err.stack),
-                attempt: 1,
-                totalAttempts: 1,
-                willRetry: false,
-                message: err.message,
-            },
-            expectedBucketOwner: options.expectedBucketOwner,
-            renderId: params.renderId,
+        overallProgress.addErrorWithoutUpload({
+            chunk: null,
+            frame: null,
+            name: err.name,
+            stack: err.stack,
+            type: 'stitcher',
+            isFatal: true,
+            tmpDir: (0, write_lambda_error_1.getTmpDirStateIfENoSp)(err.stack),
+            attempt: 1,
+            totalAttempts: 1,
+            willRetry: false,
+            message: err.message,
         });
+        await overallProgress.upload();
         renderer_1.RenderInternals.Log.error({ indent: false, logLevel: params.logLevel }, 'Wrote error to S3');
         clearTimeout(webhookDueToTimeout);
         if (params.webhook && !webhookInvoked) {
@@ -536,24 +443,20 @@ const launchHandler = async (params, options) => {
                 if (process.env.NODE_ENV === 'test') {
                     throw error;
                 }
-                await (0, write_lambda_error_1.writeLambdaError)({
-                    bucketName: params.bucketName,
-                    errorInfo: {
-                        type: 'webhook',
-                        message: err.message,
-                        name: err.name,
-                        stack: err.stack,
-                        tmpDir: null,
-                        frame: 0,
-                        chunk: 0,
-                        isFatal: false,
-                        attempt: 1,
-                        willRetry: false,
-                        totalAttempts: 1,
-                    },
-                    renderId: params.renderId,
-                    expectedBucketOwner: options.expectedBucketOwner,
+                overallProgress.addErrorWithoutUpload({
+                    type: 'webhook',
+                    message: err.message,
+                    name: err.name,
+                    stack: err.stack,
+                    tmpDir: null,
+                    frame: 0,
+                    chunk: 0,
+                    isFatal: false,
+                    attempt: 1,
+                    willRetry: false,
+                    totalAttempts: 1,
                 });
+                overallProgress.upload();
                 renderer_1.RenderInternals.Log.error({ indent: false, logLevel: params.logLevel }, 'Failed to invoke webhook:');
                 renderer_1.RenderInternals.Log.error({ indent: false, logLevel: params.logLevel }, error);
             }

package/dist/functions/renderer.d.ts

@@ -1,11 +1,11 @@
 import type { LambdaPayload } from '../shared/constants';
 import type { RequestContext } from './helpers/request-context';
+import type { OnStream } from './streaming/streaming';
 type Options = {
     expectedBucketOwner: string;
     isWarm: boolean;
 };
-export declare const ENABLE_SLOW_LEAK_DETECTION = false;
-export declare const rendererHandler: (params: LambdaPayload, options: Options, requestContext: RequestContext) => Promise<{
+export declare const rendererHandler: (params: LambdaPayload, options: Options, onStream: OnStream, requestContext: RequestContext) => Promise<{
     type: 'success';
 }>;
 export {};