@remotion/lambda-client 4.0.261

package/src/call-lambda-async.ts

@@ -0,0 +1,39 @@
+import {InvokeCommand} from '@aws-sdk/client-lambda';
+import type {
+	CallFunctionOptions,
+	CloudProvider,
+	ServerlessRoutines,
+} from '@remotion/serverless-client';
+import {getLambdaClient} from './aws-clients';
+import type {AwsRegion} from './regions';
+
+export const callFunctionAsyncImplementation = async <
+	T extends ServerlessRoutines,
+	Provider extends CloudProvider,
+>({
+	functionName,
+	payload,
+	region,
+	timeoutInTest,
+}: CallFunctionOptions<T, Provider>): Promise<void> => {
+	const stringifiedPayload = JSON.stringify(payload);
+	if (stringifiedPayload.length > 256 * 1024) {
+		throw new Error(
+			`Payload is too big: ${stringifiedPayload.length} bytes. Maximum size is 256 KB. This should not happen, please report this to the Remotion team. Payload: ${stringifiedPayload}`,
+		);
+	}
+
+	const result = await getLambdaClient(region as AwsRegion, timeoutInTest).send(
+		new InvokeCommand({
+			FunctionName: functionName,
+			Payload: stringifiedPayload,
+			InvocationType: 'Event',
+		}),
+	);
+
+	if (result.FunctionError) {
+		throw new Error(
+			`Lambda function returned error: ${result.FunctionError} ${result.LogResult}`,
+		);
+	}
+};

package/src/call-lambda-streaming.ts

@@ -0,0 +1,219 @@
+/* eslint-disable no-console */
+import type {InvokeWithResponseStreamCommandOutput} from '@aws-sdk/client-lambda';
+import {
+	InvokeWithResponseStreamCommand,
+	type InvokeWithResponseStreamResponseEvent,
+} from '@aws-sdk/client-lambda';
+import type {
+	CallFunctionOptions,
+	CloudProvider,
+	MessageTypeId,
+	OnMessage,
+	ServerlessRoutines,
+	StreamingMessage,
+} from '@remotion/serverless-client';
+import {
+	formatMap,
+	makeStreamer,
+	messageTypeIdToMessageType,
+} from '@remotion/serverless-client';
+import {getLambdaClient} from './aws-clients';
+import type {AwsRegion} from './regions';
+
+const STREAM_STALL_TIMEOUT = 30000;
+const LAMBDA_STREAM_STALL = `AWS did not invoke Lambda in ${STREAM_STALL_TIMEOUT}ms`;
+
+export const parseJsonOrThrowSource = (data: Uint8Array, type: string) => {
+	const asString = new TextDecoder('utf-8').decode(data);
+	try {
+		return JSON.parse(asString);
+	} catch {
+		throw new Error(`Invalid JSON (${type}): ${asString}`);
+	}
+};
+
+const invokeStreamOrTimeout = async <Provider extends CloudProvider>({
+	region,
+	timeoutInTest,
+	functionName,
+	type,
+	payload,
+}: {
+	region: Provider['region'];
+	timeoutInTest: number;
+	functionName: string;
+	type: string;
+	payload: Record<string, unknown>;
+}) => {
+	const resProm = getLambdaClient(region as AwsRegion, timeoutInTest).send(
+		new InvokeWithResponseStreamCommand({
+			FunctionName: functionName,
+			Payload: JSON.stringify({type, ...payload}),
+		}),
+	);
+
+	let cleanup = () => undefined;
+
+	const timeout = new Promise<InvokeWithResponseStreamCommandOutput>(
+		(_resolve, reject) => {
+			const int = setTimeout(() => {
+				reject(new Error(LAMBDA_STREAM_STALL));
+			}, STREAM_STALL_TIMEOUT);
+			cleanup = () => {
+				clearTimeout(int);
+			};
+		},
+	);
+
+	const res = await Promise.race([resProm, timeout]);
+
+	cleanup();
+
+	return res;
+};
+
+const INVALID_JSON_MESSAGE = 'Cannot parse Lambda response as JSON';
+
+const callLambdaWithStreamingWithoutRetry = async <
+	T extends ServerlessRoutines,
+	Provider extends CloudProvider,
+>({
+	functionName,
+	type,
+	payload,
+	region,
+	timeoutInTest,
+	receivedStreamingPayload,
+}: CallFunctionOptions<T, Provider> & {
+	receivedStreamingPayload: OnMessage<Provider>;
+}): Promise<void> => {
+	const res = await invokeStreamOrTimeout({
+		functionName,
+		payload,
+		region,
+		timeoutInTest,
+		type,
+	});
+
+	const {onData, clear} = makeStreamer((status, messageTypeId, data) => {
+		const messageType = messageTypeIdToMessageType(
+			messageTypeId as MessageTypeId,
+		);
+		const innerPayload =
+			formatMap[messageType] === 'json'
+				? parseJsonOrThrowSource(data, messageType)
+				: data;
+
+		const message: StreamingMessage<Provider> = {
+			successType: status,
+			message: {
+				type: messageType,
+				payload: innerPayload,
+			},
+		};
+
+		receivedStreamingPayload(message);
+	});
+
+	const dumpBuffers = () => {
+		clear();
+	};
+
+	// @ts-expect-error - We are adding a listener to a global variable
+	if (globalThis._dumpUnreleasedBuffers) {
+		// @ts-expect-error - We are adding a listener to a global variable
+		(globalThis._dumpUnreleasedBuffers as EventEmitter).addListener(
+			'dump-unreleased-buffers',
+			dumpBuffers,
+		);
+	}
+
+	const events =
+		res.EventStream as AsyncIterable<InvokeWithResponseStreamResponseEvent>;
+
+	for await (const event of events) {
+		// There are two types of events you can get on a stream.
+
+		// `PayloadChunk`: These contain the actual raw bytes of the chunk
+		// It has a single property: `Payload`
+		if (event.PayloadChunk && event.PayloadChunk.Payload) {
+			onData(event.PayloadChunk.Payload);
+		}
+
+		if (event.InvokeComplete) {
+			if (event.InvokeComplete.ErrorCode) {
+				const logs = `https://${region}.console.aws.amazon.com/cloudwatch/home?region=${region}#logsV2:logs-insights$3FqueryDetail$3D~(end~0~start~-3600~timeType~'RELATIVE~unit~'seconds~editorString~'fields*20*40timestamp*2c*20*40requestId*2c*20*40message*0a*7c*20filter*20*40requestId*20like*20*${res.$metadata.requestId}*22*0a*7c*20sort*20*40timestamp*20asc~source~(~'*2faws*2flambda*2f${functionName}))`;
+				if (event.InvokeComplete.ErrorCode === 'Unhandled') {
+					throw new Error(
+						`Lambda function ${functionName} failed with an unhandled error: ${
+							event.InvokeComplete.ErrorDetails as string
+						} See ${logs} to see the logs of this invocation.`,
+					);
+				}
+
+				throw new Error(
+					`Lambda function ${functionName} failed with error code ${event.InvokeComplete.ErrorCode}: ${event.InvokeComplete.ErrorDetails}. See ${logs} to see the logs of this invocation.`,
+				);
+			}
+		}
+
+		// Don't put a `break` statement here, as it will cause the socket to not properly exit.
+	}
+
+	// @ts-expect-error - We are adding a listener to a global variable
+	if (globalThis._dumpUnreleasedBuffers) {
+		// @ts-expect-error - We are adding a listener to a global variable
+		(globalThis._dumpUnreleasedBuffers as EventEmitter).removeListener(
+			'dump-unreleased-buffers',
+			dumpBuffers,
+		);
+	}
+
+	clear();
+};
+
+export const callFunctionWithStreamingImplementation = async <
+	Provider extends CloudProvider,
+	T extends ServerlessRoutines,
+>(
+	options: CallFunctionOptions<T, Provider> & {
+		receivedStreamingPayload: OnMessage<Provider>;
+		retriesRemaining: number;
+	},
+): Promise<void> => {
+	// As of August 2023, Lambda streaming sometimes misses parts of the JSON response.
+	// Handling this for now by applying a retry mechanism.
+
+	try {
+		// Do not remove this await
+		await callLambdaWithStreamingWithoutRetry<T, Provider>(options);
+	} catch (err) {
+		if ((err as Error).stack?.includes('TooManyRequestsException')) {
+			throw new Error(
+				`AWS Concurrency limit reached (Original Error: ${(err as Error).message}). See https://www.remotion.dev/docs/lambda/troubleshooting/rate-limit for tips to fix this.`,
+			);
+		}
+
+		if (
+			!(err as Error).message.includes(INVALID_JSON_MESSAGE) &&
+			!(err as Error).message.includes(LAMBDA_STREAM_STALL) &&
+			// https://discord.com/channels/809501355504959528/1332166561242288220/1332166561242288220
+			!(err as Error).message.includes('Runtime.TruncatedResponse') &&
+			!(err as Error).message.includes('aborted')
+		) {
+			throw err;
+		}
+
+		console.error('Retries remaining:', options.retriesRemaining);
+		if (options.retriesRemaining === 0) {
+			console.error('Throwing error:');
+			throw err;
+		}
+
+		console.error(err);
+		return callFunctionWithStreamingImplementation({
+			...options,
+			retriesRemaining: options.retriesRemaining - 1,
+		});
+	}
+};

package/src/call-lambda-sync.ts

@@ -0,0 +1,55 @@
+import {InvokeCommand} from '@aws-sdk/client-lambda';
+import type {
+	CallFunctionOptions,
+	CloudProvider,
+	OrError,
+	ServerlessReturnValues,
+	ServerlessRoutines,
+} from '@remotion/serverless-client';
+import {getLambdaClient} from './aws-clients';
+import type {AwsRegion} from './regions';
+
+const callLambdaSyncWithoutRetry = async <
+	T extends ServerlessRoutines,
+	Provider extends CloudProvider,
+>({
+	functionName,
+	payload,
+	region,
+	timeoutInTest,
+}: CallFunctionOptions<T, Provider>): Promise<
+	OrError<ServerlessReturnValues<Provider>[T]>
+> => {
+	const Payload = JSON.stringify(payload);
+	const res = await getLambdaClient(region as AwsRegion, timeoutInTest).send(
+		new InvokeCommand({
+			FunctionName: functionName,
+			Payload,
+			InvocationType: 'RequestResponse',
+		}),
+	);
+
+	const decoded = new TextDecoder('utf-8').decode(res.Payload);
+
+	try {
+		return JSON.parse(decoded) as OrError<ServerlessReturnValues<Provider>[T]>;
+	} catch {
+		throw new Error(`Invalid JSON: ${JSON.stringify(decoded)}`);
+	}
+};
+
+export const callFunctionSyncImplementation = async <
+	Provider extends CloudProvider,
+	T extends ServerlessRoutines,
+>(
+	options: CallFunctionOptions<T, Provider>,
+): Promise<ServerlessReturnValues<Provider>[T]> => {
+	const res = await callLambdaSyncWithoutRetry<T, Provider>(options);
+	if (res.type === 'error') {
+		const err = new Error(res.message);
+		err.stack = res.stack;
+		throw err;
+	}
+
+	return res;
+};

package/src/check-credentials.ts

@@ -0,0 +1,51 @@
+import {DOCS_URL, truthy} from '@remotion/serverless-client';
+import {getEnvVariable} from './get-env-variable';
+import {getIsCli} from './is-cli';
+import {isLikelyToHaveAwsProfile} from './is-likely-to-have-aws-profile';
+
+const messageForVariable = (variable: string) => {
+	return [
+		`You have tried to call a Remotion Lambda function, but have not set the environment variable ${variable}.`,
+		getIsCli()
+			? null
+			: `- Environment variables from a '.env' file are not automatically read if you are calling the Node.JS APIs, in that case you need to load the file yourself or set the environment variables manually.`,
+		`- Please refer to the Remotion Lambda docs (${DOCS_URL}/docs/lambda/setup) to see how to generate the credentials for your AWS account and then set the environment variables.`,
+		`- For more reasons see the troubleshooting page: ${DOCS_URL}/docs/lambda/troubleshooting/permissions`,
+	]
+		.filter(truthy)
+		.join('\n');
+};
+
+export const checkCredentials = () => {
+	if (getEnvVariable('REMOTION_SKIP_AWS_CREDENTIALS_CHECK')) {
+		return;
+	}
+
+	if (getEnvVariable('REMOTION_AWS_PROFILE') || getEnvVariable('AWS_PROFILE')) {
+		return;
+	}
+
+	if (isLikelyToHaveAwsProfile()) {
+		return;
+	}
+
+	if (
+		!getEnvVariable('AWS_ACCESS_KEY_ID') &&
+		!getEnvVariable('REMOTION_AWS_ACCESS_KEY_ID')
+	) {
+		throw new Error(
+			messageForVariable('AWS_ACCESS_KEY_ID or REMOTION_AWS_ACCESS_KEY_ID'),
+		);
+	}
+
+	if (
+		!getEnvVariable('AWS_SECRET_ACCESS_KEY') &&
+		!getEnvVariable('REMOTION_AWS_SECRET_ACCESS_KEY')
+	) {
+		throw new Error(
+			messageForVariable(
+				'AWS_SECRET_ACCESS_KEY or REMOTION_AWS_SECRET_ACCESS_KEY',
+			),
+		);
+	}
+};

package/src/clean-items.ts

@@ -0,0 +1,47 @@
+import type {
+	CloudProvider,
+	ProviderSpecifics,
+} from '@remotion/serverless-client';
+import {pLimit} from './p-limit';
+
+const limit = pLimit(10);
+
+export const cleanItems = <Provider extends CloudProvider>({
+	bucket,
+	onAfterItemDeleted,
+	onBeforeItemDeleted,
+	region,
+	list,
+	providerSpecifics,
+	forcePathStyle,
+}: {
+	bucket: string;
+	region: Provider['region'];
+	list: string[];
+	onBeforeItemDeleted: (data: {bucketName: string; itemName: string}) => void;
+	onAfterItemDeleted: (data: {bucketName: string; itemName: string}) => void;
+	providerSpecifics: ProviderSpecifics<Provider>;
+	forcePathStyle: boolean;
+}) => {
+	return Promise.all(
+		list.map((object) =>
+			limit(async () => {
+				onBeforeItemDeleted({
+					bucketName: bucket,
+					itemName: object,
+				});
+				await providerSpecifics.deleteFile({
+					bucketName: bucket,
+					key: object,
+					region,
+					customCredentials: null,
+					forcePathStyle,
+				});
+				onAfterItemDeleted({
+					bucketName: bucket,
+					itemName: object,
+				});
+			}),
+		),
+	);
+};

package/src/constants.ts

@@ -0,0 +1,38 @@
+import type {GenericRenderProgress, Privacy} from '@remotion/serverless-client';
+import type {AwsProvider} from './aws-provider';
+import type {AwsRegion} from './regions';
+
+export const MIN_MEMORY = 512;
+export const MAX_MEMORY = 10240;
+export const DEFAULT_MEMORY_SIZE = 2048;
+
+export const DEFAULT_TIMEOUT = 120;
+export const MIN_TIMEOUT = 15;
+export const MAX_TIMEOUT = 900;
+
+export const DEFAULT_FRAMES_PER_LAMBDA = 20;
+
+export const BINARY_NAME = 'remotion lambda';
+export const DEFAULT_REGION: AwsRegion = 'us-east-1';
+export const DEFAULT_MAX_RETRIES = 1;
+
+export const MAX_EPHEMERAL_STORAGE_IN_MB = 10240;
+// TODO: In V5, Enable set this to 10240
+export const DEFAULT_EPHEMERAL_STORAGE_IN_MB = 2048;
+export const MIN_EPHEMERAL_STORAGE_IN_MB = 512;
+
+export const DEFAULT_OUTPUT_PRIVACY: Privacy = 'public';
+
+export const DEFAULT_CLOUDWATCH_RETENTION_PERIOD = 14;
+
+export const RENDER_FN_PREFIX = 'remotion-render-';
+export const LOG_GROUP_PREFIX = '/aws/lambda/';
+export const LAMBDA_INSIGHTS_PREFIX = '/aws/lambda-insights';
+
+export const getSitesKey = (siteId: string) => `sites/${siteId}`;
+
+export type RenderProgress = GenericRenderProgress<AwsProvider>;
+
+export const LAMBDA_CONCURRENCY_LIMIT_QUOTA = 'L-B99A9384';
+
+export const REMOTION_BUCKET_PREFIX = 'remotionlambda-';

package/src/content-disposition-header.ts

@@ -0,0 +1,64 @@
+import type {DownloadBehavior} from '@remotion/serverless-client';
+
+// By setting the Content-Disposition header in an S3 object,
+// you can control if the user downloads the item if you
+// visit the link
+
+const problematicCharacters = {
+	'%3A': ':',
+	'%2F': '/',
+	'%3F': '?',
+	'%23': '#',
+	'%5B': '[',
+	'%5D': ']',
+	'%40': '@',
+	'%21': '!',
+	'%24': '$',
+	'%26': '&',
+	'%27': "'",
+	'%28': '(',
+	'%29': ')',
+	'%2A': '*',
+	'%2B': '+',
+	'%2C': ',',
+	'%3B': ';',
+};
+
+type HexInfo = {
+	containsHex: boolean;
+};
+
+const includesHexOfUnsafeChar = (path: string): HexInfo => {
+	for (const key of Object.keys(
+		problematicCharacters,
+	) as (keyof typeof problematicCharacters)[]) {
+		if (path.includes(key)) {
+			return {containsHex: true};
+		}
+	}
+
+	return {containsHex: false};
+};
+
+export const getContentDispositionHeader = (
+	behavior: DownloadBehavior | null,
+): string | undefined => {
+	if (behavior === null) {
+		return undefined;
+	}
+
+	if (behavior.type === 'play-in-browser') {
+		return undefined;
+	}
+
+	if (behavior.fileName === null) {
+		return `attachment`;
+	}
+
+	const {containsHex} = includesHexOfUnsafeChar(behavior.fileName);
+	if (containsHex) {
+		return `attachment; filename="${behavior.fileName}"`;
+	}
+
+	return `attachment; filename="${encodeURIComponent(behavior.fileName)}"`;
+};

package/src/convert-to-serve-url.ts

@@ -0,0 +1,24 @@
+import {DOCS_URL} from '@remotion/serverless-client';
+import type {AwsRegion} from './regions';
+
+export const convertToServeUrlImplementation = ({
+	urlOrId,
+	region,
+	bucketName,
+}: {
+	urlOrId: string;
+	region: AwsRegion;
+	bucketName: string;
+}) => {
+	if (urlOrId.startsWith('src/')) {
+		throw new Error(
+			`Remotion Lambda can only render based on a URL in the cloud. It seems like you passed a local file: ${urlOrId}. Read the setup guide for Remotion Lambda ${DOCS_URL}/docs/lambda/setup`,
+		);
+	}
+
+	if (urlOrId.startsWith('http://') || urlOrId.startsWith('https://')) {
+		return urlOrId;
+	}
+
+	return `https://${bucketName}.s3.${region}.amazonaws.com/sites/${urlOrId}/index.html`;
+};

package/src/create-bucket.ts

@@ -0,0 +1,67 @@
+import {
+	CreateBucketCommand,
+	DeleteBucketOwnershipControlsCommand,
+	DeletePublicAccessBlockCommand,
+	PutBucketAclCommand,
+} from '@aws-sdk/client-s3';
+import type {ProviderSpecifics} from '@remotion/serverless-client';
+import type {AwsProvider} from './aws-provider';
+import {getS3Client} from './get-s3-client';
+
+export const createBucket: ProviderSpecifics<AwsProvider>['createBucket'] =
+	async ({region, bucketName, forcePathStyle}) => {
+		await getS3Client({region, customCredentials: null, forcePathStyle}).send(
+			new CreateBucketCommand({
+				Bucket: bucketName,
+			}),
+		);
+
+		try {
+			await getS3Client({region, customCredentials: null, forcePathStyle}).send(
+				new DeleteBucketOwnershipControlsCommand({
+					Bucket: bucketName,
+				}),
+			);
+		} catch (err) {
+			if ((err as Error).message.includes('Access Denied')) {
+				throw new Error(
+					'Since April 2023, more AWS permissions are required to create an S3 bucket. You need to update your user policy to continue. See https://remotion.dev/docs/lambda/s3-public-access for instructions on how to resolve this issue.',
+				);
+			}
+
+			throw err;
+		}
+
+		try {
+			await getS3Client({region, customCredentials: null, forcePathStyle}).send(
+				new DeletePublicAccessBlockCommand({
+					Bucket: bucketName,
+				}),
+			);
+		} catch (err) {
+			if ((err as Error).message.includes('Access Denied')) {
+				throw new Error(
+					'PARTIAL SUCCESS: The s3:PutBucketOwnershipControls was found, but the s3:PutBucketPublicAccessBlock permission is not given. Since April 2023, more AWS permissions are required to create an S3 bucket. You need to update your user policy to continue. You need to update your user policy to continue. See https://remotion.dev/docs/lambda/s3-public-access for instructions on how to resolve this issue.',
+				);
+			}
+
+			throw err;
+		}
+
+		try {
+			await getS3Client({region, customCredentials: null, forcePathStyle}).send(
+				new PutBucketAclCommand({
+					Bucket: bucketName,
+					ACL: 'public-read',
+				}),
+			);
+		} catch (err) {
+			if ((err as Error).message.includes('The bucket does not allow ACLs')) {
+				throw new Error(
+					`Could not add an ACL to the bucket. This might have happened because the bucket was already successfully created before but then failed to configure correctly. We recommend to delete the bucket (${bucketName}) if it is empty and start over to fix the problem.`,
+				);
+			}
+
+			throw err;
+		}
+	};

package/src/delete-file.ts

@@ -0,0 +1,30 @@
+import {DeleteObjectCommand} from '@aws-sdk/client-s3';
+import type {
+	CustomCredentials,
+	ProviderSpecifics,
+} from '@remotion/serverless-client';
+import type {AwsProvider} from './aws-provider';
+import {getS3Client} from './get-s3-client';
+import type {AwsRegion} from './regions';
+
+export const lambdaDeleteFileImplementation: ProviderSpecifics<AwsProvider>['deleteFile'] =
+	async ({
+		bucketName,
+		key,
+		region,
+		customCredentials,
+		forcePathStyle,
+	}: {
+		region: AwsRegion;
+		bucketName: string;
+		key: string;
+		customCredentials: CustomCredentials<AwsProvider> | null;
+		forcePathStyle: boolean;
+	}) => {
+		await getS3Client({region, customCredentials, forcePathStyle}).send(
+			new DeleteObjectCommand({
+				Bucket: bucketName,
+				Key: key,
+			}),
+		);
+	};

package/src/delete-function.ts

@@ -0,0 +1,24 @@
+import {DeleteFunctionCommand} from '@aws-sdk/client-lambda';
+import type {
+	DeleteFunction,
+	DeleteFunctionInput as GenericDeleteFunctionInput,
+} from '@remotion/serverless-client';
+import {getLambdaClient} from './aws-clients';
+import type {AwsProvider} from './aws-provider';
+
+export type DeleteFunctionInput = GenericDeleteFunctionInput<AwsProvider>;
+
+/*
+ * @description Deletes a deployed Lambda function based on its name.
+ * @see [Documentation](https://remotion.dev/docs/lambda/deletefunction)
+ */
+export const deleteFunction: DeleteFunction<AwsProvider> = async ({
+	region,
+	functionName,
+}: DeleteFunctionInput): Promise<void> => {
+	await getLambdaClient(region).send(
+		new DeleteFunctionCommand({
+			FunctionName: functionName,
+		}),
+	);
+};