@remotion/lambda 4.0.200 → 4.0.201

package/dist/shared/compress-props.d.ts

@@ -1,22 +1,20 @@
-import type { ProviderSpecifics } from '@remotion/serverless';
-import { type SerializedInputProps } from '@remotion/serverless/client';
+import type { SerializedInputProps } from '@remotion/serverless/client';
+import type { AwsRegion } from '../client';
 type PropsType = 'input-props' | 'resolved-props';
 export declare const serializeOrThrow: (inputProps: Record<string, unknown>, propsType: PropsType) => string;
 export declare const getNeedsToUpload: (type: 'still' | 'video-or-audio', sizes: number[]) => boolean;
-export declare const compressInputProps: <Region extends string>({ stringifiedInputProps, region, userSpecifiedBucketName, propsType, needsToUpload, providerSpecifics, }: {
+export declare const compressInputProps: ({ stringifiedInputProps, region, userSpecifiedBucketName, propsType, needsToUpload, }: {
     stringifiedInputProps: string;
-    region: Region;
+    region: AwsRegion;
     userSpecifiedBucketName: string | null;
     propsType: PropsType;
     needsToUpload: boolean;
-    providerSpecifics: ProviderSpecifics<Region>;
 }) => Promise<SerializedInputProps>;
-export declare const decompressInputProps: <Region extends string>({ serialized, region, bucketName, expectedBucketOwner, propsType, providerSpecifics, }: {
+export declare const decompressInputProps: ({ serialized, region, bucketName, expectedBucketOwner, propsType, }: {
     serialized: SerializedInputProps;
-    region: Region;
+    region: AwsRegion;
     bucketName: string;
     expectedBucketOwner: string;
     propsType: PropsType;
-    providerSpecifics: ProviderSpecifics<Region>;
 }) => Promise<string>;
 export {};

package/dist/shared/compress-props.js

@@ -1,14 +1,18 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.decompressInputProps = exports.compressInputProps = exports.getNeedsToUpload = exports.serializeOrThrow = void 0;
-const client_1 = require("@remotion/serverless/client");
 const no_react_1 = require("remotion/no-react");
+const get_or_create_bucket_1 = require("../api/get-or-create-bucket");
+const io_1 = require("../functions/helpers/io");
+const constants_1 = require("./constants");
+const random_hash_1 = require("./random-hash");
+const stream_to_string_1 = require("./stream-to-string");
 const validate_webhook_1 = require("./validate-webhook");
 const makeKey = (type, hash) => {
     if (type === 'input-props') {
-        return (0, client_1.inputPropsKey)(hash);
+        return (0, constants_1.inputPropsKey)(hash);
     }
-    return (0, client_1.resolvedPropsKey)(hash);
+    return (0, constants_1.resolvedPropsKey)(hash);
 };
 const serializeOrThrow = (inputProps, propsType) => {
     try {
@@ -35,16 +39,15 @@ const getNeedsToUpload = (type, sizes) => {
     return false;
 };
 exports.getNeedsToUpload = getNeedsToUpload;
-const compressInputProps = async ({ stringifiedInputProps, region, userSpecifiedBucketName, propsType, needsToUpload, providerSpecifics, }) => {
-    const hash = providerSpecifics.randomHash();
+const compressInputProps = async ({ stringifiedInputProps, region, userSpecifiedBucketName, propsType, needsToUpload, }) => {
+    const hash = (0, random_hash_1.randomHash)();
     if (needsToUpload) {
-        const bucketName = userSpecifiedBucketName !== null && userSpecifiedBucketName !== void 0 ? userSpecifiedBucketName : (await (0, client_1.internalGetOrCreateBucket)({
+        const bucketName = userSpecifiedBucketName !== null && userSpecifiedBucketName !== void 0 ? userSpecifiedBucketName : (await (0, get_or_create_bucket_1.internalGetOrCreateBucket)({
             region,
             enableFolderExpiry: null,
             customCredentials: null,
-            providerSpecifics,
         })).bucketName;
-        await providerSpecifics.writeFile({
+        await (0, io_1.lambdaWriteFile)({
             body: stringifiedInputProps,
             bucketName,
             region,
@@ -66,18 +69,18 @@ const compressInputProps = async ({ stringifiedInputProps, region, userSpecified
     };
 };
 exports.compressInputProps = compressInputProps;
-const decompressInputProps = async ({ serialized, region, bucketName, expectedBucketOwner, propsType, providerSpecifics, }) => {
+const decompressInputProps = async ({ serialized, region, bucketName, expectedBucketOwner, propsType, }) => {
     if (serialized.type === 'payload') {
         return serialized.payload;
     }
     try {
-        const response = await providerSpecifics.readFile({
+        const response = await (0, io_1.lambdaReadFile)({
             bucketName,
             expectedBucketOwner,
             key: makeKey(propsType, serialized.hash),
             region,
         });
-        const body = await (0, client_1.streamToString)(response);
+        const body = await (0, stream_to_string_1.streamToString)(response);
         const payload = body;
         return payload;
     }

package/dist/shared/get-etag.d.ts

@@ -1 +1 @@
-export declare const getEtagOfFile: (filePath: string) => Promise<string>;
+export declare const getEtagOfFile: (filePath: string, onProgress: (bytes: number) => void) => () => Promise<string>;

package/dist/shared/get-etag.js

@@ -8,17 +8,38 @@ const node_crypto_1 = __importDefault(require("node:crypto"));
 const node_fs_1 = __importDefault(require("node:fs"));
 const chunk = 1024 * 1024 * 5; // 5MB
 const md5 = (data) => node_crypto_1.default.createHash('md5').update(data).digest('hex');
-const getEtagOfFile = async (filePath) => {
-    const stream = await node_fs_1.default.promises.readFile(filePath);
-    if (stream.length <= chunk) {
-        return `"${md5(stream)}"`;
-    }
-    const md5Chunks = [];
-    const chunksNumber = Math.ceil(stream.length / chunk);
-    for (let i = 0; i < chunksNumber; i++) {
-        const chunkStream = stream.slice(i * chunk, (i + 1) * chunk);
-        md5Chunks.push(md5(chunkStream));
-    }
-    return `"${md5(Buffer.from(md5Chunks.join(''), 'hex'))}-${chunksNumber}"`;
+const getEtagOfFile = (filePath, onProgress) => {
+    const calc = async () => {
+        const size = await node_fs_1.default.promises.stat(filePath).then((s) => s.size);
+        if (size <= chunk) {
+            const buffer = await node_fs_1.default.promises.readFile(filePath);
+            return `"${md5(buffer)}"`;
+        }
+        const stream = node_fs_1.default.createReadStream(filePath, {
+            highWaterMark: chunk,
+        });
+        const md5Chunks = [];
+        const chunksNumber = Math.ceil(size / chunk);
+        return new Promise((resolve, reject) => {
+            stream.on('data', (c) => {
+                md5Chunks.push(md5(c));
+                onProgress(c.length);
+            });
+            stream.on('end', () => {
+                resolve(`"${md5(Buffer.from(md5Chunks.join(''), 'hex'))}-${chunksNumber}"`);
+            });
+            stream.on('error', (err) => {
+                reject(err);
+            });
+        });
+    };
+    let tag = null;
+    return async () => {
+        if (tag !== null) {
+            return tag;
+        }
+        tag = await calc();
+        return tag;
+    };
 };
 exports.getEtagOfFile = getEtagOfFile;

package/dist/shared/get-s3-operations.d.ts

@@ -1,8 +1,9 @@
 import type { _Object } from '@aws-sdk/client-s3';
-export declare const getS3DiffOperations: ({ objects, bundle, prefix, }: {
+export declare const getS3DiffOperations: ({ objects, bundle, prefix, onProgress, }: {
     objects: _Object[];
     bundle: string;
     prefix: string;
+    onProgress: (bytes: number) => void;
 }) => Promise<{
     toDelete: _Object[];
     toUpload: string[];

package/dist/shared/get-s3-operations.js

@@ -2,31 +2,49 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getS3DiffOperations = void 0;
 const read_dir_1 = require("./read-dir");
-const getS3DiffOperations = async ({ objects, bundle, prefix, }) => {
-    const dir = await (0, read_dir_1.readDirectory)({
+const getS3DiffOperations = async ({ objects, bundle, prefix, onProgress, }) => {
+    var _a, _b, _c;
+    let totalBytes = 0;
+    const dir = (0, read_dir_1.readDirectory)({
         dir: bundle,
         etags: {},
         originalDir: bundle,
+        onProgress: (bytes) => {
+            totalBytes += bytes;
+            onProgress(totalBytes);
+        },
     });
-    const filesOnS3ButNotLocal = objects.filter((fileOnS3) => {
-        var _a;
+    const filesOnS3ButNotLocal = [];
+    for (const fileOnS3 of objects) {
         const key = (_a = fileOnS3.Key) === null || _a === void 0 ? void 0 : _a.substring(prefix.length + 1);
-        return !dir[key];
-    });
-    const localFilesNotOnS3 = Object.keys(dir).filter((d) => {
-        return !objects.find((o) => {
-            var _a;
-            const key = (_a = o.Key) === null || _a === void 0 ? void 0 : _a.substring(prefix.length + 1);
-            return key === d && o.ETag === dir[d];
-        });
-    });
-    const existing = Object.keys(dir).filter((d) => {
-        return objects.find((o) => {
-            var _a;
-            const key = (_a = o.Key) === null || _a === void 0 ? void 0 : _a.substring(prefix.length + 1);
-            return key === d && o.ETag === dir[d];
-        });
-    });
+        if (!dir[key]) {
+            filesOnS3ButNotLocal.push(fileOnS3);
+        }
+    }
+    const localFilesNotOnS3 = [];
+    for (const d of Object.keys(dir)) {
+        let found;
+        for (const o of objects) {
+            const key = (_b = o.Key) === null || _b === void 0 ? void 0 : _b.substring(prefix.length + 1);
+            if (key === d && o.ETag === (await dir[d]())) {
+                found = o;
+                break;
+            }
+        }
+        if (!found) {
+            localFilesNotOnS3.push(d);
+        }
+    }
+    const existing = [];
+    for (const d of Object.keys(dir)) {
+        for (const o of objects) {
+            const key = (_c = o.Key) === null || _c === void 0 ? void 0 : _c.substring(prefix.length + 1);
+            if (key === d && o.ETag === (await dir[d]())) {
+                existing.push(d);
+                break;
+            }
+        }
+    }
     return {
         toDelete: filesOnS3ButNotLocal,
         toUpload: localFilesNotOnS3,

package/dist/shared/parse-chunk-key.d.ts

@@ -0,0 +1,5 @@
+export declare const parseLambdaChunkKey: (key: string) => {
+    renderId: string;
+    chunk: number;
+    type: string;
+};

package/dist/shared/parse-chunk-key.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseLambdaChunkKey = void 0;
+const parseLambdaChunkKey = (key) => {
+    const match = key.match(/^renders\/(.*)\/chunks\/chunk:([0-9]+):(video|audio)$/);
+    if (!match) {
+        throw new Error(`Cannot parse filename ${key} into timing information. Malformed data.`);
+    }
+    return {
+        renderId: match[1],
+        chunk: Number(match[2]),
+        type: match[3],
+    };
+};
+exports.parseLambdaChunkKey = parseLambdaChunkKey;

package/dist/shared/parse-lambda-initialized-key.d.ts

@@ -0,0 +1,5 @@
+export declare const parseLambdaInitializedKey: (key: string) => {
+    renderId: string;
+    chunk: number;
+    attempt: number;
+};

package/dist/shared/parse-lambda-initialized-key.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseLambdaInitializedKey = void 0;
+const parseLambdaInitializedKey = (key) => {
+    const match = key.match(/^renders\/(.*)\/lambda-initialized-chunk:([0-9]+)-attempt:([0-9]+).txt$/);
+    if (!match) {
+        throw new Error(`Cannot parse filename ${key} into timing information. Malformed data.`);
+    }
+    return {
+        renderId: match[1],
+        chunk: Number(match[2]),
+        attempt: Number(match[3]),
+    };
+};
+exports.parseLambdaInitializedKey = parseLambdaInitializedKey;

package/dist/shared/read-dir.d.ts

@@ -1,9 +1,10 @@
-export declare function readDirectory({ dir, etags, originalDir, }: {
+export declare function readDirectory({ dir, etags, originalDir, onProgress, }: {
     dir: string;
     etags: {
-        [key: string]: string;
+        [key: string]: () => Promise<string>;
     };
     originalDir: string;
-}): Promise<{
-    [key: string]: string;
-}>;
+    onProgress: (bytes: number) => void;
+}): {
+    [key: string]: () => Promise<string>;
+};

package/dist/shared/read-dir.js

@@ -29,7 +29,7 @@ const path = __importStar(require("node:path"));
 const get_etag_1 = require("./get-etag");
 // Function to recursively read a directory and return a list of files
 // with their etags and file names
-async function readDirectory({ dir, etags, originalDir, }) {
+function readDirectory({ dir, etags, originalDir, onProgress, }) {
     const files = fs.readdirSync(dir);
     for (const file of files) {
         if (file.startsWith('.DS_Store'))
@@ -38,19 +38,22 @@ async function readDirectory({ dir, etags, originalDir, }) {
         if (fs.lstatSync(filePath).isDirectory()) {
             etags = {
                 ...etags,
-                ...(await readDirectory({ dir: filePath, etags, originalDir })),
+                ...readDirectory({
+                    dir: filePath,
+                    etags,
+                    originalDir,
+                    onProgress,
+                }),
             };
             continue;
         }
         // eslint-disable-next-line no-lonely-if
         if (fs.lstatSync(filePath).isSymbolicLink()) {
             const realPath = fs.realpathSync(filePath);
-            etags[path.relative(originalDir, filePath)] =
-                await (0, get_etag_1.getEtagOfFile)(realPath);
+            etags[path.relative(originalDir, filePath)] = (0, get_etag_1.getEtagOfFile)(realPath, onProgress);
         }
         else {
-            etags[path.relative(originalDir, filePath)] =
-                await (0, get_etag_1.getEtagOfFile)(filePath);
+            etags[path.relative(originalDir, filePath)] = (0, get_etag_1.getEtagOfFile)(filePath, onProgress);
         }
     }
     // Return the list of files with their etags and file names

package/dist/shared/serialize-props.d.ts

@@ -0,0 +1,14 @@
+type SerializedJSONWithCustomFields = {
+    serializedString: string;
+    customDateUsed: boolean;
+    customFileUsed: boolean;
+    mapUsed: boolean;
+    setUsed: boolean;
+};
+export declare const FILE_TOKEN = "remotion-file:";
+export declare const serializeJSONWithDate: ({ data, indent, staticBase, }: {
+    data: Record<string, unknown>;
+    indent: number | undefined;
+    staticBase: string | null;
+}) => SerializedJSONWithCustomFields;
+export {};

package/dist/shared/serialize-props.js

@@ -0,0 +1,36 @@
+"use strict";
+// Must keep this file in sync with the one in packages/core/src/input-props-serialization.ts!
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.serializeJSONWithDate = exports.FILE_TOKEN = void 0;
+const DATE_TOKEN = 'remotion-date:';
+exports.FILE_TOKEN = 'remotion-file:';
+const serializeJSONWithDate = ({ data, indent, staticBase, }) => {
+    let customDateUsed = false;
+    let customFileUsed = false;
+    let mapUsed = false;
+    let setUsed = false;
+    const serializedString = JSON.stringify(data, function (key, value) {
+        const item = this[key];
+        if (item instanceof Date) {
+            customDateUsed = true;
+            return `${DATE_TOKEN}${item.toISOString()}`;
+        }
+        if (item instanceof Map) {
+            mapUsed = true;
+            return value;
+        }
+        if (item instanceof Set) {
+            setUsed = true;
+            return value;
+        }
+        if (typeof item === 'string' &&
+            staticBase !== null &&
+            item.startsWith(staticBase)) {
+            customFileUsed = true;
+            return `${exports.FILE_TOKEN}${item.replace(staticBase + '/', '')}`;
+        }
+        return value;
+    }, indent);
+    return { serializedString, customDateUsed, customFileUsed, mapUsed, setUsed };
+};
+exports.serializeJSONWithDate = serializeJSONWithDate;

package/package.json

@@ -3,7 +3,7 @@
     "url": "https://github.com/remotion-dev/remotion/tree/main/packages/lambda"
   },
   "name": "@remotion/lambda",
-  "version": "4.0.200",
+  "version": "4.0.201",
   "description": "Render Remotion videos on AWS Lambda",
   "main": "dist/index.js",
   "sideEffects": false,
@@ -22,13 +22,13 @@
     "@aws-sdk/s3-request-presigner": "3.583.0",
     "mime-types": "2.1.34",
     "zod": "3.22.3",
-    "@remotion/bundler": "4.0.200",
-    "@remotion/renderer": "4.0.200",
-    "@remotion/cli": "4.0.200",
-    "@remotion/studio-server": "4.0.200",
-    "@remotion/streaming": "4.0.200",
-    "@remotion/serverless": "4.0.200",
-    "remotion": "4.0.200"
+    "@remotion/bundler": "4.0.201",
+    "@remotion/cli": "4.0.201",
+    "@remotion/renderer": "4.0.201",
+    "@remotion/studio-server": "4.0.201",
+    "@remotion/streaming": "4.0.201",
+    "@remotion/serverless": "4.0.201",
+    "remotion": "4.0.201"
   },
   "devDependencies": {
     "@types/mime-types": "2.1.1",
@@ -37,11 +37,11 @@
     "pureimage": "0.4.13",
     "vitest": "0.31.1",
     "zip-lib": "^0.7.2",
-    "@remotion/bundler": "4.0.200",
-    "@remotion/compositor-linux-arm64-gnu": "4.0.200"
+    "@remotion/bundler": "4.0.201",
+    "@remotion/compositor-linux-arm64-gnu": "4.0.201"
   },
   "peerDependencies": {
-    "@remotion/bundler": "4.0.200"
+    "@remotion/bundler": "4.0.201"
   },
   "publishConfig": {
     "access": "public"

package/dist/functions/provider-implementation.js

@@ -1 +0,0 @@
-"use strict";