@remotion/lambda 4.0.163 → 4.0.165
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/delete-render.js +7 -3
- package/dist/api/deploy-function.d.ts +0 -1
- package/dist/api/deploy-function.js +3 -3
- package/dist/api/deploy-site.d.ts +3 -3
- package/dist/api/download-media.js +6 -3
- package/dist/api/get-compositions-on-lambda.js +0 -2
- package/dist/api/get-render-progress.js +0 -2
- package/dist/api/make-lambda-payload.js +2 -1
- package/dist/api/render-media-on-lambda.js +0 -2
- package/dist/api/render-still-on-lambda.js +37 -25
- package/dist/cli/args.d.ts +1 -0
- package/dist/cli/commands/functions/ls.js +3 -1
- package/dist/cli/commands/render/progress.d.ts +3 -28
- package/dist/cli/commands/render/progress.js +89 -90
- package/dist/cli/commands/render/render.js +7 -40
- package/dist/cli/commands/sites/create.js +2 -2
- package/dist/functions/chunk-optimization/types.d.ts +0 -3
- package/dist/functions/helpers/calculate-chunk-times.d.ts +3 -4
- package/dist/functions/helpers/calculate-chunk-times.js +4 -8
- package/dist/functions/helpers/calculate-price-from-bucket.d.ts +3 -5
- package/dist/functions/helpers/calculate-price-from-bucket.js +5 -18
- package/dist/functions/helpers/clean-tmpdir.d.ts +0 -2
- package/dist/functions/helpers/clean-tmpdir.js +1 -7
- package/dist/functions/helpers/concat-videos.d.ts +1 -13
- package/dist/functions/helpers/concat-videos.js +7 -131
- package/dist/functions/helpers/create-post-render-data.d.ts +6 -6
- package/dist/functions/helpers/create-post-render-data.js +17 -37
- package/dist/functions/helpers/find-output-file-in-bucket.d.ts +1 -3
- package/dist/functions/helpers/find-output-file-in-bucket.js +1 -4
- package/dist/functions/helpers/get-encoding-progress-step-size.d.ts +1 -1
- package/dist/functions/helpers/get-encoding-progress-step-size.js +0 -3
- package/dist/functions/helpers/get-lambdas-invoked-stats.d.ts +1 -6
- package/dist/functions/helpers/get-lambdas-invoked-stats.js +0 -13
- package/dist/functions/helpers/get-overall-progress-s3.d.ts +8 -0
- package/dist/functions/helpers/get-overall-progress-s3.js +25 -0
- package/dist/functions/helpers/get-overall-progress.d.ts +1 -2
- package/dist/functions/helpers/get-overall-progress.js +2 -4
- package/dist/functions/helpers/get-progress.js +78 -154
- package/dist/functions/helpers/get-retry-stats.d.ts +0 -5
- package/dist/functions/helpers/get-retry-stats.js +0 -18
- package/dist/functions/helpers/inspect-errors.d.ts +4 -10
- package/dist/functions/helpers/inspect-errors.js +5 -27
- package/dist/functions/helpers/io.d.ts +0 -1
- package/dist/functions/helpers/io.js +2 -3
- package/dist/functions/helpers/lifecycle.d.ts +0 -4
- package/dist/functions/helpers/lifecycle.js +2 -3
- package/dist/functions/helpers/make-timeout-error.d.ts +0 -2
- package/dist/functions/helpers/merge-chunks.d.ts +5 -7
- package/dist/functions/helpers/merge-chunks.js +16 -149
- package/dist/functions/helpers/min-max.d.ts +1 -1
- package/dist/functions/helpers/min-max.js +1 -1
- package/dist/functions/helpers/overall-render-progress.d.ts +49 -0
- package/dist/functions/helpers/overall-render-progress.js +164 -0
- package/dist/functions/helpers/stream-renderer.d.ts +11 -0
- package/dist/functions/helpers/stream-renderer.js +127 -0
- package/dist/functions/helpers/streamify-response.d.ts +0 -3
- package/dist/functions/helpers/streamify-response.js +2 -14
- package/dist/functions/helpers/streaming-payloads.d.ts +3 -3
- package/dist/functions/helpers/write-lambda-error.d.ts +3 -6
- package/dist/functions/helpers/write-lambda-error.js +1 -21
- package/dist/functions/index.d.ts +5 -0
- package/dist/functions/index.js +70 -51
- package/dist/functions/launch.js +103 -200
- package/dist/functions/renderer.d.ts +2 -2
- package/dist/functions/renderer.js +70 -123
- package/dist/functions/start.d.ts +1 -0
- package/dist/functions/start.js +3 -2
- package/dist/functions/still.d.ts +7 -2
- package/dist/functions/still.js +17 -34
- package/dist/functions/streaming/stream-writer.d.ts +6 -0
- package/dist/functions/streaming/stream-writer.js +35 -0
- package/dist/functions/streaming/streaming.d.ts +92 -0
- package/dist/functions/streaming/streaming.js +58 -0
- package/dist/internals.d.ts +7 -6
- package/dist/shared/aws-clients.js +8 -0
- package/dist/shared/call-lambda.d.ts +5 -4
- package/dist/shared/call-lambda.js +53 -44
- package/dist/shared/constants.d.ts +15 -49
- package/dist/shared/constants.js +3 -32
- package/dist/shared/content-disposition-header.d.ts +0 -3
- package/dist/shared/get-function-version.js +0 -2
- package/dist/shared/is-flaky-error.js +4 -0
- package/dist/shared/parse-lambda-timings-key.d.ts +0 -2
- package/dist/shared/parse-lambda-timings-key.js +0 -14
- package/dist/shared/return-values.d.ts +0 -7
- package/package.json +18 -18
- package/remotionlambda-arm64.zip +0 -0
|
@@ -4,114 +4,31 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.mergeChunksAndFinishRender = void 0;
|
|
7
|
-
const renderer_1 = require("@remotion/renderer");
|
|
8
7
|
const fs_1 = __importDefault(require("fs"));
|
|
9
|
-
const node_fs_1 = require("node:fs");
|
|
10
|
-
const node_path_1 = require("node:path");
|
|
11
8
|
const cleanup_serialized_input_props_1 = require("../../shared/cleanup-serialized-input-props");
|
|
12
|
-
const constants_1 = require("../../shared/constants");
|
|
13
|
-
const truthy_1 = require("../../shared/truthy");
|
|
14
9
|
const concat_videos_1 = require("./concat-videos");
|
|
15
10
|
const create_post_render_data_1 = require("./create-post-render-data");
|
|
16
|
-
const delete_chunks_1 = require("./delete-chunks");
|
|
17
11
|
const get_current_region_1 = require("./get-current-region");
|
|
18
|
-
const get_encoding_progress_step_size_1 = require("./get-encoding-progress-step-size");
|
|
19
|
-
const get_files_to_delete_1 = require("./get-files-to-delete");
|
|
20
12
|
const get_output_url_from_metadata_1 = require("./get-output-url-from-metadata");
|
|
21
13
|
const inspect_errors_1 = require("./inspect-errors");
|
|
22
14
|
const io_1 = require("./io");
|
|
23
|
-
const render_has_audio_video_1 = require("./render-has-audio-video");
|
|
24
15
|
const timer_1 = require("./timer");
|
|
25
|
-
const write_lambda_error_1 = require("./write-lambda-error");
|
|
26
|
-
const write_post_render_data_1 = require("./write-post-render-data");
|
|
27
16
|
const mergeChunksAndFinishRender = async (options) => {
|
|
28
|
-
let lastProgressUploaded = Date.now();
|
|
29
17
|
const onProgress = (framesEncoded) => {
|
|
30
|
-
|
|
31
|
-
if (deltaSinceLastProgressUploaded < 1500 &&
|
|
32
|
-
framesEncoded !== options.numberOfFrames) {
|
|
33
|
-
return;
|
|
34
|
-
}
|
|
35
|
-
lastProgressUploaded = Date.now();
|
|
36
|
-
(0, io_1.lambdaWriteFile)({
|
|
37
|
-
bucketName: options.bucketName,
|
|
38
|
-
key: (0, constants_1.encodingProgressKey)(options.renderId),
|
|
39
|
-
body: String(Math.round(framesEncoded / (0, get_encoding_progress_step_size_1.getEncodingProgressStepSize)(options.numberOfFrames))),
|
|
40
|
-
region: (0, get_current_region_1.getCurrentRegionInFunction)(),
|
|
41
|
-
privacy: 'private',
|
|
42
|
-
expectedBucketOwner: options.expectedBucketOwner,
|
|
43
|
-
downloadBehavior: null,
|
|
44
|
-
customCredentials: null,
|
|
45
|
-
}).catch((err) => {
|
|
46
|
-
(0, write_lambda_error_1.writeLambdaError)({
|
|
47
|
-
bucketName: options.bucketName,
|
|
48
|
-
errorInfo: {
|
|
49
|
-
chunk: null,
|
|
50
|
-
frame: null,
|
|
51
|
-
isFatal: false,
|
|
52
|
-
name: err.name,
|
|
53
|
-
message: err.message,
|
|
54
|
-
stack: `Could not upload stitching progress ${err.stack}`,
|
|
55
|
-
tmpDir: null,
|
|
56
|
-
type: 'stitcher',
|
|
57
|
-
attempt: 1,
|
|
58
|
-
totalAttempts: 1,
|
|
59
|
-
willRetry: false,
|
|
60
|
-
},
|
|
61
|
-
renderId: options.renderId,
|
|
62
|
-
expectedBucketOwner: options.expectedBucketOwner,
|
|
63
|
-
});
|
|
64
|
-
});
|
|
18
|
+
options.overallProgress.setCombinedFrames(framesEncoded);
|
|
65
19
|
};
|
|
66
|
-
const onErrors = (errors) => {
|
|
67
|
-
renderer_1.RenderInternals.Log.error({ indent: false, logLevel: options.logLevel }, 'Found Errors', errors);
|
|
68
|
-
const firstError = errors[0];
|
|
69
|
-
if (firstError.chunk !== null) {
|
|
70
|
-
throw new Error(`Stopping Lambda function because error occurred while rendering chunk ${firstError.chunk}:\n${errors[0].stack
|
|
71
|
-
.split('\n')
|
|
72
|
-
.map((s) => ` ${s}`)
|
|
73
|
-
.join('\n')}`);
|
|
74
|
-
}
|
|
75
|
-
throw new Error(`Stopping Lambda function because error occurred: ${errors[0].stack}`);
|
|
76
|
-
};
|
|
77
|
-
const outdir = (0, node_path_1.join)(renderer_1.RenderInternals.tmpDir(constants_1.CONCAT_FOLDER_TOKEN), 'bucket');
|
|
78
|
-
if ((0, node_fs_1.existsSync)(outdir)) {
|
|
79
|
-
(0, node_fs_1.rmSync)(outdir, {
|
|
80
|
-
recursive: true,
|
|
81
|
-
});
|
|
82
|
-
}
|
|
83
|
-
(0, node_fs_1.mkdirSync)(outdir);
|
|
84
|
-
const { hasAudio, hasVideo } = (0, render_has_audio_video_1.lambdaRenderHasAudioVideo)(options.renderMetadata);
|
|
85
|
-
const chunkMultiplier = [hasAudio, hasVideo].filter(truthy_1.truthy).length;
|
|
86
|
-
const expectedFiles = chunkMultiplier * options.chunkCount;
|
|
87
|
-
const files = await (0, concat_videos_1.getAllFilesS3)({
|
|
88
|
-
bucket: options.bucketName,
|
|
89
|
-
expectedFiles,
|
|
90
|
-
outdir,
|
|
91
|
-
renderId: options.renderId,
|
|
92
|
-
region: (0, get_current_region_1.getCurrentRegionInFunction)(),
|
|
93
|
-
expectedBucketOwner: options.expectedBucketOwner,
|
|
94
|
-
onErrors,
|
|
95
|
-
logLevel: options.logLevel,
|
|
96
|
-
});
|
|
97
|
-
options.onAllChunks({
|
|
98
|
-
inputProps: options.inputProps,
|
|
99
|
-
serializedResolvedProps: options.serializedResolvedProps,
|
|
100
|
-
framesPerLambda: options.framesPerLambda,
|
|
101
|
-
compositionStart: options.compositionStart,
|
|
102
|
-
});
|
|
103
20
|
const encodingStart = Date.now();
|
|
104
21
|
if (options.renderMetadata.type === 'still') {
|
|
105
22
|
throw new Error('Cannot merge stills');
|
|
106
23
|
}
|
|
107
|
-
const { outfile, cleanupChunksProm } = await (0, concat_videos_1.
|
|
24
|
+
const { outfile, cleanupChunksProm } = await (0, concat_videos_1.concatVideos)({
|
|
108
25
|
onProgress,
|
|
109
26
|
numberOfFrames: options.numberOfFrames,
|
|
110
27
|
codec: options.codec,
|
|
111
28
|
fps: options.fps,
|
|
112
29
|
numberOfGifLoops: options.numberOfGifLoops,
|
|
113
|
-
files,
|
|
114
|
-
outdir,
|
|
30
|
+
files: options.files,
|
|
31
|
+
outdir: options.outdir,
|
|
115
32
|
audioCodec: options.audioCodec,
|
|
116
33
|
audioBitrate: options.audioBitrate,
|
|
117
34
|
logLevel: options.logLevel,
|
|
@@ -122,8 +39,9 @@ const mergeChunksAndFinishRender = async (options) => {
|
|
|
122
39
|
muted: options.renderMetadata.muted,
|
|
123
40
|
});
|
|
124
41
|
const encodingStop = Date.now();
|
|
125
|
-
|
|
126
|
-
const
|
|
42
|
+
options.overallProgress.setTimeToCombine(encodingStop - encodingStart);
|
|
43
|
+
const outputSize = fs_1.default.statSync(outfile).size;
|
|
44
|
+
const writeToS3 = (0, timer_1.timer)(`Writing to S3 (${outputSize} bytes)`, options.logLevel);
|
|
127
45
|
await (0, io_1.lambdaWriteFile)({
|
|
128
46
|
bucketName: options.renderBucketName,
|
|
129
47
|
key: options.key,
|
|
@@ -135,44 +53,9 @@ const mergeChunksAndFinishRender = async (options) => {
|
|
|
135
53
|
customCredentials: options.customCredentials,
|
|
136
54
|
});
|
|
137
55
|
writeToS3.end();
|
|
138
|
-
const
|
|
139
|
-
|
|
140
|
-
prefix: (0, constants_1.rendersPrefix)(options.renderId),
|
|
141
|
-
expectedBucketOwner: options.expectedBucketOwner,
|
|
142
|
-
region: (0, get_current_region_1.getCurrentRegionInFunction)(),
|
|
56
|
+
const errorExplanations = (0, inspect_errors_1.inspectErrors)({
|
|
57
|
+
errors: options.overallProgress.get().errors,
|
|
143
58
|
});
|
|
144
|
-
const finalEncodingProgressProm = (0, io_1.lambdaWriteFile)({
|
|
145
|
-
bucketName: options.bucketName,
|
|
146
|
-
key: (0, constants_1.encodingProgressKey)(options.renderId),
|
|
147
|
-
body: String(Math.ceil(options.numberOfFrames /
|
|
148
|
-
(0, get_encoding_progress_step_size_1.getEncodingProgressStepSize)(options.numberOfFrames))),
|
|
149
|
-
region: (0, get_current_region_1.getCurrentRegionInFunction)(),
|
|
150
|
-
privacy: 'private',
|
|
151
|
-
expectedBucketOwner: options.expectedBucketOwner,
|
|
152
|
-
downloadBehavior: null,
|
|
153
|
-
customCredentials: null,
|
|
154
|
-
});
|
|
155
|
-
const errorExplanationsProm = (0, inspect_errors_1.inspectErrors)({
|
|
156
|
-
contents,
|
|
157
|
-
renderId: options.renderId,
|
|
158
|
-
bucket: options.bucketName,
|
|
159
|
-
region: (0, get_current_region_1.getCurrentRegionInFunction)(),
|
|
160
|
-
expectedBucketOwner: options.expectedBucketOwner,
|
|
161
|
-
});
|
|
162
|
-
const jobs = (0, get_files_to_delete_1.getFilesToDelete)({
|
|
163
|
-
chunkCount: options.chunkCount,
|
|
164
|
-
renderId: options.renderId,
|
|
165
|
-
hasAudio,
|
|
166
|
-
hasVideo,
|
|
167
|
-
});
|
|
168
|
-
const deletProm = options.logLevel === 'verbose'
|
|
169
|
-
? Promise.resolve(0)
|
|
170
|
-
: (0, delete_chunks_1.cleanupFiles)({
|
|
171
|
-
region: (0, get_current_region_1.getCurrentRegionInFunction)(),
|
|
172
|
-
bucket: options.bucketName,
|
|
173
|
-
contents,
|
|
174
|
-
jobs,
|
|
175
|
-
});
|
|
176
59
|
const cleanupSerializedInputPropsProm = (0, cleanup_serialized_input_props_1.cleanupSerializedInputProps)({
|
|
177
60
|
bucketName: options.bucketName,
|
|
178
61
|
region: (0, get_current_region_1.getCurrentRegionInFunction)(),
|
|
@@ -185,39 +68,23 @@ const mergeChunksAndFinishRender = async (options) => {
|
|
|
185
68
|
});
|
|
186
69
|
const { url: outputUrl } = (0, get_output_url_from_metadata_1.getOutputUrlFromMetadata)(options.renderMetadata, options.bucketName, options.customCredentials);
|
|
187
70
|
const postRenderData = (0, create_post_render_data_1.createPostRenderData)({
|
|
188
|
-
expectedBucketOwner: options.expectedBucketOwner,
|
|
189
71
|
region: (0, get_current_region_1.getCurrentRegionInFunction)(),
|
|
190
|
-
renderId: options.renderId,
|
|
191
72
|
memorySizeInMb: Number(process.env.AWS_LAMBDA_FUNCTION_MEMORY_SIZE),
|
|
192
73
|
renderMetadata: options.renderMetadata,
|
|
193
|
-
|
|
194
|
-
errorExplanations: await errorExplanationsProm,
|
|
195
|
-
timeToEncode: encodingStop - encodingStart,
|
|
74
|
+
errorExplanations,
|
|
196
75
|
timeToDelete: (await Promise.all([
|
|
197
|
-
deletProm,
|
|
198
76
|
cleanupSerializedInputPropsProm,
|
|
199
77
|
cleanupResolvedInputPropsProm,
|
|
200
|
-
])).reduce((a, b) => a
|
|
78
|
+
])).reduce((a, b) => Math.max(a, b), 0),
|
|
201
79
|
outputFile: {
|
|
202
|
-
lastModified: Date.now(),
|
|
203
|
-
size: outputSize.size,
|
|
204
80
|
url: outputUrl,
|
|
205
81
|
},
|
|
82
|
+
outputSize,
|
|
83
|
+
timeToCombine: encodingStop - encodingStart,
|
|
84
|
+
overallProgress: options.overallProgress.get(),
|
|
85
|
+
timeToFinish: Date.now() - options.startTime,
|
|
206
86
|
});
|
|
207
|
-
|
|
208
|
-
await (0, write_post_render_data_1.writePostRenderData)({
|
|
209
|
-
bucketName: options.bucketName,
|
|
210
|
-
expectedBucketOwner: options.expectedBucketOwner,
|
|
211
|
-
postRenderData,
|
|
212
|
-
region: (0, get_current_region_1.getCurrentRegionInFunction)(),
|
|
213
|
-
renderId: options.renderId,
|
|
214
|
-
});
|
|
215
|
-
await (0, io_1.lambdaDeleteFile)({
|
|
216
|
-
bucketName: options.bucketName,
|
|
217
|
-
key: (0, constants_1.initalizedMetadataKey)(options.renderId),
|
|
218
|
-
region: (0, get_current_region_1.getCurrentRegionInFunction)(),
|
|
219
|
-
customCredentials: null,
|
|
220
|
-
});
|
|
87
|
+
options.overallProgress.setPostRenderData(postRenderData);
|
|
221
88
|
await Promise.all([cleanupChunksProm, fs_1.default.promises.rm(outfile)]);
|
|
222
89
|
return postRenderData;
|
|
223
90
|
};
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
export declare const min: (arr: number[]) => number;
|
|
2
|
-
export declare const max: (arr: number[]) => number
|
|
2
|
+
export declare const max: (arr: number[]) => number;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { LogLevel } from '@remotion/renderer';
|
|
2
|
+
import type { AwsRegion } from '../../client';
|
|
3
|
+
import type { PostRenderData, RenderMetadata } from '../../shared/constants';
|
|
4
|
+
import type { ParsedTiming } from '../../shared/parse-lambda-timings-key';
|
|
5
|
+
import type { ChunkRetry } from './get-retry-stats';
|
|
6
|
+
import type { LambdaErrorInfo } from './write-lambda-error';
|
|
7
|
+
export type OverallRenderProgress = {
|
|
8
|
+
chunks: number[];
|
|
9
|
+
framesRendered: number;
|
|
10
|
+
framesEncoded: number;
|
|
11
|
+
combinedFrames: number;
|
|
12
|
+
timeToCombine: number | null;
|
|
13
|
+
timeToEncode: number | null;
|
|
14
|
+
timeToRenderFrames: number | null;
|
|
15
|
+
lambdasInvoked: number;
|
|
16
|
+
retries: ChunkRetry[];
|
|
17
|
+
postRenderData: PostRenderData | null;
|
|
18
|
+
timings: ParsedTiming[];
|
|
19
|
+
renderMetadata: RenderMetadata | null;
|
|
20
|
+
errors: LambdaErrorInfo[];
|
|
21
|
+
timeoutTimestamp: number;
|
|
22
|
+
};
|
|
23
|
+
export type OverallProgressHelper = {
|
|
24
|
+
upload: () => Promise<void>;
|
|
25
|
+
setFrames: ({ encoded, rendered, index, }: {
|
|
26
|
+
rendered: number;
|
|
27
|
+
encoded: number;
|
|
28
|
+
index: number;
|
|
29
|
+
}) => void;
|
|
30
|
+
setLambdaInvoked: (chunk: number) => void;
|
|
31
|
+
addChunkCompleted: (chunkIndex: number, start: number, rendered: number) => void;
|
|
32
|
+
setCombinedFrames: (framesEncoded: number) => void;
|
|
33
|
+
setTimeToCombine: (timeToCombine: number) => void;
|
|
34
|
+
addRetry: (retry: ChunkRetry) => void;
|
|
35
|
+
setPostRenderData: (postRenderData: PostRenderData) => void;
|
|
36
|
+
setRenderMetadata: (renderMetadata: RenderMetadata) => void;
|
|
37
|
+
addErrorWithoutUpload: (errorInfo: LambdaErrorInfo) => void;
|
|
38
|
+
setExpectedChunks: (expectedChunks: number) => void;
|
|
39
|
+
get: () => OverallRenderProgress;
|
|
40
|
+
};
|
|
41
|
+
export declare const makeInitialOverallRenderProgress: (timeoutTimestamp: number) => OverallRenderProgress;
|
|
42
|
+
export declare const makeOverallRenderProgress: ({ renderId, bucketName, expectedBucketOwner, region, timeoutTimestamp, logLevel, }: {
|
|
43
|
+
renderId: string;
|
|
44
|
+
bucketName: string;
|
|
45
|
+
expectedBucketOwner: string;
|
|
46
|
+
region: AwsRegion;
|
|
47
|
+
timeoutTimestamp: number;
|
|
48
|
+
logLevel: LogLevel;
|
|
49
|
+
}) => OverallProgressHelper;
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.makeOverallRenderProgress = exports.makeInitialOverallRenderProgress = void 0;
|
|
4
|
+
const renderer_1 = require("@remotion/renderer");
|
|
5
|
+
const constants_1 = require("../../shared/constants");
|
|
6
|
+
const io_1 = require("./io");
|
|
7
|
+
const makeInitialOverallRenderProgress = (timeoutTimestamp) => {
|
|
8
|
+
return {
|
|
9
|
+
chunks: [],
|
|
10
|
+
framesRendered: 0,
|
|
11
|
+
framesEncoded: 0,
|
|
12
|
+
combinedFrames: 0,
|
|
13
|
+
timeToCombine: null,
|
|
14
|
+
timeToEncode: null,
|
|
15
|
+
lambdasInvoked: 0,
|
|
16
|
+
retries: [],
|
|
17
|
+
postRenderData: null,
|
|
18
|
+
timings: [],
|
|
19
|
+
renderMetadata: null,
|
|
20
|
+
errors: [],
|
|
21
|
+
timeToRenderFrames: null,
|
|
22
|
+
timeoutTimestamp,
|
|
23
|
+
};
|
|
24
|
+
};
|
|
25
|
+
exports.makeInitialOverallRenderProgress = makeInitialOverallRenderProgress;
|
|
26
|
+
const makeOverallRenderProgress = ({ renderId, bucketName, expectedBucketOwner, region, timeoutTimestamp, logLevel, }) => {
|
|
27
|
+
let framesRendered = [];
|
|
28
|
+
let framesEncoded = [];
|
|
29
|
+
let lambdasInvoked = [];
|
|
30
|
+
const renderProgress = (0, exports.makeInitialOverallRenderProgress)(timeoutTimestamp);
|
|
31
|
+
let currentUploadPromise = null;
|
|
32
|
+
const getCurrentProgress = () => renderProgress;
|
|
33
|
+
let latestUploadRequest = 0;
|
|
34
|
+
const getLatestRequestId = () => latestUploadRequest;
|
|
35
|
+
let encodeStartTime = null;
|
|
36
|
+
let renderFramesStartTime = null;
|
|
37
|
+
const upload = async () => {
|
|
38
|
+
const uploadRequestId = ++latestUploadRequest;
|
|
39
|
+
if (currentUploadPromise) {
|
|
40
|
+
await currentUploadPromise;
|
|
41
|
+
}
|
|
42
|
+
const toWrite = JSON.stringify(getCurrentProgress());
|
|
43
|
+
// Deduplicate two fast incoming requests
|
|
44
|
+
await new Promise((resolve) => {
|
|
45
|
+
setImmediate(() => resolve());
|
|
46
|
+
});
|
|
47
|
+
// If request has been replaced by a new one
|
|
48
|
+
if (getLatestRequestId() !== uploadRequestId) {
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
const start = Date.now();
|
|
52
|
+
currentUploadPromise = (0, io_1.lambdaWriteFile)({
|
|
53
|
+
body: toWrite,
|
|
54
|
+
bucketName,
|
|
55
|
+
customCredentials: null,
|
|
56
|
+
downloadBehavior: null,
|
|
57
|
+
expectedBucketOwner,
|
|
58
|
+
key: (0, constants_1.overallProgressKey)(renderId),
|
|
59
|
+
privacy: 'private',
|
|
60
|
+
region,
|
|
61
|
+
})
|
|
62
|
+
.then(() => {
|
|
63
|
+
// By default, upload is way too fast (~20 requests per second)
|
|
64
|
+
// Space out the requests a bit
|
|
65
|
+
return new Promise((resolve) => {
|
|
66
|
+
setTimeout(resolve, 500 - (Date.now() - start));
|
|
67
|
+
});
|
|
68
|
+
})
|
|
69
|
+
.catch((err) => {
|
|
70
|
+
// If an error occurs in uploading the state that contains the errors,
|
|
71
|
+
// that is unfortunate. We just log it.
|
|
72
|
+
renderer_1.RenderInternals.Log.error({ indent: false, logLevel }, 'Error uploading progress', err);
|
|
73
|
+
});
|
|
74
|
+
await currentUploadPromise;
|
|
75
|
+
currentUploadPromise = null;
|
|
76
|
+
};
|
|
77
|
+
return {
|
|
78
|
+
upload,
|
|
79
|
+
setFrames: ({ encoded, rendered, index, }) => {
|
|
80
|
+
if (framesRendered.length === 0) {
|
|
81
|
+
throw new Error('Expected chunks to be set before frames are set');
|
|
82
|
+
}
|
|
83
|
+
if (framesEncoded.length === 0) {
|
|
84
|
+
throw new Error('Expected chunks to be set before frames are set');
|
|
85
|
+
}
|
|
86
|
+
framesRendered[index] = rendered;
|
|
87
|
+
framesEncoded[index] = encoded;
|
|
88
|
+
const totalFramesEncoded = framesEncoded.reduce((a, b) => a + b, 0);
|
|
89
|
+
const totalFramesRendered = framesRendered.reduce((a, b) => a + b, 0);
|
|
90
|
+
if (renderProgress.framesEncoded === 0 && totalFramesEncoded > 0) {
|
|
91
|
+
encodeStartTime = Date.now();
|
|
92
|
+
}
|
|
93
|
+
if (renderProgress.framesRendered === 0 && totalFramesRendered > 0) {
|
|
94
|
+
renderFramesStartTime = Date.now();
|
|
95
|
+
}
|
|
96
|
+
if (renderProgress.timeToRenderFrames === null) {
|
|
97
|
+
const frameCount = renderProgress.renderMetadata &&
|
|
98
|
+
renderProgress.renderMetadata.type === 'video'
|
|
99
|
+
? renderer_1.RenderInternals.getFramesToRender(renderProgress.renderMetadata.frameRange, renderProgress.renderMetadata.everyNthFrame).length
|
|
100
|
+
: null;
|
|
101
|
+
if (frameCount === totalFramesRendered) {
|
|
102
|
+
const timeToRenderFrames = Date.now() - (renderFramesStartTime !== null && renderFramesStartTime !== void 0 ? renderFramesStartTime : Date.now());
|
|
103
|
+
renderProgress.timeToRenderFrames = timeToRenderFrames;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
renderProgress.framesRendered = totalFramesRendered;
|
|
107
|
+
renderProgress.framesEncoded = totalFramesEncoded;
|
|
108
|
+
upload();
|
|
109
|
+
},
|
|
110
|
+
addChunkCompleted: (chunkIndex, start, rendered) => {
|
|
111
|
+
var _a;
|
|
112
|
+
renderProgress.chunks.push(chunkIndex);
|
|
113
|
+
if (renderProgress.chunks.length ===
|
|
114
|
+
((_a = renderProgress.renderMetadata) === null || _a === void 0 ? void 0 : _a.totalChunks)) {
|
|
115
|
+
const timeToEncode = Date.now() - (encodeStartTime !== null && encodeStartTime !== void 0 ? encodeStartTime : Date.now());
|
|
116
|
+
renderProgress.timeToEncode = timeToEncode;
|
|
117
|
+
}
|
|
118
|
+
renderProgress.timings.push({
|
|
119
|
+
chunk: chunkIndex,
|
|
120
|
+
start,
|
|
121
|
+
rendered,
|
|
122
|
+
});
|
|
123
|
+
upload();
|
|
124
|
+
},
|
|
125
|
+
setCombinedFrames: (frames) => {
|
|
126
|
+
renderProgress.combinedFrames = frames;
|
|
127
|
+
upload();
|
|
128
|
+
},
|
|
129
|
+
setTimeToCombine: (timeToCombine) => {
|
|
130
|
+
renderProgress.timeToCombine = timeToCombine;
|
|
131
|
+
upload();
|
|
132
|
+
},
|
|
133
|
+
setLambdaInvoked(chunk) {
|
|
134
|
+
if (lambdasInvoked.length === 0) {
|
|
135
|
+
throw new Error('Expected chunks to be set before lambdas are set');
|
|
136
|
+
}
|
|
137
|
+
lambdasInvoked[chunk] = true;
|
|
138
|
+
renderProgress.lambdasInvoked = lambdasInvoked.reduce((a, b) => a + Number(b), 0);
|
|
139
|
+
upload();
|
|
140
|
+
},
|
|
141
|
+
setPostRenderData(postRenderData) {
|
|
142
|
+
renderProgress.postRenderData = postRenderData;
|
|
143
|
+
upload();
|
|
144
|
+
},
|
|
145
|
+
setRenderMetadata: (renderMetadata) => {
|
|
146
|
+
renderProgress.renderMetadata = renderMetadata;
|
|
147
|
+
upload();
|
|
148
|
+
},
|
|
149
|
+
addErrorWithoutUpload: (errorInfo) => {
|
|
150
|
+
renderProgress.errors.push(errorInfo);
|
|
151
|
+
},
|
|
152
|
+
setExpectedChunks: (expectedChunks) => {
|
|
153
|
+
framesRendered = new Array(expectedChunks).fill(0);
|
|
154
|
+
framesEncoded = new Array(expectedChunks).fill(0);
|
|
155
|
+
lambdasInvoked = new Array(expectedChunks).fill(false);
|
|
156
|
+
},
|
|
157
|
+
addRetry(retry) {
|
|
158
|
+
renderProgress.retries.push(retry);
|
|
159
|
+
upload();
|
|
160
|
+
},
|
|
161
|
+
get: () => renderProgress,
|
|
162
|
+
};
|
|
163
|
+
};
|
|
164
|
+
exports.makeOverallRenderProgress = makeOverallRenderProgress;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { LogLevel } from '@remotion/renderer';
|
|
2
|
+
import type { LambdaPayload } from '../../defaults';
|
|
3
|
+
import type { OverallProgressHelper } from './overall-render-progress';
|
|
4
|
+
export declare const streamRendererFunctionWithRetry: ({ payload, files, functionName, outdir, overallProgress, logLevel, }: {
|
|
5
|
+
payload: LambdaPayload;
|
|
6
|
+
functionName: string;
|
|
7
|
+
outdir: string;
|
|
8
|
+
overallProgress: OverallProgressHelper;
|
|
9
|
+
files: string[];
|
|
10
|
+
logLevel: LogLevel;
|
|
11
|
+
}) => Promise<unknown>;
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.streamRendererFunctionWithRetry = void 0;
|
|
4
|
+
const renderer_1 = require("@remotion/renderer");
|
|
5
|
+
const fs_1 = require("fs");
|
|
6
|
+
const path_1 = require("path");
|
|
7
|
+
const defaults_1 = require("../../defaults");
|
|
8
|
+
const call_lambda_1 = require("../../shared/call-lambda");
|
|
9
|
+
const get_current_region_1 = require("./get-current-region");
|
|
10
|
+
const streamRenderer = ({ payload, functionName, outdir, overallProgress, files, logLevel, }) => {
|
|
11
|
+
if (payload.type !== defaults_1.LambdaRoutines.renderer) {
|
|
12
|
+
throw new Error('Expected renderer type');
|
|
13
|
+
}
|
|
14
|
+
return new Promise((resolve) => {
|
|
15
|
+
const receivedStreamingPayload = ({ message }) => {
|
|
16
|
+
if (message.type === 'lambda-invoked') {
|
|
17
|
+
overallProgress.setLambdaInvoked(payload.chunk);
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
20
|
+
if (message.type === 'frames-rendered') {
|
|
21
|
+
overallProgress.setFrames({
|
|
22
|
+
index: payload.chunk,
|
|
23
|
+
encoded: message.payload.encoded,
|
|
24
|
+
rendered: message.payload.rendered,
|
|
25
|
+
});
|
|
26
|
+
return;
|
|
27
|
+
}
|
|
28
|
+
if (message.type === 'video-chunk-rendered') {
|
|
29
|
+
const filename = (0, path_1.join)(outdir, `chunk:${String(payload.chunk).padStart(8, '0')}:video`);
|
|
30
|
+
(0, fs_1.writeFileSync)(filename, message.payload);
|
|
31
|
+
files.push(filename);
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
if (message.type === 'audio-chunk-rendered') {
|
|
35
|
+
const filename = (0, path_1.join)(outdir, `chunk:${String(payload.chunk).padStart(8, '0')}:audio`);
|
|
36
|
+
(0, fs_1.writeFileSync)(filename, message.payload);
|
|
37
|
+
files.push(filename);
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
if (message.type === 'chunk-complete') {
|
|
41
|
+
renderer_1.RenderInternals.Log.verbose({ indent: false, logLevel }, `Rendered chunk ${payload.chunk}`);
|
|
42
|
+
overallProgress.addChunkCompleted(payload.chunk, message.payload.start, message.payload.rendered);
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
if (message.type === 'error-occurred') {
|
|
46
|
+
overallProgress.addErrorWithoutUpload(message.payload.errorInfo);
|
|
47
|
+
overallProgress.setFrames({
|
|
48
|
+
encoded: 0,
|
|
49
|
+
index: payload.chunk,
|
|
50
|
+
rendered: 0,
|
|
51
|
+
});
|
|
52
|
+
renderer_1.RenderInternals.Log.error({
|
|
53
|
+
indent: false,
|
|
54
|
+
logLevel,
|
|
55
|
+
}, `Renderer function of chunk ${payload.chunk} failed with error: ${message.payload.error}`);
|
|
56
|
+
renderer_1.RenderInternals.Log.error({
|
|
57
|
+
indent: false,
|
|
58
|
+
logLevel,
|
|
59
|
+
}, `Will retry chunk = ${message.payload.shouldRetry}`);
|
|
60
|
+
resolve({
|
|
61
|
+
type: 'error',
|
|
62
|
+
error: message.payload.error,
|
|
63
|
+
shouldRetry: message.payload.shouldRetry,
|
|
64
|
+
});
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
throw new Error(`Unknown message type ${message.type}`);
|
|
68
|
+
};
|
|
69
|
+
(0, call_lambda_1.callLambdaWithStreaming)({
|
|
70
|
+
functionName,
|
|
71
|
+
payload,
|
|
72
|
+
retriesRemaining: 1,
|
|
73
|
+
region: (0, get_current_region_1.getCurrentRegionInFunction)(),
|
|
74
|
+
timeoutInTest: 12000,
|
|
75
|
+
type: defaults_1.LambdaRoutines.renderer,
|
|
76
|
+
receivedStreamingPayload,
|
|
77
|
+
})
|
|
78
|
+
.then(() => {
|
|
79
|
+
resolve({
|
|
80
|
+
type: 'success',
|
|
81
|
+
});
|
|
82
|
+
})
|
|
83
|
+
.catch((err) => {
|
|
84
|
+
resolve({
|
|
85
|
+
type: 'error',
|
|
86
|
+
error: err.stack,
|
|
87
|
+
shouldRetry: false,
|
|
88
|
+
});
|
|
89
|
+
});
|
|
90
|
+
});
|
|
91
|
+
};
|
|
92
|
+
const streamRendererFunctionWithRetry = async ({ payload, files, functionName, outdir, overallProgress, logLevel, }) => {
|
|
93
|
+
if (payload.type !== defaults_1.LambdaRoutines.renderer) {
|
|
94
|
+
throw new Error('Expected renderer type');
|
|
95
|
+
}
|
|
96
|
+
const result = await streamRenderer({
|
|
97
|
+
files,
|
|
98
|
+
functionName,
|
|
99
|
+
outdir,
|
|
100
|
+
overallProgress,
|
|
101
|
+
payload,
|
|
102
|
+
logLevel,
|
|
103
|
+
});
|
|
104
|
+
if (result.type === 'error') {
|
|
105
|
+
if (!result.shouldRetry) {
|
|
106
|
+
throw result.error;
|
|
107
|
+
}
|
|
108
|
+
overallProgress.addRetry({
|
|
109
|
+
attempt: payload.attempt + 1,
|
|
110
|
+
time: Date.now(),
|
|
111
|
+
chunk: payload.chunk,
|
|
112
|
+
});
|
|
113
|
+
return (0, exports.streamRendererFunctionWithRetry)({
|
|
114
|
+
files,
|
|
115
|
+
functionName,
|
|
116
|
+
outdir,
|
|
117
|
+
overallProgress,
|
|
118
|
+
payload: {
|
|
119
|
+
...payload,
|
|
120
|
+
attempt: payload.attempt + 1,
|
|
121
|
+
retriesLeft: payload.retriesLeft - 1,
|
|
122
|
+
},
|
|
123
|
+
logLevel,
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
};
|
|
127
|
+
exports.streamRendererFunctionWithRetry = streamRendererFunctionWithRetry;
|
|
@@ -11,7 +11,4 @@ export declare class ResponseStream extends Stream.Writable {
|
|
|
11
11
|
setContentType(contentType: string): void;
|
|
12
12
|
setIsBase64Encoded(isBase64Encoded: boolean): void;
|
|
13
13
|
}
|
|
14
|
-
export declare const HANDLER_STREAMING: unique symbol;
|
|
15
|
-
export declare const STREAM_RESPONSE = "response";
|
|
16
|
-
export declare function isInAWS(handler: Function): boolean;
|
|
17
14
|
export declare function streamifyResponse(handler: Function): Function;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.streamifyResponse = exports.
|
|
3
|
+
exports.streamifyResponse = exports.ResponseStream = void 0;
|
|
4
4
|
const stream_1 = require("stream");
|
|
5
5
|
class ResponseStream extends stream_1.Stream.Writable {
|
|
6
6
|
constructor() {
|
|
@@ -29,16 +29,6 @@ function patchArgs(argList) {
|
|
|
29
29
|
}
|
|
30
30
|
return argList[1];
|
|
31
31
|
}
|
|
32
|
-
exports.HANDLER_STREAMING = Symbol.for('aws.lambda.runtime.handler.streaming');
|
|
33
|
-
exports.STREAM_RESPONSE = 'response';
|
|
34
|
-
function isInAWS(handler) {
|
|
35
|
-
return (
|
|
36
|
-
// @ts-expect-error
|
|
37
|
-
handler[exports.HANDLER_STREAMING] !== undefined &&
|
|
38
|
-
// @ts-expect-error
|
|
39
|
-
handler[exports.HANDLER_STREAMING] === exports.STREAM_RESPONSE);
|
|
40
|
-
}
|
|
41
|
-
exports.isInAWS = isInAWS;
|
|
42
32
|
function streamifyResponse(handler) {
|
|
43
33
|
// Check if we are inside Lambda
|
|
44
34
|
if (process.env.AWS_LAMBDA_FUNCTION_VERSION &&
|
|
@@ -56,9 +46,7 @@ function streamifyResponse(handler) {
|
|
|
56
46
|
EventStream: [
|
|
57
47
|
{
|
|
58
48
|
PayloadChunk: {
|
|
59
|
-
Payload: responseStream.
|
|
60
|
-
? responseStream.getBufferedData()
|
|
61
|
-
: responseStream.getBufferedData(),
|
|
49
|
+
Payload: responseStream.getBufferedData(),
|
|
62
50
|
},
|
|
63
51
|
InvokeComplete: true,
|
|
64
52
|
},
|
|
@@ -4,16 +4,16 @@ declare const streamingPayloadSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObje
|
|
|
4
4
|
type: z.ZodLiteral<"render-id-determined">;
|
|
5
5
|
renderId: z.ZodString;
|
|
6
6
|
}, "strip", z.ZodTypeAny, {
|
|
7
|
-
renderId: string;
|
|
8
7
|
type: "render-id-determined";
|
|
9
|
-
}, {
|
|
10
8
|
renderId: string;
|
|
9
|
+
}, {
|
|
11
10
|
type: "render-id-determined";
|
|
11
|
+
renderId: string;
|
|
12
12
|
}>]>;
|
|
13
13
|
export type StreamingPayloads = z.infer<typeof streamingPayloadSchema>;
|
|
14
14
|
export declare const isStreamingPayload: (str: string) => false | {
|
|
15
|
-
renderId: string;
|
|
16
15
|
type: "render-id-determined";
|
|
16
|
+
renderId: string;
|
|
17
17
|
};
|
|
18
18
|
export declare const sendProgressEvent: (responseStream: ResponseStream, payload: StreamingPayloads) => void;
|
|
19
19
|
export {};
|
|
@@ -17,12 +17,9 @@ export type LambdaErrorInfo = {
|
|
|
17
17
|
};
|
|
18
18
|
export declare const getTmpDirStateIfENoSp: (err: string) => LambdaErrorInfo['tmpDir'];
|
|
19
19
|
export type EnhancedErrorInfo = LambdaErrorInfo & {
|
|
20
|
+
/**
|
|
21
|
+
* @deprecated Will always be an empty string.
|
|
22
|
+
*/
|
|
20
23
|
s3Location: string;
|
|
21
24
|
explanation: string | null;
|
|
22
25
|
};
|
|
23
|
-
export declare const writeLambdaError: ({ bucketName, renderId, errorInfo, expectedBucketOwner, }: {
|
|
24
|
-
bucketName: string;
|
|
25
|
-
renderId: string;
|
|
26
|
-
expectedBucketOwner: string;
|
|
27
|
-
errorInfo: LambdaErrorInfo;
|
|
28
|
-
}) => Promise<void>;
|