@hyperframes/engine 0.8.177 → 0.8.179
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/services/__fixtures__/loopback.d.ts +2 -0
- package/dist/services/__fixtures__/loopback.js +15 -0
- package/dist/services/audioMixer.d.ts +1 -0
- package/dist/services/audioMixer.js +1 -0
- package/dist/services/browserManager.js +33 -25
- package/dist/services/chromeHostCeiling.d.ts +9 -0
- package/dist/services/chromeHostCeiling.js +66 -0
- package/dist/services/videoFrameExtractor.d.ts +3 -1
- package/dist/services/videoFrameExtractor.js +19 -3
- package/dist/utils/ffprobe.js +14 -2
- package/dist/utils/managedChildProcess.d.ts +5 -1
- package/dist/utils/managedChildProcess.js +30 -8
- package/dist/utils/urlInput.d.ts +7 -0
- package/dist/utils/urlInput.js +34 -0
- package/package.json +3 -3
|
@@ -3,3 +3,5 @@ import { type Browser } from "puppeteer-core";
|
|
|
3
3
|
/** Listens on a free loopback port and returns the server's origin. */
|
|
4
4
|
export declare function listenOnLoopback(server: Server): Promise<string>;
|
|
5
5
|
export declare function launchTestChrome(executablePath: string | undefined): Promise<Browser>;
|
|
6
|
+
/** Closes a test Chrome, killing it when close hangs (seen on Windows runners) so afterAll never times out. */
|
|
7
|
+
export declare function closeTestChrome(browser: Browser | undefined, timeoutMs?: number): Promise<void>;
|
|
@@ -14,3 +14,18 @@ export function launchTestChrome(executablePath) {
|
|
|
14
14
|
args: ["--no-sandbox", "--disable-dev-shm-usage"],
|
|
15
15
|
});
|
|
16
16
|
}
|
|
17
|
+
/** Closes a test Chrome, killing it when close hangs (seen on Windows runners) so afterAll never times out. */
|
|
18
|
+
export async function closeTestChrome(browser, timeoutMs = 5_000) {
|
|
19
|
+
if (!browser)
|
|
20
|
+
return;
|
|
21
|
+
let timer;
|
|
22
|
+
const closed = await Promise.race([
|
|
23
|
+
browser.close().then(() => true, () => false),
|
|
24
|
+
new Promise((resolve) => (timer = setTimeout(resolve, timeoutMs, false))),
|
|
25
|
+
]);
|
|
26
|
+
clearTimeout(timer);
|
|
27
|
+
if (closed)
|
|
28
|
+
return;
|
|
29
|
+
browser.process()?.kill("SIGKILL");
|
|
30
|
+
await browser.disconnect().catch(() => undefined);
|
|
31
|
+
}
|
|
@@ -33,4 +33,5 @@ export declare function parseAudioElements(html: string): AudioElement[];
|
|
|
33
33
|
export declare function processCompositionAudio(elements: AudioElement[], baseDir: string, workDir: string, outputPath: string, totalDuration: number, signal?: AbortSignal, config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout" | "audioGain">> & {
|
|
34
34
|
/** Serialised master bus chain, from `parseMasterFxChain`. */
|
|
35
35
|
masterFxChain?: string;
|
|
36
|
+
urlByAbsentLocalPath?: ReadonlyMap<string, string>;
|
|
36
37
|
}, compiledDir?: string): Promise<MixResult>;
|
|
@@ -935,6 +935,7 @@ export async function processCompositionAudio(elements, baseDir, workDir, output
|
|
|
935
935
|
if (!isHttpUrl(srcPath)) {
|
|
936
936
|
// Same browser-URL path semantics as videos.
|
|
937
937
|
srcPath = resolveProjectRelativeSrc(element.src, baseDir, compiledDir);
|
|
938
|
+
srcPath = config?.urlByAbsentLocalPath?.get(srcPath) ?? srcPath;
|
|
938
939
|
}
|
|
939
940
|
if (isHttpUrl(srcPath)) {
|
|
940
941
|
try {
|
|
@@ -10,7 +10,7 @@ import { execSync } from "child_process";
|
|
|
10
10
|
import { existsSync, readdirSync, statSync } from "fs";
|
|
11
11
|
import { join } from "path";
|
|
12
12
|
import { homedir } from "os";
|
|
13
|
-
import { chromeMajorCeiling, exceedsChromeCeiling } from "./chromeHostCeiling.js";
|
|
13
|
+
import { chromeMajorCeiling, compareBrowserVersionsDescending, exceedsChromeCeiling, isManagedChromeDir, pickSharedShell, } from "./chromeHostCeiling.js";
|
|
14
14
|
import { DEFAULT_CONFIG } from "../config.js";
|
|
15
15
|
import { getSystemTotalMb, LOW_MEMORY_TOTAL_MB_THRESHOLD } from "./systemMemory.js";
|
|
16
16
|
import { describeWslGpuAccess, detectWsl } from "./wslHost.js";
|
|
@@ -103,28 +103,6 @@ async function readWebGlInfo(browser) {
|
|
|
103
103
|
};
|
|
104
104
|
});
|
|
105
105
|
}
|
|
106
|
-
function compareBrowserVersionsDescending(left, right) {
|
|
107
|
-
const parse = (value) => {
|
|
108
|
-
const version = value.slice(value.indexOf("-") + 1);
|
|
109
|
-
const segments = [];
|
|
110
|
-
for (const segment of version.split(".")) {
|
|
111
|
-
const parsed = Number.parseInt(segment, 10);
|
|
112
|
-
if (!Number.isFinite(parsed))
|
|
113
|
-
break;
|
|
114
|
-
segments.push(parsed);
|
|
115
|
-
}
|
|
116
|
-
return segments;
|
|
117
|
-
};
|
|
118
|
-
const leftSegments = parse(left);
|
|
119
|
-
const rightSegments = parse(right);
|
|
120
|
-
const length = Math.max(leftSegments.length, rightSegments.length);
|
|
121
|
-
for (let index = 0; index < length; index += 1) {
|
|
122
|
-
const difference = (rightSegments[index] ?? 0) - (leftSegments[index] ?? 0);
|
|
123
|
-
if (difference !== 0)
|
|
124
|
-
return difference;
|
|
125
|
-
}
|
|
126
|
-
return 0;
|
|
127
|
-
}
|
|
128
106
|
const CACHED_HEADLESS_SHELL_EXECUTABLES = {
|
|
129
107
|
"darwin/arm64": ["chrome-headless-shell-mac-arm64", "chrome-headless-shell"],
|
|
130
108
|
"darwin/x64": ["chrome-headless-shell-mac-x64", "chrome-headless-shell"],
|
|
@@ -172,6 +150,33 @@ function findCachedHeadlessShell(baseDir) {
|
|
|
172
150
|
}
|
|
173
151
|
return undefined;
|
|
174
152
|
}
|
|
153
|
+
function findSharedHeadlessShell(baseDir) {
|
|
154
|
+
const executable = cachedHeadlessShellExecutable();
|
|
155
|
+
if (!executable || !existsSync(baseDir))
|
|
156
|
+
return undefined;
|
|
157
|
+
try {
|
|
158
|
+
return pickSharedShell(readdirSync(baseDir), (version) => {
|
|
159
|
+
const binary = join(baseDir, version, ...executable);
|
|
160
|
+
return isNonEmptyFile(binary) ? binary : undefined;
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
catch {
|
|
164
|
+
return undefined;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
function findManagedHeadlessShell(baseDir) {
|
|
168
|
+
const executable = cachedHeadlessShellExecutable();
|
|
169
|
+
if (!executable || !existsSync(baseDir))
|
|
170
|
+
return undefined;
|
|
171
|
+
try {
|
|
172
|
+
const version = readdirSync(baseDir).find(isManagedChromeDir);
|
|
173
|
+
const binary = version && join(baseDir, version, ...executable);
|
|
174
|
+
return binary && isNonEmptyFile(binary) ? binary : undefined;
|
|
175
|
+
}
|
|
176
|
+
catch {
|
|
177
|
+
return undefined;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
175
180
|
/**
|
|
176
181
|
* Resolve chrome-headless-shell binary for deterministic BeginFrame rendering.
|
|
177
182
|
* Checks config.chromePath, then PRODUCER_HEADLESS_SHELL_PATH env var,
|
|
@@ -197,8 +202,11 @@ export function resolveHeadlessShellPath(config) {
|
|
|
197
202
|
}
|
|
198
203
|
return envPath;
|
|
199
204
|
}
|
|
200
|
-
const
|
|
201
|
-
|
|
205
|
+
const ours = join(homedir(), ".cache", "hyperframes", "chrome", "chrome-headless-shell");
|
|
206
|
+
const shared = join(homedir(), ".cache", "puppeteer", "chrome-headless-shell");
|
|
207
|
+
return (findManagedHeadlessShell(ours) ??
|
|
208
|
+
findSharedHeadlessShell(shared) ??
|
|
209
|
+
findCachedHeadlessShell(ours));
|
|
202
210
|
}
|
|
203
211
|
/** Full Chrome for Testing in Puppeteer's cache (`@puppeteer/browsers install chrome@...`), newest first. */
|
|
204
212
|
export function resolveFullChromePath() {
|
|
@@ -2,3 +2,12 @@
|
|
|
2
2
|
export declare function chromeMajorCeiling(hostPlatform?: string, hostRelease?: string): number | undefined;
|
|
3
3
|
/** True when a cache dir name (`mac-152.0.1.2`) or build id (`152.0.1.2`) is newer than the ceiling. */
|
|
4
4
|
export declare function exceedsChromeCeiling(versionName: string, ceiling: number | undefined): boolean;
|
|
5
|
+
export declare function managedChromeVersion(hostPlatform?: string, hostRelease?: string): string;
|
|
6
|
+
export declare const isManagedChromeDir: (versionName: string) => boolean;
|
|
7
|
+
/** Newest first, by numeric build (`linux-148.0.1` before `linux-99.0.1`). */
|
|
8
|
+
export declare function compareBrowserVersionsDescending(left: string, right: string): number;
|
|
9
|
+
/** Whether a shell loads a blank page and exits: an old build can die at launch on a newer OS. Asked once per path. */
|
|
10
|
+
export declare function shellStarts(binary: string): boolean;
|
|
11
|
+
/** The shared Puppeteer cache's shell: the managed build, else the newest under the host's ceiling that starts.
|
|
12
|
+
* `shell` names a version dir's executable, or undefined when it holds no usable file. */
|
|
13
|
+
export declare function pickSharedShell(versions: readonly string[], shell: (version: string) => string | undefined): string | undefined;
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { spawnSync } from "node:child_process";
|
|
2
|
+
import { debuglog } from "node:util";
|
|
1
3
|
import { release } from "node:os";
|
|
2
4
|
/** Chrome 151 dropped macOS 12 and older (Darwin < 22); 150 is the last major that launches there. */
|
|
3
5
|
const MACOS_12_LAST_CHROME_MAJOR = 150;
|
|
@@ -14,3 +16,67 @@ export function exceedsChromeCeiling(versionName, ceiling) {
|
|
|
14
16
|
const major = Number.parseInt(versionName.slice(versionName.indexOf("-") + 1), 10);
|
|
15
17
|
return major > ceiling;
|
|
16
18
|
}
|
|
19
|
+
// Deliberately Beta, not Canary: 153.0.8000.0 measured identical to this build on every probe. macOS 12 gets 150,
|
|
20
|
+
// the last headless shell that launches there; it exposes canvas.drawElementImage under CanvasDrawElement.
|
|
21
|
+
const CHROME_VERSION = "152.0.7977.30";
|
|
22
|
+
const MACOS_12_CHROME_VERSION = "150.0.7871.124";
|
|
23
|
+
export function managedChromeVersion(hostPlatform = process.platform, hostRelease = release()) {
|
|
24
|
+
return chromeMajorCeiling(hostPlatform, hostRelease) === undefined
|
|
25
|
+
? CHROME_VERSION
|
|
26
|
+
: MACOS_12_CHROME_VERSION;
|
|
27
|
+
}
|
|
28
|
+
export const isManagedChromeDir = (versionName) => versionName.slice(versionName.indexOf("-") + 1) === managedChromeVersion();
|
|
29
|
+
/** Newest first, by numeric build (`linux-148.0.1` before `linux-99.0.1`). */
|
|
30
|
+
export function compareBrowserVersionsDescending(left, right) {
|
|
31
|
+
const parse = (value) => {
|
|
32
|
+
const version = value.slice(value.indexOf("-") + 1);
|
|
33
|
+
const segments = [];
|
|
34
|
+
for (const segment of version.split(".")) {
|
|
35
|
+
const parsed = Number.parseInt(segment, 10);
|
|
36
|
+
if (!Number.isFinite(parsed))
|
|
37
|
+
break;
|
|
38
|
+
segments.push(parsed);
|
|
39
|
+
}
|
|
40
|
+
return segments;
|
|
41
|
+
};
|
|
42
|
+
const leftSegments = parse(left);
|
|
43
|
+
const rightSegments = parse(right);
|
|
44
|
+
const length = Math.max(leftSegments.length, rightSegments.length);
|
|
45
|
+
for (let index = 0; index < length; index += 1) {
|
|
46
|
+
const difference = (rightSegments[index] ?? 0) - (leftSegments[index] ?? 0);
|
|
47
|
+
if (difference !== 0)
|
|
48
|
+
return difference;
|
|
49
|
+
}
|
|
50
|
+
return 0;
|
|
51
|
+
}
|
|
52
|
+
const starts = new Map();
|
|
53
|
+
const debug = debuglog("hyperframes:browser");
|
|
54
|
+
/** Whether a shell loads a blank page and exits: an old build can die at launch on a newer OS. Asked once per path. */
|
|
55
|
+
export function shellStarts(binary) {
|
|
56
|
+
let answer = starts.get(binary);
|
|
57
|
+
if (answer === undefined) {
|
|
58
|
+
const probe = ["--no-sandbox", "--dump-dom", "about:blank"];
|
|
59
|
+
answer = spawnSync(binary, probe, { stdio: "ignore", timeout: 10_000 })?.status === 0;
|
|
60
|
+
if (!answer)
|
|
61
|
+
debug("skipping %s: it did not start", binary);
|
|
62
|
+
starts.set(binary, answer);
|
|
63
|
+
}
|
|
64
|
+
return answer;
|
|
65
|
+
}
|
|
66
|
+
/** The shared Puppeteer cache's shell: the managed build, else the newest under the host's ceiling that starts.
|
|
67
|
+
* `shell` names a version dir's executable, or undefined when it holds no usable file. */
|
|
68
|
+
export function pickSharedShell(versions, shell) {
|
|
69
|
+
const managed = versions.find(isManagedChromeDir);
|
|
70
|
+
const pinned = managed && shell(managed);
|
|
71
|
+
if (pinned)
|
|
72
|
+
return pinned;
|
|
73
|
+
const ceiling = chromeMajorCeiling();
|
|
74
|
+
for (const version of [...versions].sort(compareBrowserVersionsDescending)) {
|
|
75
|
+
if (version === managed || exceedsChromeCeiling(version, ceiling))
|
|
76
|
+
continue;
|
|
77
|
+
const binary = shell(version);
|
|
78
|
+
if (binary && shellStarts(binary))
|
|
79
|
+
return binary;
|
|
80
|
+
}
|
|
81
|
+
return undefined;
|
|
82
|
+
}
|
|
@@ -107,6 +107,7 @@ export interface ExtractionOptions {
|
|
|
107
107
|
deferRangeExtraction?: boolean;
|
|
108
108
|
/** Directories the caller rewrites on every render (downloaded media); their sources are cached by content. */
|
|
109
109
|
contentKeyedDirs?: readonly string[];
|
|
110
|
+
urlByAbsentLocalPath?: ReadonlyMap<string, string>;
|
|
110
111
|
}
|
|
111
112
|
export declare const EXTRACT_CACHE_MIN_AGE_MS: number;
|
|
112
113
|
/**
|
|
@@ -240,7 +241,8 @@ export interface ImageElement {
|
|
|
240
241
|
end: number;
|
|
241
242
|
}
|
|
242
243
|
export declare function parseImageElements(html: string): ImageElement[];
|
|
243
|
-
export declare function extractVideoFramesRange(
|
|
244
|
+
export declare function extractVideoFramesRange(...args: Parameters<typeof extractVideoFramesRangeUnredacted>): Promise<ExtractedFrames>;
|
|
245
|
+
declare function extractVideoFramesRangeUnredacted(videoPath: string, videoId: string, startTime: number, duration: number, options: ExtractionOptions, signal?: AbortSignal, config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">>,
|
|
244
246
|
/** Frames go straight here, with no per-videoId subdir (the cache layer's keyed entry). */
|
|
245
247
|
outputDirOverride?: string): Promise<ExtractedFrames>;
|
|
246
248
|
export declare function classifyFfmpegSpawnError(error: unknown, stderr?: string): VideoSourceExtractionError;
|
|
@@ -18,6 +18,7 @@ import { resolveReferencedStart } from "./referenceResolver.js";
|
|
|
18
18
|
import { isKnownInactiveTimelineWindow } from "./mediaTimelineWindow.js";
|
|
19
19
|
import { extractFinalVideoFrameTimestamp, extractMediaMetadata, } from "../utils/ffprobe.js";
|
|
20
20
|
import { inputAlphaOpaqueWarning, probeInputAlphaPlane } from "../utils/alphaPlaneProbe.js";
|
|
21
|
+
import { redactUrlQueriesInError, urlInputArgs } from "../utils/urlInput.js";
|
|
21
22
|
import { analyzeCompositionHdr, isHdrColorSpace as isHdrColorSpaceUtil, } from "../utils/hdr.js";
|
|
22
23
|
import { downloadToTemp, isHttpUrl, safeDownloadUrlIdentity, UrlDownloadError, writeUrlDownloadTelemetry, } from "../utils/urlDownloader.js";
|
|
23
24
|
import { runFfmpeg } from "../utils/runFfmpeg.js";
|
|
@@ -418,7 +419,9 @@ export function parseImageElements(html) {
|
|
|
418
419
|
}
|
|
419
420
|
/** The first shown frame's colour tags; see {@link firstFrameColourArgs}. */
|
|
420
421
|
async function readFirstFrameColour(videoPath, signal) {
|
|
421
|
-
const result = await runFfmpeg(firstFrameColourArgs(videoPath), {
|
|
422
|
+
const result = await runFfmpeg([...urlInputArgs(videoPath), ...firstFrameColourArgs(videoPath)], {
|
|
423
|
+
signal,
|
|
424
|
+
});
|
|
422
425
|
return parseFirstFrameColour(result.stderr);
|
|
423
426
|
}
|
|
424
427
|
/** Chrome plays untagged VP9 and AV1 as BT.601, H.264 and VP8 as BT.709 from 720 coded lines (assumed for the rest). */
|
|
@@ -516,7 +519,15 @@ async function runSegmentedExtraction(job) {
|
|
|
516
519
|
}
|
|
517
520
|
return ran[ran.length - 1];
|
|
518
521
|
}
|
|
519
|
-
export async function extractVideoFramesRange(
|
|
522
|
+
export async function extractVideoFramesRange(...args) {
|
|
523
|
+
try {
|
|
524
|
+
return await extractVideoFramesRangeUnredacted(...args);
|
|
525
|
+
}
|
|
526
|
+
catch (error) {
|
|
527
|
+
throw redactUrlQueriesInError(error);
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
async function extractVideoFramesRangeUnredacted(videoPath, videoId, startTime, duration, options, signal, config,
|
|
520
531
|
/** Frames go straight here, with no per-videoId subdir (the cache layer's keyed entry). */
|
|
521
532
|
outputDirOverride) {
|
|
522
533
|
const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
|
|
@@ -555,7 +566,7 @@ outputDirOverride) {
|
|
|
555
566
|
if (options.frameRanges && (!sampleAtOutputFps || metadata.isVFR)) {
|
|
556
567
|
throw new Error("frameRanges extraction needs a constant-frame-rate source");
|
|
557
568
|
}
|
|
558
|
-
const args =
|
|
569
|
+
const args = urlInputArgs(videoPath);
|
|
559
570
|
if (decodeWithVideoToolbox) {
|
|
560
571
|
args.push("-hwaccel", "videotoolbox");
|
|
561
572
|
}
|
|
@@ -1129,6 +1140,11 @@ export async function extractAllVideoFrames(videos, baseDir, options, signal, co
|
|
|
1129
1140
|
if (!isHttpUrl(videoPath)) {
|
|
1130
1141
|
videoPath = resolveProjectRelativeSrc(video.src, baseDir, compiledDir);
|
|
1131
1142
|
}
|
|
1143
|
+
const remoteUrl = options.urlByAbsentLocalPath?.get(videoPath);
|
|
1144
|
+
if (remoteUrl) {
|
|
1145
|
+
resolvedVideos.push({ video, videoPath: remoteUrl });
|
|
1146
|
+
continue;
|
|
1147
|
+
}
|
|
1132
1148
|
if (isHttpUrl(videoPath)) {
|
|
1133
1149
|
const downloadDir = join(options.outputDir, DOWNLOAD_SUBDIR);
|
|
1134
1150
|
mkdirSync(downloadDir, { recursive: true });
|
package/dist/utils/ffprobe.js
CHANGED
|
@@ -5,6 +5,7 @@ import { chunkCrc32 } from "./crc32.js";
|
|
|
5
5
|
import { StringDecoder } from "node:string_decoder";
|
|
6
6
|
import { basename } from "path";
|
|
7
7
|
import { redactTelemetryString } from "@hyperframes/core";
|
|
8
|
+
import { urlInputArgs } from "./urlInput.js";
|
|
8
9
|
import { FFPROBE_PATH_ENV, getFfprobeBinary } from "./ffmpegBinaries.js";
|
|
9
10
|
import { ManagedChildProcess } from "./managedChildProcess.js";
|
|
10
11
|
import { trackChildProcess } from "./processTracker.js";
|
|
@@ -56,7 +57,15 @@ async function runFfprobe(filePath, argsWithoutInput, signal, stdoutOptions) {
|
|
|
56
57
|
throw new Error('[FFmpeg] Refusing to probe "-": stdin is not a supported input path.');
|
|
57
58
|
}
|
|
58
59
|
const command = getFfprobeBinary();
|
|
59
|
-
const
|
|
60
|
+
const args = [
|
|
61
|
+
"-v",
|
|
62
|
+
"error",
|
|
63
|
+
...urlInputArgs(filePath, { tool: "ffprobe" }),
|
|
64
|
+
...argsWithoutInput,
|
|
65
|
+
"--",
|
|
66
|
+
filePath,
|
|
67
|
+
];
|
|
68
|
+
const proc = spawn(command, args, {
|
|
60
69
|
// Nothing is ever written to the child's stdin; leaving it as a pipe is
|
|
61
70
|
// what lets a stdin-reading invocation block indefinitely.
|
|
62
71
|
stdio: ["ignore", "pipe", "pipe"],
|
|
@@ -593,7 +602,10 @@ export async function extractMediaMetadata(filePath) {
|
|
|
593
602
|
// ffprobe ran at all, which is exactly the build it exists to cover: one
|
|
594
603
|
// that reports gbr but does not decode cICP. An HDR PQ PNG then resolved
|
|
595
604
|
// colorTransfer "" and graded SDR.
|
|
596
|
-
const
|
|
605
|
+
const isPng = output !== null && isPngImageProbe(output);
|
|
606
|
+
const cicp = !isPng || (colorTransfer && colorPrimaries && colorSpaceVal)
|
|
607
|
+
? null
|
|
608
|
+
: stillImage()?.colorSpace;
|
|
597
609
|
const merged = {
|
|
598
610
|
colorTransfer: colorTransfer || cicp?.colorTransfer || "",
|
|
599
611
|
colorPrimaries: colorPrimaries || cicp?.colorPrimaries || "",
|
|
@@ -26,7 +26,9 @@ export declare class ManagedChildProcess {
|
|
|
26
26
|
private readonly outcomePromise;
|
|
27
27
|
private resolveOutcome;
|
|
28
28
|
private requestedReason;
|
|
29
|
-
private
|
|
29
|
+
private rawStderrTail;
|
|
30
|
+
private rawStderrTailCut;
|
|
31
|
+
private readonly stderrDecoder;
|
|
30
32
|
private spawned;
|
|
31
33
|
private settled;
|
|
32
34
|
private deadlineTimer;
|
|
@@ -37,6 +39,8 @@ export declare class ManagedChildProcess {
|
|
|
37
39
|
get isSettled(): boolean;
|
|
38
40
|
markActivity(): void;
|
|
39
41
|
private readonly onStderr;
|
|
42
|
+
private get maxBytes();
|
|
43
|
+
private redactedStderrTail;
|
|
40
44
|
private readonly onSpawn;
|
|
41
45
|
private readonly onClose;
|
|
42
46
|
private readonly onError;
|
|
@@ -1,5 +1,8 @@
|
|
|
1
|
+
import { StringDecoder } from "node:string_decoder";
|
|
2
|
+
import { redactUrlQueryStrings } from "@hyperframes/core";
|
|
1
3
|
const DEFAULT_TERMINATION_GRACE_MS = 2_000;
|
|
2
4
|
const DEFAULT_STDERR_MAX_BYTES = 64 * 1024;
|
|
5
|
+
const RAW_STDERR_SLACK_FACTOR = 2;
|
|
3
6
|
/** Owns cancellation, escalation, stderr and reaping for one child process. */
|
|
4
7
|
export class ManagedChildProcess {
|
|
5
8
|
child;
|
|
@@ -9,7 +12,9 @@ export class ManagedChildProcess {
|
|
|
9
12
|
outcomePromise;
|
|
10
13
|
resolveOutcome;
|
|
11
14
|
requestedReason = null;
|
|
12
|
-
|
|
15
|
+
rawStderrTail = "";
|
|
16
|
+
rawStderrTailCut = false;
|
|
17
|
+
stderrDecoder = new StringDecoder("utf8");
|
|
13
18
|
spawned = false;
|
|
14
19
|
settled = false;
|
|
15
20
|
deadlineTimer = null;
|
|
@@ -47,13 +52,21 @@ export class ManagedChildProcess {
|
|
|
47
52
|
}
|
|
48
53
|
onStderr = (data) => {
|
|
49
54
|
const chunk = Buffer.isBuffer(data) ? data : Buffer.from(data);
|
|
50
|
-
const
|
|
51
|
-
this.
|
|
52
|
-
if (this.
|
|
53
|
-
this.
|
|
54
|
-
}
|
|
55
|
+
const joined = this.rawStderrTail + this.stderrDecoder.write(chunk);
|
|
56
|
+
this.rawStderrTail = lastBytes(joined, RAW_STDERR_SLACK_FACTOR * this.maxBytes);
|
|
57
|
+
if (this.rawStderrTail !== joined)
|
|
58
|
+
this.rawStderrTailCut = true;
|
|
55
59
|
this.options.onStderr?.(chunk.toString());
|
|
56
60
|
};
|
|
61
|
+
get maxBytes() {
|
|
62
|
+
return this.options.stderrMaxBytes ?? DEFAULT_STDERR_MAX_BYTES;
|
|
63
|
+
}
|
|
64
|
+
redactedStderrTail() {
|
|
65
|
+
const raw = this.rawStderrTailCut
|
|
66
|
+
? withoutTokenSeveredByCut(this.rawStderrTail)
|
|
67
|
+
: this.rawStderrTail;
|
|
68
|
+
return lastBytes(redactUrlQueryStrings(raw), this.maxBytes);
|
|
69
|
+
}
|
|
57
70
|
onSpawn = () => {
|
|
58
71
|
this.spawned = true;
|
|
59
72
|
};
|
|
@@ -62,7 +75,7 @@ export class ManagedChildProcess {
|
|
|
62
75
|
reason: this.requestedReason ?? "exit",
|
|
63
76
|
exitCode,
|
|
64
77
|
signal,
|
|
65
|
-
stderr: this.
|
|
78
|
+
stderr: this.redactedStderrTail(),
|
|
66
79
|
durationMs: this.now() - this.startedAtMs,
|
|
67
80
|
});
|
|
68
81
|
};
|
|
@@ -73,7 +86,7 @@ export class ManagedChildProcess {
|
|
|
73
86
|
reason: "spawn_error",
|
|
74
87
|
exitCode: null,
|
|
75
88
|
signal: null,
|
|
76
|
-
stderr: this.
|
|
89
|
+
stderr: this.rawStderrTail.length > 0 ? this.redactedStderrTail() : error.message,
|
|
77
90
|
durationMs: this.now() - this.startedAtMs,
|
|
78
91
|
error,
|
|
79
92
|
});
|
|
@@ -145,3 +158,12 @@ export class ManagedChildProcess {
|
|
|
145
158
|
this.escalationTimer = null;
|
|
146
159
|
}
|
|
147
160
|
}
|
|
161
|
+
function lastBytes(text, maxBytes) {
|
|
162
|
+
return Buffer.byteLength(text) > maxBytes
|
|
163
|
+
? Buffer.from(text).subarray(-maxBytes).toString()
|
|
164
|
+
: text;
|
|
165
|
+
}
|
|
166
|
+
function withoutTokenSeveredByCut(text) {
|
|
167
|
+
const firstSpace = text.search(/\s/);
|
|
168
|
+
return firstSpace < 0 ? "" : text.slice(firstSpace);
|
|
169
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/** ffmpeg and ffprobe skip TLS certificate checks unless told to; an https input must be verified. */
|
|
2
|
+
export declare function urlInputArgs(input: string, { tool, caBundles, }?: {
|
|
3
|
+
tool?: "ffmpeg" | "ffprobe";
|
|
4
|
+
caBundles?: readonly string[];
|
|
5
|
+
}): string[];
|
|
6
|
+
/** A signed URL grants access until it expires: strip query strings from every string field of an error and its causes. */
|
|
7
|
+
export declare function redactUrlQueriesInError<T>(error: T): T;
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
import { redactUrlQueryStrings } from "@hyperframes/core";
|
|
3
|
+
// The Lambda nodejs image (Amazon Linux 2023) ships the first; Debian/Ubuntu the second; macOS the third.
|
|
4
|
+
const CA_BUNDLES = [
|
|
5
|
+
"/etc/pki/tls/certs/ca-bundle.crt",
|
|
6
|
+
"/etc/ssl/certs/ca-certificates.crt",
|
|
7
|
+
"/etc/ssl/cert.pem",
|
|
8
|
+
];
|
|
9
|
+
// Lambda's ffprobe 4.0 accepts these but not -reconnect_on_network_error; a stalled read fails after 30 s.
|
|
10
|
+
const RECONNECT_ARGS = ["-rw_timeout", "30000000", "-reconnect", "1", "-reconnect_delay_max", "2"];
|
|
11
|
+
/** ffmpeg and ffprobe skip TLS certificate checks unless told to; an https input must be verified. */
|
|
12
|
+
export function urlInputArgs(input, { tool = "ffmpeg", caBundles = CA_BUNDLES, } = {}) {
|
|
13
|
+
if (!/^https:\/\//i.test(input))
|
|
14
|
+
return [];
|
|
15
|
+
const caFile = caBundles.find((path) => existsSync(path));
|
|
16
|
+
if (!caFile) {
|
|
17
|
+
throw new Error(`[FFmpeg] cannot verify TLS for an https input: no CA bundle at ${caBundles.join(", ")}`);
|
|
18
|
+
}
|
|
19
|
+
const networkRetry = tool === "ffmpeg" ? ["-reconnect_on_network_error", "1"] : [];
|
|
20
|
+
return ["-tls_verify", "1", "-ca_file", caFile, ...RECONNECT_ARGS, ...networkRetry];
|
|
21
|
+
}
|
|
22
|
+
/** A signed URL grants access until it expires: strip query strings from every string field of an error and its causes. */
|
|
23
|
+
export function redactUrlQueriesInError(error) {
|
|
24
|
+
if (!(error instanceof Error))
|
|
25
|
+
return error;
|
|
26
|
+
const fields = error;
|
|
27
|
+
for (const key of ["message", "stack", ...Object.keys(error)]) {
|
|
28
|
+
if (typeof fields[key] === "string")
|
|
29
|
+
fields[key] = redactUrlQueryStrings(fields[key]);
|
|
30
|
+
}
|
|
31
|
+
if (error.cause !== error)
|
|
32
|
+
redactUrlQueriesInError(error.cause);
|
|
33
|
+
return error;
|
|
34
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hyperframes/engine",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.179",
|
|
4
4
|
"description": "Seekable web page to video rendering engine (Puppeteer + FFmpeg)",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -50,8 +50,8 @@
|
|
|
50
50
|
"linkedom": "^0.18.12",
|
|
51
51
|
"puppeteer": "^25.8.0",
|
|
52
52
|
"puppeteer-core": "^25.8.0",
|
|
53
|
-
"@hyperframes/core": "^0.8.
|
|
54
|
-
"@hyperframes/parsers": "^0.8.
|
|
53
|
+
"@hyperframes/core": "^0.8.179",
|
|
54
|
+
"@hyperframes/parsers": "^0.8.179"
|
|
55
55
|
},
|
|
56
56
|
"devDependencies": {
|
|
57
57
|
"@types/node": "^25.0.10",
|