@hyperframes/engine 0.8.213 → 0.8.216
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config.d.ts +2 -0
- package/dist/config.js +4 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.js +2 -2
- package/dist/services/chunkEncoder.js +1 -1
- package/dist/services/frameCapture.d.ts +4 -0
- package/dist/services/frameCapture.js +6 -0
- package/dist/services/parallelCoordinator.js +16 -8
- package/dist/services/systemMemory.d.ts +4 -0
- package/dist/services/systemMemory.js +32 -0
- package/dist/services/videoFrameInjector.d.ts +1 -1
- package/dist/services/videoFrameInjector.js +16 -5
- package/dist/types.d.ts +3 -0
- package/dist/utils/renderProvenance.js +18 -0
- package/dist/utils/urlDownloader.d.ts +4 -0
- package/dist/utils/urlDownloader.js +52 -1
- package/package.json +4 -3
package/dist/config.d.ts
CHANGED
|
@@ -92,6 +92,8 @@ export interface EngineConfig {
|
|
|
92
92
|
* the runtime probe fires). Not intended to be set by callers.
|
|
93
93
|
*/
|
|
94
94
|
forceScreenshotExplicitlyOptedOut?: boolean;
|
|
95
|
+
/** How long a BeginFrame capture may take to draw its first frame before the render retries by screenshot. */
|
|
96
|
+
beginFrameFirstFrameDeadlineMs?: number;
|
|
95
97
|
/**
|
|
96
98
|
* Low-memory render profile. When `true`, the orchestrator collapses the
|
|
97
99
|
* pipeline to its cheapest shape on memory-constrained hosts: it skips the
|
package/dist/config.js
CHANGED
|
@@ -61,6 +61,7 @@ const OPTIONAL_ENGINE_CONFIG_FIELDS = [
|
|
|
61
61
|
"expectedChromiumMajor",
|
|
62
62
|
"pageSideCompositingAutoDisabled",
|
|
63
63
|
"forceScreenshotExplicitlyOptedOut",
|
|
64
|
+
"beginFrameFirstFrameDeadlineMs",
|
|
64
65
|
"streamingEncodeAutoDisabledOnWin32Compound",
|
|
65
66
|
"runtimeManifestPath",
|
|
66
67
|
"extractCacheDir",
|
|
@@ -148,6 +149,9 @@ function validateEngineConfigScalars(config) {
|
|
|
148
149
|
if (typeof config.jpegQuality === "number" && config.jpegQuality > 100) {
|
|
149
150
|
throw new Error("Engine config jpegQuality must be <= 100");
|
|
150
151
|
}
|
|
152
|
+
if (config.beginFrameFirstFrameDeadlineMs !== undefined) {
|
|
153
|
+
assertEngineConfigNumber(config, "beginFrameFirstFrameDeadlineMs", 1, true);
|
|
154
|
+
}
|
|
151
155
|
}
|
|
152
156
|
function validateEngineConfigParallelism(config) {
|
|
153
157
|
if (config.concurrency !== "auto" &&
|
package/dist/index.d.ts
CHANGED
|
@@ -39,7 +39,7 @@ export { acquireBrowser, setHostHandlesSigint, releaseBrowser, drainBrowserPool,
|
|
|
39
39
|
export { augmentProtocolTimeoutError, isProtocolTimeoutError, } from "./services/protocolTimeoutErrorHint.js";
|
|
40
40
|
export { augmentPageNavigationTimeoutError, isPageNavigationTimeoutError, type NavigationTimeoutHintContext, } from "./services/pageNavigationTimeoutErrorHint.js";
|
|
41
41
|
export { forcePreservedDrawingBuffer } from "./services/drawElementService.js";
|
|
42
|
-
export { createCaptureSession, initializeSession, closeCaptureSession, captureFrame, captureFrameToBuffer, captureFrameToBufferPipelined, captureFramesBatchPipelined, DrawElementVerificationError, isDrawElementVerificationError, getDrawElementVerificationDetails, type DrawElementVerificationDetails, recaptureDrawElementFrameForVerify, completeDeferredDrawElementInit, writeCapturedFrame, discardWarmupCapture, getCompositionDuration, getCapturePerfSummary, percentileOf, prepareCaptureSessionForReuse, deriveBeginFrameProbeTimeTicks, type CaptureSession, isTransientBrowserError, isMemoryExhaustionError, type BeforeCaptureHook, type DiscardWarmupInnerCapture, type StaticVerificationOutcome, VfxFailureError, waitForImagesReady, evaluateHfDiagnostic, buildZeroDurationDiagnostic, } from "./services/frameCapture.js";
|
|
42
|
+
export { createCaptureSession, initializeSession, closeCaptureSession, captureFrame, captureFrameToBuffer, captureFrameToBufferPipelined, captureFramesBatchPipelined, DrawElementVerificationError, isDrawElementVerificationError, getDrawElementVerificationDetails, type DrawElementVerificationDetails, recaptureDrawElementFrameForVerify, completeDeferredDrawElementInit, writeCapturedFrame, discardWarmupCapture, getCompositionDuration, getCompositionLengthSource, type CompositionLengthSource, getCapturePerfSummary, percentileOf, prepareCaptureSessionForReuse, deriveBeginFrameProbeTimeTicks, type CaptureSession, isTransientBrowserError, isMemoryExhaustionError, type BeforeCaptureHook, type DiscardWarmupInnerCapture, type StaticVerificationOutcome, VfxFailureError, waitForImagesReady, evaluateHfDiagnostic, buildZeroDurationDiagnostic, } from "./services/frameCapture.js";
|
|
43
43
|
export { CaptureFailure, classifyCaptureFailure, isFatalCaptureFailure, type CaptureFailureKind, type CaptureWorkerDiagnostic, } from "./services/captureFailure.js";
|
|
44
44
|
export { createChromeMemorySampler, type ChromeMemoryStats, } from "./services/chromeMemorySampler.js";
|
|
45
45
|
export { installMediaRenderIdBridge } from "./services/mediaRenderIdBridge.js";
|
|
@@ -69,7 +69,7 @@ export { isBrowserSafePort, listenOnBrowserSafePort, settleOnBrowserSafePort, }
|
|
|
69
69
|
export { readWebGlVendorInfoFromCanvas } from "./utils/readWebGlVendorInfoFromCanvas.js";
|
|
70
70
|
export { extractMediaMetadata, extractVideoMetadata, extractFinalVideoFrameTimestamp, extractAudioMetadata, declareContentAddressedUrl, declareProbeFactsUrl, PROBE_FACTS_VERSION, buildProbeFactsText, probeMediaProfile, runFfprobe, analyzeKeyframeIntervals, type VideoMetadata, type AudioMetadata, type MediaProbeProfile, type KeyframeAnalysis, } from "./utils/ffprobe.js";
|
|
71
71
|
export { NOT_MEDIA_PAYLOAD, NotMediaPayloadError, assertMediaPayload, fingerprintElementId, isNotMediaPayload, } from "./utils/notMediaPayload.js";
|
|
72
|
-
export { assertPublicHttpsUrl, isBlockedNetworkHost, downloadToTemp, fetchPublicHttpsText, isHttpUrl, safeDownloadUrlIdentity, writeUrlDownloadTelemetry, type SafeDownloadUrlIdentity, type UrlDownloadOptions, type UrlDownloadTelemetry, type PublicHttpsTextOptions, } from "./utils/urlDownloader.js";
|
|
72
|
+
export { _setDownloadFetchForTests, assertPublicHttpsUrl, isBlockedNetworkHost, downloadToTemp, fetchPublicHttpsText, isHttpUrl, safeDownloadUrlIdentity, writeUrlDownloadTelemetry, type SafeDownloadUrlIdentity, type UrlDownloadOptions, type UrlDownloadTelemetry, type PublicHttpsTextOptions, } from "./utils/urlDownloader.js";
|
|
73
73
|
export { runFfmpeg, formatFfmpegError, describeFfmpegFailure, isExternalFfmpegInterruption, type RunFfmpegOptions, type RunFfmpegResult, } from "./utils/runFfmpeg.js";
|
|
74
74
|
export { aacSegmentCount, encodeAac } from "./services/aacEncode.js";
|
|
75
75
|
export { ManagedChildProcess, type ManagedChildProcessOptions, type ManagedChildProcessOutcome, type ManagedProcessTerminationReason, } from "./utils/managedChildProcess.js";
|
package/dist/index.js
CHANGED
|
@@ -41,7 +41,7 @@ export { augmentProtocolTimeoutError, isProtocolTimeoutError, } from "./services
|
|
|
41
41
|
export { augmentPageNavigationTimeoutError, isPageNavigationTimeoutError, } from "./services/pageNavigationTimeoutErrorHint.js";
|
|
42
42
|
// ── Frame capture pipeline ──────────────────────────────────────────────────────
|
|
43
43
|
export { forcePreservedDrawingBuffer } from "./services/drawElementService.js";
|
|
44
|
-
export { createCaptureSession, initializeSession, closeCaptureSession, captureFrame, captureFrameToBuffer, captureFrameToBufferPipelined, captureFramesBatchPipelined, DrawElementVerificationError, isDrawElementVerificationError, getDrawElementVerificationDetails, recaptureDrawElementFrameForVerify, completeDeferredDrawElementInit, writeCapturedFrame, discardWarmupCapture, getCompositionDuration, getCapturePerfSummary, percentileOf, prepareCaptureSessionForReuse, deriveBeginFrameProbeTimeTicks, isTransientBrowserError, isMemoryExhaustionError, VfxFailureError, waitForImagesReady, evaluateHfDiagnostic, buildZeroDurationDiagnostic, } from "./services/frameCapture.js";
|
|
44
|
+
export { createCaptureSession, initializeSession, closeCaptureSession, captureFrame, captureFrameToBuffer, captureFrameToBufferPipelined, captureFramesBatchPipelined, DrawElementVerificationError, isDrawElementVerificationError, getDrawElementVerificationDetails, recaptureDrawElementFrameForVerify, completeDeferredDrawElementInit, writeCapturedFrame, discardWarmupCapture, getCompositionDuration, getCompositionLengthSource, getCapturePerfSummary, percentileOf, prepareCaptureSessionForReuse, deriveBeginFrameProbeTimeTicks, isTransientBrowserError, isMemoryExhaustionError, VfxFailureError, waitForImagesReady, evaluateHfDiagnostic, buildZeroDurationDiagnostic, } from "./services/frameCapture.js";
|
|
45
45
|
export { CaptureFailure, classifyCaptureFailure, isFatalCaptureFailure, } from "./services/captureFailure.js";
|
|
46
46
|
export { createChromeMemorySampler, } from "./services/chromeMemorySampler.js";
|
|
47
47
|
// ── Screenshot (BeginFrame) ─────────────────────────────────────────────────────
|
|
@@ -73,7 +73,7 @@ export { isBrowserSafePort, listenOnBrowserSafePort, settleOnBrowserSafePort, }
|
|
|
73
73
|
export { readWebGlVendorInfoFromCanvas } from "./utils/readWebGlVendorInfoFromCanvas.js";
|
|
74
74
|
export { extractMediaMetadata, extractVideoMetadata, extractFinalVideoFrameTimestamp, extractAudioMetadata, declareContentAddressedUrl, declareProbeFactsUrl, PROBE_FACTS_VERSION, buildProbeFactsText, probeMediaProfile, runFfprobe, analyzeKeyframeIntervals, } from "./utils/ffprobe.js";
|
|
75
75
|
export { NOT_MEDIA_PAYLOAD, NotMediaPayloadError, assertMediaPayload, fingerprintElementId, isNotMediaPayload, } from "./utils/notMediaPayload.js";
|
|
76
|
-
export { assertPublicHttpsUrl, isBlockedNetworkHost, downloadToTemp, fetchPublicHttpsText, isHttpUrl, safeDownloadUrlIdentity, writeUrlDownloadTelemetry, } from "./utils/urlDownloader.js";
|
|
76
|
+
export { _setDownloadFetchForTests, assertPublicHttpsUrl, isBlockedNetworkHost, downloadToTemp, fetchPublicHttpsText, isHttpUrl, safeDownloadUrlIdentity, writeUrlDownloadTelemetry, } from "./utils/urlDownloader.js";
|
|
77
77
|
export { runFfmpeg, formatFfmpegError, describeFfmpegFailure, isExternalFfmpegInterruption, } from "./utils/runFfmpeg.js";
|
|
78
78
|
export { aacSegmentCount, encodeAac } from "./services/aacEncode.js";
|
|
79
79
|
export { ManagedChildProcess, } from "./utils/managedChildProcess.js";
|
|
@@ -22,7 +22,7 @@ import { appendRenderProvenanceArgs } from "../utils/renderProvenance.js";
|
|
|
22
22
|
export const ENCODER_PRESETS = {
|
|
23
23
|
draft: { preset: "ultrafast", quality: 28, codec: "h264" },
|
|
24
24
|
standard: { preset: "medium", quality: 18, codec: "h264" },
|
|
25
|
-
high: { preset: "
|
|
25
|
+
high: { preset: "fast", quality: 15, codec: "h264" },
|
|
26
26
|
};
|
|
27
27
|
function appendEncodeTimeoutMessage(error, timedOut, timeoutMs) {
|
|
28
28
|
if (!timedOut)
|
|
@@ -14,6 +14,7 @@ import { pageScreenshotCapture } from "./screenshotService.js";
|
|
|
14
14
|
import { type EngineConfig } from "../config.js";
|
|
15
15
|
import type { CaptureOptions, CaptureVideoMetadataHint, CaptureResult, CaptureBufferResult, CapturePerfSummary, CaptureWarning, SubTimelineWaitMemo, SubTimelineWaitOutcome } from "../types.js";
|
|
16
16
|
import { type ChromeMemorySampler, type ChromePids } from "./chromeMemorySampler.js";
|
|
17
|
+
import type { CompositionLengthSource } from "@hyperframes/core/runtime/composition-length";
|
|
17
18
|
export { isMemoryExhaustionError, isTransientBrowserError } from "./captureFailure.js";
|
|
18
19
|
export type { CaptureOptions, CaptureResult, CaptureBufferResult, CapturePerfSummary };
|
|
19
20
|
/** Called after seeking, before screenshot. Use for video frame injection or other pre-capture work.
|
|
@@ -748,6 +749,9 @@ export declare function discardWarmupCapture(session: CaptureSession, frameIndex
|
|
|
748
749
|
export declare function closeCaptureSession(session: CaptureSession): Promise<void>;
|
|
749
750
|
export declare function prepareCaptureSessionForReuse(session: CaptureSession, outputDir: string, onBeforeCapture: BeforeCaptureHook | null): void;
|
|
750
751
|
export declare function getCompositionDuration(session: CaptureSession): Promise<number>;
|
|
752
|
+
export type { CompositionLengthSource };
|
|
753
|
+
/** Which input set the composition's length, as the runtime published it; null when it did not. */
|
|
754
|
+
export declare function getCompositionLengthSource(session: CaptureSession): Promise<CompositionLengthSource | null>;
|
|
751
755
|
/**
|
|
752
756
|
* Ungated-release safety net, part 1: capture K screenshot ground-truth frames
|
|
753
757
|
* BEFORE the drawElement canvas is injected (the only window where a page
|
|
@@ -3956,6 +3956,12 @@ export async function getCompositionDuration(session) {
|
|
|
3956
3956
|
return window.__hf?.duration ?? 0;
|
|
3957
3957
|
});
|
|
3958
3958
|
}
|
|
3959
|
+
/** Which input set the composition's length, as the runtime published it; null when it did not. */
|
|
3960
|
+
export async function getCompositionLengthSource(session) {
|
|
3961
|
+
if (!session.isInitialized)
|
|
3962
|
+
throw new Error("[FrameCapture] Session not initialized");
|
|
3963
|
+
return session.page.evaluate(() => window.__hf?.lengthSource ?? null);
|
|
3964
|
+
}
|
|
3959
3965
|
/**
|
|
3960
3966
|
* Ungated-release safety net, part 1: capture K screenshot ground-truth frames
|
|
3961
3967
|
* BEFORE the drawElement canvas is injected (the only window where a page
|
|
@@ -16,7 +16,7 @@ import { DEFAULT_CONFIG } from "../config.js";
|
|
|
16
16
|
import { assertSwiftShader } from "../utils/assertSwiftShader.js";
|
|
17
17
|
import { readWebGlVendorInfoFromCanvas } from "../utils/readWebGlVendorInfoFromCanvas.js";
|
|
18
18
|
import { resolveHeadlessShellPath } from "./browserManager.js";
|
|
19
|
-
import { getSystemTotalMb } from "./systemMemory.js";
|
|
19
|
+
import { getCgroupCpuLimit, getSystemTotalMb } from "./systemMemory.js";
|
|
20
20
|
import { CaptureFailure, classifyCaptureFailure, isFatalCaptureFailure, } from "./captureFailure.js";
|
|
21
21
|
// System-memory budget per parallel worker. Each worker is a full Chrome
|
|
22
22
|
// process (SwiftShader compositor + raster threads) plus its share of parent-
|
|
@@ -27,11 +27,13 @@ const MEMORY_PER_WORKER_MB = 1536;
|
|
|
27
27
|
// Parent-process V8 heap the coordinator itself needs regardless of worker
|
|
28
28
|
// count (compile artifacts, puppeteer sessions, encoder state).
|
|
29
29
|
const HEAP_RESERVED_MB = 1024;
|
|
30
|
-
// Parent
|
|
31
|
-
//
|
|
32
|
-
|
|
33
|
-
//
|
|
34
|
-
|
|
30
|
+
// Parent heap per worker, by --trace-gc in a 10 GB container (10-28): 21 MB on a short film, 228 MB on a 1080×1920
|
|
31
|
+
// video-heavy one (427 → 1,110 MB from 1 to 4 workers). 320 keeps 1.4× over that; `--workers N` still overrides.
|
|
32
|
+
const HEAP_PER_WORKER_MB = 320;
|
|
33
|
+
// The desktop app keeps the old figure: its 4,192 MB heap is where 5 workers ran out (#5149). A packaged app runs the
|
|
34
|
+
// CLI on its own plain Node, so it is known by the client tag it sets on every CLI call, or by Electron itself.
|
|
35
|
+
const DESKTOP_HEAP_PER_WORKER_MB = 640;
|
|
36
|
+
const inDesktopApp = () => Boolean(process.versions.electron) || /^desktop\//.test(process.env.HYPERFRAMES_CLIENT ?? "");
|
|
35
37
|
// Apple-GPU disk capture measured ~15-25 MB parent heap per worker; 14 cores kept gaining to 8.
|
|
36
38
|
const APPLE_DISK_HEAP_PER_WORKER_MB = 128;
|
|
37
39
|
const APPLE_DISK_CORES_PER_WORKER = 1.75;
|
|
@@ -139,8 +141,14 @@ export function computeWorkerSizing(totalFrames, requested, config) {
|
|
|
139
141
|
const heapLimitMb = Math.round(getHeapStatistics().heap_size_limit / (1024 * 1024));
|
|
140
142
|
const appleDisk = config?.appleGpuDiskCapture === true;
|
|
141
143
|
const heapBasedWorkers = Math.max(1, Math.floor((heapLimitMb - HEAP_RESERVED_MB) /
|
|
142
|
-
(appleDisk
|
|
143
|
-
|
|
144
|
+
(appleDisk
|
|
145
|
+
? APPLE_DISK_HEAP_PER_WORKER_MB
|
|
146
|
+
: inDesktopApp()
|
|
147
|
+
? DESKTOP_HEAP_PER_WORKER_MB
|
|
148
|
+
: HEAP_PER_WORKER_MB)));
|
|
149
|
+
// A CPU quota (docker --cpus) bounds workers at its whole CPUs: at 3.5, 3 beat 2 and 4 (johnny, 10-28).
|
|
150
|
+
const cpuQuota = getCgroupCpuLimit();
|
|
151
|
+
const cpuBasedWorkers = Math.max(1, cpuQuota === null ? cpuCount - 2 : Math.min(cpuCount - 2, Math.floor(cpuQuota)));
|
|
144
152
|
const memoryBasedWorkers = Math.max(1, Math.floor((totalMemoryMb * 0.5) / MEMORY_PER_WORKER_MB));
|
|
145
153
|
const frameBasedWorkers = Math.floor(totalFrames / MIN_FRAMES_PER_WORKER);
|
|
146
154
|
const captureCostMultiplier = Math.max(1, config?.captureCostMultiplier ?? 1);
|
|
@@ -9,8 +9,12 @@
|
|
|
9
9
|
*/
|
|
10
10
|
/** Parse cgroup v2/v1 memory limits from sysfs file contents into MiB. */
|
|
11
11
|
export declare function parseCgroupLimitMb(v2Content: string | null, v1Content: string | null): number | null;
|
|
12
|
+
/** The CPUs a cgroup quota allows (v2 `cpu.max`, else v1 quota over period), or null when it sets none. */
|
|
13
|
+
export declare function parseCgroupCpus(v2Content: string | null, v1Quota: string | null, v1Period: string | null): number | null;
|
|
12
14
|
/** Test-only: reset the cached cgroup memory probe. */
|
|
13
15
|
export declare function _resetCgroupLimitCacheForTests(): void;
|
|
16
|
+
/** The CPUs this process's cgroup may use (docker `--cpus`), or null off Linux or without a quota. */
|
|
17
|
+
export declare function getCgroupCpuLimit(): number | null;
|
|
14
18
|
/**
|
|
15
19
|
* Actual Linux cgroup memory ceiling in MiB, or null when the process is not
|
|
16
20
|
* cgroup-limited. Unlike getSystemTotalMb this never falls back to host RAM.
|
|
@@ -23,7 +23,11 @@ const CGROUP_V2_MEMORY_MAX_PATH = "/sys/fs/cgroup/memory.max";
|
|
|
23
23
|
const CGROUP_V1_MEMORY_LIMIT_PATH = "/sys/fs/cgroup/memory/memory.limit_in_bytes";
|
|
24
24
|
// Kernel no-limit sentinel is page-rounded 2^63-1 (~9223372036854771712); >= 2^60 is implausible as a real limit.
|
|
25
25
|
const CGROUP_V1_NO_LIMIT_CUTOFF_BYTES = 2n ** 60n;
|
|
26
|
+
const CGROUP_V2_CPU_MAX_PATH = "/sys/fs/cgroup/cpu.max";
|
|
27
|
+
const CGROUP_V1_CPU_QUOTA_PATH = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us";
|
|
28
|
+
const CGROUP_V1_CPU_PERIOD_PATH = "/sys/fs/cgroup/cpu/cpu.cfs_period_us";
|
|
26
29
|
let _cachedCgroupLimitMb;
|
|
30
|
+
let _cachedCgroupCpus;
|
|
27
31
|
let _warnedCgroupReadFailure = false;
|
|
28
32
|
/** Parse cgroup v2/v1 memory limits from sysfs file contents into MiB. */
|
|
29
33
|
export function parseCgroupLimitMb(v2Content, v1Content) {
|
|
@@ -58,11 +62,39 @@ function parsePositiveByteLimitMb(content, noLimitCutoffBytes) {
|
|
|
58
62
|
}
|
|
59
63
|
return Number(bytes / BYTES_PER_MIB_BIGINT);
|
|
60
64
|
}
|
|
65
|
+
/** The CPUs a cgroup quota allows (v2 `cpu.max`, else v1 quota over period), or null when it sets none. */
|
|
66
|
+
export function parseCgroupCpus(v2Content, v1Quota, v1Period) {
|
|
67
|
+
if (v2Content !== null) {
|
|
68
|
+
const [quota, period] = v2Content.trim().split(/\s+/);
|
|
69
|
+
return quotaCpus(quota, period);
|
|
70
|
+
}
|
|
71
|
+
return quotaCpus(v1Quota?.trim(), v1Period?.trim());
|
|
72
|
+
}
|
|
73
|
+
function quotaCpus(quota, period) {
|
|
74
|
+
if (!quota || !period || !/^\d+$/.test(quota) || !/^\d+$/.test(period))
|
|
75
|
+
return null;
|
|
76
|
+
const [q, p] = [Number(quota), Number(period)];
|
|
77
|
+
return q > 0 && p > 0 ? q / p : null;
|
|
78
|
+
}
|
|
61
79
|
/** Test-only: reset the cached cgroup memory probe. */
|
|
62
80
|
export function _resetCgroupLimitCacheForTests() {
|
|
63
81
|
_cachedCgroupLimitMb = undefined;
|
|
82
|
+
_cachedCgroupCpus = undefined;
|
|
64
83
|
_warnedCgroupReadFailure = false;
|
|
65
84
|
}
|
|
85
|
+
/** The CPUs this process's cgroup may use (docker `--cpus`), or null off Linux or without a quota. */
|
|
86
|
+
export function getCgroupCpuLimit() {
|
|
87
|
+
if (_cachedCgroupCpus !== undefined)
|
|
88
|
+
return _cachedCgroupCpus;
|
|
89
|
+
if (process.platform !== "linux")
|
|
90
|
+
return (_cachedCgroupCpus = null);
|
|
91
|
+
const v2 = readCgroupFile(CGROUP_V2_CPU_MAX_PATH);
|
|
92
|
+
_cachedCgroupCpus =
|
|
93
|
+
v2 !== null
|
|
94
|
+
? parseCgroupCpus(v2, null, null)
|
|
95
|
+
: parseCgroupCpus(null, readCgroupFile(CGROUP_V1_CPU_QUOTA_PATH), readCgroupFile(CGROUP_V1_CPU_PERIOD_PATH));
|
|
96
|
+
return _cachedCgroupCpus;
|
|
97
|
+
}
|
|
66
98
|
/**
|
|
67
99
|
* Actual Linux cgroup memory ceiling in MiB, or null when the process is not
|
|
68
100
|
* cgroup-limited. Unlike getSystemTotalMb this never falls back to host RAM.
|
|
@@ -89,7 +89,7 @@ export interface ShownBox {
|
|
|
89
89
|
}
|
|
90
90
|
/**
|
|
91
91
|
* Seeks to each of `times`; returns each video's largest drawn box in device pixels. Videos that are
|
|
92
|
-
* graded, sampled by a canvas, `object-fit: none | scale-down` or never laid out get none (full size).
|
|
92
|
+
* graded (directly or through a grade group), sampled by a canvas, `object-fit: none | scale-down` or never laid out get none (full size).
|
|
93
93
|
*/
|
|
94
94
|
export declare function measureVideoShownBoxes(page: Page, videoIds: readonly string[], times: readonly number[]): Promise<Map<string, ShownBox>>;
|
|
95
95
|
/**
|
|
@@ -10,7 +10,7 @@ import { promises as fs } from "fs";
|
|
|
10
10
|
import { touchCacheDir } from "./extractionCache.js";
|
|
11
11
|
import { injectVideoFramesBatch, syncVideoFrameVisibility } from "./screenshotService.js";
|
|
12
12
|
import { DEFAULT_CONFIG } from "../config.js";
|
|
13
|
-
import { HF_COLOR_GRADING_ATTR, HF_COLOR_GRADING_CANVAS_ID_PREFIX, RENDER_FRAME_ID_PREFIX, RENDER_FRAME_ID_SUFFIX, renderFrameIdForRenderId, } from "@hyperframes/core";
|
|
13
|
+
import { HF_COLOR_GRADING_ATTR, HF_COLOR_GRADING_CANVAS_ID_PREFIX, HF_COLOR_GROUP_ATTR, HF_COLOR_GROUP_TAG, RENDER_FRAME_ID_PREFIX, RENDER_FRAME_ID_SUFFIX, renderFrameIdForRenderId, } from "@hyperframes/core";
|
|
14
14
|
/**
|
|
15
15
|
* Two-bound LRU keyed by frame path. Either bound triggers eviction of the
|
|
16
16
|
* oldest entry — entry count protects against pathological many-tiny-frames
|
|
@@ -255,12 +255,12 @@ async function setVideoElementsVisibility(page, videoIds, visible) {
|
|
|
255
255
|
}
|
|
256
256
|
/**
|
|
257
257
|
* Seeks to each of `times`; returns each video's largest drawn box in device pixels. Videos that are
|
|
258
|
-
* graded, sampled by a canvas, `object-fit: none | scale-down` or never laid out get none (full size).
|
|
258
|
+
* graded (directly or through a grade group), sampled by a canvas, `object-fit: none | scale-down` or never laid out get none (full size).
|
|
259
259
|
*/
|
|
260
260
|
export async function measureVideoShownBoxes(page, videoIds, times) {
|
|
261
261
|
if (videoIds.length === 0 || times.length === 0)
|
|
262
262
|
return new Map();
|
|
263
|
-
const boxes = await page.evaluate(async (ids, seekTimes, gradingAttr, gradingCanvasPrefix) => {
|
|
263
|
+
const boxes = await page.evaluate(async (ids, seekTimes, gradingAttr, gradingCanvasPrefix, groupAttr, groupTag) => {
|
|
264
264
|
const hf = window.__hf;
|
|
265
265
|
if (typeof hf?.seek !== "function")
|
|
266
266
|
return [];
|
|
@@ -268,8 +268,19 @@ export async function measureVideoShownBoxes(page, videoIds, times) {
|
|
|
268
268
|
.__hfWaitForSeekCompletion;
|
|
269
269
|
const largest = new Map();
|
|
270
270
|
const fullSize = new Set();
|
|
271
|
+
const gradedThroughGroup = (video) => {
|
|
272
|
+
const groupId = video.getAttribute(groupAttr);
|
|
273
|
+
if (!groupId)
|
|
274
|
+
return false;
|
|
275
|
+
// the member's own composition first: ids are unique only per composition file
|
|
276
|
+
const scope = video.closest("[data-composition-id]");
|
|
277
|
+
const group = (scope &&
|
|
278
|
+
Array.from(scope.getElementsByTagName(groupTag)).find((g) => g.id === groupId)) ||
|
|
279
|
+
document.getElementById(groupId);
|
|
280
|
+
return group?.localName === groupTag && group.hasAttribute(gradingAttr);
|
|
281
|
+
};
|
|
271
282
|
const keepsFullSize = (video) => {
|
|
272
|
-
if (video.hasAttribute(gradingAttr))
|
|
283
|
+
if (video.hasAttribute(gradingAttr) || gradedThroughGroup(video))
|
|
273
284
|
return true;
|
|
274
285
|
const fit = window.getComputedStyle(video).objectFit;
|
|
275
286
|
return fit === "none" || fit === "scale-down";
|
|
@@ -297,7 +308,7 @@ export async function measureVideoShownBoxes(page, videoIds, times) {
|
|
|
297
308
|
measure(id);
|
|
298
309
|
}
|
|
299
310
|
return Array.from(largest).filter(([id, box]) => !fullSize.has(id) && box.width > 0 && box.height > 0);
|
|
300
|
-
}, [...videoIds], [...times], HF_COLOR_GRADING_ATTR, HF_COLOR_GRADING_CANVAS_ID_PREFIX);
|
|
311
|
+
}, [...videoIds], [...times], HF_COLOR_GRADING_ATTR, HF_COLOR_GRADING_CANVAS_ID_PREFIX, HF_COLOR_GROUP_ATTR, HF_COLOR_GROUP_TAG);
|
|
301
312
|
return new Map(boxes);
|
|
302
313
|
}
|
|
303
314
|
/**
|
package/dist/types.d.ts
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
*/
|
|
7
7
|
import type { AudioFailureDiagnostic } from "./services/audioMixer.types.js";
|
|
8
8
|
import type { Fps } from "@hyperframes/core";
|
|
9
|
+
import type { CompositionLengthSource } from "@hyperframes/core/runtime/composition-length";
|
|
9
10
|
import type { ChromeMemoryStats } from "./services/chromeMemorySampler.js";
|
|
10
11
|
import type { MotionBlurOptions } from "./services/motionBlur.js";
|
|
11
12
|
export type PageMotionBlurOff = "off" | "hdr-render" | "layered-composite" | "distributed-render";
|
|
@@ -114,6 +115,8 @@ export interface HfSeekOptions {
|
|
|
114
115
|
export interface HfProtocol {
|
|
115
116
|
/** Total duration of the composition in seconds */
|
|
116
117
|
duration: number;
|
|
118
|
+
/** Which input set `duration`. */
|
|
119
|
+
lengthSource?: CompositionLengthSource;
|
|
117
120
|
/** Seek to a specific time. Must produce deterministic visual output. */
|
|
118
121
|
seek(time: number, options?: HfSeekOptions): void;
|
|
119
122
|
/** Optional: media elements the engine should handle */
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { readFileSync } from "node:fs";
|
|
2
|
+
import { createRequire } from "node:module";
|
|
2
3
|
import { dirname, join } from "node:path";
|
|
3
4
|
import { fileURLToPath } from "node:url";
|
|
4
5
|
import { readTagCI } from "./ffprobe.js";
|
|
@@ -91,9 +92,26 @@ export function resolveOwnVersionFrom(startDir) {
|
|
|
91
92
|
break;
|
|
92
93
|
dir = parent;
|
|
93
94
|
}
|
|
95
|
+
// An app that bundles the engine into its own dist has no package of ours above the bundle; the
|
|
96
|
+
// @hyperframes/engine installed beside it is the one it was built from.
|
|
97
|
+
const installed = installedEngineVersion(startDir);
|
|
98
|
+
if (installed)
|
|
99
|
+
return installed;
|
|
94
100
|
reportUnresolved(`no package.json matching ${OWN_PACKAGE_NAME} within ${MAX_WALK_DEPTH} levels`);
|
|
95
101
|
return UNRESOLVED_VERSION;
|
|
96
102
|
}
|
|
103
|
+
function installedEngineVersion(fromDir) {
|
|
104
|
+
try {
|
|
105
|
+
const path = createRequire(join(fromDir, "index.js")).resolve("@hyperframes/engine/package.json");
|
|
106
|
+
const pkg = JSON.parse(readFileSync(path, "utf8"));
|
|
107
|
+
if (pkg.name === "@hyperframes/engine" && typeof pkg.version === "string" && pkg.version)
|
|
108
|
+
return pkg.version;
|
|
109
|
+
}
|
|
110
|
+
catch {
|
|
111
|
+
// Not installed beside us: the caller reports it.
|
|
112
|
+
}
|
|
113
|
+
return undefined;
|
|
114
|
+
}
|
|
97
115
|
/**
|
|
98
116
|
* Make the failure audible. The point of this change is that a resolution
|
|
99
117
|
* failure stops being indistinguishable from data - a silent fallback is what
|
|
@@ -54,7 +54,11 @@ export declare function isBlockedNetworkHost(hostname: string): boolean;
|
|
|
54
54
|
* address range (SSRF guard).
|
|
55
55
|
*/
|
|
56
56
|
export declare function assertPublicHttpsUrl(url: string): void;
|
|
57
|
+
type DownloadFetch = (url: string, init: RequestInit) => Promise<Response>;
|
|
58
|
+
/** Test seam: replaces the transport every download goes through; no argument restores it. */
|
|
59
|
+
export declare function _setDownloadFetchForTests(replacement?: DownloadFetch): void;
|
|
57
60
|
/** Fetch bounded UTF-8 text while applying the downloader's redirect and SSRF policy to every hop. */
|
|
58
61
|
export declare function fetchPublicHttpsText(url: string, options: PublicHttpsTextOptions): Promise<string>;
|
|
59
62
|
export declare function downloadToTemp(url: string, destDir: string, timeoutMs?: number, signal?: AbortSignal, onTransientRetry?: (error: UrlDownloadError) => void, options?: UrlDownloadOptions): Promise<string>;
|
|
60
63
|
export declare function isHttpUrl(path: string): boolean;
|
|
64
|
+
export {};
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { closeSync, createReadStream, createWriteStream, existsSync, fsyncSync, linkSync, mkdtempSync, mkdirSync, lstatSync, openSync, readdirSync, rmdirSync, rmSync, statSync, unlinkSync, } from "fs";
|
|
2
2
|
import { createHash, randomUUID } from "crypto";
|
|
3
|
+
import dns from "node:dns";
|
|
4
|
+
import { fetchGoesThroughEnvProxy } from "@hyperframes/core/env-proxy";
|
|
3
5
|
import { BlockList, isIP } from "node:net";
|
|
4
6
|
import { dirname, extname, join } from "path";
|
|
5
7
|
import { Readable, Transform } from "stream";
|
|
@@ -360,6 +362,49 @@ async function acquireCachePathLock(localPath, timeoutMs, signal) {
|
|
|
360
362
|
}
|
|
361
363
|
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
|
|
362
364
|
const MAX_REDIRECTS = 5;
|
|
365
|
+
const NON_PUBLIC_ADDRESS_CODE = "HF_NON_PUBLIC_ADDRESS";
|
|
366
|
+
/** Resolves a host and refuses to connect when any answer is a non-public address. */
|
|
367
|
+
const publicOnlyLookup = (hostname, options, callback) => {
|
|
368
|
+
dns.lookup(hostname, { ...options, all: true }, (error, answers) => {
|
|
369
|
+
if (error)
|
|
370
|
+
return callback(error, "", 0);
|
|
371
|
+
if (answers.length === 0 ||
|
|
372
|
+
answers.some((a) => isBlockedNetworkHost(a.address.split("%")[0]))) {
|
|
373
|
+
const refused = Object.assign(new Error("host resolves to a non-public address"), {
|
|
374
|
+
code: NON_PUBLIC_ADDRESS_CODE,
|
|
375
|
+
});
|
|
376
|
+
return callback(refused, "", 0);
|
|
377
|
+
}
|
|
378
|
+
if (options.all)
|
|
379
|
+
return callback(null, answers);
|
|
380
|
+
return callback(null, answers[0].address, answers[0].family);
|
|
381
|
+
});
|
|
382
|
+
};
|
|
383
|
+
let undiciTransport;
|
|
384
|
+
const fetchPublic = async (url, init) => {
|
|
385
|
+
// Behind an env proxy the proxy resolves the name; the URL and redirect checks are the guard then.
|
|
386
|
+
if (fetchGoesThroughEnvProxy())
|
|
387
|
+
return fetch(url, init);
|
|
388
|
+
undiciTransport ??= import("undici").then(({ Agent, fetch: undiciFetch }) => {
|
|
389
|
+
const dispatcher = new Agent({ connect: { lookup: publicOnlyLookup } });
|
|
390
|
+
return (target, options) => undiciFetch(target, { ...options, dispatcher });
|
|
391
|
+
});
|
|
392
|
+
return (await undiciTransport)(url, init);
|
|
393
|
+
};
|
|
394
|
+
let downloadFetch = fetchPublic;
|
|
395
|
+
/** Test seam: replaces the transport every download goes through; no argument restores it. */
|
|
396
|
+
export function _setDownloadFetchForTests(replacement) {
|
|
397
|
+
downloadFetch = replacement ?? fetchPublic;
|
|
398
|
+
}
|
|
399
|
+
function refusedNonPublicAddress(error) {
|
|
400
|
+
for (let depth = 0; error && depth < 4; depth += 1) {
|
|
401
|
+
if (typeof error === "object" && "code" in error && error.code === NON_PUBLIC_ADDRESS_CODE) {
|
|
402
|
+
return true;
|
|
403
|
+
}
|
|
404
|
+
error = typeof error === "object" && "cause" in error ? error.cause : undefined;
|
|
405
|
+
}
|
|
406
|
+
return false;
|
|
407
|
+
}
|
|
363
408
|
function assertAllowedDownloadUrl(url, redirect) {
|
|
364
409
|
try {
|
|
365
410
|
assertPublicHttpsUrl(url);
|
|
@@ -398,10 +443,16 @@ async function fetchWithValidatedRedirects(initialUrl, controller) {
|
|
|
398
443
|
// lgtm[js/file-access-to-http] — every redirect hop is fetched manually
|
|
399
444
|
// only after the HTTPS/private-host guard above; automatic redirect
|
|
400
445
|
// following is disabled so an allowed host cannot bounce into IMDS.
|
|
401
|
-
const response = await
|
|
446
|
+
const response = await downloadFetch(currentUrl, {
|
|
402
447
|
signal: controller.signal,
|
|
403
448
|
redirect: "manual",
|
|
404
449
|
headers: { "accept-encoding": "identity" },
|
|
450
|
+
}).catch((error) => {
|
|
451
|
+
if (!refusedNonPublicAddress(error))
|
|
452
|
+
throw error;
|
|
453
|
+
throw new UrlDownloadError("http_rejected", false, redirects > 0
|
|
454
|
+
? "Download redirect target is not permitted"
|
|
455
|
+
: "Download URL is not permitted");
|
|
405
456
|
});
|
|
406
457
|
if (!REDIRECT_STATUSES.has(response.status))
|
|
407
458
|
return { response, finalUrl: currentUrl };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hyperframes/engine",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.216",
|
|
4
4
|
"description": "Seekable web page to video rendering engine (Puppeteer + FFmpeg)",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -54,8 +54,9 @@
|
|
|
54
54
|
"linkedom": "^0.18.12",
|
|
55
55
|
"puppeteer": "~25.11.0",
|
|
56
56
|
"puppeteer-core": "^25.11.0",
|
|
57
|
-
"
|
|
58
|
-
"@hyperframes/
|
|
57
|
+
"undici": "7.25.0",
|
|
58
|
+
"@hyperframes/core": "^0.8.216",
|
|
59
|
+
"@hyperframes/parsers": "^0.8.216"
|
|
59
60
|
},
|
|
60
61
|
"devDependencies": {
|
|
61
62
|
"@types/node": "^25.0.10",
|