@hyperframes/engine 0.7.27 → 0.7.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config.d.ts +21 -9
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +21 -1
- package/dist/config.js.map +1 -1
- package/dist/services/extractionCache.d.ts +73 -18
- package/dist/services/extractionCache.d.ts.map +1 -1
- package/dist/services/extractionCache.js +254 -22
- package/dist/services/extractionCache.js.map +1 -1
- package/dist/services/frameCapture.d.ts +36 -0
- package/dist/services/frameCapture.d.ts.map +1 -1
- package/dist/services/frameCapture.js +56 -18
- package/dist/services/frameCapture.js.map +1 -1
- package/dist/services/videoFrameExtractor.d.ts +23 -5
- package/dist/services/videoFrameExtractor.d.ts.map +1 -1
- package/dist/services/videoFrameExtractor.js +392 -200
- package/dist/services/videoFrameExtractor.js.map +1 -1
- package/package.json +2 -2
package/dist/config.d.ts
CHANGED
|
@@ -153,16 +153,19 @@ export interface EngineConfig {
|
|
|
153
153
|
/**
|
|
154
154
|
* Directory where the content-addressed extraction cache persists frame
|
|
155
155
|
* bundles keyed on (path, mtime, size, mediaStart, duration, fps, format).
|
|
156
|
-
*
|
|
157
|
-
*
|
|
158
|
-
* behaviour.
|
|
156
|
+
* Defaults on under the OS temp directory:
|
|
157
|
+
* `<tmpdir>/hyperframes-extract-cache-<uid>`.
|
|
159
158
|
*
|
|
160
|
-
*
|
|
161
|
-
*
|
|
162
|
-
*
|
|
163
|
-
*
|
|
164
|
-
*
|
|
165
|
-
*
|
|
159
|
+
* New entries publish atomically: frames are extracted into a unique
|
|
160
|
+
* partial directory, the `.hf-complete` sentinel is written there, and the
|
|
161
|
+
* partial directory is renamed into the final key directory. Concurrent
|
|
162
|
+
* renders against the same cache are safe; at worst, two renders duplicate
|
|
163
|
+
* ffmpeg work and one rehydrates from the winner.
|
|
164
|
+
*
|
|
165
|
+
* Set `HYPERFRAMES_EXTRACT_CACHE_DIR` to a path to override the default, or
|
|
166
|
+
* to `off`, `none`, `false`, or `0` to disable caching for the process.
|
|
167
|
+
* When disabled, extraction runs into the render's workDir and cleanup
|
|
168
|
+
* removes it when the render ends, preserving the pre-cache behaviour.
|
|
166
169
|
*
|
|
167
170
|
* **Network filesystems.** `mtime` resolution on NFS/SMB mounts can be
|
|
168
171
|
* coarser than expected (seconds rather than nanoseconds), which may
|
|
@@ -172,6 +175,15 @@ export interface EngineConfig {
|
|
|
172
175
|
* Env fallback: `HYPERFRAMES_EXTRACT_CACHE_DIR`.
|
|
173
176
|
*/
|
|
174
177
|
extractCacheDir?: string;
|
|
178
|
+
/**
|
|
179
|
+
* Soft disk budget for `extractCacheDir`, in bytes. The renderer runs a
|
|
180
|
+
* best-effort LRU sweep after extraction and evicts oldest sentineled
|
|
181
|
+
* entries until the cache is under this cap, while protecting young entries
|
|
182
|
+
* that may belong to live renders.
|
|
183
|
+
*
|
|
184
|
+
* Env fallback: `HYPERFRAMES_EXTRACT_CACHE_MAX_MB` (megabytes).
|
|
185
|
+
*/
|
|
186
|
+
extractCacheMaxBytes: number;
|
|
175
187
|
debug: boolean;
|
|
176
188
|
}
|
|
177
189
|
/** Default configuration — sensible for Hyperframes compositions. */
|
package/dist/config.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAWH;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAE3B,GAAG,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;IAClB,OAAO,EAAE,OAAO,GAAG,UAAU,GAAG,MAAM,CAAC;IACvC,MAAM,EAAE,MAAM,GAAG,KAAK,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IAGpB,yDAAyD;IACzD,WAAW,EAAE,MAAM,GAAG,MAAM,CAAC;IAC7B,sCAAsC;IACtC,cAAc,EAAE,MAAM,CAAC;IACvB,uDAAuD;IACvD,iBAAiB,EAAE,MAAM,CAAC;IAC1B,2DAA2D;IAC3D,oBAAoB,EAAE,MAAM,CAAC;IAG7B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,OAAO,CAAC;IACpB;;;;;;;;OAQG;IACH,cAAc,EAAE,UAAU,GAAG,UAAU,GAAG,MAAM,CAAC;IACjD,iBAAiB,EAAE,OAAO,CAAC;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,6DAA6D;IAC7D,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,qEAAqE;IACrE,eAAe,EAAE,OAAO,CAAC;IACzB;;;;OAIG;IACH,gBAAgB,EAAE,OAAO,CAAC;IAC1B;;;;;;;;OAQG;IACH,aAAa,EAAE,OAAO,CAAC;IACvB;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,yBAAyB,EAAE,OAAO,CAAC;IAGnC;;;OAGG;IACH,UAAU,EAAE,MAAM,CAAC;IACnB,mBAAmB,EAAE,OAAO,CAAC;IAC7B,eAAe,EAAE,MAAM,CAAC;IACxB,qBAAqB,EAAE,OAAO,CAAC;IAC/B;;;;OAIG;IACH,iCAAiC,EAAE,MAAM,CAAC;IAG1C,+DAA+D;IAC/D,mBAAmB,EAAE,MAAM,CAAC;IAC5B,wEAAwE;IACxE,oBAAoB,EAAE,MAAM,CAAC;IAC7B;;;;;OAKG;IACH,sBAAsB,EAAE,MAAM,CAAC;IAG/B,kEAAkE;IAClE,GAAG,EAAE;QAAE,QAAQ,EAAE,KAAK,GAAG,IAAI,CAAA;KAAE,GAAG,KAAK,CAAC;IACxC,yEAAyE;IACzE,aAAa,EAAE,OAAO,CAAC;IAGvB,SAAS,EAAE,MAAM,CAAC;IAClB;;;;;;OAMG;IACH,sBAAsB,EAAE,MAAM,CAAC;IAC/B;;;;;OAKG;IACH,6BAA6B,EAAE,MAAM,CAAC;IAGtC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,MAAM,CAAC;IAC3B;;;;;;;;OAQG;IACH,qBAAqB,EAAE,MAAM,CAAC;IAG9B,kDAAkD;IAClD,aAAa,EAAE,OAAO,CAAC;IACvB,mDAAmD;IACnD,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAG7B;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;;;OAOG;IACH,oBAAoB,EAAE,MAAM,CAAC;IAG7B,KAAK,EAAE,OAAO,CAAC;CAChB;AAED,qEAAqE;AACrE,eAAO,MAAM,cAAc,EAAE,YAiD5B,CAAC;AAqBF;;;;;;;;;;GAUG;AACH,wBAAgB,kCAAkC,CAChD,aAAa,EAAE,MAAM,EACrB,IAAI,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACtC,MAAM,CAYR;AAgBD;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,YAAY,CAmK7E"}
|
package/dist/config.js
CHANGED
|
@@ -5,6 +5,8 @@
|
|
|
5
5
|
* env var sprawl with a structured interface. Env vars still work as
|
|
6
6
|
* fallbacks for backward compatibility during migration.
|
|
7
7
|
*/
|
|
8
|
+
import { tmpdir } from "node:os";
|
|
9
|
+
import { join } from "node:path";
|
|
8
10
|
import { getSystemTotalMb, isLowMemorySystem, LOW_MEMORY_TOTAL_MB_THRESHOLD, } from "./services/systemMemory.js";
|
|
9
11
|
import { DEFAULT_VP9_CPU_USED, normalizeVp9CpuUsed } from "./services/vp9Options.js";
|
|
10
12
|
/** Default configuration — sensible for Hyperframes compositions. */
|
|
@@ -45,6 +47,7 @@ export const DEFAULT_CONFIG = {
|
|
|
45
47
|
renderReadyTimeout: 15_000,
|
|
46
48
|
pageNavigationTimeout: 60_000,
|
|
47
49
|
verifyRuntime: true,
|
|
50
|
+
extractCacheMaxBytes: 2 * 1024 ** 3,
|
|
48
51
|
debug: false,
|
|
49
52
|
};
|
|
50
53
|
/**
|
|
@@ -151,6 +154,21 @@ export function resolveConfig(overrides) {
|
|
|
151
154
|
const raw = env("HF_STATIC_DEDUP")?.trim().toLowerCase();
|
|
152
155
|
return !(raw === "false" || raw === "off" || raw === "0");
|
|
153
156
|
};
|
|
157
|
+
const resolveExtractCacheDir = () => {
|
|
158
|
+
const raw = env("HYPERFRAMES_EXTRACT_CACHE_DIR");
|
|
159
|
+
if (raw === undefined) {
|
|
160
|
+
return join(tmpdir(), `hyperframes-extract-cache-${process.getuid?.() ?? "u"}`);
|
|
161
|
+
}
|
|
162
|
+
const trimmed = raw.trim();
|
|
163
|
+
const normalized = trimmed.toLowerCase();
|
|
164
|
+
if (normalized === "off" ||
|
|
165
|
+
normalized === "none" ||
|
|
166
|
+
normalized === "false" ||
|
|
167
|
+
normalized === "0") {
|
|
168
|
+
return undefined;
|
|
169
|
+
}
|
|
170
|
+
return raw;
|
|
171
|
+
};
|
|
154
172
|
// Env-var layer (backward compat)
|
|
155
173
|
const fromEnv = {
|
|
156
174
|
concurrency: env("PRODUCER_MAX_WORKERS") ? Number(env("PRODUCER_MAX_WORKERS")) : undefined,
|
|
@@ -193,7 +211,9 @@ export function resolveConfig(overrides) {
|
|
|
193
211
|
pageNavigationTimeout: envNum("PRODUCER_PAGE_NAVIGATION_TIMEOUT_MS", DEFAULT_CONFIG.pageNavigationTimeout),
|
|
194
212
|
verifyRuntime: env("PRODUCER_VERIFY_HYPERFRAME_RUNTIME") !== "false",
|
|
195
213
|
runtimeManifestPath: env("PRODUCER_HYPERFRAME_MANIFEST_PATH"),
|
|
196
|
-
extractCacheDir:
|
|
214
|
+
extractCacheDir: resolveExtractCacheDir(),
|
|
215
|
+
extractCacheMaxBytes: envNum("HYPERFRAMES_EXTRACT_CACHE_MAX_MB", DEFAULT_CONFIG.extractCacheMaxBytes / 1024 ** 2) *
|
|
216
|
+
1024 ** 2,
|
|
197
217
|
};
|
|
198
218
|
// Remove undefined values so they don't override defaults
|
|
199
219
|
const cleanEnv = Object.fromEntries(Object.entries(fromEnv).filter(([, v]) => v !== undefined));
|
package/dist/config.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EACL,gBAAgB,EAChB,iBAAiB,EACjB,6BAA6B,GAC9B,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EACL,gBAAgB,EAChB,iBAAiB,EACjB,6BAA6B,GAC9B,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AA2MrF,qEAAqE;AACrE,MAAM,CAAC,MAAM,cAAc,GAAiB;IAC1C,GAAG,EAAE,EAAE;IACP,OAAO,EAAE,UAAU;IACnB,MAAM,EAAE,MAAM;IACd,WAAW,EAAE,EAAE;IAEf,WAAW,EAAE,MAAM;IACnB,cAAc,EAAE,GAAG;IACnB,iBAAiB,EAAE,GAAG;IACtB,oBAAoB,EAAE,IAAI;IAE1B,UAAU,EAAE,KAAK;IACjB,cAAc,EAAE,UAAU;IAC1B,iBAAiB,EAAE,IAAI;IACvB,cAAc,EAAE,OAAO;IACvB,eAAe,EAAE,OAAO;IACxB,eAAe,EAAE,KAAK;IACtB,gBAAgB,EAAE,IAAI;IACtB,sEAAsE;IACtE,uEAAuE;IACvE,aAAa,EAAE,KAAK;IACpB,yBAAyB,EAAE,IAAI;IAE/B,UAAU,EAAE,oBAAoB;IAChC,mBAAmB,EAAE,KAAK;IAC1B,eAAe,EAAE,GAAG;IACpB,qBAAqB,EAAE,IAAI;IAC3B,iCAAiC,EAAE,GAAG;IAEtC,mBAAmB,EAAE,OAAO;IAC5B,oBAAoB,EAAE,OAAO;IAC7B,sBAAsB,EAAE,OAAO;IAE/B,GAAG,EAAE,KAAK;IACV,aAAa,EAAE,IAAI;IAEnB,SAAS,EAAE,CAAC;IACZ,sBAAsB,EAAE,GAAG;IAC3B,6BAA6B,EAAE,IAAI;IAEnC,kBAAkB,EAAE,MAAM;IAC1B,kBAAkB,EAAE,MAAM;IAC1B,qBAAqB,EAAE,MAAM;IAE7B,aAAa,EAAE,IAAI;IAEnB,oBAAoB,EAAE,CAAC,GAAG,IAAI,IAAI,CAAC;IAEnC,KAAK,EAAE,KAAK;CACb,CAAC;AAEF;;;;;;;;;GASG;AACH,MAAM,iCAAiC,GAAG,IAAI,GAAG,IAAI,CAAC;AAEtD;;;;GAIG;AACH,MAAM,8BAA8B,GAAG,SAAS,CAAC;AAEjD;;;;;;;;;;GAUG;AACH,MAAM,UAAU,kCAAkC,CAChD,aAAqB,EACrB,IAAuC;IAEvC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IAC/B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QACrF,OAAO,aAAa,CAAC;IACvB,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,KAAK,GAAG,MAAM,CAAC,GAAG,iCAAiC,CAAC;IACpE,IAAI,MAAM,IAAI,CAAC;QAAE,OAAO,aAAa,CAAC;IACtC,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,CAAC;IACjD,6EAA6E;IAC7E,sEAAsE;IACtE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,8BAA8B,CAAC,CAAC;IACxE,OAAO,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,SAAS,wBAAwB;IAC/B,MAAM,KAAK,GAAG,gBAAgB,EAAE,CAAC;IACjC,IAAI,KAAK,GAAG,IAAI;QAAE,OAAO,EAAE,CAAC;IAC5B,IAAI,KAAK,IAAI,6BAA6B;QAAE,OAAO,EAAE,CAAC;IACtD,OAAO,cAAc,CAAC,sBAAsB,CAAC;AAC/C,CAAC;AAED,SAAS,0BAA0B;IACjC,MAAM,KAAK,GAAG,gBAAgB,EAAE,CAAC;IACjC,IAAI,KAAK,GAAG,IAAI;QAAE,OAAO,GAAG,CAAC;IAC7B,IAAI,KAAK,IAAI,6BAA6B;QAAE,OAAO,GAAG,CAAC;IACvD,OAAO,cAAc,CAAC,6BAA6B,CAAC;AACtD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,SAAiC;IAC7D,MAAM,GAAG,GAAG,CAAC,GAAW,EAAsB,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAClE,MAAM,MAAM,GAAG,CAAC,GAAW,EAAE,QAAgB,EAAU,EAAE;QACvD,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QACrB,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,EAAE;YAAE,OAAO,QAAQ,CAAC;QACrD,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QACtB,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC3C,CAAC,CAAC;IACF,MAAM,OAAO,GAAG,CAAC,GAAW,EAAE,QAAiB,EAAW,EAAE;QAC1D,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QACrB,IAAI,GAAG,KAAK,SAAS;YAAE,OAAO,QAAQ,CAAC;QACvC,OAAO,GAAG,KAAK,MAAM,CAAC;IACxB,CAAC,CAAC;IACF,MAAM,aAAa,GAAG,GAAW,EAAE;QACjC,MAAM,GAAG,GAAG,GAAG,CAAC,uBAAuB,CAAC,CAAC;QACzC,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,EAAE;YAAE,OAAO,cAAc,CAAC,UAAU,CAAC;QACtE,OAAO,mBAAmB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1C,CAAC,CAAC;IACF,MAAM,iBAAiB,GAAG,GAAmC,EAAE;QAC7D,MAAM,GAAG,GAAG,GAAG,CAAC,2BAA2B,CAAC,CAAC;QAC7C,IAAI,GAAG,KAAK,UAAU,IAAI,GAAG,KAAK,UAAU,IAAI,GAAG,KAAK,MAAM;YAAE,OAAO,GAAG,CAAC;QAC3E,OAAO,cAAc,CAAC,cAAc,CAAC;IACvC,CAAC,CAAC;IACF,4EAA4E;IAC5E,MAAM,oBAAoB,GAAG,GAAY,EAAE;QACzC,MAAM,GAAG,GAAG,GAAG,CAAC,0BAA0B,CAAC,EAAE,WAAW,EAAE,CAAC;QAC3D,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,GAAG;YAAE,OAAO,IAAI,CAAC;QAC/D,IAAI,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,KAAK,GAAG;YAAE,OAAO,KAAK,CAAC;QAClE,OAAO,iBAAiB,EAAE,CAAC;IAC7B,CAAC,CAAC;IACF,kEAAkE;IAClE,MAAM,uBAAuB,GAAG,GAAY,EAAE;QAC5C,MAAM,GAAG,GAAG,GAAG,CAAC,iBAAiB,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACzD,OAAO,CAAC,CAAC,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,KAAK,GAAG,CAAC,CAAC;IAC5D,CAAC,CAAC;IACF,MAAM,sBAAsB,GAAG,GAAuB,EAAE;QACtD,MAAM,GAAG,GAAG,GAAG,CAAC,+BAA+B,CAAC,CAAC;QACjD,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC,MAAM,EAAE,EAAE,6BAA6B,OAAO,CAAC,MAAM,EAAE,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QAClF,CAAC;QACD,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;QACzC,IACE,UAAU,KAAK,KAAK;YACpB,UAAU,KAAK,MAAM;YACrB,UAAU,KAAK,OAAO;YACtB,UAAU,KAAK,GAAG,EAClB,CAAC;YACD,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC,CAAC;IAEF,kCAAkC;IAClC,MAAM,OAAO,GAA0B;QACrC,WAAW,EAAE,GAAG,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;QAC1F,cAAc,EAAE,MAAM,CAAC,2BAA2B,EAAE,cAAc,CAAC,cAAc,CAAC;QAClF,iBAAiB,EAAE,MAAM,CAAC,8BAA8B,EAAE,cAAc,CAAC,iBAAiB,CAAC;QAC3F,oBAAoB,EAAE,MAAM,CAC1B,iCAAiC,EACjC,cAAc,CAAC,oBAAoB,CACpC;QAED,UAAU,EAAE,GAAG,CAAC,8BAA8B,CAAC;QAC/C,UAAU,EAAE,OAAO,CAAC,sBAAsB,EAAE,cAAc,CAAC,UAAU,CAAC;QACtE,cAAc,EAAE,iBAAiB,EAAE;QACnC,iBAAiB,EAAE,OAAO,CAAC,8BAA8B,EAAE,cAAc,CAAC,iBAAiB,CAAC;QAC5F,cAAc,EAAE,MAAM,CAAC,sCAAsC,EAAE,cAAc,CAAC,cAAc,CAAC;QAC7F,eAAe,EAAE,MAAM,CACrB,wCAAwC,EACxC,cAAc,CAAC,eAAe,CAC/B;QACD,qBAAqB,EAAE,GAAG,CAAC,kCAAkC,CAAC;YAC5D,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;YACjD,CAAC,CAAC,SAAS;QAEb,eAAe,EAAE,OAAO,CAAC,2BAA2B,EAAE,cAAc,CAAC,eAAe,CAAC;QACrF,gBAAgB,EAAE,uBAAuB,EAAE;QAC3C,aAAa,EAAE,oBAAoB,EAAE;QACrC,yBAAyB,EAAE,OAAO,CAChC,0BAA0B,EAC1B,cAAc,CAAC,yBAAyB,CACzC;QAED,UAAU,EAAE,aAAa,EAAE;QAC3B,mBAAmB,EAAE,OAAO,CAC1B,gCAAgC,EAChC,cAAc,CAAC,mBAAmB,CACnC;QACD,eAAe,EAAE,IAAI,CAAC,GAAG,CACvB,GAAG,EACH,MAAM,CAAC,4BAA4B,EAAE,cAAc,CAAC,eAAe,CAAC,CACrE;QACD,qBAAqB,EAAE,OAAO,CAC5B,kCAAkC,EAClC,cAAc,CAAC,qBAAqB,CACrC;QACD,iCAAiC,EAAE,IAAI,CAAC,GAAG,CACzC,CAAC,EACD,MAAM,CACJ,gDAAgD,EAChD,cAAc,CAAC,iCAAiC,CACjD,CACF;QAED,mBAAmB,EAAE,MAAM,CAAC,0BAA0B,EAAE,cAAc,CAAC,mBAAmB,CAAC;QAC3F,oBAAoB,EAAE,MAAM,CAAC,2BAA2B,EAAE,cAAc,CAAC,oBAAoB,CAAC;QAC9F,sBAAsB,EAAE,MAAM,CAC5B,6BAA6B,EAC7B,cAAc,CAAC,sBAAsB,CACtC;QAED,GAAG,EAAE,CAAC,GAAG,EAAE;YACT,MAAM,GAAG,GAAG,GAAG,CAAC,uBAAuB,CAAC,CAAC;YACzC,IAAI,GAAG,KAAK,KAAK,IAAI,GAAG,KAAK,IAAI;gBAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC;YAC5D,OAAO,KAAK,CAAC;QACf,CAAC,CAAC,EAAE;QACJ,aAAa,EAAE,OAAO,CAAC,0BAA0B,EAAE,cAAc,CAAC,aAAa,CAAC;QAEhF,SAAS,EAAE,MAAM,CAAC,qBAAqB,EAAE,cAAc,CAAC,SAAS,CAAC;QAClE,sBAAsB,EAAE,IAAI,CAAC,GAAG,CAC9B,EAAE,EACF,MAAM,CAAC,qCAAqC,EAAE,wBAAwB,EAAE,CAAC,CAC1E;QACD,6BAA6B,EAAE,IAAI,CAAC,GAAG,CACrC,EAAE,EACF,MAAM,CAAC,wCAAwC,EAAE,0BAA0B,EAAE,CAAC,CAC/E;QAED,kBAAkB,EAAE,MAAM,CACxB,kCAAkC,EAClC,cAAc,CAAC,kBAAkB,CAClC;QACD,kBAAkB,EAAE,MAAM,CACxB,kCAAkC,EAClC,cAAc,CAAC,kBAAkB,CAClC;QACD,qBAAqB,EAAE,MAAM,CAC3B,qCAAqC,EACrC,cAAc,CAAC,qBAAqB,CACrC;QAED,aAAa,EAAE,GAAG,CAAC,oCAAoC,CAAC,KAAK,OAAO;QACpE,mBAAmB,EAAE,GAAG,CAAC,mCAAmC,CAAC;QAE7D,eAAe,EAAE,sBAAsB,EAAE;QACzC,oBAAoB,EAClB,MAAM,CAAC,kCAAkC,EAAE,cAAc,CAAC,oBAAoB,GAAG,IAAI,IAAI,CAAC,CAAC;YAC3F,IAAI,IAAI,CAAC;KACZ,CAAC;IAEF,0DAA0D;IAC1D,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC;IAEhG,MAAM,MAAM,GAAG;QACb,GAAG,cAAc;QACjB,GAAG,QAAQ;QACX,GAAG,SAAS;KACb,CAAC;IACF,OAAO;QACL,GAAG,MAAM;QACT,UAAU,EAAE,mBAAmB,CAAC,MAAM,CAAC,UAAU,CAAC;KACnD,CAAC;AACJ,CAAC"}
|
|
@@ -5,7 +5,8 @@
|
|
|
5
5
|
* after capture. Repeat renders of the same composition (preview → final,
|
|
6
6
|
* studio iteration) re-extract identical frames from the same source file,
|
|
7
7
|
* burning ffmpeg time that adds no value. This module keys extracted frame
|
|
8
|
-
* bundles on the (path, mtime, size, mediaStart, duration, fps, format
|
|
8
|
+
* bundles on the (path, mtime, size, mediaStart, duration, fps, format,
|
|
9
|
+
* optional transform)
|
|
9
10
|
* tuple so re-renders resolve to a pre-extracted directory instead of
|
|
10
11
|
* re-invoking ffmpeg.
|
|
11
12
|
*
|
|
@@ -15,10 +16,14 @@
|
|
|
15
16
|
* - Cache entries live under `<rootDir>/<SCHEMA_PREFIX><key[0..16]>/` so
|
|
16
17
|
* `ls` output and tracing logs stay short. Truncation to 16 hex chars
|
|
17
18
|
* leaves 64 bits of entropy — collision risk at cache scale is negligible.
|
|
18
|
-
* -
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
19
|
+
* - Frames are extracted into a unique `<entry>.partial-<pid>-<uuid>/` dir.
|
|
20
|
+
* Once all frames are written, the partial dir receives the `.hf-complete`
|
|
21
|
+
* sentinel and is atomically renamed to the final key dir. Concurrent
|
|
22
|
+
* same-key writers may duplicate ffmpeg work, but readers only ever serve
|
|
23
|
+
* complete entries.
|
|
24
|
+
* - The sentinel mtime is touched on hits and used as the cache's LRU clock.
|
|
25
|
+
* `gcExtractionCache` evicts by that mtime and also clears old partial dirs
|
|
26
|
+
* left behind by crashed writers.
|
|
22
27
|
*
|
|
23
28
|
* ### Versioning
|
|
24
29
|
*
|
|
@@ -31,8 +36,16 @@ import type { VideoMetadata } from "../utils/ffprobe.js";
|
|
|
31
36
|
export declare const FRAME_FILENAME_PREFIX = "frame_";
|
|
32
37
|
/** Sentinel filename written after a cache entry is fully populated. */
|
|
33
38
|
export declare const COMPLETE_SENTINEL = ".hf-complete";
|
|
34
|
-
/**
|
|
35
|
-
export declare const
|
|
39
|
+
/** Marker file stamped after each GC sweep; drives the staleness fallback. */
|
|
40
|
+
export declare const GC_MARKER = ".hf-last-gc";
|
|
41
|
+
/**
|
|
42
|
+
* Current schema version. Bump when the cache-contents invariant changes.
|
|
43
|
+
* v2 -> v3: one-pass VFR extraction (-fps_mode cfr) replaces the two-pass
|
|
44
|
+
* VFR-to-CFR re-encode, changing frame contents for VFR sources under
|
|
45
|
+
* identical key tuples. Without the bump, warm v2 entries (two-pass frames)
|
|
46
|
+
* would keep being served across the deploy boundary.
|
|
47
|
+
*/
|
|
48
|
+
export declare const SCHEMA_PREFIX = "hfcache-v3-";
|
|
36
49
|
export type CacheFrameFormat = "jpg" | "png";
|
|
37
50
|
export interface CacheKeyInput {
|
|
38
51
|
/** Absolute path to the source video file. Part of the key so moved files
|
|
@@ -52,6 +65,8 @@ export interface CacheKeyInput {
|
|
|
52
65
|
fps: number;
|
|
53
66
|
/** Output image format. */
|
|
54
67
|
format: CacheFrameFormat;
|
|
68
|
+
/** Optional source transform applied during extraction. */
|
|
69
|
+
transform?: string;
|
|
55
70
|
}
|
|
56
71
|
export interface CacheEntry {
|
|
57
72
|
/** Absolute path to the cache entry directory. */
|
|
@@ -61,11 +76,15 @@ export interface CacheEntry {
|
|
|
61
76
|
}
|
|
62
77
|
export interface CacheLookup {
|
|
63
78
|
/** Cache entry information — always returned even on a miss so the caller
|
|
64
|
-
* can
|
|
79
|
+
* can derive a partial dir and publish it after extraction. */
|
|
65
80
|
entry: CacheEntry;
|
|
66
81
|
/** True when the entry exists AND carries the completion sentinel. */
|
|
67
82
|
hit: boolean;
|
|
68
83
|
}
|
|
84
|
+
export interface CachePublishResult {
|
|
85
|
+
dir: string;
|
|
86
|
+
published: boolean;
|
|
87
|
+
}
|
|
69
88
|
/**
|
|
70
89
|
* Read `(mtimeMs, size)` for a path. Returns `null` if the file is missing —
|
|
71
90
|
* callers should skip the cache path for that entry so the extractor surfaces
|
|
@@ -88,9 +107,9 @@ export declare function computeCacheKey(input: CacheKeyInput): string;
|
|
|
88
107
|
export declare function cacheEntryDirName(keyHash: string): string;
|
|
89
108
|
/**
|
|
90
109
|
* Look up a cache entry by key input. Returns the resolved entry path plus a
|
|
91
|
-
* `hit` flag. On miss, callers should extract frames into
|
|
92
|
-
* (
|
|
93
|
-
* once
|
|
110
|
+
* `hit` flag. On miss, callers should extract frames into a
|
|
111
|
+
* `partialCacheEntryDir(entry)` directory and publish it with
|
|
112
|
+
* `publishCacheEntry` once extraction succeeds.
|
|
94
113
|
*/
|
|
95
114
|
export declare function lookupCacheEntry(rootDir: string, input: CacheKeyInput): CacheLookup;
|
|
96
115
|
/**
|
|
@@ -98,18 +117,54 @@ export declare function lookupCacheEntry(rootDir: string, input: CacheKeyInput):
|
|
|
98
117
|
* Idempotent: `mkdirSync({recursive:true})` is a no-op when the dir exists.
|
|
99
118
|
*/
|
|
100
119
|
export declare function ensureCacheEntryDir(entry: CacheEntry): void;
|
|
120
|
+
/**
|
|
121
|
+
* Unique render-owned directory used to populate a cache entry before the
|
|
122
|
+
* atomic publish rename.
|
|
123
|
+
*/
|
|
124
|
+
export declare function partialCacheEntryDir(entry: CacheEntry): string;
|
|
125
|
+
export declare function publishCacheEntry(entry: CacheEntry, partialDir: string): CachePublishResult;
|
|
126
|
+
/**
|
|
127
|
+
* Update the LRU clock for a complete cache entry. Misses and filesystem
|
|
128
|
+
* races are harmless: the caller can still use the entry it already found.
|
|
129
|
+
*/
|
|
130
|
+
export declare function touchCacheEntry(entry: CacheEntry): void;
|
|
101
131
|
/**
|
|
102
132
|
* Write the completion sentinel so subsequent lookups treat this entry as a
|
|
103
|
-
* hit. Must be called only after every frame has been written.
|
|
133
|
+
* hit. Must be called only after every frame has been written. The extractor
|
|
134
|
+
* now publishes new entries via `publishCacheEntry`; this helper remains
|
|
135
|
+
* exported for tests and legacy callers that materialize entries directly.
|
|
104
136
|
*
|
|
105
|
-
* Concurrency:
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
* anyone running concurrent renders against a shared cache root should front
|
|
110
|
-
* it with an external lock.
|
|
137
|
+
* Concurrency: direct mark is non-atomic and should not be used for shared
|
|
138
|
+
* writer paths. `publishCacheEntry` writes the sentinel inside a partial dir
|
|
139
|
+
* and atomically renames it into place, so concurrent writers duplicate work
|
|
140
|
+
* but never serve torn frames.
|
|
111
141
|
*/
|
|
112
142
|
export declare function markCacheEntryComplete(entry: CacheEntry): void;
|
|
143
|
+
export interface GcStats {
|
|
144
|
+
/** Complete entries evicted by the LRU size sweep. */
|
|
145
|
+
evictedEntries: number;
|
|
146
|
+
/** Bytes reclaimed by evicted entries. */
|
|
147
|
+
evictedBytes: number;
|
|
148
|
+
/** Aged `.partial-*` dirs (crashed writers) removed. */
|
|
149
|
+
agedPartialsRemoved: number;
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Opportunistic size-capped LRU cleanup for extracted video frames.
|
|
153
|
+
*
|
|
154
|
+
* Scans only direct cache-looking children and never throws. The age guard is
|
|
155
|
+
* a liveness heuristic, not a lock. Returns counts so the caller can surface
|
|
156
|
+
* eviction pressure in render observability.
|
|
157
|
+
*/
|
|
158
|
+
/**
|
|
159
|
+
* Whether the staleness fallback should force a sweep: true when no sweep
|
|
160
|
+
* marker exists or the last sweep is older than `maxAgeMs`. Lets 100%-warm
|
|
161
|
+
* workloads (which skip the per-miss sweep) still reclaim space eventually.
|
|
162
|
+
*/
|
|
163
|
+
export declare function gcSweepDue(rootDir: string, maxAgeMs: number): boolean;
|
|
164
|
+
export declare function gcExtractionCache(rootDir: string, opts: {
|
|
165
|
+
maxBytes: number;
|
|
166
|
+
minAgeMs: number;
|
|
167
|
+
}): GcStats;
|
|
113
168
|
/**
|
|
114
169
|
* Rebuild the in-memory frame index for a cached entry. Called on cache hits
|
|
115
170
|
* so the extractor's caller receives the same `ExtractedFrames` shape it
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extractionCache.d.ts","sourceRoot":"","sources":["../../src/services/extractionCache.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"extractionCache.d.ts","sourceRoot":"","sources":["../../src/services/extractionCache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAeH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEzD,uEAAuE;AACvE,eAAO,MAAM,qBAAqB,WAAW,CAAC;AAE9C,wEAAwE;AACxE,eAAO,MAAM,iBAAiB,iBAAiB,CAAC;AAEhD,8EAA8E;AAC9E,eAAO,MAAM,SAAS,gBAAgB,CAAC;AAEvC;;;;;;GAMG;AACH,eAAO,MAAM,aAAa,gBAAgB,CAAC;AAK3C,MAAM,MAAM,gBAAgB,GAAG,KAAK,GAAG,KAAK,CAAC;AAE7C,MAAM,WAAW,aAAa;IAC5B;+DAC2D;IAC3D,SAAS,EAAE,MAAM,CAAC;IAClB,kFAAkF;IAClF,OAAO,EAAE,MAAM,CAAC;IAChB,wEAAwE;IACxE,IAAI,EAAE,MAAM,CAAC;IACb,6EAA6E;IAC7E,UAAU,EAAE,MAAM,CAAC;IACnB;;yCAEqC;IACrC,QAAQ,EAAE,MAAM,CAAC;IACjB,uCAAuC;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,2BAA2B;IAC3B,MAAM,EAAE,gBAAgB,CAAC;IACzB,2DAA2D;IAC3D,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,UAAU;IACzB,kDAAkD;IAClD,GAAG,EAAE,MAAM,CAAC;IACZ,qEAAqE;IACrE,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,WAAW;IAC1B;oEACgE;IAChE,KAAK,EAAE,UAAU,CAAC;IAClB,sEAAsE;IACtE,GAAG,EAAE,OAAO,CAAC;CACd;AAED,MAAM,WAAW,kBAAkB;IACjC,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,OAAO,CAAC;CACpB;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAOvF;AA0BD;;GAEG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,aAAa,GAAG,MAAM,CAE5D;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAEzD;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,GAAG,WAAW,CAKnF;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI,CAE3D;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAE9D;AA0BD,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,GAAG,kBAAkB,CAgC3F;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI,CAOvD;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI,CAE9D;AAiGD,MAAM,WAAW,OAAO;IACtB,sDAAsD;IACtD,cAAc,EAAE,MAAM,CAAC;IACvB,0CAA0C;IAC1C,YAAY,EAAE,MAAM,CAAC;IACrB,wDAAwD;IACxD,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAED;;;;;;GAMG;AACH;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAMrE;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,MAAM,EACf,IAAI,EAAE;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,GAC3C,OAAO,CAuCT;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,gBAAgB,CAAC;IACzB,QAAQ,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,aAAa,CAAC;IACxB,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,wBAAgB,mBAAmB,CACjC,KAAK,EAAE,UAAU,EACjB,OAAO,EAAE,gBAAgB,GACxB,gBAAgB,CAoBlB"}
|
|
@@ -5,7 +5,8 @@
|
|
|
5
5
|
* after capture. Repeat renders of the same composition (preview → final,
|
|
6
6
|
* studio iteration) re-extract identical frames from the same source file,
|
|
7
7
|
* burning ffmpeg time that adds no value. This module keys extracted frame
|
|
8
|
-
* bundles on the (path, mtime, size, mediaStart, duration, fps, format
|
|
8
|
+
* bundles on the (path, mtime, size, mediaStart, duration, fps, format,
|
|
9
|
+
* optional transform)
|
|
9
10
|
* tuple so re-renders resolve to a pre-extracted directory instead of
|
|
10
11
|
* re-invoking ffmpeg.
|
|
11
12
|
*
|
|
@@ -15,10 +16,14 @@
|
|
|
15
16
|
* - Cache entries live under `<rootDir>/<SCHEMA_PREFIX><key[0..16]>/` so
|
|
16
17
|
* `ls` output and tracing logs stay short. Truncation to 16 hex chars
|
|
17
18
|
* leaves 64 bits of entropy — collision risk at cache scale is negligible.
|
|
18
|
-
* -
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
19
|
+
* - Frames are extracted into a unique `<entry>.partial-<pid>-<uuid>/` dir.
|
|
20
|
+
* Once all frames are written, the partial dir receives the `.hf-complete`
|
|
21
|
+
* sentinel and is atomically renamed to the final key dir. Concurrent
|
|
22
|
+
* same-key writers may duplicate ffmpeg work, but readers only ever serve
|
|
23
|
+
* complete entries.
|
|
24
|
+
* - The sentinel mtime is touched on hits and used as the cache's LRU clock.
|
|
25
|
+
* `gcExtractionCache` evicts by that mtime and also clears old partial dirs
|
|
26
|
+
* left behind by crashed writers.
|
|
22
27
|
*
|
|
23
28
|
* ### Versioning
|
|
24
29
|
*
|
|
@@ -26,16 +31,23 @@
|
|
|
26
31
|
* extraction format, frame layout). Old entries under the previous prefix
|
|
27
32
|
* become inert and can be gc'd by the caller.
|
|
28
33
|
*/
|
|
29
|
-
import { createHash } from "node:crypto";
|
|
30
|
-
import { mkdirSync, readdirSync, statSync, writeFileSync } from "node:fs";
|
|
31
|
-
import { existsSync } from "node:fs";
|
|
34
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
35
|
+
import { existsSync, lstatSync, mkdirSync, readdirSync, renameSync, rmSync, statSync, utimesSync, writeFileSync, } from "node:fs";
|
|
32
36
|
import { join } from "node:path";
|
|
33
37
|
/** Filename prefix for extracted frames. Shared with the extractor. */
|
|
34
38
|
export const FRAME_FILENAME_PREFIX = "frame_";
|
|
35
39
|
/** Sentinel filename written after a cache entry is fully populated. */
|
|
36
40
|
export const COMPLETE_SENTINEL = ".hf-complete";
|
|
37
|
-
/**
|
|
38
|
-
export const
|
|
41
|
+
/** Marker file stamped after each GC sweep; drives the staleness fallback. */
|
|
42
|
+
export const GC_MARKER = ".hf-last-gc";
|
|
43
|
+
/**
|
|
44
|
+
* Current schema version. Bump when the cache-contents invariant changes.
|
|
45
|
+
* v2 -> v3: one-pass VFR extraction (-fps_mode cfr) replaces the two-pass
|
|
46
|
+
* VFR-to-CFR re-encode, changing frame contents for VFR sources under
|
|
47
|
+
* identical key tuples. Without the bump, warm v2 entries (two-pass frames)
|
|
48
|
+
* would keep being served across the deploy boundary.
|
|
49
|
+
*/
|
|
50
|
+
export const SCHEMA_PREFIX = "hfcache-v3-";
|
|
39
51
|
/** Truncated hex chars of SHA-256 used for the entry directory name. */
|
|
40
52
|
const KEY_HEX_CHARS = 16;
|
|
41
53
|
/**
|
|
@@ -56,7 +68,7 @@ export function readKeyStat(videoPath) {
|
|
|
56
68
|
}
|
|
57
69
|
function canonicalKeyBlob(input) {
|
|
58
70
|
const durationForKey = Number.isFinite(input.duration) ? input.duration : -1;
|
|
59
|
-
|
|
71
|
+
const blob = {
|
|
60
72
|
p: input.videoPath,
|
|
61
73
|
m: input.mtimeMs,
|
|
62
74
|
s: input.size,
|
|
@@ -64,7 +76,10 @@ function canonicalKeyBlob(input) {
|
|
|
64
76
|
d: durationForKey,
|
|
65
77
|
f: input.fps,
|
|
66
78
|
fmt: input.format,
|
|
67
|
-
}
|
|
79
|
+
};
|
|
80
|
+
if (input.transform !== undefined)
|
|
81
|
+
blob.t = input.transform;
|
|
82
|
+
return JSON.stringify(blob);
|
|
68
83
|
}
|
|
69
84
|
/**
|
|
70
85
|
* Compute the SHA-256 hex digest for a cache key input.
|
|
@@ -81,9 +96,9 @@ export function cacheEntryDirName(keyHash) {
|
|
|
81
96
|
}
|
|
82
97
|
/**
|
|
83
98
|
* Look up a cache entry by key input. Returns the resolved entry path plus a
|
|
84
|
-
* `hit` flag. On miss, callers should extract frames into
|
|
85
|
-
* (
|
|
86
|
-
* once
|
|
99
|
+
* `hit` flag. On miss, callers should extract frames into a
|
|
100
|
+
* `partialCacheEntryDir(entry)` directory and publish it with
|
|
101
|
+
* `publishCacheEntry` once extraction succeeds.
|
|
87
102
|
*/
|
|
88
103
|
export function lookupCacheEntry(rootDir, input) {
|
|
89
104
|
const keyHash = computeCacheKey(input);
|
|
@@ -98,20 +113,237 @@ export function lookupCacheEntry(rootDir, input) {
|
|
|
98
113
|
export function ensureCacheEntryDir(entry) {
|
|
99
114
|
mkdirSync(entry.dir, { recursive: true });
|
|
100
115
|
}
|
|
116
|
+
/**
|
|
117
|
+
* Unique render-owned directory used to populate a cache entry before the
|
|
118
|
+
* atomic publish rename.
|
|
119
|
+
*/
|
|
120
|
+
export function partialCacheEntryDir(entry) {
|
|
121
|
+
return `${entry.dir}.partial-${process.pid}-${randomUUID().slice(0, 8)}`;
|
|
122
|
+
}
|
|
123
|
+
function isTargetExistsRenameError(err) {
|
|
124
|
+
const code = err.code;
|
|
125
|
+
return code === "EEXIST" || code === "ENOTEMPTY" || code === "EPERM";
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* Publish an extracted partial directory as the final cache entry.
|
|
129
|
+
*
|
|
130
|
+
* Same-filesystem directory rename is atomic: readers either see no entry or
|
|
131
|
+
* a complete sentineled entry. When another writer wins the race, the caller
|
|
132
|
+
* should rehydrate from the final dir. If publish cannot complete safely, the
|
|
133
|
+
* partial remains render-owned and must be cleaned up by the render cleanup.
|
|
134
|
+
*/
|
|
135
|
+
/**
|
|
136
|
+
* If a concurrent writer's completed entry is visible, discard our partial
|
|
137
|
+
* and serve theirs. Identical keys produce identical frames, so adopting the
|
|
138
|
+
* winner is always correct. Returns null when no winner is present.
|
|
139
|
+
*/
|
|
140
|
+
function adoptPublishedWinner(entry, partialDir) {
|
|
141
|
+
if (!existsSync(join(entry.dir, COMPLETE_SENTINEL)))
|
|
142
|
+
return null;
|
|
143
|
+
removeDir(partialDir);
|
|
144
|
+
return { dir: entry.dir, published: true };
|
|
145
|
+
}
|
|
146
|
+
export function publishCacheEntry(entry, partialDir) {
|
|
147
|
+
try {
|
|
148
|
+
writeFileSync(join(partialDir, COMPLETE_SENTINEL), "", "utf-8");
|
|
149
|
+
}
|
|
150
|
+
catch {
|
|
151
|
+
return { dir: partialDir, published: false };
|
|
152
|
+
}
|
|
153
|
+
try {
|
|
154
|
+
renameSync(partialDir, entry.dir);
|
|
155
|
+
return { dir: entry.dir, published: true };
|
|
156
|
+
}
|
|
157
|
+
catch (err) {
|
|
158
|
+
if (!isTargetExistsRenameError(err))
|
|
159
|
+
return { dir: partialDir, published: false };
|
|
160
|
+
}
|
|
161
|
+
const winner = adoptPublishedWinner(entry, partialDir);
|
|
162
|
+
if (winner)
|
|
163
|
+
return winner;
|
|
164
|
+
try {
|
|
165
|
+
rmSync(entry.dir, { recursive: true, force: true });
|
|
166
|
+
}
|
|
167
|
+
catch {
|
|
168
|
+
return { dir: partialDir, published: false };
|
|
169
|
+
}
|
|
170
|
+
try {
|
|
171
|
+
renameSync(partialDir, entry.dir);
|
|
172
|
+
return { dir: entry.dir, published: true };
|
|
173
|
+
}
|
|
174
|
+
catch {
|
|
175
|
+
// TOCTOU: a concurrent writer can publish between the winner check, the
|
|
176
|
+
// rm above, and this retry. Re-run the adopt check so a winner that
|
|
177
|
+
// landed inside that window is served rather than reported as a failure.
|
|
178
|
+
return adoptPublishedWinner(entry, partialDir) ?? { dir: partialDir, published: false };
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Update the LRU clock for a complete cache entry. Misses and filesystem
|
|
183
|
+
* races are harmless: the caller can still use the entry it already found.
|
|
184
|
+
*/
|
|
185
|
+
export function touchCacheEntry(entry) {
|
|
186
|
+
try {
|
|
187
|
+
const now = new Date();
|
|
188
|
+
utimesSync(join(entry.dir, COMPLETE_SENTINEL), now, now);
|
|
189
|
+
}
|
|
190
|
+
catch {
|
|
191
|
+
// Best effort LRU touch.
|
|
192
|
+
}
|
|
193
|
+
}
|
|
101
194
|
/**
|
|
102
195
|
* Write the completion sentinel so subsequent lookups treat this entry as a
|
|
103
|
-
* hit. Must be called only after every frame has been written.
|
|
196
|
+
* hit. Must be called only after every frame has been written. The extractor
|
|
197
|
+
* now publishes new entries via `publishCacheEntry`; this helper remains
|
|
198
|
+
* exported for tests and legacy callers that materialize entries directly.
|
|
104
199
|
*
|
|
105
|
-
* Concurrency:
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
* anyone running concurrent renders against a shared cache root should front
|
|
110
|
-
* it with an external lock.
|
|
200
|
+
* Concurrency: direct mark is non-atomic and should not be used for shared
|
|
201
|
+
* writer paths. `publishCacheEntry` writes the sentinel inside a partial dir
|
|
202
|
+
* and atomically renames it into place, so concurrent writers duplicate work
|
|
203
|
+
* but never serve torn frames.
|
|
111
204
|
*/
|
|
112
205
|
export function markCacheEntryComplete(entry) {
|
|
113
206
|
writeFileSync(join(entry.dir, COMPLETE_SENTINEL), "", "utf-8");
|
|
114
207
|
}
|
|
208
|
+
/** Any generation of this cache's entries ("hfcache-v*"), current or superseded. */
|
|
209
|
+
const CACHE_GENERATION_PREFIX = "hfcache-v";
|
|
210
|
+
function isCacheLikeChild(name) {
|
|
211
|
+
// Match every schema generation, not just SCHEMA_PREFIX: after a schema
|
|
212
|
+
// bump, superseded-version entries would otherwise be invisible to the
|
|
213
|
+
// sweep and orphan their disk forever. Old-generation entries never get
|
|
214
|
+
// sentinel touches, so the LRU evicts them first.
|
|
215
|
+
return name.startsWith(CACHE_GENERATION_PREFIX) || name.includes(".partial-");
|
|
216
|
+
}
|
|
217
|
+
function isPartialChild(name) {
|
|
218
|
+
return name.includes(".partial-");
|
|
219
|
+
}
|
|
220
|
+
function directorySizeBytes(path) {
|
|
221
|
+
try {
|
|
222
|
+
const stat = lstatSync(path);
|
|
223
|
+
if (!stat.isDirectory())
|
|
224
|
+
return stat.size;
|
|
225
|
+
}
|
|
226
|
+
catch {
|
|
227
|
+
return 0;
|
|
228
|
+
}
|
|
229
|
+
let total = 0;
|
|
230
|
+
let children;
|
|
231
|
+
try {
|
|
232
|
+
children = readdirSync(path);
|
|
233
|
+
}
|
|
234
|
+
catch {
|
|
235
|
+
return 0;
|
|
236
|
+
}
|
|
237
|
+
for (const child of children) {
|
|
238
|
+
const childPath = join(path, child);
|
|
239
|
+
try {
|
|
240
|
+
const stat = lstatSync(childPath);
|
|
241
|
+
if (stat.isDirectory()) {
|
|
242
|
+
total += directorySizeBytes(childPath);
|
|
243
|
+
}
|
|
244
|
+
else {
|
|
245
|
+
total += stat.size;
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
catch {
|
|
249
|
+
// Ignore entries deleted or made unreadable during the sweep.
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
return total;
|
|
253
|
+
}
|
|
254
|
+
function removeDir(path) {
|
|
255
|
+
try {
|
|
256
|
+
rmSync(path, { recursive: true, force: true });
|
|
257
|
+
}
|
|
258
|
+
catch {
|
|
259
|
+
// Cache GC is opportunistic; one bad entry must not abort the sweep.
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* Stat one cache-looking child for the GC sweep. Aged partial dirs (crashed
|
|
264
|
+
* writers) are removed immediately and yield `null`; entries that disappear
|
|
265
|
+
* or fail to stat mid-sweep also yield `null`.
|
|
266
|
+
*/
|
|
267
|
+
function collectGcEntry(dir, name, now, minAgeMs, stats) {
|
|
268
|
+
try {
|
|
269
|
+
const dirStat = statSync(dir);
|
|
270
|
+
if (isPartialChild(name) && now - dirStat.mtimeMs >= minAgeMs) {
|
|
271
|
+
removeDir(dir);
|
|
272
|
+
stats.agedPartialsRemoved += 1;
|
|
273
|
+
return null;
|
|
274
|
+
}
|
|
275
|
+
let lastUseMs = dirStat.mtimeMs;
|
|
276
|
+
try {
|
|
277
|
+
lastUseMs = statSync(join(dir, COMPLETE_SENTINEL)).mtimeMs;
|
|
278
|
+
}
|
|
279
|
+
catch {
|
|
280
|
+
// Unsentineled entries use directory mtime as a stale-entry clock.
|
|
281
|
+
}
|
|
282
|
+
return { dir, size: directorySizeBytes(dir), lastUseMs, ageMs: now - lastUseMs };
|
|
283
|
+
}
|
|
284
|
+
catch {
|
|
285
|
+
return null;
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
/**
|
|
289
|
+
* Opportunistic size-capped LRU cleanup for extracted video frames.
|
|
290
|
+
*
|
|
291
|
+
* Scans only direct cache-looking children and never throws. The age guard is
|
|
292
|
+
* a liveness heuristic, not a lock. Returns counts so the caller can surface
|
|
293
|
+
* eviction pressure in render observability.
|
|
294
|
+
*/
|
|
295
|
+
/**
|
|
296
|
+
* Whether the staleness fallback should force a sweep: true when no sweep
|
|
297
|
+
* marker exists or the last sweep is older than `maxAgeMs`. Lets 100%-warm
|
|
298
|
+
* workloads (which skip the per-miss sweep) still reclaim space eventually.
|
|
299
|
+
*/
|
|
300
|
+
export function gcSweepDue(rootDir, maxAgeMs) {
|
|
301
|
+
try {
|
|
302
|
+
return Date.now() - statSync(join(rootDir, GC_MARKER)).mtimeMs > maxAgeMs;
|
|
303
|
+
}
|
|
304
|
+
catch {
|
|
305
|
+
return true;
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
export function gcExtractionCache(rootDir, opts) {
|
|
309
|
+
const stats = { evictedEntries: 0, evictedBytes: 0, agedPartialsRemoved: 0 };
|
|
310
|
+
try {
|
|
311
|
+
writeFileSync(join(rootDir, GC_MARKER), "", "utf-8");
|
|
312
|
+
}
|
|
313
|
+
catch {
|
|
314
|
+
// Unwritable root: the sweep below will no-op on the same root anyway.
|
|
315
|
+
}
|
|
316
|
+
try {
|
|
317
|
+
const now = Date.now();
|
|
318
|
+
const entries = [];
|
|
319
|
+
for (const child of readdirSync(rootDir, { withFileTypes: true })) {
|
|
320
|
+
if (!child.isDirectory() || !isCacheLikeChild(child.name))
|
|
321
|
+
continue;
|
|
322
|
+
const entry = collectGcEntry(join(rootDir, child.name), child.name, now, opts.minAgeMs, stats);
|
|
323
|
+
if (entry)
|
|
324
|
+
entries.push(entry);
|
|
325
|
+
}
|
|
326
|
+
let totalBytes = entries.reduce((sum, e) => sum + e.size, 0);
|
|
327
|
+
if (totalBytes <= opts.maxBytes)
|
|
328
|
+
return stats;
|
|
329
|
+
entries.sort((a, b) => a.lastUseMs - b.lastUseMs);
|
|
330
|
+
for (const entry of entries) {
|
|
331
|
+
// ponytail: age-based liveness guard, not a lock; a render longer than minAge with a full cache could lose entries mid-read - acceptable, next render re-extracts.
|
|
332
|
+
if (entry.ageMs < opts.minAgeMs)
|
|
333
|
+
continue;
|
|
334
|
+
removeDir(entry.dir);
|
|
335
|
+
stats.evictedEntries += 1;
|
|
336
|
+
stats.evictedBytes += entry.size;
|
|
337
|
+
totalBytes -= entry.size;
|
|
338
|
+
if (totalBytes <= opts.maxBytes)
|
|
339
|
+
break;
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
catch {
|
|
343
|
+
// Missing root or unreadable cache: no cleanup this sweep.
|
|
344
|
+
}
|
|
345
|
+
return stats;
|
|
346
|
+
}
|
|
115
347
|
export function rehydrateCacheEntry(entry, options) {
|
|
116
348
|
const framePattern = `${FRAME_FILENAME_PREFIX}%05d.${options.format}`;
|
|
117
349
|
const framePaths = new Map();
|