cozyclay 1.7.1 → 1.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/CHANGELOG.md +86 -0
  2. package/README.md +11 -3
  3. package/bin/agent/agent-routes.mjs +8 -2
  4. package/bin/agent/h3-preservation.mjs +218 -0
  5. package/bin/agent/video-adapters.mjs +223 -10
  6. package/bin/cozyclay.mjs +11 -1
  7. package/dist/ai-camera-control/index.html +17 -43
  8. package/dist/app/index.html +5 -4
  9. package/dist/assets/analytics-Dc9ooyn1.js +1 -0
  10. package/dist/assets/app-CCw5KRC0.css +1 -0
  11. package/dist/assets/app-CG7OIkz0.js +4859 -0
  12. package/dist/assets/{demo-CEleUnpD.js → demo-BENMp24T.js} +1 -1
  13. package/dist/assets/shot-prompt-CuqtYxB7.css +1 -0
  14. package/dist/assets/shot-prompt-F2C_YN3J.js +15 -0
  15. package/dist/assets/{ticket-CX73jjoc.js → ticket-BDieXjZs.js} +1 -1
  16. package/dist/assets/workflow-BHvP2fxY.js +179 -0
  17. package/dist/assets/workflow-CeQxy8Uq.css +1 -0
  18. package/dist/ca5b3d011bb7f59fe4232e1e31aec2f0.txt +1 -0
  19. package/dist/cozyclay-package.json +1 -1
  20. package/dist/greybox-to-video/index.html +220 -0
  21. package/dist/index.html +324 -207
  22. package/dist/llms.txt +17 -0
  23. package/dist/media/cozyclay-seedance-vs-greybox-poster.jpg +0 -0
  24. package/dist/media/playground-poster.jpg +0 -0
  25. package/dist/previs-software/index.html +220 -0
  26. package/dist/privacy/index.html +153 -0
  27. package/dist/robots.txt +10 -0
  28. package/dist/scenes/city-block.cclayproject +1 -0
  29. package/dist/seedance-camera-control/index.html +199 -0
  30. package/dist/site.css +118 -0
  31. package/dist/sitemap.xml +18 -6
  32. package/dist/sw.js +3 -0
  33. package/dist/workflow/index.html +7 -4
  34. package/package.json +2 -2
  35. package/src/App.jsx +813 -478
  36. package/src/analytics.js +3 -1
  37. package/src/app-stage.jsx +16 -1
  38. package/src/ardy/client.js +3 -1
  39. package/src/ardy/motion-calibration.js +122 -0
  40. package/src/ardy/root-drop.js +197 -7
  41. package/src/ardy/timeline.jsx +65 -41
  42. package/src/camera-tutorial.jsx +475 -0
  43. package/src/controls.jsx +11 -0
  44. package/src/dualview.jsx +8 -0
  45. package/src/hierarchy-panel.jsx +183 -138
  46. package/src/locale.js +4 -0
  47. package/src/main.jsx +20 -7
  48. package/src/multimodel-ingest.js +20 -2
  49. package/src/planview.jsx +9 -6
  50. package/src/playground.js +61 -0
  51. package/src/project-browser.jsx +19 -1
  52. package/src/scene-objects.js +13 -2
  53. package/src/scenes.js +9 -1
  54. package/src/settings-menu.jsx +117 -0
  55. package/src/starter-scenes.js +25 -0
  56. package/src/styles.css +852 -297
  57. package/src/workflow/AgentPanel.jsx +10 -2
  58. package/src/workflow/WorkflowBuilder.jsx +22 -5
  59. package/src/workflow/agent-client.js +13 -1
  60. package/src/workflow/agent-panel.css +20 -3
  61. package/src/workflow/workflow.css +1 -0
  62. package/tools/ardy/BRIDGE.md +10 -2
  63. package/tools/ardy/README.md +23 -6
  64. package/tools/ardy/__pycache__/cclay_gvhmr_worker.cpython-313.pyc +0 -0
  65. package/tools/ardy/bridge.mjs +6 -2
  66. package/tools/ardy/cclay_gvhmr_worker.py +139 -8
  67. package/tools/ardy/extract.mjs +56 -51
  68. package/tools/ardy/mocap-metrics.mjs +67 -33
  69. package/tools/ardy/mocap-quality-gate.mjs +110 -0
  70. package/tools/ardy/motion-stabilize.mjs +243 -0
  71. package/tools/ardy/runners/gvhmr-worker.mjs +15 -12
  72. package/tools/dev/page-shell.py +117 -0
  73. package/tools/dev/pages/greybox-to-video.html +147 -0
  74. package/tools/dev/pages/previs-software.html +146 -0
  75. package/tools/dev/pages/privacy.html +93 -0
  76. package/tools/dev/pages/seedance-camera-control.html +126 -0
  77. package/tools/indexnow.key +1 -0
  78. package/tools/indexnow.mjs +22 -0
  79. package/tools/qa/studio-control-count.mjs +63 -0
  80. package/tools/run-tests.mjs +17 -1
  81. package/tools/sitemap.mjs +34 -0
  82. package/dist/assets/analytics-Ce3YvFby.js +0 -1
  83. package/dist/assets/app-DNYMNzNh.css +0 -1
  84. package/dist/assets/app-jcR6lsws.js +0 -4859
  85. package/dist/assets/shot-prompt-s4R0Y9R7.js +0 -14
  86. package/dist/assets/vision_bundle-jFkh-fIS.js +0 -41
  87. package/dist/assets/workflow-BcZSgt5C.js +0 -180
  88. package/dist/assets/workflow-CKCpHJGL.css +0 -1
  89. package/src/analytics-toggle.jsx +0 -24
  90. package/src/locale-toggle.jsx +0 -18
package/CHANGELOG.md CHANGED
@@ -1,5 +1,91 @@
1
1
  # Changelog
2
2
 
3
+ ## 1.8.1
4
+
5
+ The Studio camera tutorial now starts where the landing page does and shows
6
+ where to click.
7
+
8
+ - Opening the tutorial (Settings ▾ → Camera tutorial or `/app/?tutorial=camera`)
9
+ loads the City Block starter scene with the walk take on its character, at
10
+ frame 0 in the free camera, so Shot / Rail / Play have something to frame.
11
+ A scene with unsaved changes is replaced only after a confirm.
12
+ - Each step points at its control: a pulsing spotlight plus a numbered beacon
13
+ and caption pinned to `+ Add shot`, the shot block and `Draw rail`, the
14
+ top view, and the look-through button; the Look / Walk / Dolly / Orbit steps
15
+ show an animated mouse-and-keys cue in the viewport, with the walk keycaps
16
+ turning green as they are pressed. The hint card names the region ("↓
17
+ Timeline, Shots lane", "→ Viewport"). Overlays never take the pointer and go
18
+ static under `prefers-reduced-motion`.
19
+
20
+ ## 1.8.0
21
+
22
+ The Studio's chrome gets a research-backed cut: the same capabilities, a third
23
+ fewer controls on screen at once, and one place for each of them. A first-time
24
+ creator can now learn the camera inside the Studio itself, or on cozyclay.org
25
+ before installing, and keep the scene they made. Video mocap gets a quality
26
+ gate, and AI video takes are checked against the scene they were supposed to
27
+ preserve.
28
+
29
+ ### Studio UI simplification
30
+
31
+ - Simultaneously visible controls drop from 49 / 54 / 65 (Scene / Camera /
32
+ Motion mode) to 35 / 38 / 51, measured by `tools/qa/studio-control-count.mjs`
33
+ and recorded with the per-control rationale in `docs/studio-ui-ia.md`.
34
+ - The top bar keeps five actions. One **Export ▾** menu leads with the keyframe
35
+ pack and folds the mp4, depth/normal passes, storyboard and blocking frame
36
+ under it; the three Record buttons and the inspector's duplicate FOV/Recenter
37
+ are gone. **Settings ▾** holds language and analytics.
38
+ - **View ▾** on the viewport bar owns the reference grid, Auto Color and body
39
+ part colours; Move/Rotate/Scale stay on the transform strip only.
40
+ - The scene switcher sits on the hierarchy's root row; the Projects… button,
41
+ the root fold caret and the Characters group row are removed and characters
42
+ sit directly under the scene.
43
+ - Timeline motion tools (IK, Foot, Body, speed, Clear, take bar) render only in
44
+ Motion mode; Foot snap and Body contact appear only while IK is on.
45
+ - The Scene/PlayView tabs are replaced by the viewport's look-through button:
46
+ it enters an explicit preview of the shot camera and Esc returns.
47
+ - Selecting the camera switches to Camera mode; the Placement row appears in
48
+ Motion mode; Generate all waits for at least one prompt block.
49
+ - The `advancedMode` / `beginnerMode` gates that had been hardcoded on are
50
+ removed.
51
+
52
+ ### Learn the camera
53
+
54
+ - A seven-step camera tutorial runs inside the Studio: Look, Walk (W A S D
55
+ Q E), Dolly, Orbit, Shot, Rail, Play. Open it from Settings ▾ → Camera
56
+ tutorial or `/app/?tutorial=camera`; each step completes only when the
57
+ gesture actually happens, and the strip never takes the pointer.
58
+ - cozyclay.org embeds a live Studio playground on a preset city block with the
59
+ same seven steps, a second demo reel, and leaner copy. The tutorial ends by
60
+ handing over `npx cozyclay --scene city-block`, which opens the local Studio
61
+ on that starter scene; starter scenes are also offered from the project
62
+ browser, and the visitor can download the scene they made.
63
+
64
+ ### Agent panel in the Studio
65
+
66
+ - The Workflow page's Agent chat column is available in the Studio, toggled
67
+ from View ▾ → Panels → Agent panel or Cmd/Ctrl+B. It boots collapsed so the
68
+ mode budgets above are unchanged.
69
+
70
+ ### Video mocap and AI video takes
71
+
72
+ - GVHMR is the only extraction backend; the bridge and the Studio return a
73
+ named error instead of silently falling back to browser MediaPipe.
74
+ - GVHMR output goes through adaptive position/rotation stabilisation, explicit
75
+ contact correction, support-surface height (a chair lifts the motion), a
76
+ mocap quality gate, and scene calibration that survives restore. Palette
77
+ segmentation diagnostics are exposed for AI-rendered mannequin clips.
78
+ - H3 video takes are checked against the requested scene and camera: the
79
+ Workflow shows a lock receipt on success and rejects a take whose background
80
+ or camera drifted, keeping no stale preview.
81
+
82
+ ### Site
83
+
84
+ - Four search-facing pages (greybox to video, previs software, Seedance camera
85
+ control, privacy), `robots.txt` with an explicit AI-crawler policy,
86
+ `llms.txt`, a generated sitemap with real last-modified dates, and IndexNow
87
+ pings on deploy.
88
+
3
89
  ## 1.7.1
4
90
 
5
91
  - Open the complete Studio at the development server root; the unfinished
package/README.md CHANGED
@@ -18,7 +18,9 @@
18
18
  </p>
19
19
 
20
20
  <p align="center">
21
- <a href="https://cozyclay.org/">Demo reel</a> ·
21
+ <a href="https://cozyclay.org/#try">Try it in the browser</a> ·
22
+ <a href="https://cozyclay.org/greybox-to-video/">Greybox to AI video</a> ·
23
+ <a href="https://cozyclay.org/previs-software/">Previs software compared</a> ·
22
24
  <a href="#quick-start">Quick start</a> ·
23
25
  <a href="#what-you-can-do">Features</a> ·
24
26
  <a href="#ai-control-mcp">AI control</a> ·
@@ -34,7 +36,13 @@ CozyClay is a browser-based 3D staging studio built with Three.js and React Thre
34
36
  npx cozyclay
35
37
  ```
36
38
 
37
- That is the whole install. **[cozyclay.org](https://cozyclay.org/)** has the demo reel and a walkthrough of what the studio does; to use it, run it on your own machine. It ships seeded with a pre-generated motion clip, so you can scrub the timeline, drive the cameras and draw a dolly rail straight away — generating *new* motion is optional and uses the Kimodo bridge when configured.
39
+ That is the whole install. Not sure yet? **[Try it in the browser first](https://cozyclay.org/#try)** a seven-step camera tutorial on a live scene then keep going on your machine with the same set:
40
+
41
+ ```bash
42
+ npx cozyclay --scene city-block
43
+ ```
44
+
45
+ The studio ships seeded with a pre-generated motion clip, so you can scrub the timeline, drive the cameras and draw a dolly rail straight away — generating *new* motion is optional and uses the Kimodo bridge when configured.
38
46
 
39
47
  ## Demo
40
48
 
@@ -65,7 +73,7 @@ npx cozyclay
65
73
  bunx cozyclay
66
74
  ```
67
75
 
68
- That downloads the built studio and opens it at `http://127.0.0.1:5180/app/`. Nothing to compile, no dependency tree to install. Useful flags: `--port 5200`, `--no-open`, `--no-motion`.
76
+ That downloads the built studio and opens it at `http://127.0.0.1:5180/app/`. Nothing to compile, no dependency tree to install. Useful flags: `--port 5200`, `--no-open`, `--no-motion`, `--scene city-block` (start on the bundled starter scene instead of an empty room; the first-run dialog offers the same under **Start from a scene**, and a `.cclayproject` downloaded from the browser tutorial opens with **Open a project**).
69
77
 
70
78
  A global install gives you `cclay`, the same command with less typing. Once a day the launcher checks npm for a newer release and prints a one-line notice after the studio is up; it stays quiet when you're current or offline. `cclay update` installs the latest release, and `--no-update-check` skips the check entirely.
71
79
 
@@ -196,8 +196,14 @@ export function createAgentHandler({ auth = defaultAuth, codex, handlers, liveHu
196
196
  if (!adapter || !adapter.configured()) { json(res, 409, { error: "video provider is not configured" }); return true; }
197
197
  try {
198
198
  const result = await adapter.generate({ ...value, durationSeconds: Number(value.durationSeconds) });
199
- json(res, 200, { ...(result.mp4Base64 ? { dataUrl: `data:video/mp4;base64,${result.mp4Base64}` } : { url: result.url }), width: result.width, height: result.height, seconds: result.seconds });
200
- } catch (error) { json(res, 502, { error: error?.message || "video provider failed" }); }
199
+ json(res, 200, { ...(result.mp4Base64 ? { dataUrl: `data:video/mp4;base64,${result.mp4Base64}` } : { url: result.url }), width: result.width, height: result.height, seconds: result.seconds, ...(result.preservation ? { preservation: result.preservation } : {}) });
200
+ } catch (error) {
201
+ // A generated H3 take that fails the plate check is unsafe to show as
202
+ // a locked shot. Keep the distinction visible to the client so it can
203
+ // ask for a retry instead of silently accepting a drifting set.
204
+ const status = error?.code === "h3-preservation-failed" ? 422 : 502;
205
+ json(res, status, { error: error?.message || "video provider failed", ...(error?.preservation ? { preservation: error.preservation } : {}) });
206
+ }
201
207
  return true;
202
208
  }
203
209
  if (req.method !== "POST" || !["/agent/turn", "/agent/stop"].includes(path)) {
@@ -0,0 +1,218 @@
1
+ import { mkdtemp, rm, writeFile } from "node:fs/promises";
2
+ import { tmpdir } from "node:os";
3
+ import { join } from "node:path";
4
+ import { spawn } from "node:child_process";
5
+
6
+ /**
7
+ * H3 is an image-to-video model: a prompt can request a locked camera, but it
8
+ * cannot prove that the returned pixels stayed on that camera. This module
9
+ * is the output-side guard. It compares several decoded frames with the
10
+ * uploaded plate in a border band where the animated performer normally is
11
+ * absent. A request is accepted only when the output dimensions/aspect and
12
+ * the measured plate drift are inside the contract.
13
+ */
14
+
15
+ export const H3_OUTPUT_LIMITS = Object.freeze({
16
+ // JPEG/AVC quantisation and the model's first-frame reconstruction create a
17
+ // small amount of noise. A real camera move changes many edge pixels by much
18
+ // more than this value.
19
+ edgeP95Rgb: 14,
20
+ edgeMeanRgb: 6,
21
+ // The full-frame 80th percentile catches a camera translation in the
22
+ // middle of the set even when the border happens to be covered by a subject.
23
+ globalP80Rgb: 16,
24
+ // Once the output's first decoded frame has established the model's codec
25
+ // reconstruction, every later frame must keep the static scene band stable.
26
+ // This catches a camera move that happens to reconstruct to a similar value
27
+ // relative to the uploaded plate (and avoids treating subject motion as set
28
+ // motion).
29
+ temporalEdgeP95Rgb: 14,
30
+ temporalEdgeMeanRgb: 6,
31
+ temporalGlobalP80Rgb: 16,
32
+ cameraDriftPx: 2,
33
+ aspectError: 0.01,
34
+ borderFraction: 0.12,
35
+ });
36
+
37
+ /** Estimate a rigid camera translation against the plate in the static band. */
38
+ export function estimateH3CameraShift(reference, frame, width, height, { borderFraction = H3_OUTPUT_LIMITS.borderFraction, maxShift = 4 } = {}) {
39
+ const borderX = Math.max(1, Math.floor(width * borderFraction));
40
+ const borderY = Math.max(1, Math.floor(height * borderFraction));
41
+ if (width <= borderX * 2 + 2 || height <= borderY * 2 + 2) return { x: 0, y: 0, errorRgb: 0, distancePx: 0 };
42
+ let best = { x: 0, y: 0, error: Infinity }; let baseline = Infinity;
43
+ for (let dy = -maxShift; dy <= maxShift; dy += 1) for (let dx = -maxShift; dx <= maxShift; dx += 1) {
44
+ let sum = 0; let count = 0;
45
+ for (let y = 0; y < height; y += 4) for (let x = 0; x < width; x += 4) {
46
+ // Estimate rigid movement only from the static perimeter. Sampling the
47
+ // interior lets a moving performer win the alignment search and can
48
+ // incorrectly call subject motion a camera correction.
49
+ if (!(x < borderX || x >= width - borderX || y < borderY || y >= height - borderY)) continue;
50
+ if (x + dx < 0 || x + dx >= width || y + dy < 0 || y + dy >= height) continue;
51
+ const a = (y * width + x) * 3; const b = ((y + dy) * width + x + dx) * 3;
52
+ sum += (Math.abs(reference[a] - frame[b]) + Math.abs(reference[a + 1] - frame[b + 1]) + Math.abs(reference[a + 2] - frame[b + 2])) / 3;
53
+ count += 1;
54
+ }
55
+ const error = sum / Math.max(1, count);
56
+ if (dx === 0 && dy === 0) baseline = error;
57
+ if (error < best.error) best = { x: dx, y: dy, error };
58
+ }
59
+ const improvementRgb = Math.max(0, baseline - best.error);
60
+ // A flat/softly lit border has many equally good translations. Only treat a
61
+ // shift as camera evidence when the alignment materially improves error.
62
+ const distancePx = improvementRgb >= 8 ? Math.hypot(best.x, best.y) : 0;
63
+ return { x: best.x, y: best.y, errorRgb: best.error, baselineErrorRgb: baseline, improvementRgb, distancePx };
64
+ }
65
+
66
+ /** Compare two generated frames in the static perimeter and full frame. */
67
+ export function compareH3FrameStability(reference, frame, width, height, { borderFraction = H3_OUTPUT_LIMITS.borderFraction } = {}) {
68
+ if (!(reference instanceof Uint8Array) || !(frame instanceof Uint8Array) || reference.length < width * height * 3 || frame.length < width * height * 3) throw new TypeError("compareH3FrameStability needs two decoded RGB frames");
69
+ const borderX = Math.max(1, Math.floor(width * borderFraction));
70
+ const borderY = Math.max(1, Math.floor(height * borderFraction));
71
+ const edge = []; const global = [];
72
+ for (let y = 0; y < height; y += 2) for (let x = 0; x < width; x += 2) {
73
+ const i = (y * width + x) * 3;
74
+ const d = (Math.abs(reference[i] - frame[i]) + Math.abs(reference[i + 1] - frame[i + 1]) + Math.abs(reference[i + 2] - frame[i + 2])) / 3;
75
+ global.push(d);
76
+ if (x < borderX || x >= width - borderX || y < borderY || y >= height - borderY) edge.push(d);
77
+ }
78
+ edge.sort((a, b) => a - b); global.sort((a, b) => a - b);
79
+ const meanRgb = edge.reduce((sum, value) => sum + value, 0) / Math.max(1, edge.length);
80
+ const p95Rgb = edge[Math.min(edge.length - 1, Math.floor(edge.length * 0.95))] ?? 255;
81
+ const globalP80Rgb = global[Math.min(global.length - 1, Math.floor(global.length * 0.8))] ?? 255;
82
+ const camera = estimateH3CameraShift(reference, frame, width, height, { borderFraction });
83
+ return { meanRgb, p95Rgb, globalP80Rgb, cameraDriftPx: camera.distancePx, cameraShift: { x: camera.x, y: camera.y }, cameraErrorRgb: camera.errorRgb, cameraImprovementRgb: camera.improvementRgb, samples: edge.length, globalSamples: global.length };
84
+ }
85
+
86
+ function run(command, args, input = null) {
87
+ return new Promise((resolve, reject) => {
88
+ const child = spawn(command, args, { stdio: ["pipe", "pipe", "pipe"] });
89
+ const out = []; const err = [];
90
+ child.stdout.on("data", (chunk) => out.push(chunk));
91
+ child.stderr.on("data", (chunk) => err.push(chunk));
92
+ child.once("error", reject);
93
+ child.once("close", (code) => {
94
+ if (code === 0) resolve({ stdout: Buffer.concat(out), stderr: Buffer.concat(err).toString("utf8") });
95
+ else reject(new Error(`${command} exited ${code}: ${Buffer.concat(err).toString("utf8").trim()}`));
96
+ });
97
+ if (input) child.stdin.end(input); else child.stdin.end();
98
+ });
99
+ }
100
+
101
+ async function decode(path, width, height, seek = null, inputFormat = null) {
102
+ const args = ["-hide_banner", "-loglevel", "error"];
103
+ if (inputFormat) args.push("-f", inputFormat);
104
+ args.push("-i", path);
105
+ // Place -ss after the input so ffmpeg decodes through the preceding GOP and
106
+ // returns the requested frame. Fast input seeking would repeatedly return
107
+ // the first keyframe on long-GOP H3 clips, making temporal camera drift look
108
+ // stable when it was never inspected.
109
+ if (seek !== null) args.push("-ss", String(Math.max(0, seek)));
110
+ args.push("-vf", `scale=${width}:${height}:flags=bicubic`, "-frames:v", "1", "-f", "rawvideo", "-pix_fmt", "rgb24", "pipe:1");
111
+ const { stdout } = await run("ffmpeg", args);
112
+ const expected = width * height * 3;
113
+ if (stdout.length < expected) throw new Error(`decoded frame is ${stdout.length} bytes; expected ${expected}`);
114
+ return stdout.subarray(0, expected);
115
+ }
116
+
117
+ async function decodeNearest(path, width, height, seek, duration) {
118
+ // Short test clips and variable frame rate outputs may have no frame at an
119
+ // arbitrary timestamp near the end. Walk backwards to the nearest decoded
120
+ // frame instead of treating a valid clip as an inspection failure.
121
+ for (let offset = 0; offset <= Math.max(1, duration); offset += 1 / 12) {
122
+ try { return await decode(path, width, height, Math.max(0, seek - offset)); } catch { /* try the preceding frame */ }
123
+ }
124
+ throw new Error("H3 output has no decodable video frame.");
125
+ }
126
+
127
+ async function probe(path) {
128
+ const { stdout } = await run("ffprobe", ["-v", "error", "-select_streams", "v:0", "-show_entries", "stream=width,height,duration,nb_frames,r_frame_rate", "-of", "json", path]);
129
+ const stream = JSON.parse(stdout.toString("utf8"))?.streams?.[0];
130
+ if (!stream || !Number.isFinite(Number(stream.width)) || !Number.isFinite(Number(stream.height))) throw new Error("H3 output has no video stream metadata.");
131
+ const [rateNumerator, rateDenominator] = String(stream.r_frame_rate || "").split("/").map(Number);
132
+ const fps = rateNumerator > 0 && rateDenominator > 0 ? rateNumerator / rateDenominator : null;
133
+ return { width: Number(stream.width), height: Number(stream.height), seconds: Number.isFinite(Number(stream.duration)) ? Number(stream.duration) : null, frames: Number.isFinite(Number(stream.nb_frames)) ? Number(stream.nb_frames) : null, fps };
134
+ }
135
+
136
+ /** Compare a decoded RGB frame to the plate. Returns mean and p95 RGB error. */
137
+ export function compareH3Plate(reference, frame, width, height, { borderFraction = H3_OUTPUT_LIMITS.borderFraction } = {}) {
138
+ if (!(reference instanceof Uint8Array) || !(frame instanceof Uint8Array) || reference.length < width * height * 3 || frame.length < width * height * 3) throw new TypeError("compareH3Plate needs two decoded RGB frames");
139
+ const borderX = Math.max(1, Math.floor(width * borderFraction));
140
+ const borderY = Math.max(1, Math.floor(height * borderFraction));
141
+ const errors = []; const globalErrors = [];
142
+ for (let y = 0; y < height; y += 2) for (let x = 0; x < width; x += 2) {
143
+ const i = (y * width + x) * 3;
144
+ const d = (Math.abs(reference[i] - frame[i]) + Math.abs(reference[i + 1] - frame[i + 1]) + Math.abs(reference[i + 2] - frame[i + 2])) / 3;
145
+ globalErrors.push(d);
146
+ if (x < borderX || x >= width - borderX || y < borderY || y >= height - borderY) errors.push(d);
147
+ }
148
+ errors.sort((a, b) => a - b);
149
+ globalErrors.sort((a, b) => a - b);
150
+ const mean = errors.reduce((sum, value) => sum + value, 0) / Math.max(1, errors.length);
151
+ const p95 = errors[Math.min(errors.length - 1, Math.floor(errors.length * 0.95))] ?? 255;
152
+ const globalP80 = globalErrors[Math.min(globalErrors.length - 1, Math.floor(globalErrors.length * 0.8))] ?? 255;
153
+ const camera = estimateH3CameraShift(reference, frame, width, height, { borderFraction });
154
+ return { meanRgb: mean, p95Rgb: p95, globalP80Rgb: globalP80, cameraDriftPx: camera.distancePx, cameraShift: { x: camera.x, y: camera.y }, cameraErrorRgb: camera.errorRgb, cameraImprovementRgb: camera.improvementRgb, samples: errors.length, globalSamples: globalErrors.length };
155
+ }
156
+
157
+ /**
158
+ * Decode and validate an H3 output against the uploaded first frame. This is
159
+ * deliberately fail-closed: if ffmpeg/ffprobe cannot inspect the result, the
160
+ * caller must not present it as a scene-locked take.
161
+ */
162
+ export async function inspectH3Output({ imageDataUrl, videoBytes, expectedWidth, expectedHeight, limits = H3_OUTPUT_LIMITS, compositorVerified = false }) {
163
+ if (!imageDataUrl?.startsWith("data:image/")) throw new Error("H3 preservation requires the uploaded first image.");
164
+ if (!Buffer.isBuffer(videoBytes) || videoBytes.length === 0) throw new Error("H3 preservation requires non-empty video bytes.");
165
+ const dir = await mkdtemp(join(tmpdir(), "cozyclay-h3-"));
166
+ const mime = /^data:([^;,]+)/.exec(imageDataUrl)?.[1] || "image/png";
167
+ const extension = mime.includes("jpeg") || mime.includes("jpg") ? "jpg" : mime.includes("webp") ? "webp" : "png";
168
+ const imagePath = join(dir, `plate.${extension}`); const videoPath = join(dir, "output.mp4");
169
+ try {
170
+ const match = /^data:[^;,]+;base64,(.+)$/.exec(imageDataUrl);
171
+ if (!match) throw new Error("H3 preservation received an invalid image data URL.");
172
+ await writeFile(imagePath, Buffer.from(match[1], "base64"));
173
+ await writeFile(videoPath, videoBytes);
174
+ const meta = await probe(videoPath);
175
+ const plateMeta = await probe(imagePath);
176
+ const width = meta.width; const height = meta.height;
177
+ // H3 may quantise the canvas to its own 32-pixel grid. The uploaded plate
178
+ // is the source of truth for a locked camera; accepting a second requested
179
+ // ratio would allow a crop/reframe to pass the preservation gate.
180
+ const plateAspect = Number(plateMeta.width) / Number(plateMeta.height);
181
+ const requestedAspect = Number(expectedWidth) / Number(expectedHeight);
182
+ const outputAspect = width / height;
183
+ const targetAspect = Number.isFinite(plateAspect) && plateAspect > 0 ? plateAspect : requestedAspect;
184
+ const aspectError = Number.isFinite(targetAspect) && targetAspect > 0 ? Math.abs(outputAspect - targetAspect) / targetAspect : 0;
185
+ if (aspectError > limits.aspectError) throw new Error(`H3 preservation failed: output aspect drift ${(aspectError * 100).toFixed(2)}%.`);
186
+ const reference = await decode(imagePath, width, height);
187
+ const duration = Number.isFinite(meta.seconds) && meta.seconds > 0 ? meta.seconds : 1;
188
+ // Container duration points just past the final decoded frame. Clamp all
189
+ // seeks to that frame so short clips (and VFR outputs) cannot fail closed
190
+ // merely because a probe sample landed in the tail gap.
191
+ const frameStep = Number.isFinite(meta.fps) && meta.fps > 0 ? 1 / meta.fps : 1 / 24;
192
+ const lastFrameTime = Math.max(0, duration - frameStep);
193
+ const times = [...new Set([0, lastFrameTime * 0.25, lastFrameTime * 0.5, lastFrameTime * 0.75, lastFrameTime])];
194
+ const frames = []; const decoded = [];
195
+ for (const time of times) {
196
+ const frame = await decodeNearest(videoPath, width, height, time, duration);
197
+ decoded.push(frame); frames.push(compareH3Plate(reference, frame, width, height, limits));
198
+ }
199
+ const temporal = decoded.slice(1).map((frame) => compareH3FrameStability(decoded[0], frame, width, height, limits));
200
+ const worst = frames.reduce((acc, item) => ({ meanRgb: Math.max(acc.meanRgb, item.meanRgb), p95Rgb: Math.max(acc.p95Rgb, item.p95Rgb), globalP80Rgb: Math.max(acc.globalP80Rgb, item.globalP80Rgb), cameraDriftPx: Math.max(acc.cameraDriftPx, item.cameraDriftPx) }), { meanRgb: 0, p95Rgb: 0, globalP80Rgb: 0, cameraDriftPx: 0 });
201
+ const temporalWorst = temporal.reduce((acc, item) => ({ meanRgb: Math.max(acc.meanRgb, item.meanRgb), p95Rgb: Math.max(acc.p95Rgb, item.p95Rgb), globalP80Rgb: Math.max(acc.globalP80Rgb, item.globalP80Rgb), cameraDriftPx: Math.max(acc.cameraDriftPx, item.cameraDriftPx) }), { meanRgb: 0, p95Rgb: 0, globalP80Rgb: 0, cameraDriftPx: 0 });
202
+ // A verified SAM3/ImageCompositeMasked graph owns the foreground mask and
203
+ // copies the uploaded plate into every non-subject pixel. In that mode a
204
+ // full-frame percentile would mistake legitimate actor motion for a set
205
+ // change (especially in close-up shots). Keep the hard perimeter checks for
206
+ // camera/background drift; retain full-frame checks for unverified callers.
207
+ const plateGlobalPass = compositorVerified || worst.globalP80Rgb <= limits.globalP80Rgb;
208
+ const temporalGlobalPass = compositorVerified || temporalWorst.globalP80Rgb <= limits.temporalGlobalP80Rgb;
209
+ const pass = worst.p95Rgb <= limits.edgeP95Rgb && worst.meanRgb <= limits.edgeMeanRgb && plateGlobalPass && worst.cameraDriftPx <= limits.cameraDriftPx && temporalWorst.p95Rgb <= limits.temporalEdgeP95Rgb && temporalWorst.meanRgb <= limits.temporalEdgeMeanRgb && temporalGlobalPass && temporalWorst.cameraDriftPx <= limits.cameraDriftPx;
210
+ return { pass, width, height, seconds: meta.seconds, aspectError, frames, temporal, worst, temporalWorst, compositorVerified, limits };
211
+ } finally {
212
+ await rm(dir, { recursive: true, force: true });
213
+ }
214
+ }
215
+
216
+ export async function inspectH3OutputFromData({ imageDataUrl, videoBytes, expectedWidth, expectedHeight, limits, compositorVerified = false }) {
217
+ return inspectH3Output({ imageDataUrl, videoBytes, expectedWidth, expectedHeight, limits, compositorVerified });
218
+ }