cozyclay 1.7.1 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +69 -0
- package/README.md +11 -3
- package/bin/agent/agent-routes.mjs +8 -2
- package/bin/agent/h3-preservation.mjs +218 -0
- package/bin/agent/video-adapters.mjs +223 -10
- package/bin/cozyclay.mjs +11 -1
- package/dist/ai-camera-control/index.html +17 -43
- package/dist/app/index.html +5 -4
- package/dist/assets/analytics-Dc9ooyn1.js +1 -0
- package/dist/assets/app-CHc6SS4r.js +4859 -0
- package/dist/assets/app-q2B32RKM.css +1 -0
- package/dist/assets/{demo-CEleUnpD.js → demo-BENMp24T.js} +1 -1
- package/dist/assets/shot-prompt-CuqtYxB7.css +1 -0
- package/dist/assets/shot-prompt-F2C_YN3J.js +15 -0
- package/dist/assets/{ticket-CX73jjoc.js → ticket-BDieXjZs.js} +1 -1
- package/dist/assets/workflow-BHvP2fxY.js +179 -0
- package/dist/assets/workflow-CeQxy8Uq.css +1 -0
- package/dist/ca5b3d011bb7f59fe4232e1e31aec2f0.txt +1 -0
- package/dist/cozyclay-package.json +1 -1
- package/dist/greybox-to-video/index.html +220 -0
- package/dist/index.html +324 -207
- package/dist/llms.txt +17 -0
- package/dist/media/cozyclay-seedance-vs-greybox-poster.jpg +0 -0
- package/dist/media/playground-poster.jpg +0 -0
- package/dist/previs-software/index.html +220 -0
- package/dist/privacy/index.html +153 -0
- package/dist/robots.txt +10 -0
- package/dist/scenes/city-block.cclayproject +1 -0
- package/dist/seedance-camera-control/index.html +199 -0
- package/dist/site.css +118 -0
- package/dist/sitemap.xml +18 -6
- package/dist/sw.js +3 -0
- package/dist/workflow/index.html +7 -4
- package/package.json +2 -2
- package/src/App.jsx +724 -478
- package/src/analytics.js +3 -1
- package/src/app-stage.jsx +16 -1
- package/src/ardy/client.js +3 -1
- package/src/ardy/motion-calibration.js +122 -0
- package/src/ardy/root-drop.js +197 -7
- package/src/ardy/timeline.jsx +65 -41
- package/src/camera-tutorial.jsx +197 -0
- package/src/controls.jsx +11 -0
- package/src/dualview.jsx +8 -0
- package/src/hierarchy-panel.jsx +183 -138
- package/src/locale.js +4 -0
- package/src/main.jsx +20 -7
- package/src/multimodel-ingest.js +20 -2
- package/src/planview.jsx +9 -6
- package/src/playground.js +61 -0
- package/src/project-browser.jsx +19 -1
- package/src/scene-objects.js +13 -2
- package/src/scenes.js +9 -1
- package/src/settings-menu.jsx +117 -0
- package/src/starter-scenes.js +25 -0
- package/src/styles.css +475 -290
- package/src/workflow/AgentPanel.jsx +10 -2
- package/src/workflow/WorkflowBuilder.jsx +22 -5
- package/src/workflow/agent-client.js +13 -1
- package/src/workflow/agent-panel.css +20 -3
- package/src/workflow/workflow.css +1 -0
- package/tools/ardy/BRIDGE.md +10 -2
- package/tools/ardy/README.md +23 -6
- package/tools/ardy/__pycache__/cclay_gvhmr_worker.cpython-313.pyc +0 -0
- package/tools/ardy/bridge.mjs +6 -2
- package/tools/ardy/cclay_gvhmr_worker.py +139 -8
- package/tools/ardy/extract.mjs +56 -51
- package/tools/ardy/mocap-metrics.mjs +67 -33
- package/tools/ardy/mocap-quality-gate.mjs +110 -0
- package/tools/ardy/motion-stabilize.mjs +243 -0
- package/tools/ardy/runners/gvhmr-worker.mjs +15 -12
- package/tools/dev/page-shell.py +117 -0
- package/tools/dev/pages/greybox-to-video.html +147 -0
- package/tools/dev/pages/previs-software.html +146 -0
- package/tools/dev/pages/privacy.html +93 -0
- package/tools/dev/pages/seedance-camera-control.html +126 -0
- package/tools/indexnow.key +1 -0
- package/tools/indexnow.mjs +22 -0
- package/tools/qa/studio-control-count.mjs +63 -0
- package/tools/run-tests.mjs +17 -1
- package/tools/sitemap.mjs +34 -0
- package/dist/assets/analytics-Ce3YvFby.js +0 -1
- package/dist/assets/app-DNYMNzNh.css +0 -1
- package/dist/assets/app-jcR6lsws.js +0 -4859
- package/dist/assets/shot-prompt-s4R0Y9R7.js +0 -14
- package/dist/assets/vision_bundle-jFkh-fIS.js +0 -41
- package/dist/assets/workflow-BcZSgt5C.js +0 -180
- package/dist/assets/workflow-CKCpHJGL.css +0 -1
- package/src/analytics-toggle.jsx +0 -24
- package/src/locale-toggle.jsx +0 -18
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,74 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 1.8.0
|
|
4
|
+
|
|
5
|
+
The Studio's chrome gets a research-backed cut: the same capabilities, a third
|
|
6
|
+
fewer controls on screen at once, and one place for each of them. A first-time
|
|
7
|
+
creator can now learn the camera inside the Studio itself, or on cozyclay.org
|
|
8
|
+
before installing, and keep the scene they made. Video mocap gets a quality
|
|
9
|
+
gate, and AI video takes are checked against the scene they were supposed to
|
|
10
|
+
preserve.
|
|
11
|
+
|
|
12
|
+
### Studio UI simplification
|
|
13
|
+
|
|
14
|
+
- Simultaneously visible controls drop from 49 / 54 / 65 (Scene / Camera /
|
|
15
|
+
Motion mode) to 35 / 38 / 51, measured by `tools/qa/studio-control-count.mjs`
|
|
16
|
+
and recorded with the per-control rationale in `docs/studio-ui-ia.md`.
|
|
17
|
+
- The top bar keeps five actions. One **Export ▾** menu leads with the keyframe
|
|
18
|
+
pack and folds the mp4, depth/normal passes, storyboard and blocking frame
|
|
19
|
+
under it; the three Record buttons and the inspector's duplicate FOV/Recenter
|
|
20
|
+
are gone. **Settings ▾** holds language and analytics.
|
|
21
|
+
- **View ▾** on the viewport bar owns the reference grid, Auto Color and body
|
|
22
|
+
part colours; Move/Rotate/Scale stay on the transform strip only.
|
|
23
|
+
- The scene switcher sits on the hierarchy's root row; the Projects… button,
|
|
24
|
+
the root fold caret and the Characters group row are removed and characters
|
|
25
|
+
sit directly under the scene.
|
|
26
|
+
- Timeline motion tools (IK, Foot, Body, speed, Clear, take bar) render only in
|
|
27
|
+
Motion mode; Foot snap and Body contact appear only while IK is on.
|
|
28
|
+
- The Scene/PlayView tabs are replaced by the viewport's look-through button:
|
|
29
|
+
it enters an explicit preview of the shot camera and Esc returns.
|
|
30
|
+
- Selecting the camera switches to Camera mode; the Placement row appears in
|
|
31
|
+
Motion mode; Generate all waits for at least one prompt block.
|
|
32
|
+
- The `advancedMode` / `beginnerMode` gates that had been hardcoded on are
|
|
33
|
+
removed.
|
|
34
|
+
|
|
35
|
+
### Learn the camera
|
|
36
|
+
|
|
37
|
+
- A seven-step camera tutorial runs inside the Studio: Look, Walk (W A S D
|
|
38
|
+
Q E), Dolly, Orbit, Shot, Rail, Play. Open it from Settings ▾ → Camera
|
|
39
|
+
tutorial or `/app/?tutorial=camera`; each step completes only when the
|
|
40
|
+
gesture actually happens, and the strip never takes the pointer.
|
|
41
|
+
- cozyclay.org embeds a live Studio playground on a preset city block with the
|
|
42
|
+
same seven steps, a second demo reel, and leaner copy. The tutorial ends by
|
|
43
|
+
handing over `npx cozyclay --scene city-block`, which opens the local Studio
|
|
44
|
+
on that starter scene; starter scenes are also offered from the project
|
|
45
|
+
browser, and the visitor can download the scene they made.
|
|
46
|
+
|
|
47
|
+
### Agent panel in the Studio
|
|
48
|
+
|
|
49
|
+
- The Workflow page's Agent chat column is available in the Studio, toggled
|
|
50
|
+
from View ▾ → Panels → Agent panel or Cmd/Ctrl+B. It boots collapsed so the
|
|
51
|
+
mode budgets above are unchanged.
|
|
52
|
+
|
|
53
|
+
### Video mocap and AI video takes
|
|
54
|
+
|
|
55
|
+
- GVHMR is the only extraction backend; the bridge and the Studio return a
|
|
56
|
+
named error instead of silently falling back to browser MediaPipe.
|
|
57
|
+
- GVHMR output goes through adaptive position/rotation stabilisation, explicit
|
|
58
|
+
contact correction, support-surface height (a chair lifts the motion), a
|
|
59
|
+
mocap quality gate, and scene calibration that survives restore. Palette
|
|
60
|
+
segmentation diagnostics are exposed for AI-rendered mannequin clips.
|
|
61
|
+
- H3 video takes are checked against the requested scene and camera: the
|
|
62
|
+
Workflow shows a lock receipt on success and rejects a take whose background
|
|
63
|
+
or camera drifted, keeping no stale preview.
|
|
64
|
+
|
|
65
|
+
### Site
|
|
66
|
+
|
|
67
|
+
- Four search-facing pages (greybox to video, previs software, Seedance camera
|
|
68
|
+
control, privacy), `robots.txt` with an explicit AI-crawler policy,
|
|
69
|
+
`llms.txt`, a generated sitemap with real last-modified dates, and IndexNow
|
|
70
|
+
pings on deploy.
|
|
71
|
+
|
|
3
72
|
## 1.7.1
|
|
4
73
|
|
|
5
74
|
- Open the complete Studio at the development server root; the unfinished
|
package/README.md
CHANGED
|
@@ -18,7 +18,9 @@
|
|
|
18
18
|
</p>
|
|
19
19
|
|
|
20
20
|
<p align="center">
|
|
21
|
-
<a href="https://cozyclay.org
|
|
21
|
+
<a href="https://cozyclay.org/#try">Try it in the browser</a> ·
|
|
22
|
+
<a href="https://cozyclay.org/greybox-to-video/">Greybox to AI video</a> ·
|
|
23
|
+
<a href="https://cozyclay.org/previs-software/">Previs software compared</a> ·
|
|
22
24
|
<a href="#quick-start">Quick start</a> ·
|
|
23
25
|
<a href="#what-you-can-do">Features</a> ·
|
|
24
26
|
<a href="#ai-control-mcp">AI control</a> ·
|
|
@@ -34,7 +36,13 @@ CozyClay is a browser-based 3D staging studio built with Three.js and React Thre
|
|
|
34
36
|
npx cozyclay
|
|
35
37
|
```
|
|
36
38
|
|
|
37
|
-
That is the whole install. **[
|
|
39
|
+
That is the whole install. Not sure yet? **[Try it in the browser first](https://cozyclay.org/#try)** — a seven-step camera tutorial on a live scene — then keep going on your machine with the same set:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
npx cozyclay --scene city-block
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
The studio ships seeded with a pre-generated motion clip, so you can scrub the timeline, drive the cameras and draw a dolly rail straight away — generating *new* motion is optional and uses the Kimodo bridge when configured.
|
|
38
46
|
|
|
39
47
|
## Demo
|
|
40
48
|
|
|
@@ -65,7 +73,7 @@ npx cozyclay
|
|
|
65
73
|
bunx cozyclay
|
|
66
74
|
```
|
|
67
75
|
|
|
68
|
-
That downloads the built studio and opens it at `http://127.0.0.1:5180/app/`. Nothing to compile, no dependency tree to install. Useful flags: `--port 5200`, `--no-open`, `--no-motion`.
|
|
76
|
+
That downloads the built studio and opens it at `http://127.0.0.1:5180/app/`. Nothing to compile, no dependency tree to install. Useful flags: `--port 5200`, `--no-open`, `--no-motion`, `--scene city-block` (start on the bundled starter scene instead of an empty room; the first-run dialog offers the same under **Start from a scene**, and a `.cclayproject` downloaded from the browser tutorial opens with **Open a project**).
|
|
69
77
|
|
|
70
78
|
A global install gives you `cclay`, the same command with less typing. Once a day the launcher checks npm for a newer release and prints a one-line notice after the studio is up; it stays quiet when you're current or offline. `cclay update` installs the latest release, and `--no-update-check` skips the check entirely.
|
|
71
79
|
|
|
@@ -196,8 +196,14 @@ export function createAgentHandler({ auth = defaultAuth, codex, handlers, liveHu
|
|
|
196
196
|
if (!adapter || !adapter.configured()) { json(res, 409, { error: "video provider is not configured" }); return true; }
|
|
197
197
|
try {
|
|
198
198
|
const result = await adapter.generate({ ...value, durationSeconds: Number(value.durationSeconds) });
|
|
199
|
-
json(res, 200, { ...(result.mp4Base64 ? { dataUrl: `data:video/mp4;base64,${result.mp4Base64}` } : { url: result.url }), width: result.width, height: result.height, seconds: result.seconds });
|
|
200
|
-
} catch (error) {
|
|
199
|
+
json(res, 200, { ...(result.mp4Base64 ? { dataUrl: `data:video/mp4;base64,${result.mp4Base64}` } : { url: result.url }), width: result.width, height: result.height, seconds: result.seconds, ...(result.preservation ? { preservation: result.preservation } : {}) });
|
|
200
|
+
} catch (error) {
|
|
201
|
+
// A generated H3 take that fails the plate check is unsafe to show as
|
|
202
|
+
// a locked shot. Keep the distinction visible to the client so it can
|
|
203
|
+
// ask for a retry instead of silently accepting a drifting set.
|
|
204
|
+
const status = error?.code === "h3-preservation-failed" ? 422 : 502;
|
|
205
|
+
json(res, status, { error: error?.message || "video provider failed", ...(error?.preservation ? { preservation: error.preservation } : {}) });
|
|
206
|
+
}
|
|
201
207
|
return true;
|
|
202
208
|
}
|
|
203
209
|
if (req.method !== "POST" || !["/agent/turn", "/agent/stop"].includes(path)) {
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
2
|
+
import { tmpdir } from "node:os";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { spawn } from "node:child_process";
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* H3 is an image-to-video model: a prompt can request a locked camera, but it
|
|
8
|
+
* cannot prove that the returned pixels stayed on that camera. This module
|
|
9
|
+
* is the output-side guard. It compares several decoded frames with the
|
|
10
|
+
* uploaded plate in a border band where the animated performer normally is
|
|
11
|
+
* absent. A request is accepted only when the output dimensions/aspect and
|
|
12
|
+
* the measured plate drift are inside the contract.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
export const H3_OUTPUT_LIMITS = Object.freeze({
|
|
16
|
+
// JPEG/AVC quantisation and the model's first-frame reconstruction create a
|
|
17
|
+
// small amount of noise. A real camera move changes many edge pixels by much
|
|
18
|
+
// more than this value.
|
|
19
|
+
edgeP95Rgb: 14,
|
|
20
|
+
edgeMeanRgb: 6,
|
|
21
|
+
// The full-frame 80th percentile catches a camera translation in the
|
|
22
|
+
// middle of the set even when the border happens to be covered by a subject.
|
|
23
|
+
globalP80Rgb: 16,
|
|
24
|
+
// Once the output's first decoded frame has established the model's codec
|
|
25
|
+
// reconstruction, every later frame must keep the static scene band stable.
|
|
26
|
+
// This catches a camera move that happens to reconstruct to a similar value
|
|
27
|
+
// relative to the uploaded plate (and avoids treating subject motion as set
|
|
28
|
+
// motion).
|
|
29
|
+
temporalEdgeP95Rgb: 14,
|
|
30
|
+
temporalEdgeMeanRgb: 6,
|
|
31
|
+
temporalGlobalP80Rgb: 16,
|
|
32
|
+
cameraDriftPx: 2,
|
|
33
|
+
aspectError: 0.01,
|
|
34
|
+
borderFraction: 0.12,
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
/** Estimate a rigid camera translation against the plate in the static band. */
|
|
38
|
+
export function estimateH3CameraShift(reference, frame, width, height, { borderFraction = H3_OUTPUT_LIMITS.borderFraction, maxShift = 4 } = {}) {
|
|
39
|
+
const borderX = Math.max(1, Math.floor(width * borderFraction));
|
|
40
|
+
const borderY = Math.max(1, Math.floor(height * borderFraction));
|
|
41
|
+
if (width <= borderX * 2 + 2 || height <= borderY * 2 + 2) return { x: 0, y: 0, errorRgb: 0, distancePx: 0 };
|
|
42
|
+
let best = { x: 0, y: 0, error: Infinity }; let baseline = Infinity;
|
|
43
|
+
for (let dy = -maxShift; dy <= maxShift; dy += 1) for (let dx = -maxShift; dx <= maxShift; dx += 1) {
|
|
44
|
+
let sum = 0; let count = 0;
|
|
45
|
+
for (let y = 0; y < height; y += 4) for (let x = 0; x < width; x += 4) {
|
|
46
|
+
// Estimate rigid movement only from the static perimeter. Sampling the
|
|
47
|
+
// interior lets a moving performer win the alignment search and can
|
|
48
|
+
// incorrectly call subject motion a camera correction.
|
|
49
|
+
if (!(x < borderX || x >= width - borderX || y < borderY || y >= height - borderY)) continue;
|
|
50
|
+
if (x + dx < 0 || x + dx >= width || y + dy < 0 || y + dy >= height) continue;
|
|
51
|
+
const a = (y * width + x) * 3; const b = ((y + dy) * width + x + dx) * 3;
|
|
52
|
+
sum += (Math.abs(reference[a] - frame[b]) + Math.abs(reference[a + 1] - frame[b + 1]) + Math.abs(reference[a + 2] - frame[b + 2])) / 3;
|
|
53
|
+
count += 1;
|
|
54
|
+
}
|
|
55
|
+
const error = sum / Math.max(1, count);
|
|
56
|
+
if (dx === 0 && dy === 0) baseline = error;
|
|
57
|
+
if (error < best.error) best = { x: dx, y: dy, error };
|
|
58
|
+
}
|
|
59
|
+
const improvementRgb = Math.max(0, baseline - best.error);
|
|
60
|
+
// A flat/softly lit border has many equally good translations. Only treat a
|
|
61
|
+
// shift as camera evidence when the alignment materially improves error.
|
|
62
|
+
const distancePx = improvementRgb >= 8 ? Math.hypot(best.x, best.y) : 0;
|
|
63
|
+
return { x: best.x, y: best.y, errorRgb: best.error, baselineErrorRgb: baseline, improvementRgb, distancePx };
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Compare two generated frames in the static perimeter and full frame. */
|
|
67
|
+
export function compareH3FrameStability(reference, frame, width, height, { borderFraction = H3_OUTPUT_LIMITS.borderFraction } = {}) {
|
|
68
|
+
if (!(reference instanceof Uint8Array) || !(frame instanceof Uint8Array) || reference.length < width * height * 3 || frame.length < width * height * 3) throw new TypeError("compareH3FrameStability needs two decoded RGB frames");
|
|
69
|
+
const borderX = Math.max(1, Math.floor(width * borderFraction));
|
|
70
|
+
const borderY = Math.max(1, Math.floor(height * borderFraction));
|
|
71
|
+
const edge = []; const global = [];
|
|
72
|
+
for (let y = 0; y < height; y += 2) for (let x = 0; x < width; x += 2) {
|
|
73
|
+
const i = (y * width + x) * 3;
|
|
74
|
+
const d = (Math.abs(reference[i] - frame[i]) + Math.abs(reference[i + 1] - frame[i + 1]) + Math.abs(reference[i + 2] - frame[i + 2])) / 3;
|
|
75
|
+
global.push(d);
|
|
76
|
+
if (x < borderX || x >= width - borderX || y < borderY || y >= height - borderY) edge.push(d);
|
|
77
|
+
}
|
|
78
|
+
edge.sort((a, b) => a - b); global.sort((a, b) => a - b);
|
|
79
|
+
const meanRgb = edge.reduce((sum, value) => sum + value, 0) / Math.max(1, edge.length);
|
|
80
|
+
const p95Rgb = edge[Math.min(edge.length - 1, Math.floor(edge.length * 0.95))] ?? 255;
|
|
81
|
+
const globalP80Rgb = global[Math.min(global.length - 1, Math.floor(global.length * 0.8))] ?? 255;
|
|
82
|
+
const camera = estimateH3CameraShift(reference, frame, width, height, { borderFraction });
|
|
83
|
+
return { meanRgb, p95Rgb, globalP80Rgb, cameraDriftPx: camera.distancePx, cameraShift: { x: camera.x, y: camera.y }, cameraErrorRgb: camera.errorRgb, cameraImprovementRgb: camera.improvementRgb, samples: edge.length, globalSamples: global.length };
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function run(command, args, input = null) {
|
|
87
|
+
return new Promise((resolve, reject) => {
|
|
88
|
+
const child = spawn(command, args, { stdio: ["pipe", "pipe", "pipe"] });
|
|
89
|
+
const out = []; const err = [];
|
|
90
|
+
child.stdout.on("data", (chunk) => out.push(chunk));
|
|
91
|
+
child.stderr.on("data", (chunk) => err.push(chunk));
|
|
92
|
+
child.once("error", reject);
|
|
93
|
+
child.once("close", (code) => {
|
|
94
|
+
if (code === 0) resolve({ stdout: Buffer.concat(out), stderr: Buffer.concat(err).toString("utf8") });
|
|
95
|
+
else reject(new Error(`${command} exited ${code}: ${Buffer.concat(err).toString("utf8").trim()}`));
|
|
96
|
+
});
|
|
97
|
+
if (input) child.stdin.end(input); else child.stdin.end();
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
async function decode(path, width, height, seek = null, inputFormat = null) {
|
|
102
|
+
const args = ["-hide_banner", "-loglevel", "error"];
|
|
103
|
+
if (inputFormat) args.push("-f", inputFormat);
|
|
104
|
+
args.push("-i", path);
|
|
105
|
+
// Place -ss after the input so ffmpeg decodes through the preceding GOP and
|
|
106
|
+
// returns the requested frame. Fast input seeking would repeatedly return
|
|
107
|
+
// the first keyframe on long-GOP H3 clips, making temporal camera drift look
|
|
108
|
+
// stable when it was never inspected.
|
|
109
|
+
if (seek !== null) args.push("-ss", String(Math.max(0, seek)));
|
|
110
|
+
args.push("-vf", `scale=${width}:${height}:flags=bicubic`, "-frames:v", "1", "-f", "rawvideo", "-pix_fmt", "rgb24", "pipe:1");
|
|
111
|
+
const { stdout } = await run("ffmpeg", args);
|
|
112
|
+
const expected = width * height * 3;
|
|
113
|
+
if (stdout.length < expected) throw new Error(`decoded frame is ${stdout.length} bytes; expected ${expected}`);
|
|
114
|
+
return stdout.subarray(0, expected);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
async function decodeNearest(path, width, height, seek, duration) {
|
|
118
|
+
// Short test clips and variable frame rate outputs may have no frame at an
|
|
119
|
+
// arbitrary timestamp near the end. Walk backwards to the nearest decoded
|
|
120
|
+
// frame instead of treating a valid clip as an inspection failure.
|
|
121
|
+
for (let offset = 0; offset <= Math.max(1, duration); offset += 1 / 12) {
|
|
122
|
+
try { return await decode(path, width, height, Math.max(0, seek - offset)); } catch { /* try the preceding frame */ }
|
|
123
|
+
}
|
|
124
|
+
throw new Error("H3 output has no decodable video frame.");
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
async function probe(path) {
|
|
128
|
+
const { stdout } = await run("ffprobe", ["-v", "error", "-select_streams", "v:0", "-show_entries", "stream=width,height,duration,nb_frames,r_frame_rate", "-of", "json", path]);
|
|
129
|
+
const stream = JSON.parse(stdout.toString("utf8"))?.streams?.[0];
|
|
130
|
+
if (!stream || !Number.isFinite(Number(stream.width)) || !Number.isFinite(Number(stream.height))) throw new Error("H3 output has no video stream metadata.");
|
|
131
|
+
const [rateNumerator, rateDenominator] = String(stream.r_frame_rate || "").split("/").map(Number);
|
|
132
|
+
const fps = rateNumerator > 0 && rateDenominator > 0 ? rateNumerator / rateDenominator : null;
|
|
133
|
+
return { width: Number(stream.width), height: Number(stream.height), seconds: Number.isFinite(Number(stream.duration)) ? Number(stream.duration) : null, frames: Number.isFinite(Number(stream.nb_frames)) ? Number(stream.nb_frames) : null, fps };
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** Compare a decoded RGB frame to the plate. Returns mean and p95 RGB error. */
|
|
137
|
+
export function compareH3Plate(reference, frame, width, height, { borderFraction = H3_OUTPUT_LIMITS.borderFraction } = {}) {
|
|
138
|
+
if (!(reference instanceof Uint8Array) || !(frame instanceof Uint8Array) || reference.length < width * height * 3 || frame.length < width * height * 3) throw new TypeError("compareH3Plate needs two decoded RGB frames");
|
|
139
|
+
const borderX = Math.max(1, Math.floor(width * borderFraction));
|
|
140
|
+
const borderY = Math.max(1, Math.floor(height * borderFraction));
|
|
141
|
+
const errors = []; const globalErrors = [];
|
|
142
|
+
for (let y = 0; y < height; y += 2) for (let x = 0; x < width; x += 2) {
|
|
143
|
+
const i = (y * width + x) * 3;
|
|
144
|
+
const d = (Math.abs(reference[i] - frame[i]) + Math.abs(reference[i + 1] - frame[i + 1]) + Math.abs(reference[i + 2] - frame[i + 2])) / 3;
|
|
145
|
+
globalErrors.push(d);
|
|
146
|
+
if (x < borderX || x >= width - borderX || y < borderY || y >= height - borderY) errors.push(d);
|
|
147
|
+
}
|
|
148
|
+
errors.sort((a, b) => a - b);
|
|
149
|
+
globalErrors.sort((a, b) => a - b);
|
|
150
|
+
const mean = errors.reduce((sum, value) => sum + value, 0) / Math.max(1, errors.length);
|
|
151
|
+
const p95 = errors[Math.min(errors.length - 1, Math.floor(errors.length * 0.95))] ?? 255;
|
|
152
|
+
const globalP80 = globalErrors[Math.min(globalErrors.length - 1, Math.floor(globalErrors.length * 0.8))] ?? 255;
|
|
153
|
+
const camera = estimateH3CameraShift(reference, frame, width, height, { borderFraction });
|
|
154
|
+
return { meanRgb: mean, p95Rgb: p95, globalP80Rgb: globalP80, cameraDriftPx: camera.distancePx, cameraShift: { x: camera.x, y: camera.y }, cameraErrorRgb: camera.errorRgb, cameraImprovementRgb: camera.improvementRgb, samples: errors.length, globalSamples: globalErrors.length };
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Decode and validate an H3 output against the uploaded first frame. This is
|
|
159
|
+
* deliberately fail-closed: if ffmpeg/ffprobe cannot inspect the result, the
|
|
160
|
+
* caller must not present it as a scene-locked take.
|
|
161
|
+
*/
|
|
162
|
+
export async function inspectH3Output({ imageDataUrl, videoBytes, expectedWidth, expectedHeight, limits = H3_OUTPUT_LIMITS, compositorVerified = false }) {
|
|
163
|
+
if (!imageDataUrl?.startsWith("data:image/")) throw new Error("H3 preservation requires the uploaded first image.");
|
|
164
|
+
if (!Buffer.isBuffer(videoBytes) || videoBytes.length === 0) throw new Error("H3 preservation requires non-empty video bytes.");
|
|
165
|
+
const dir = await mkdtemp(join(tmpdir(), "cozyclay-h3-"));
|
|
166
|
+
const mime = /^data:([^;,]+)/.exec(imageDataUrl)?.[1] || "image/png";
|
|
167
|
+
const extension = mime.includes("jpeg") || mime.includes("jpg") ? "jpg" : mime.includes("webp") ? "webp" : "png";
|
|
168
|
+
const imagePath = join(dir, `plate.${extension}`); const videoPath = join(dir, "output.mp4");
|
|
169
|
+
try {
|
|
170
|
+
const match = /^data:[^;,]+;base64,(.+)$/.exec(imageDataUrl);
|
|
171
|
+
if (!match) throw new Error("H3 preservation received an invalid image data URL.");
|
|
172
|
+
await writeFile(imagePath, Buffer.from(match[1], "base64"));
|
|
173
|
+
await writeFile(videoPath, videoBytes);
|
|
174
|
+
const meta = await probe(videoPath);
|
|
175
|
+
const plateMeta = await probe(imagePath);
|
|
176
|
+
const width = meta.width; const height = meta.height;
|
|
177
|
+
// H3 may quantise the canvas to its own 32-pixel grid. The uploaded plate
|
|
178
|
+
// is the source of truth for a locked camera; accepting a second requested
|
|
179
|
+
// ratio would allow a crop/reframe to pass the preservation gate.
|
|
180
|
+
const plateAspect = Number(plateMeta.width) / Number(plateMeta.height);
|
|
181
|
+
const requestedAspect = Number(expectedWidth) / Number(expectedHeight);
|
|
182
|
+
const outputAspect = width / height;
|
|
183
|
+
const targetAspect = Number.isFinite(plateAspect) && plateAspect > 0 ? plateAspect : requestedAspect;
|
|
184
|
+
const aspectError = Number.isFinite(targetAspect) && targetAspect > 0 ? Math.abs(outputAspect - targetAspect) / targetAspect : 0;
|
|
185
|
+
if (aspectError > limits.aspectError) throw new Error(`H3 preservation failed: output aspect drift ${(aspectError * 100).toFixed(2)}%.`);
|
|
186
|
+
const reference = await decode(imagePath, width, height);
|
|
187
|
+
const duration = Number.isFinite(meta.seconds) && meta.seconds > 0 ? meta.seconds : 1;
|
|
188
|
+
// Container duration points just past the final decoded frame. Clamp all
|
|
189
|
+
// seeks to that frame so short clips (and VFR outputs) cannot fail closed
|
|
190
|
+
// merely because a probe sample landed in the tail gap.
|
|
191
|
+
const frameStep = Number.isFinite(meta.fps) && meta.fps > 0 ? 1 / meta.fps : 1 / 24;
|
|
192
|
+
const lastFrameTime = Math.max(0, duration - frameStep);
|
|
193
|
+
const times = [...new Set([0, lastFrameTime * 0.25, lastFrameTime * 0.5, lastFrameTime * 0.75, lastFrameTime])];
|
|
194
|
+
const frames = []; const decoded = [];
|
|
195
|
+
for (const time of times) {
|
|
196
|
+
const frame = await decodeNearest(videoPath, width, height, time, duration);
|
|
197
|
+
decoded.push(frame); frames.push(compareH3Plate(reference, frame, width, height, limits));
|
|
198
|
+
}
|
|
199
|
+
const temporal = decoded.slice(1).map((frame) => compareH3FrameStability(decoded[0], frame, width, height, limits));
|
|
200
|
+
const worst = frames.reduce((acc, item) => ({ meanRgb: Math.max(acc.meanRgb, item.meanRgb), p95Rgb: Math.max(acc.p95Rgb, item.p95Rgb), globalP80Rgb: Math.max(acc.globalP80Rgb, item.globalP80Rgb), cameraDriftPx: Math.max(acc.cameraDriftPx, item.cameraDriftPx) }), { meanRgb: 0, p95Rgb: 0, globalP80Rgb: 0, cameraDriftPx: 0 });
|
|
201
|
+
const temporalWorst = temporal.reduce((acc, item) => ({ meanRgb: Math.max(acc.meanRgb, item.meanRgb), p95Rgb: Math.max(acc.p95Rgb, item.p95Rgb), globalP80Rgb: Math.max(acc.globalP80Rgb, item.globalP80Rgb), cameraDriftPx: Math.max(acc.cameraDriftPx, item.cameraDriftPx) }), { meanRgb: 0, p95Rgb: 0, globalP80Rgb: 0, cameraDriftPx: 0 });
|
|
202
|
+
// A verified SAM3/ImageCompositeMasked graph owns the foreground mask and
|
|
203
|
+
// copies the uploaded plate into every non-subject pixel. In that mode a
|
|
204
|
+
// full-frame percentile would mistake legitimate actor motion for a set
|
|
205
|
+
// change (especially in close-up shots). Keep the hard perimeter checks for
|
|
206
|
+
// camera/background drift; retain full-frame checks for unverified callers.
|
|
207
|
+
const plateGlobalPass = compositorVerified || worst.globalP80Rgb <= limits.globalP80Rgb;
|
|
208
|
+
const temporalGlobalPass = compositorVerified || temporalWorst.globalP80Rgb <= limits.temporalGlobalP80Rgb;
|
|
209
|
+
const pass = worst.p95Rgb <= limits.edgeP95Rgb && worst.meanRgb <= limits.edgeMeanRgb && plateGlobalPass && worst.cameraDriftPx <= limits.cameraDriftPx && temporalWorst.p95Rgb <= limits.temporalEdgeP95Rgb && temporalWorst.meanRgb <= limits.temporalEdgeMeanRgb && temporalGlobalPass && temporalWorst.cameraDriftPx <= limits.cameraDriftPx;
|
|
210
|
+
return { pass, width, height, seconds: meta.seconds, aspectError, frames, temporal, worst, temporalWorst, compositorVerified, limits };
|
|
211
|
+
} finally {
|
|
212
|
+
await rm(dir, { recursive: true, force: true });
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
export async function inspectH3OutputFromData({ imageDataUrl, videoBytes, expectedWidth, expectedHeight, limits, compositorVerified = false }) {
|
|
217
|
+
return inspectH3Output({ imageDataUrl, videoBytes, expectedWidth, expectedHeight, limits, compositorVerified });
|
|
218
|
+
}
|
|
@@ -1,7 +1,82 @@
|
|
|
1
|
+
import { inspectH3Output } from "./h3-preservation.mjs";
|
|
2
|
+
|
|
1
3
|
const COMFY_DEFAULT_WIDTH = 1024;
|
|
2
4
|
const COMFY_DEFAULT_HEIGHT = 576;
|
|
3
5
|
const MAX_INLINE_VIDEO = 24 * 1024 * 1024;
|
|
4
6
|
|
|
7
|
+
export function comfyDimensionsForAspect(aspect) {
|
|
8
|
+
if (aspect === "9:16") return { width: 576, height: 1024 };
|
|
9
|
+
if (aspect === "1:1") return { width: 768, height: 768 };
|
|
10
|
+
// H3's latent canvas is quantized to 32-pixel axes; 1024x598 is not a
|
|
11
|
+
// valid canvas and makes MiniMaxH3ImageToVideo reject the graph. 1312x768
|
|
12
|
+
// is the nearest valid 12:7 canvas under H3's 768-short-edge pixel cap.
|
|
13
|
+
if (aspect === "12:7") return { width: 1312, height: 768 };
|
|
14
|
+
// Keep every axis on H3's 32-pixel latent grid. These are the closest
|
|
15
|
+
// representable canvases to the requested cinematic ratios.
|
|
16
|
+
if (aspect === "2.39:1") return { width: 1152, height: 480 };
|
|
17
|
+
if (aspect === "21:9") return { width: 1120, height: 480 };
|
|
18
|
+
if (aspect === "4:3") return { width: 768, height: 576 };
|
|
19
|
+
return { width: COMFY_DEFAULT_WIDTH, height: COMFY_DEFAULT_HEIGHT };
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
// MiniMax H3 is conditioned on the uploaded first frame, but the model is
|
|
23
|
+
// still free to invent a new set or camera unless the contract is repeated in
|
|
24
|
+
// every request. Keep this wording positive and structural: mentioning things
|
|
25
|
+
// such as windows or curtains in a negative prompt has repeatedly caused H3 to
|
|
26
|
+
// hallucinate those objects into the set.
|
|
27
|
+
export const H3_PRESERVATION_CONTRACT =
|
|
28
|
+
"Treat the provided first image as an immutable scene plate for the entire video. Keep the exact background, floor, set geometry, lighting, horizon, lens, viewpoint, framing, and object positions unchanged from the first frame through the last frame. Animate only the foreground human subject. Use one continuous shot with a locked camera: no pan, tilt, orbit, dolly, zoom, crop, reframing, cut, transition, or time jump. Preserve all non-subject pixels and keep the subject's silhouette edges sharp. Motion priority: honor the requested subject action, including climbing, sitting, hanging, jumping, or any other vertical movement; never flatten that movement or force the subject to remain on the floor. This subject-motion rule changes only the performer, never the scene plate or camera.";
|
|
29
|
+
|
|
30
|
+
export function isH3Workflow(workflow) {
|
|
31
|
+
let found = false;
|
|
32
|
+
const walk = (value) => {
|
|
33
|
+
if (found || !value || typeof value !== "object") return;
|
|
34
|
+
if (typeof value.class_type === "string" && /minimax.?h3|h3.*video/i.test(value.class_type)) found = true;
|
|
35
|
+
for (const [key, child] of Object.entries(value)) {
|
|
36
|
+
if (key !== "prompt" && typeof child === "string" && /minimax.?h3|h3.*video/i.test(child)) found = true;
|
|
37
|
+
else if (key !== "prompt") walk(child);
|
|
38
|
+
}
|
|
39
|
+
};
|
|
40
|
+
walk(workflow);
|
|
41
|
+
return found;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function buildH3LockedPrompt(prompt) {
|
|
45
|
+
const source = String(prompt ?? "").trim();
|
|
46
|
+
if (!source) return H3_PRESERVATION_CONTRACT;
|
|
47
|
+
if (source.includes("immutable scene plate for the entire video")) return source;
|
|
48
|
+
return `${source}\n\n${H3_PRESERVATION_CONTRACT}`;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** A Comfy API link is [node id, output index]. A truthy value in
|
|
52
|
+
* `first_frame` is not enough: a stale graph can point H3 at an empty latent
|
|
53
|
+
* or another generated image while a separate LoadImage node sits unused.
|
|
54
|
+
* Follow the link back to the LoadImage node that the adapter will rewrite to
|
|
55
|
+
* the uploaded clay frame. This makes the upload and the H3 condition one
|
|
56
|
+
* connected path, like checking that a film reel is actually threaded through
|
|
57
|
+
* the projector rather than merely present on the desk. */
|
|
58
|
+
function hasFirstFrameCondition(workflow) {
|
|
59
|
+
if (!isH3Workflow(workflow)) return true;
|
|
60
|
+
const nodes = workflow && typeof workflow === "object" ? workflow : {};
|
|
61
|
+
const refId = (value) => Array.isArray(value) && typeof value[0] === "string" ? value[0] : null;
|
|
62
|
+
const reachesLoadImage = (id, visiting = new Set()) => {
|
|
63
|
+
if (!id || visiting.has(id)) return false;
|
|
64
|
+
const node = nodes[id];
|
|
65
|
+
if (!node || typeof node !== "object") return false;
|
|
66
|
+
if (/loadimage/i.test(String(node.class_type || "")) && typeof node.inputs?.image === "string" && node.inputs.image.trim()) return true;
|
|
67
|
+
// Keep each link traversal independent. A dead branch must not mark a
|
|
68
|
+
// shared node as visited and hide a later valid path to LoadImage.
|
|
69
|
+
const nextVisiting = new Set(visiting);
|
|
70
|
+
nextVisiting.add(id);
|
|
71
|
+
return Object.values(node.inputs || {}).some((value) => reachesLoadImage(refId(value), nextVisiting));
|
|
72
|
+
};
|
|
73
|
+
return Object.values(nodes).some((node) => {
|
|
74
|
+
if (!node || typeof node !== "object" || !/minimax.?h3|h3.*video/i.test(String(node.class_type || ""))) return false;
|
|
75
|
+
const value = (node.inputs || {}).first_frame ?? (node.inputs || {}).image;
|
|
76
|
+
return reachesLoadImage(refId(value));
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
|
|
5
80
|
const sleep = (ms, signal) => new Promise((resolve, reject) => {
|
|
6
81
|
if (signal?.aborted) return reject(Object.assign(new Error("Aborted"), { name: "AbortError" }));
|
|
7
82
|
const timer = setTimeout(resolve, ms);
|
|
@@ -33,18 +108,131 @@ function findDimensions(value, fallbackWidth, fallbackHeight) {
|
|
|
33
108
|
return { width, height, seconds };
|
|
34
109
|
}
|
|
35
110
|
|
|
36
|
-
function
|
|
37
|
-
|
|
111
|
+
function preferredVideoOutputNodes(workflow) {
|
|
112
|
+
return new Set(Object.entries(workflow || {})
|
|
113
|
+
.filter(([, node]) => /savevideo|videocombine|videooutput/i.test(String(node?.class_type || "")))
|
|
114
|
+
.map(([id]) => id));
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const isH3Node = (node) => /minimax.?h3|h3.*video/i.test(String(node?.class_type || ""));
|
|
118
|
+
const isLoadImageNode = (node) => /loadimage/i.test(String(node?.class_type || ""));
|
|
119
|
+
const isCompositeNode = (node) => /imagecomposite|compositeimage|imagemattecomposite/i.test(String(node?.class_type || ""));
|
|
120
|
+
const isMaskNode = (node) => {
|
|
121
|
+
if (isCompositeNode(node)) return false;
|
|
122
|
+
// A generic/static mask can hide a broken tracker and would not preserve a
|
|
123
|
+
// moving performer across frames. Only accept a temporal SAM3/tracked-mask
|
|
124
|
+
// node on the compositor path.
|
|
125
|
+
return /sam3[_-]?(videotrack|tracktomask)|tracktomask|videomask/i.test(String(node?.class_type || ""));
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
function h3GuardError(message, reason) {
|
|
129
|
+
return Object.assign(new Error(message), {
|
|
130
|
+
code: "h3-preservation-failed",
|
|
131
|
+
preservation: { pass: false, reason },
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
function linkedNodeIds(value, result = []) {
|
|
136
|
+
if (Array.isArray(value)) {
|
|
137
|
+
if (typeof value[0] === "string") result.push(value[0]);
|
|
138
|
+
else for (const child of value) linkedNodeIds(child, result);
|
|
139
|
+
} else if (value && typeof value === "object") {
|
|
140
|
+
for (const child of Object.values(value)) linkedNodeIds(child, result);
|
|
141
|
+
}
|
|
142
|
+
return result;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function reachesNode(nodes, startId, predicate, visiting = new Set()) {
|
|
146
|
+
if (!startId || visiting.has(startId)) return false;
|
|
147
|
+
const node = nodes?.[startId];
|
|
148
|
+
if (!node || typeof node !== "object") return false;
|
|
149
|
+
if (predicate(node)) return true;
|
|
150
|
+
const nextVisiting = new Set(visiting);
|
|
151
|
+
nextVisiting.add(startId);
|
|
152
|
+
return Object.values(node.inputs || {}).some((value) => linkedNodeIds(value).some((id) => reachesNode(nodes, id, predicate, nextVisiting)));
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* H3 itself is an image-to-video sampler and cannot guarantee that generated
|
|
157
|
+
* pixels outside the performer stay unchanged. A valid production graph must
|
|
158
|
+
* therefore composite the generated subject over the uploaded plate before
|
|
159
|
+
* its final video writer. Keep this structural check alongside the pixel
|
|
160
|
+
* validator: the pixel check catches a bad run, while this check prevents a
|
|
161
|
+
* graph with no deterministic preservation stage from running at all.
|
|
162
|
+
*/
|
|
163
|
+
export function hasH3SceneComposite(workflow, outputNodeIds) {
|
|
164
|
+
const nodes = workflow && typeof workflow === "object" ? workflow : {};
|
|
165
|
+
const outputIds = [...(outputNodeIds || [])].map(String);
|
|
166
|
+
if (!outputIds.length) return { pass: false, outputs: [] };
|
|
167
|
+
const checkOutput = (outputId) => {
|
|
168
|
+
const compositors = new Set();
|
|
169
|
+
const collect = (id, visiting = new Set()) => {
|
|
170
|
+
if (!id || visiting.has(id)) return;
|
|
171
|
+
const node = nodes[id];
|
|
172
|
+
if (!node || typeof node !== "object") return;
|
|
173
|
+
if (isCompositeNode(node)) compositors.add(String(id));
|
|
174
|
+
const nextVisiting = new Set(visiting);
|
|
175
|
+
nextVisiting.add(id);
|
|
176
|
+
for (const value of Object.values(node.inputs || {})) for (const linkedId of linkedNodeIds(value)) collect(linkedId, nextVisiting);
|
|
177
|
+
};
|
|
178
|
+
collect(outputId);
|
|
179
|
+
for (const compositorId of compositors) {
|
|
180
|
+
const node = nodes[compositorId];
|
|
181
|
+
const entries = Object.entries(node.inputs || {});
|
|
182
|
+
const background = entries.filter(([key]) => /destination|background|plate|base/i.test(key));
|
|
183
|
+
const source = entries.filter(([key]) => /source|foreground|overlay|subject|image2|video/i.test(key));
|
|
184
|
+
const mask = entries.filter(([key]) => /mask|alpha|matte/i.test(key));
|
|
185
|
+
const reaches = (entries, predicate) => entries.some(([, value]) => linkedNodeIds(value).some((id) => reachesNode(nodes, id, predicate)));
|
|
186
|
+
// A TrackToMask node by itself is not proof of a subject mask: a stale
|
|
187
|
+
// graph can feed it unrelated track data (or a static all-white mask).
|
|
188
|
+
// Require its SAM3_VideoTrack input to be derived from the H3 frames so
|
|
189
|
+
// the compositor cannot silently paste an untracked/generated scene.
|
|
190
|
+
const reachesTrackedMask = (id, visiting = new Set()) => {
|
|
191
|
+
if (!id || visiting.has(id)) return false;
|
|
192
|
+
const candidate = nodes[id];
|
|
193
|
+
if (!candidate || typeof candidate !== "object") return false;
|
|
194
|
+
const nextVisiting = new Set(visiting);
|
|
195
|
+
nextVisiting.add(id);
|
|
196
|
+
if (/sam3[_-]?videotrack/i.test(String(candidate.class_type || ""))) {
|
|
197
|
+
return linkedNodeIds(candidate.inputs?.images).some((imageId) => reachesNode(nodes, imageId, isH3Node));
|
|
198
|
+
}
|
|
199
|
+
if (/sam3[_-]?tracktomask/i.test(String(candidate.class_type || ""))) {
|
|
200
|
+
return linkedNodeIds(candidate.inputs?.track_data).some((trackId) => reachesTrackedMask(trackId, nextVisiting));
|
|
201
|
+
}
|
|
202
|
+
return Object.values(candidate.inputs || {}).some((value) => linkedNodeIds(value).some((linkedId) => reachesTrackedMask(linkedId, nextVisiting)));
|
|
203
|
+
};
|
|
204
|
+
const hasTrackedMask = mask.some(([, value]) => linkedNodeIds(value).some((id) => reachesTrackedMask(id)));
|
|
205
|
+
if (reaches(background, isLoadImageNode) && reaches(source, isH3Node) && hasTrackedMask) {
|
|
206
|
+
return { pass: true, outputId, compositorId, classType: String(node.class_type || "") };
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
return { pass: false, outputId };
|
|
210
|
+
};
|
|
211
|
+
const outputs = outputIds.map(checkOutput);
|
|
212
|
+
return { pass: outputs.every((output) => output.pass), outputs, ...(outputs.length === 1 ? outputs[0] : {}) };
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
function replaceWorkflowInputs(value, prompt, imageName, width, height, { h3Only = false, h3Context = false } = {}) {
|
|
216
|
+
if (Array.isArray(value)) return value.map((item) => replaceWorkflowInputs(item, prompt, imageName, width, height, { h3Only, h3Context }));
|
|
38
217
|
if (!value || typeof value !== "object") {
|
|
39
218
|
if (typeof value !== "string") return value;
|
|
40
219
|
if (/PROMPT|paste your/i.test(value)) return prompt;
|
|
41
220
|
return value;
|
|
42
221
|
}
|
|
43
222
|
const output = {};
|
|
223
|
+
const h3Node = /minimax.?h3|h3.*video/i.test(String(value.class_type || ""));
|
|
224
|
+
// The prompt lives under the H3 node's `inputs` object in API exports. Carry
|
|
225
|
+
// a narrow target flag into that child object, while leaving prompts on
|
|
226
|
+
// unrelated SAM3/tracker nodes untouched.
|
|
227
|
+
const targetH3Prompt = h3Context || h3Node;
|
|
44
228
|
for (const [key, item] of Object.entries(value)) {
|
|
45
229
|
if (key === "image" && typeof item === "string" && /loadimage/i.test(String(value.class_type || ""))) output[key] = imageName;
|
|
230
|
+
// Workflow exports often contain a real example sentence rather than the
|
|
231
|
+
// literal PROMPT marker. Replace every node prompt so the H3 preservation
|
|
232
|
+
// contract cannot be bypassed by a stale saved sentence.
|
|
233
|
+
else if (key === "prompt" && typeof item === "string" && (!h3Only || targetH3Prompt)) output[key] = prompt;
|
|
46
234
|
else if ((key === "length" || key === "width" || key === "height") && Number.isInteger(Number(item))) output[key] = key === "length" ? Number(item) : (key === "width" ? width : height);
|
|
47
|
-
else output[key] = replaceWorkflowInputs(item, prompt, imageName, width, height);
|
|
235
|
+
else output[key] = replaceWorkflowInputs(item, prompt, imageName, width, height, { h3Only, h3Context: targetH3Prompt });
|
|
48
236
|
}
|
|
49
237
|
return output;
|
|
50
238
|
}
|
|
@@ -52,21 +240,35 @@ function replaceWorkflowInputs(value, prompt, imageName, width, height) {
|
|
|
52
240
|
function createComfy(env, fetchImpl) {
|
|
53
241
|
const base = env.COZYCLAY_COMFY_URL?.replace(/\/$/, "");
|
|
54
242
|
const workflowPath = env.COZYCLAY_COMFY_WORKFLOW;
|
|
55
|
-
let workflow;
|
|
56
243
|
return {
|
|
57
244
|
id: "comfy", name: "ComfyUI",
|
|
58
245
|
configured: () => Boolean(base && workflowPath),
|
|
59
246
|
async generate({ prompt, imageDataUrl, durationSeconds, aspect, fps = 24, signal }) {
|
|
60
|
-
|
|
247
|
+
// Read the graph for every request. Comfy users commonly save a new
|
|
248
|
+
// H3 graph (for example, after adding the compositor lock) while this
|
|
249
|
+
// server stays alive; caching would silently keep submitting the old
|
|
250
|
+
// topology and bypass the current first-frame/output contract.
|
|
251
|
+
const workflow = JSON.parse(await (await import("node:fs/promises")).readFile(workflowPath, "utf8"));
|
|
252
|
+
const h3 = isH3Workflow(workflow);
|
|
253
|
+
const preferredOutputs = h3 ? preferredVideoOutputNodes(workflow) : new Set();
|
|
254
|
+
if (h3 && !hasFirstFrameCondition(workflow)) {
|
|
255
|
+
throw h3GuardError("H3 workflow must connect the uploaded image to the first_frame input; refusing an unlocked camera/background run.", "missing-first-frame");
|
|
256
|
+
}
|
|
257
|
+
if (h3 && !preferredOutputs.size) {
|
|
258
|
+
throw h3GuardError("H3 workflow must expose a final SaveVideo/VideoCombine output; refusing an unverified camera/background run.", "missing-final-output");
|
|
259
|
+
}
|
|
260
|
+
const sceneComposite = h3 ? hasH3SceneComposite(workflow, preferredOutputs) : { pass: true };
|
|
261
|
+
if (h3 && !sceneComposite.pass) {
|
|
262
|
+
throw h3GuardError("H3 workflow must composite the generated subject over the uploaded plate with a tracked mask before final video output; refusing an unlocked camera/background run.", "missing-compositor");
|
|
263
|
+
}
|
|
61
264
|
const form = new FormData();
|
|
62
265
|
form.append("image", dataUrlBlob(imageDataUrl), "cozyclay-frame.png");
|
|
63
266
|
form.append("overwrite", "true");
|
|
64
267
|
const uploaded = await fetchImpl(`${base}/upload/image`, { method: "POST", body: form, signal }).then(jsonResponse);
|
|
65
268
|
const imageName = uploaded.name || uploaded.filename;
|
|
66
269
|
if (!imageName) throw new Error("ComfyUI did not return an uploaded filename.");
|
|
67
|
-
const width
|
|
68
|
-
const
|
|
69
|
-
const promptGraph = replaceWorkflowInputs(workflow, prompt, imageName, width, height);
|
|
270
|
+
const { width, height } = comfyDimensionsForAspect(aspect);
|
|
271
|
+
const promptGraph = replaceWorkflowInputs(workflow, h3 ? buildH3LockedPrompt(prompt) : prompt, imageName, width, height, { h3Only: h3, h3Context: false });
|
|
70
272
|
const queued = await fetchImpl(`${base}/prompt`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ prompt: promptGraph, client_id: `cozyclay-${Date.now()}` }), signal }).then(jsonResponse);
|
|
71
273
|
if (!queued.prompt_id) throw new Error("ComfyUI did not return a prompt id.");
|
|
72
274
|
const deadline = Date.now() + 15 * 60 * 1000;
|
|
@@ -75,7 +277,9 @@ function createComfy(env, fetchImpl) {
|
|
|
75
277
|
history = await fetchImpl(`${base}/history/${encodeURIComponent(queued.prompt_id)}`, { signal }).then(jsonResponse);
|
|
76
278
|
const entry = history[queued.prompt_id] || history;
|
|
77
279
|
if (entry?.outputs && Object.keys(entry.outputs).length) {
|
|
78
|
-
for (const node of Object.
|
|
280
|
+
for (const [nodeId, node] of Object.entries(entry.outputs)) {
|
|
281
|
+
if (preferredOutputs.size && !preferredOutputs.has(String(nodeId))) continue;
|
|
282
|
+
for (const output of Object.values(node || {})) {
|
|
79
283
|
const files = Array.isArray(output) ? output : [output];
|
|
80
284
|
for (const file of files) if (file?.filename && /\.(mp4|webm|gif|mov)$/i.test(file.filename)) {
|
|
81
285
|
const query = new URLSearchParams({ filename: file.filename, subfolder: file.subfolder || "", type: file.type || "output" });
|
|
@@ -83,7 +287,16 @@ function createComfy(env, fetchImpl) {
|
|
|
83
287
|
if (!response.ok) throw new Error(`ComfyUI video fetch failed (${response.status}).`);
|
|
84
288
|
const bytes = Buffer.from(await response.arrayBuffer());
|
|
85
289
|
const dimensions = findDimensions(entry, width, height);
|
|
86
|
-
|
|
290
|
+
let preservation;
|
|
291
|
+
if (h3) {
|
|
292
|
+
preservation = await inspectH3Output({ imageDataUrl, videoBytes: bytes, expectedWidth: width, expectedHeight: height, compositorVerified: sceneComposite.pass });
|
|
293
|
+
if (!preservation.pass) {
|
|
294
|
+
const { worst } = preservation;
|
|
295
|
+
throw Object.assign(new Error(`H3 preservation failed: background/camera drift p95=${worst.p95Rgb.toFixed(1)} RGB, mean=${worst.meanRgb.toFixed(1)} RGB.`), { code: "h3-preservation-failed", preservation });
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
return { mp4Base64: bytes.length <= MAX_INLINE_VIDEO ? bytes.toString("base64") : undefined, url: bytes.length > MAX_INLINE_VIDEO ? `${base}/view?${query}` : undefined, width: dimensions.width, height: dimensions.height, seconds: dimensions.seconds ?? durationSeconds, ...(preservation ? { preservation: { ...preservation, compositor: sceneComposite } } : {}) };
|
|
299
|
+
}
|
|
87
300
|
}
|
|
88
301
|
}
|
|
89
302
|
}
|