cozyclay 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +85 -0
- package/README.md +91 -25
- package/THIRD_PARTY_NOTICES.md +68 -1
- package/bin/cozyclay.mjs +71 -8
- package/dist/CNAME +1 -0
- package/dist/ai-camera-control/index.html +406 -0
- package/dist/app/index.html +57 -0
- package/dist/assets/app-B3U5aut1.js +4811 -0
- package/dist/assets/app-BrRF0wso.css +1 -0
- package/dist/assets/vision_bundle-jFkh-fIS.js +41 -0
- package/dist/favicon.ico +0 -0
- package/dist/fonts/InstrumentSerif-OFL.txt +93 -0
- package/dist/fonts/Inter-OFL.txt +92 -0
- package/dist/fonts/README.md +15 -0
- package/dist/icons/apple-touch-icon.png +0 -0
- package/dist/icons/icon-192.png +0 -0
- package/dist/icons/icon-32.png +0 -0
- package/dist/icons/icon-512.png +0 -0
- package/dist/icons/icon-maskable-512.png +0 -0
- package/dist/index.html +450 -5
- package/dist/manifest.webmanifest +42 -0
- package/dist/media/cozyclay-demo-poster.jpg +0 -0
- package/dist/media/cozyclay-logo.png +0 -0
- package/dist/robots.txt +4 -0
- package/dist/sitemap.xml +15 -0
- package/dist/sw.js +127 -0
- package/mcp/LIVE-PROTOCOL.md +63 -0
- package/mcp/README.md +142 -0
- package/mcp/ardy-prompts.mjs +170 -0
- package/mcp/live-hub.mjs +105 -0
- package/mcp/package.json +24 -0
- package/mcp/server.mjs +1394 -0
- package/package.json +122 -78
- package/src/App.jsx +4235 -1047
- package/src/ardy/client.js +5 -4
- package/src/ardy/cskel27.js +7 -2
- package/src/ardy/ik.js +25 -15
- package/src/ardy/mp4-duration.js +102 -0
- package/src/ardy/npz.js +64 -3
- package/src/ardy/playback.js +31 -1
- package/src/ardy/prompt-clips.js +7 -2
- package/src/ardy/retime.js +211 -0
- package/src/ardy/timeline-coordinates.js +24 -0
- package/src/ardy/timeline.jsx +650 -70
- package/src/ardy/to-cskel27.js +34 -12
- package/src/ardy/trim.js +33 -0
- package/src/ardy/waypoints.js +14 -9
- package/src/asset-pane.jsx +36 -0
- package/src/camera-block.js +60 -0
- package/src/camera-follow.js +49 -10
- package/src/camera-rail-schedule.js +176 -0
- package/src/controls.jsx +16 -2
- package/src/cuts.js +184 -0
- package/src/dualview.jsx +14 -8
- package/src/generation/client.js +56 -0
- package/src/generation/session.js +47 -0
- package/src/generation/shot-spec.js +40 -0
- package/src/generation/use-generation.js +55 -0
- package/src/hierarchy-model.js +104 -30
- package/src/hierarchy-panel.jsx +306 -26
- package/src/live-control.js +122 -0
- package/src/locale-toggle.jsx +18 -0
- package/src/locale.js +34 -0
- package/src/main.jsx +5 -0
- package/src/matte-editor.js +543 -0
- package/src/matte.js +503 -0
- package/src/multimodel-ingest.js +344 -0
- package/src/object-catalog.jsx +32 -3
- package/src/object-gizmo.jsx +43 -15
- package/src/planview.jsx +208 -89
- package/src/pose-extract/browser.js +42 -0
- package/src/pose-extract/detector.js +75 -0
- package/src/pose-extract/fit.js +194 -0
- package/src/pose-extract/index.js +56 -0
- package/src/pose-extract/keyframes.js +74 -0
- package/src/pose-extract/landmarks.js +162 -0
- package/src/pose-extract/take.js +87 -0
- package/src/pose-extract/video-frames.js +91 -0
- package/src/pose-thumbs.js +152 -0
- package/src/poses.js +114 -0
- package/src/posestudio.jsx +434 -22
- package/src/project-browser.jsx +135 -0
- package/src/project.js +289 -0
- package/src/props.jsx +69 -3
- package/src/pwa.js +42 -0
- package/src/result-modal.jsx +124 -0
- package/src/room.jsx +14 -35
- package/src/scene-asset-cache.js +125 -0
- package/src/scene-assets.js +288 -0
- package/src/scene-objects.js +245 -7
- package/src/scenes.js +404 -0
- package/src/shot-authoring.js +246 -77
- package/src/shot.js +8 -4
- package/src/styles.css +2151 -183
- package/src/ui.jsx +54 -5
- package/tools/ardy/BRIDGE.md +73 -6
- package/tools/ardy/README.md +64 -5
- package/tools/ardy/bridge.mjs +146 -390
- package/tools/ardy/bvh-cskel27.mjs +1209 -0
- package/tools/ardy/cclay_constrained_generate.py +561 -0
- package/tools/ardy/cclay_motion_edit.py +74 -9
- package/tools/ardy/cclay_sequence_generate.py +117 -7
- package/tools/ardy/cclay_worker.py +304 -0
- package/tools/ardy/extract.mjs +367 -0
- package/tools/ardy/footage.mjs +462 -0
- package/tools/ardy/merge-text-encoder.py +160 -0
- package/tools/ardy/npz.mjs +74 -9
- package/tools/ardy/run-local.mjs +233 -0
- package/tools/ardy/run-on-box.sh +25 -0
- package/tools/ardy/run-sequence-on-box.sh +15 -0
- package/tools/ardy/runners/index.mjs +23 -0
- package/tools/ardy/runners/local.mjs +452 -0
- package/tools/ardy/runners/proc.mjs +125 -0
- package/tools/ardy/runners/remote.mjs +252 -0
- package/tools/ardy/setup-local.mjs +189 -0
- package/tools/ardy/setup-text-encoder-on-box.sh +93 -0
- package/tools/ardy/setup-text-encoder.py +296 -0
- package/tools/generation/README.md +29 -0
- package/tools/generation/bridge.mjs +149 -0
- package/tools/generation/job-store.mjs +51 -0
- package/tools/generation/providers/kling.mjs +76 -0
- package/tools/generation/providers/runway.mjs +101 -0
- package/tools/generation/providers/seedance.mjs +87 -0
- package/tools/generation/providers/shared.mjs +49 -0
- package/tools/generation/providers/veo.mjs +84 -0
- package/tools/metrics/archive-traffic.mjs +138 -0
- package/dist/assets/index-BW_S1YRy.js +0 -4395
- package/dist/assets/index-nRmmJgap.css +0 -1
- package/tools/ardy/__pycache__/cclay_sequence_generate.cpython-313.pyc +0 -0
- package/tools/ardy/out/gen-1786443326924-c6019e-generated.npz +0 -0
- package/tools/ardy/out/gen-1786443609325-0053db-generated.npz +0 -0
- package/tools/ardy/out/gen-1786443835628-749ed1-generated.npz +0 -0
- package/tools/ardy/out/gen-1786462605247-03cb19-generated.npz +0 -0
- package/tools/qa-browser.mjs +0 -84
- package/tools/qa-crop3.mjs +0 -22
- package/tools/qa-playview.mjs +0 -49
- package/tools/qa-screenshot.mjs +0 -24
- package/tools/qa-visual.mjs +0 -51
|
@@ -0,0 +1,462 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* footage.mjs — the bridge's footage-fetch side: platform pages (YouTube,
|
|
3
|
+
* Vimeo, …) serve a player, not a file, and block cross-origin reads, so the
|
|
4
|
+
* BROWSER can never ingest them. This process can: yt-dlp downloads the
|
|
5
|
+
* video stream, ffmpeg locks it to a constant rate (platform video is often
|
|
6
|
+
* VFR — frame numbers would drift against seconds) no higher than extraction
|
|
7
|
+
* can use, and the result is served back over loopback where the ordinary
|
|
8
|
+
* ingest takes over.
|
|
9
|
+
*
|
|
10
|
+
* Same safety posture as motion delivery: served paths come only from an
|
|
11
|
+
* allowlist this process populated, a client disconnect kills the child
|
|
12
|
+
* process group, and every failure is a NAMED reason the UI can act on.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { spawn } from "node:child_process";
|
|
16
|
+
import { randomBytes } from "node:crypto";
|
|
17
|
+
import { createReadStream, existsSync, mkdirSync, readdirSync, rmSync, statSync } from "node:fs";
|
|
18
|
+
import { dirname, join, sep } from "node:path";
|
|
19
|
+
import { fileURLToPath } from "node:url";
|
|
20
|
+
import { killGroup, track } from "./runners/proc.mjs";
|
|
21
|
+
|
|
22
|
+
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
23
|
+
const FOOTAGE_DIR = join(HERE, "out", "footage");
|
|
24
|
+
// Normalisation locks the clip to a CONSTANT rate and keeps the source's own
|
|
25
|
+
// rate up to the extraction ceiling below: platform video is often VFR (frame
|
|
26
|
+
// numbers drift against seconds), but resampling 30 → 24 would drop every 5th
|
|
27
|
+
// frame and turn smooth motion into a 4-step-1-jump judder that extraction
|
|
28
|
+
// reads as real velocity. The take is retimed to the 24 fps timeline AFTER
|
|
29
|
+
// extraction, in rotation space.
|
|
30
|
+
const FOOTAGE_FPS_FALLBACK = 30;
|
|
31
|
+
const FOOTAGE_FPS_MIN = 10;
|
|
32
|
+
const FOOTAGE_FPS_MAX = 60;
|
|
33
|
+
// The ceiling extraction is allowed to see. SAM-3D-Body solves every frame
|
|
34
|
+
// INDEPENDENTLY and errs a little each time, so twice the rate feeds twice the
|
|
35
|
+
// estimation noise per second while the smoothing windows behind it are
|
|
36
|
+
// counted in FRAMES and therefore span half as much time. Measured on two
|
|
37
|
+
// takes of the same session: a 60 fps / 1632-frame extract trembled 3.36 and
|
|
38
|
+
// 3.54 cm at the feet, a 30 fps / 270-frame one 0.60 and 0.72 cm — five times
|
|
39
|
+
// the wobble, plus double the GPU cost, for frames a 24 fps timeline can
|
|
40
|
+
// never show. Both intake paths clamp here; this is the single edit that
|
|
41
|
+
// moves the ceiling.
|
|
42
|
+
export const EXTRACT_FPS_MAX = 30;
|
|
43
|
+
const FOOTAGE_MAX_S = 900; // 15 min: previs reference, not an archive mirror
|
|
44
|
+
const FOOTAGE_MAX_HEIGHT = 1080;
|
|
45
|
+
const FOOTAGE_ALLOWLIST_MAX = 16; // newest downloads only; evicted ids 404
|
|
46
|
+
const FOOTAGE_ID = /^[0-9]+-[0-9a-f]{6}$/;
|
|
47
|
+
// Prefer the Homebrew binaries over whatever shadows them on PATH: a stale
|
|
48
|
+
// pip-installed yt-dlp is the classic cause of "The page needs to be
|
|
49
|
+
// reloaded" refusals from YouTube, and it sorts FIRST in a default PATH.
|
|
50
|
+
function firstExisting(override, candidates, fallback) {
|
|
51
|
+
if (override) return override;
|
|
52
|
+
for (const candidate of candidates) if (existsSync(candidate)) return candidate;
|
|
53
|
+
return fallback;
|
|
54
|
+
}
|
|
55
|
+
const YTDLP = firstExisting(process.env.CCLAY_YTDLP, ["/opt/homebrew/bin/yt-dlp", "/usr/local/bin/yt-dlp"], "yt-dlp");
|
|
56
|
+
const FFMPEG = firstExisting(process.env.CCLAY_FFMPEG, ["/opt/homebrew/bin/ffmpeg", "/usr/local/bin/ffmpeg"], "ffmpeg");
|
|
57
|
+
const FFPROBE = firstExisting(process.env.CCLAY_FFPROBE, ["/opt/homebrew/bin/ffprobe", "/usr/local/bin/ffprobe"], "ffprobe");
|
|
58
|
+
// The default (web) client serves full resolution but hits YouTube's
|
|
59
|
+
// "confirm you're not a bot" wall on music/mix content; the android client
|
|
60
|
+
// always passes but is capped around 360p. So every yt-dlp call goes web
|
|
61
|
+
// first and falls back to android ONLY on that specific refusal — quality
|
|
62
|
+
// when possible, availability when not.
|
|
63
|
+
const YTDLP_BASE = ["--no-playlist", "--no-warnings"];
|
|
64
|
+
const YTDLP_BOTFIX = ["--extractor-args", "youtube:player_client=android"];
|
|
65
|
+
// Two ways YouTube refuses the web client: the bot wall up front, or a 403
|
|
66
|
+
// on the media URLs partway through a download. Both mean "switch client".
|
|
67
|
+
const WEB_CLIENT_REFUSED = /confirm you.re not a bot|HTTP Error 403/i;
|
|
68
|
+
|
|
69
|
+
const footageAllowlist = new Map(); // id -> absolute mp4 path under FOOTAGE_DIR
|
|
70
|
+
|
|
71
|
+
/** `[download] 42.3% of ...` -> 0.423; null for any other line. */
|
|
72
|
+
export function parseDownloadRatio(line) {
|
|
73
|
+
const match = /\[download\]\s+([\d.]+)%/.exec(line);
|
|
74
|
+
if (!match) return null;
|
|
75
|
+
const percent = Number(match[1]);
|
|
76
|
+
if (!Number.isFinite(percent)) return null;
|
|
77
|
+
return Math.min(1, Math.max(0, percent / 100));
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** ffmpeg's `... time=00:00:12.34 ...` against the clip length -> 0..1. */
|
|
81
|
+
export function parseEncodeRatio(line, durationS) {
|
|
82
|
+
if (!(durationS > 0)) return null;
|
|
83
|
+
const match = /time=(\d+):(\d\d):(\d\d(?:\.\d+)?)/.exec(line);
|
|
84
|
+
if (!match) return null;
|
|
85
|
+
const seconds = Number(match[1]) * 3600 + Number(match[2]) * 60 + Number(match[3]);
|
|
86
|
+
if (!Number.isFinite(seconds)) return null;
|
|
87
|
+
return Math.min(1, Math.max(0, seconds / durationS));
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** yt-dlp's probe line `"123.45\t29.97\tSome title"` -> { durationS, fps,
|
|
91
|
+
* title }. A missing/absurd rate falls back to 30 and is clamped to a sane
|
|
92
|
+
* band — the ffmpeg pass hard-locks whatever rate is returned here. */
|
|
93
|
+
export function parseProbeLine(line) {
|
|
94
|
+
const parts = line.split("\t");
|
|
95
|
+
if (parts.length < 3) return null;
|
|
96
|
+
const durationS = Number(parts[0]);
|
|
97
|
+
const rawFps = Number(parts[1]);
|
|
98
|
+
const title = parts.slice(2).join("\t").trim();
|
|
99
|
+
if (!Number.isFinite(durationS) || durationS <= 0) return null;
|
|
100
|
+
const fps = Number.isFinite(rawFps) && rawFps > 0
|
|
101
|
+
? Math.min(FOOTAGE_FPS_MAX, Math.max(FOOTAGE_FPS_MIN, Math.round(rawFps)))
|
|
102
|
+
: FOOTAGE_FPS_FALLBACK;
|
|
103
|
+
return { durationS, fps, title };
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* The rate a source of `sourceFps` may be fed to extraction at. Never
|
|
108
|
+
* upsamples — 24 stays 24, 25 stays 25 — because inventing frames invents
|
|
109
|
+
* motion. Above the ceiling it prefers an INTEGER divisor of the source
|
|
110
|
+
* (60→30, 50→25, 48→24, 120→30): dropping every n-th frame keeps the
|
|
111
|
+
* surviving frames exactly as they were shot, while landing on a foreign
|
|
112
|
+
* rate has to duplicate frames at uneven intervals and reads as judder. A
|
|
113
|
+
* rate with no divisor that lands in range (35, 45, …) takes the ceiling
|
|
114
|
+
* itself, and an unreadable rate takes it too — the ceiling is the safe
|
|
115
|
+
* answer, never the source's word.
|
|
116
|
+
*/
|
|
117
|
+
export function capExtractFps(sourceFps) {
|
|
118
|
+
const fps = Math.round(Number(sourceFps));
|
|
119
|
+
if (!Number.isFinite(fps) || fps <= 0) return EXTRACT_FPS_MAX;
|
|
120
|
+
if (fps <= EXTRACT_FPS_MAX) return fps;
|
|
121
|
+
const divisor = Math.ceil(fps / EXTRACT_FPS_MAX);
|
|
122
|
+
return fps % divisor === 0 ? fps / divisor : EXTRACT_FPS_MAX;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* The one ffmpeg pass both intake paths use: a CONSTANT `fps`, no audio, a
|
|
127
|
+
* faststart mp4. Nothing here touches presentation timestamps as a whole —
|
|
128
|
+
* `fps=N` only drops or repeats frames, so the clip comes out the same
|
|
129
|
+
* length and the same speed it went in.
|
|
130
|
+
*/
|
|
131
|
+
export function normalizeArgs(inputPath, outputPath, fps) {
|
|
132
|
+
return [
|
|
133
|
+
"-y", "-i", inputPath,
|
|
134
|
+
"-vf", `fps=${fps}`,
|
|
135
|
+
"-an",
|
|
136
|
+
"-c:v", "libx264", "-preset", "veryfast", "-crf", "20",
|
|
137
|
+
"-movflags", "+faststart",
|
|
138
|
+
outputPath,
|
|
139
|
+
];
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** ffprobe's `key=num/den` block -> { fps, durationS }, either field null when
|
|
143
|
+
* the file would not say. avg_frame_rate is frames ÷ length over the whole
|
|
144
|
+
* stream, which is what extraction actually meets; r_frame_rate is only the
|
|
145
|
+
* container's base tick and reads as 90000 on some VFR muxes, so it is the
|
|
146
|
+
* fallback, not the answer. */
|
|
147
|
+
export function parseProbeStreamInfo(text) {
|
|
148
|
+
const source = typeof text === "string" ? text : "";
|
|
149
|
+
const ratio = (key) => {
|
|
150
|
+
const match = new RegExp(`^${key}=([\\d.]+)(?:/([\\d.]+))?$`, "m").exec(source);
|
|
151
|
+
if (!match) return null;
|
|
152
|
+
const value = match[2] === undefined ? Number(match[1]) : Number(match[1]) / Number(match[2]);
|
|
153
|
+
return Number.isFinite(value) && value > 0 ? value : null;
|
|
154
|
+
};
|
|
155
|
+
return { fps: ratio("avg_frame_rate") ?? ratio("r_frame_rate"), durationS: ratio("duration") };
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function runCollect(command, args, { children, timeoutMs, onLine }) {
|
|
159
|
+
return new Promise((resolvePromise, reject) => {
|
|
160
|
+
const child = spawn(command, args, { detached: true, stdio: ["ignore", "pipe", "pipe"] });
|
|
161
|
+
children.add(child);
|
|
162
|
+
track(child);
|
|
163
|
+
let out = "";
|
|
164
|
+
let err = "";
|
|
165
|
+
let buffered = "";
|
|
166
|
+
const timer = timeoutMs
|
|
167
|
+
? setTimeout(() => {
|
|
168
|
+
killGroup(child);
|
|
169
|
+
reject(new Error("footage-timeout"));
|
|
170
|
+
}, timeoutMs)
|
|
171
|
+
: null;
|
|
172
|
+
const feed = (text) => {
|
|
173
|
+
buffered += text;
|
|
174
|
+
// yt-dlp redraws progress with \r, ffmpeg logs stats to stderr the
|
|
175
|
+
// same way; both count as line breaks for the progress parsers.
|
|
176
|
+
const lines = buffered.split(/[\r\n]/);
|
|
177
|
+
buffered = lines.pop() ?? "";
|
|
178
|
+
for (const line of lines) if (line) onLine?.(line);
|
|
179
|
+
};
|
|
180
|
+
child.stdout.on("data", (chunk) => {
|
|
181
|
+
out += chunk;
|
|
182
|
+
feed(String(chunk));
|
|
183
|
+
});
|
|
184
|
+
child.stderr.on("data", (chunk) => {
|
|
185
|
+
err += chunk;
|
|
186
|
+
feed(String(chunk));
|
|
187
|
+
});
|
|
188
|
+
child.on("error", (error) => {
|
|
189
|
+
if (timer) clearTimeout(timer);
|
|
190
|
+
children.delete(child);
|
|
191
|
+
reject(new Error(`spawn ${command}: ${error.message}`));
|
|
192
|
+
});
|
|
193
|
+
child.on("close", (code) => {
|
|
194
|
+
if (timer) clearTimeout(timer);
|
|
195
|
+
children.delete(child);
|
|
196
|
+
if (buffered) onLine?.(buffered);
|
|
197
|
+
if (code === 0) resolvePromise({ out, err });
|
|
198
|
+
else reject(new Error(err.split("\n").filter(Boolean).pop() || `${command} exited ${code}`));
|
|
199
|
+
});
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/** yt-dlp with the web client, retried via the android client only when
|
|
204
|
+
* YouTube's bot wall is the stated reason for the failure. */
|
|
205
|
+
async function runYtdlp(args, options) {
|
|
206
|
+
try {
|
|
207
|
+
return await runCollect(YTDLP, [...YTDLP_BASE, ...args], options);
|
|
208
|
+
} catch (error) {
|
|
209
|
+
if (!WEB_CLIENT_REFUSED.test(error?.message ?? "")) throw error;
|
|
210
|
+
console.error("[bridge] youtube refused the web client; retrying with the android client (≤360p)");
|
|
211
|
+
return runCollect(YTDLP, [...YTDLP_BASE, ...YTDLP_BOTFIX, ...args], options);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
function cleanup(paths) {
|
|
216
|
+
for (const path of paths) {
|
|
217
|
+
try {
|
|
218
|
+
rmSync(path, { force: true });
|
|
219
|
+
} catch {
|
|
220
|
+
/* best effort */
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/**
|
|
226
|
+
* Bring a local video down to EXTRACT_FPS_MAX, if it is over it. Resolves to
|
|
227
|
+
* `{ path, fps, capped }`: `path` is the INPUT untouched whenever the source
|
|
228
|
+
* is already at or under the ceiling, so a bridge download — normalized at
|
|
229
|
+
* this same ceiling on the way in — is never re-encoded a second time for
|
|
230
|
+
* nothing. `fps` is what the file at `path` now runs at, i.e. what extraction
|
|
231
|
+
* will see; it is null only when ffprobe would not name a rate, in which case
|
|
232
|
+
* the clip goes through as it is rather than be refused over a probe hiccup.
|
|
233
|
+
* The ffmpeg failure itself throws, for the caller to name.
|
|
234
|
+
*/
|
|
235
|
+
export async function conformToExtractFps(inputPath, outputPath, { children = new Set(), timeoutMs = 600000, onCap, onProgress } = {}) {
|
|
236
|
+
let probed = { fps: null, durationS: null };
|
|
237
|
+
try {
|
|
238
|
+
const { out } = await runCollect(
|
|
239
|
+
FFPROBE,
|
|
240
|
+
[
|
|
241
|
+
"-v", "error", "-select_streams", "v:0",
|
|
242
|
+
"-show_entries", "stream=avg_frame_rate,r_frame_rate",
|
|
243
|
+
"-show_entries", "format=duration",
|
|
244
|
+
"-of", "default=noprint_wrappers=1",
|
|
245
|
+
inputPath,
|
|
246
|
+
],
|
|
247
|
+
{ children, timeoutMs: 60000 }
|
|
248
|
+
);
|
|
249
|
+
probed = parseProbeStreamInfo(out);
|
|
250
|
+
} catch (err) {
|
|
251
|
+
console.error(`[bridge] fps probe failed for ${inputPath}: ${err.message}`);
|
|
252
|
+
}
|
|
253
|
+
if (probed.fps === null) return { path: inputPath, fps: null, capped: false };
|
|
254
|
+
const targetFps = capExtractFps(probed.fps);
|
|
255
|
+
if (targetFps >= Math.round(probed.fps)) return { path: inputPath, fps: targetFps, capped: false };
|
|
256
|
+
onCap?.(targetFps);
|
|
257
|
+
await runCollect(FFMPEG, normalizeArgs(inputPath, outputPath, targetFps), {
|
|
258
|
+
children,
|
|
259
|
+
timeoutMs,
|
|
260
|
+
onLine: (line) => {
|
|
261
|
+
const ratio = parseEncodeRatio(line, probed.durationS ?? 0);
|
|
262
|
+
if (ratio !== null) onProgress?.(ratio);
|
|
263
|
+
},
|
|
264
|
+
});
|
|
265
|
+
return { path: outputPath, fps: targetFps, capped: true };
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* POST /ardy/footage — body {url}, answer ndjson:
|
|
270
|
+
* {event:"status", message} stage announcements
|
|
271
|
+
* {event:"progress", stage, ratio} download / normalize movement
|
|
272
|
+
* {event:"done", url, title, durationS, fps}
|
|
273
|
+
* {event:"error", message} message is a NAMED reason
|
|
274
|
+
*/
|
|
275
|
+
export async function handleFootage(req, res, readBody) {
|
|
276
|
+
let body;
|
|
277
|
+
try {
|
|
278
|
+
body = JSON.parse(await readBody(req));
|
|
279
|
+
} catch (err) {
|
|
280
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
281
|
+
res.end(`${JSON.stringify({ ok: false, reason: err.message })}\n`);
|
|
282
|
+
return;
|
|
283
|
+
}
|
|
284
|
+
const url = typeof body?.url === "string" ? body.url.trim() : "";
|
|
285
|
+
if (!/^https?:\/\/\S{1,2000}$/.test(url)) {
|
|
286
|
+
res.writeHead(400, { "Content-Type": "application/json" });
|
|
287
|
+
res.end(`${JSON.stringify({ ok: false, reason: "footage-url-invalid" })}\n`);
|
|
288
|
+
return;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
res.writeHead(200, { "Content-Type": "application/x-ndjson", "Cache-Control": "no-store" });
|
|
292
|
+
const send = (obj) => {
|
|
293
|
+
if (res.writableEnded) return;
|
|
294
|
+
try {
|
|
295
|
+
res.write(`${JSON.stringify(obj)}\n`);
|
|
296
|
+
} catch {
|
|
297
|
+
/* socket gone */
|
|
298
|
+
}
|
|
299
|
+
};
|
|
300
|
+
const fail = (message) => {
|
|
301
|
+
send({ event: "error", message });
|
|
302
|
+
res.end();
|
|
303
|
+
};
|
|
304
|
+
|
|
305
|
+
const children = new Set();
|
|
306
|
+
res.on("close", () => {
|
|
307
|
+
if (!res.writableEnded) {
|
|
308
|
+
console.error(`[bridge] client disconnected mid-footage; killing ${children.size} child group(s)`);
|
|
309
|
+
for (const child of children) killGroup(child);
|
|
310
|
+
children.clear();
|
|
311
|
+
}
|
|
312
|
+
});
|
|
313
|
+
|
|
314
|
+
mkdirSync(FOOTAGE_DIR, { recursive: true });
|
|
315
|
+
const stamp = `${Date.now()}-${randomBytes(3).toString("hex")}`;
|
|
316
|
+
const rawPrefix = `raw-${stamp}`;
|
|
317
|
+
const outPath = join(FOOTAGE_DIR, `${stamp}.mp4`);
|
|
318
|
+
|
|
319
|
+
// 1) probe: refuse lives (no duration) and long videos before any bytes.
|
|
320
|
+
send({ event: "status", message: "probing" });
|
|
321
|
+
let probe;
|
|
322
|
+
try {
|
|
323
|
+
const { out } = await runYtdlp(
|
|
324
|
+
["--print", "%(duration)s\t%(fps)s\t%(title)s", url],
|
|
325
|
+
{ children, timeoutMs: 60000 }
|
|
326
|
+
);
|
|
327
|
+
probe = parseProbeLine(out.split("\n").find((line) => line.includes("\t")) ?? "");
|
|
328
|
+
} catch (err) {
|
|
329
|
+
console.error(`[bridge] footage probe failed: ${err.message}`);
|
|
330
|
+
fail("footage-probe-failed");
|
|
331
|
+
return;
|
|
332
|
+
}
|
|
333
|
+
if (!probe) {
|
|
334
|
+
fail("footage-live-unsupported");
|
|
335
|
+
return;
|
|
336
|
+
}
|
|
337
|
+
if (probe.durationS > FOOTAGE_MAX_S) {
|
|
338
|
+
fail("footage-too-long");
|
|
339
|
+
return;
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
// 2) download the video stream only (audio never reaches the ingest).
|
|
343
|
+
send({ event: "status", message: "downloading" });
|
|
344
|
+
try {
|
|
345
|
+
await runYtdlp(
|
|
346
|
+
[
|
|
347
|
+
"--newline",
|
|
348
|
+
"-f", `bv*[height<=${FOOTAGE_MAX_HEIGHT}]/b[height<=${FOOTAGE_MAX_HEIGHT}]/b`,
|
|
349
|
+
"-o", join(FOOTAGE_DIR, `${rawPrefix}.%(ext)s`),
|
|
350
|
+
url,
|
|
351
|
+
],
|
|
352
|
+
{
|
|
353
|
+
children,
|
|
354
|
+
timeoutMs: 10 * 60 * 1000,
|
|
355
|
+
onLine: (line) => {
|
|
356
|
+
const ratio = parseDownloadRatio(line);
|
|
357
|
+
if (ratio !== null) send({ event: "progress", stage: "download", ratio });
|
|
358
|
+
},
|
|
359
|
+
}
|
|
360
|
+
);
|
|
361
|
+
} catch (err) {
|
|
362
|
+
console.error(`[bridge] footage download failed: ${err.message}`);
|
|
363
|
+
cleanup(readdirSync(FOOTAGE_DIR).filter((f) => f.startsWith(rawPrefix)).map((f) => join(FOOTAGE_DIR, f)));
|
|
364
|
+
fail("footage-download-failed");
|
|
365
|
+
return;
|
|
366
|
+
}
|
|
367
|
+
const rawName = readdirSync(FOOTAGE_DIR).find((f) => f.startsWith(rawPrefix));
|
|
368
|
+
if (!rawName) {
|
|
369
|
+
fail("footage-download-failed");
|
|
370
|
+
return;
|
|
371
|
+
}
|
|
372
|
+
const rawPath = join(FOOTAGE_DIR, rawName);
|
|
373
|
+
|
|
374
|
+
// 3) lock to a CONSTANT rate at the source's own fps, capped at the rate
|
|
375
|
+
// extraction can use: VFR platform video would otherwise make "frame 385"
|
|
376
|
+
// a different instant than the timeline believes, and resampling to a
|
|
377
|
+
// foreign rate would judder the motion. Everything downstream — the done
|
|
378
|
+
// event below, the browser probe, the retime — is told THIS number, so the
|
|
379
|
+
// clip's declared rate is the rate SAM will actually meet.
|
|
380
|
+
const targetFps = capExtractFps(probe.fps);
|
|
381
|
+
send({ event: "status", message: "normalizing" });
|
|
382
|
+
try {
|
|
383
|
+
await runCollect(
|
|
384
|
+
FFMPEG,
|
|
385
|
+
normalizeArgs(rawPath, outPath, targetFps),
|
|
386
|
+
{
|
|
387
|
+
children,
|
|
388
|
+
timeoutMs: 10 * 60 * 1000,
|
|
389
|
+
onLine: (line) => {
|
|
390
|
+
const ratio = parseEncodeRatio(line, probe.durationS);
|
|
391
|
+
if (ratio !== null) send({ event: "progress", stage: "normalize", ratio });
|
|
392
|
+
},
|
|
393
|
+
}
|
|
394
|
+
);
|
|
395
|
+
} catch (err) {
|
|
396
|
+
console.error(`[bridge] footage normalize failed: ${err.message}`);
|
|
397
|
+
cleanup([rawPath, outPath]);
|
|
398
|
+
fail("footage-normalize-failed");
|
|
399
|
+
return;
|
|
400
|
+
}
|
|
401
|
+
cleanup([rawPath]);
|
|
402
|
+
|
|
403
|
+
footageAllowlist.set(stamp, outPath);
|
|
404
|
+
if (footageAllowlist.size > FOOTAGE_ALLOWLIST_MAX) {
|
|
405
|
+
const oldest = footageAllowlist.keys().next().value;
|
|
406
|
+
cleanup([footageAllowlist.get(oldest)]);
|
|
407
|
+
footageAllowlist.delete(oldest);
|
|
408
|
+
}
|
|
409
|
+
send({
|
|
410
|
+
event: "done",
|
|
411
|
+
url: `/ardy/footage/${stamp}.mp4`,
|
|
412
|
+
footage: stamp,
|
|
413
|
+
title: probe.title,
|
|
414
|
+
durationS: probe.durationS,
|
|
415
|
+
fps: targetFps,
|
|
416
|
+
});
|
|
417
|
+
res.end();
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
/** Absolute path for an allowlisted footage id, or null — the extract route
|
|
421
|
+
* reuses a bridge-downloaded clip without a second trip through the browser. */
|
|
422
|
+
export function footagePath(id) {
|
|
423
|
+
if (typeof id !== "string" || !FOOTAGE_ID.test(id)) return null;
|
|
424
|
+
return footageAllowlist.get(id) ?? null;
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
/** GET /ardy/footage/<id>.mp4 — allowlist only, exactly like motion delivery. */
|
|
428
|
+
export function serveFootage(req, res, pathname) {
|
|
429
|
+
const sendJson = (status, obj) => {
|
|
430
|
+
res.writeHead(status, { "Content-Type": "application/json", "Cache-Control": "no-store" });
|
|
431
|
+
res.end(`${JSON.stringify(obj)}\n`);
|
|
432
|
+
};
|
|
433
|
+
const match = /^\/ardy\/footage\/([^/]+)\.mp4$/.exec(pathname);
|
|
434
|
+
if (!match || !FOOTAGE_ID.test(match[1]) || !footageAllowlist.has(match[1])) {
|
|
435
|
+
sendJson(404, { ok: false, reason: `unknown or expired footage "${match?.[1] ?? pathname}"` });
|
|
436
|
+
return 404;
|
|
437
|
+
}
|
|
438
|
+
const absPath = footageAllowlist.get(match[1]);
|
|
439
|
+
if (!absPath.startsWith(`${FOOTAGE_DIR}${sep}`)) {
|
|
440
|
+
sendJson(404, { ok: false, reason: "footage path is outside the footage directory" });
|
|
441
|
+
return 404;
|
|
442
|
+
}
|
|
443
|
+
let size;
|
|
444
|
+
try {
|
|
445
|
+
size = statSync(absPath).size;
|
|
446
|
+
} catch {
|
|
447
|
+
sendJson(404, { ok: false, reason: "footage is no longer on disk" });
|
|
448
|
+
return 404;
|
|
449
|
+
}
|
|
450
|
+
res.writeHead(200, {
|
|
451
|
+
"Content-Type": "video/mp4",
|
|
452
|
+
"Content-Length": size,
|
|
453
|
+
"Cache-Control": "no-store",
|
|
454
|
+
});
|
|
455
|
+
createReadStream(absPath)
|
|
456
|
+
.on("error", (err) => {
|
|
457
|
+
console.error(`[bridge] error streaming ${absPath}: ${err.message}`);
|
|
458
|
+
res.destroy();
|
|
459
|
+
})
|
|
460
|
+
.pipe(res);
|
|
461
|
+
return 200;
|
|
462
|
+
}
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""merge-text-encoder.py - bake the MNTP LoRA into the base weights.
|
|
3
|
+
|
|
4
|
+
Run under the ARDY text-encoder venv (needs torch, peft, transformers, and
|
|
5
|
+
the `ardy` package importable — i.e. from the ARDY checkout):
|
|
6
|
+
|
|
7
|
+
cd ~/ardy && .venv/bin/python merge-text-encoder.py --dest ~/cclay-text-encoders
|
|
8
|
+
|
|
9
|
+
Why this exists: transformers v5 removed the automatic PEFT-adapter
|
|
10
|
+
resolution that `from_pretrained` did in v4 (`find_adapter_config_file` is
|
|
11
|
+
gone from modeling_utils). ARDY's vendored LLM2Vec loader relies on that
|
|
12
|
+
behavior — it points `from_pretrained` at the MNTP *adapter* repo and lets
|
|
13
|
+
transformers chase `adapter_config.json` to the (gated) base. Under v5 that
|
|
14
|
+
path simply fails with "no model.safetensors found".
|
|
15
|
+
|
|
16
|
+
So instead of an adapter directory, the local MNTP path becomes a full
|
|
17
|
+
model: this script loads the ungated base (NousResearch mirror, already
|
|
18
|
+
hash-verified by setup-text-encoder.py), applies the MNTP LoRA with peft,
|
|
19
|
+
`merge_and_unload()`s it — the exact math upstream LLM2Vec performs at load
|
|
20
|
+
time — and saves the merged weights to the path ARDY's TEXT_ENCODERS_DIR
|
|
21
|
+
override resolves. Plain full-model loading works identically under
|
|
22
|
+
transformers v4 and v5; the supervised LoRA keeps loading through peft
|
|
23
|
+
directly, which both versions support.
|
|
24
|
+
|
|
25
|
+
Layout before: <dest>/McGill-NLP/...-mntp (downloaded adapter files)
|
|
26
|
+
Layout after: <dest>/McGill-NLP/...-mntp-src (the downloads, kept for
|
|
27
|
+
provenance + re-merges)
|
|
28
|
+
<dest>/McGill-NLP/...-mntp (merged full model +
|
|
29
|
+
tokenizer + MERGE_MANIFEST)
|
|
30
|
+
|
|
31
|
+
The merged config.json keeps `_name_or_path` =
|
|
32
|
+
"meta-llama/Meta-Llama-3-8B-Instruct": it is a metadata string (no download
|
|
33
|
+
or gate involved) that LLM2Vec's `prepare_for_tokenization` matches to pick
|
|
34
|
+
the Llama-3 chat template — losing it would silently change every prompt
|
|
35
|
+
embedding.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
import argparse
|
|
39
|
+
import hashlib
|
|
40
|
+
import json
|
|
41
|
+
import os
|
|
42
|
+
import shutil
|
|
43
|
+
import sys
|
|
44
|
+
|
|
45
|
+
BASE = "NousResearch/Meta-Llama-3-8B-Instruct"
|
|
46
|
+
MNTP = "McGill-NLP/LLM2Vec-Meta-Llama-3-8B-Instruct-mntp"
|
|
47
|
+
MNTP_SRC = MNTP + "-src"
|
|
48
|
+
NAME_OR_PATH = "meta-llama/Meta-Llama-3-8B-Instruct"
|
|
49
|
+
TOKENIZER_FILES = ["tokenizer.json", "tokenizer_config.json", "special_tokens_map.json"]
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def log(msg):
|
|
53
|
+
print("merge-text-encoder: %s" % msg, flush=True)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def fail(msg):
|
|
57
|
+
print("merge-text-encoder: ERROR: %s" % msg, file=sys.stderr, flush=True)
|
|
58
|
+
sys.exit(1)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def sha256_file(path):
|
|
62
|
+
h = hashlib.sha256()
|
|
63
|
+
with open(path, "rb") as f:
|
|
64
|
+
for block in iter(lambda: f.read(1024 * 1024), b""):
|
|
65
|
+
h.update(block)
|
|
66
|
+
return h.hexdigest()
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def main():
|
|
70
|
+
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
71
|
+
ap.add_argument("--dest", default=os.environ.get(
|
|
72
|
+
"TEXT_ENCODERS_DIR", os.path.expanduser("~/cclay-text-encoders")))
|
|
73
|
+
args = ap.parse_args()
|
|
74
|
+
dest = os.path.abspath(os.path.expanduser(args.dest))
|
|
75
|
+
base_dir = os.path.join(dest, BASE)
|
|
76
|
+
mntp_dir = os.path.join(dest, MNTP)
|
|
77
|
+
src_dir = os.path.join(dest, MNTP_SRC)
|
|
78
|
+
|
|
79
|
+
if not os.path.isfile(os.path.join(base_dir, "model.safetensors.index.json")):
|
|
80
|
+
fail("base model missing under %s — run setup-text-encoder.py first" % base_dir)
|
|
81
|
+
|
|
82
|
+
# Migrate a pre-merge tree: the download step used to land the adapter
|
|
83
|
+
# files directly at the mntp path.
|
|
84
|
+
if os.path.isdir(mntp_dir) and not os.path.isdir(src_dir) \
|
|
85
|
+
and os.path.isfile(os.path.join(mntp_dir, "adapter_model.safetensors")) \
|
|
86
|
+
and not os.path.isfile(os.path.join(mntp_dir, "model.safetensors.index.json")):
|
|
87
|
+
os.rename(mntp_dir, src_dir)
|
|
88
|
+
log("moved downloaded adapter layout to %s" % MNTP_SRC)
|
|
89
|
+
|
|
90
|
+
if not os.path.isfile(os.path.join(src_dir, "adapter_model.safetensors")):
|
|
91
|
+
fail("MNTP adapter missing under %s — run setup-text-encoder.py first" % src_dir)
|
|
92
|
+
|
|
93
|
+
adapter_sha = sha256_file(os.path.join(src_dir, "adapter_model.safetensors"))
|
|
94
|
+
manifest_path = os.path.join(mntp_dir, "MERGE_MANIFEST.json")
|
|
95
|
+
if os.path.isfile(manifest_path):
|
|
96
|
+
with open(manifest_path, "r", encoding="utf-8") as f:
|
|
97
|
+
manifest = json.load(f)
|
|
98
|
+
if manifest.get("mntp_adapter_sha256") == adapter_sha \
|
|
99
|
+
and os.path.isfile(os.path.join(mntp_dir, "model.safetensors.index.json")):
|
|
100
|
+
log("merged model already present and matches the adapter hash — nothing to do")
|
|
101
|
+
return
|
|
102
|
+
|
|
103
|
+
import torch # deferred: keep --help fast and errors readable
|
|
104
|
+
import peft
|
|
105
|
+
import transformers
|
|
106
|
+
from peft import PeftModel
|
|
107
|
+
from ardy.model.llm2vec.llm2vec import LLM2Vec
|
|
108
|
+
|
|
109
|
+
log("loading base (bfloat16, cpu) from %s" % base_dir)
|
|
110
|
+
model_class = LLM2Vec._get_model_class("LlamaConfig", enable_bidirectional=True)
|
|
111
|
+
model = model_class.from_pretrained(base_dir, torch_dtype=torch.bfloat16)
|
|
112
|
+
|
|
113
|
+
log("applying + merging the MNTP LoRA from %s" % src_dir)
|
|
114
|
+
model = PeftModel.from_pretrained(model, src_dir)
|
|
115
|
+
model = model.merge_and_unload()
|
|
116
|
+
|
|
117
|
+
tmp_out = mntp_dir + ".merging"
|
|
118
|
+
if os.path.isdir(tmp_out):
|
|
119
|
+
shutil.rmtree(tmp_out)
|
|
120
|
+
log("saving merged model to %s" % mntp_dir)
|
|
121
|
+
model.save_pretrained(tmp_out, safe_serialization=True)
|
|
122
|
+
del model
|
|
123
|
+
|
|
124
|
+
for name in TOKENIZER_FILES:
|
|
125
|
+
src = os.path.join(src_dir, name)
|
|
126
|
+
if os.path.isfile(src):
|
|
127
|
+
shutil.copy2(src, os.path.join(tmp_out, name))
|
|
128
|
+
|
|
129
|
+
# Preserve the chat-template key and the source architecture label.
|
|
130
|
+
cfg_path = os.path.join(tmp_out, "config.json")
|
|
131
|
+
with open(cfg_path, "r", encoding="utf-8") as f:
|
|
132
|
+
cfg = json.load(f)
|
|
133
|
+
with open(os.path.join(src_dir, "config.json"), "r", encoding="utf-8") as f:
|
|
134
|
+
src_cfg = json.load(f)
|
|
135
|
+
cfg["_name_or_path"] = NAME_OR_PATH
|
|
136
|
+
if "architectures" in src_cfg:
|
|
137
|
+
cfg["architectures"] = src_cfg["architectures"]
|
|
138
|
+
with open(cfg_path, "w", encoding="utf-8") as f:
|
|
139
|
+
json.dump(cfg, f, indent=2)
|
|
140
|
+
f.write("\n")
|
|
141
|
+
|
|
142
|
+
with open(os.path.join(tmp_out, "MERGE_MANIFEST.json"), "w", encoding="utf-8") as f:
|
|
143
|
+
json.dump({
|
|
144
|
+
"base": BASE,
|
|
145
|
+
"mntp_adapter_sha256": adapter_sha,
|
|
146
|
+
"torch": torch.__version__,
|
|
147
|
+
"transformers": transformers.__version__,
|
|
148
|
+
"peft": peft.__version__,
|
|
149
|
+
"name_or_path": NAME_OR_PATH,
|
|
150
|
+
}, f, indent=2)
|
|
151
|
+
f.write("\n")
|
|
152
|
+
|
|
153
|
+
if os.path.isdir(mntp_dir):
|
|
154
|
+
shutil.rmtree(mntp_dir)
|
|
155
|
+
os.rename(tmp_out, mntp_dir)
|
|
156
|
+
log("done — %s is a full merged model; the supervised LoRA stays a peft adapter" % MNTP)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
if __name__ == "__main__":
|
|
160
|
+
main()
|