cozyclay 1.1.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. package/CHANGELOG.md +85 -0
  2. package/README.md +91 -25
  3. package/THIRD_PARTY_NOTICES.md +68 -1
  4. package/bin/cozyclay.mjs +71 -8
  5. package/dist/CNAME +1 -0
  6. package/dist/ai-camera-control/index.html +406 -0
  7. package/dist/app/index.html +57 -0
  8. package/dist/assets/app-B3U5aut1.js +4811 -0
  9. package/dist/assets/app-BrRF0wso.css +1 -0
  10. package/dist/assets/vision_bundle-jFkh-fIS.js +41 -0
  11. package/dist/favicon.ico +0 -0
  12. package/dist/fonts/InstrumentSerif-OFL.txt +93 -0
  13. package/dist/fonts/Inter-OFL.txt +92 -0
  14. package/dist/fonts/README.md +15 -0
  15. package/dist/icons/apple-touch-icon.png +0 -0
  16. package/dist/icons/icon-192.png +0 -0
  17. package/dist/icons/icon-32.png +0 -0
  18. package/dist/icons/icon-512.png +0 -0
  19. package/dist/icons/icon-maskable-512.png +0 -0
  20. package/dist/index.html +450 -5
  21. package/dist/manifest.webmanifest +42 -0
  22. package/dist/media/cozyclay-demo-poster.jpg +0 -0
  23. package/dist/media/cozyclay-logo.png +0 -0
  24. package/dist/robots.txt +4 -0
  25. package/dist/sitemap.xml +15 -0
  26. package/dist/sw.js +127 -0
  27. package/mcp/LIVE-PROTOCOL.md +63 -0
  28. package/mcp/README.md +142 -0
  29. package/mcp/ardy-prompts.mjs +170 -0
  30. package/mcp/live-hub.mjs +105 -0
  31. package/mcp/package.json +24 -0
  32. package/mcp/server.mjs +1394 -0
  33. package/package.json +122 -78
  34. package/src/App.jsx +4235 -1047
  35. package/src/ardy/client.js +5 -4
  36. package/src/ardy/cskel27.js +7 -2
  37. package/src/ardy/ik.js +25 -15
  38. package/src/ardy/mp4-duration.js +102 -0
  39. package/src/ardy/npz.js +64 -3
  40. package/src/ardy/playback.js +31 -1
  41. package/src/ardy/prompt-clips.js +7 -2
  42. package/src/ardy/retime.js +211 -0
  43. package/src/ardy/timeline-coordinates.js +24 -0
  44. package/src/ardy/timeline.jsx +650 -70
  45. package/src/ardy/to-cskel27.js +34 -12
  46. package/src/ardy/trim.js +33 -0
  47. package/src/ardy/waypoints.js +14 -9
  48. package/src/asset-pane.jsx +36 -0
  49. package/src/camera-block.js +60 -0
  50. package/src/camera-follow.js +49 -10
  51. package/src/camera-rail-schedule.js +176 -0
  52. package/src/controls.jsx +16 -2
  53. package/src/cuts.js +184 -0
  54. package/src/dualview.jsx +14 -8
  55. package/src/generation/client.js +56 -0
  56. package/src/generation/session.js +47 -0
  57. package/src/generation/shot-spec.js +40 -0
  58. package/src/generation/use-generation.js +55 -0
  59. package/src/hierarchy-model.js +104 -30
  60. package/src/hierarchy-panel.jsx +306 -26
  61. package/src/live-control.js +122 -0
  62. package/src/locale-toggle.jsx +18 -0
  63. package/src/locale.js +34 -0
  64. package/src/main.jsx +5 -0
  65. package/src/matte-editor.js +543 -0
  66. package/src/matte.js +503 -0
  67. package/src/multimodel-ingest.js +344 -0
  68. package/src/object-catalog.jsx +32 -3
  69. package/src/object-gizmo.jsx +43 -15
  70. package/src/planview.jsx +208 -89
  71. package/src/pose-extract/browser.js +42 -0
  72. package/src/pose-extract/detector.js +75 -0
  73. package/src/pose-extract/fit.js +194 -0
  74. package/src/pose-extract/index.js +56 -0
  75. package/src/pose-extract/keyframes.js +74 -0
  76. package/src/pose-extract/landmarks.js +162 -0
  77. package/src/pose-extract/take.js +87 -0
  78. package/src/pose-extract/video-frames.js +91 -0
  79. package/src/pose-thumbs.js +152 -0
  80. package/src/poses.js +114 -0
  81. package/src/posestudio.jsx +434 -22
  82. package/src/project-browser.jsx +135 -0
  83. package/src/project.js +289 -0
  84. package/src/props.jsx +69 -3
  85. package/src/pwa.js +42 -0
  86. package/src/result-modal.jsx +124 -0
  87. package/src/room.jsx +14 -35
  88. package/src/scene-asset-cache.js +125 -0
  89. package/src/scene-assets.js +288 -0
  90. package/src/scene-objects.js +245 -7
  91. package/src/scenes.js +404 -0
  92. package/src/shot-authoring.js +246 -77
  93. package/src/shot.js +8 -4
  94. package/src/styles.css +2151 -183
  95. package/src/ui.jsx +54 -5
  96. package/tools/ardy/BRIDGE.md +73 -6
  97. package/tools/ardy/README.md +64 -5
  98. package/tools/ardy/bridge.mjs +146 -390
  99. package/tools/ardy/bvh-cskel27.mjs +1209 -0
  100. package/tools/ardy/cclay_constrained_generate.py +561 -0
  101. package/tools/ardy/cclay_motion_edit.py +74 -9
  102. package/tools/ardy/cclay_sequence_generate.py +117 -7
  103. package/tools/ardy/cclay_worker.py +304 -0
  104. package/tools/ardy/extract.mjs +367 -0
  105. package/tools/ardy/footage.mjs +462 -0
  106. package/tools/ardy/merge-text-encoder.py +160 -0
  107. package/tools/ardy/npz.mjs +74 -9
  108. package/tools/ardy/run-local.mjs +233 -0
  109. package/tools/ardy/run-on-box.sh +25 -0
  110. package/tools/ardy/run-sequence-on-box.sh +15 -0
  111. package/tools/ardy/runners/index.mjs +23 -0
  112. package/tools/ardy/runners/local.mjs +452 -0
  113. package/tools/ardy/runners/proc.mjs +125 -0
  114. package/tools/ardy/runners/remote.mjs +252 -0
  115. package/tools/ardy/setup-local.mjs +189 -0
  116. package/tools/ardy/setup-text-encoder-on-box.sh +93 -0
  117. package/tools/ardy/setup-text-encoder.py +296 -0
  118. package/tools/generation/README.md +29 -0
  119. package/tools/generation/bridge.mjs +149 -0
  120. package/tools/generation/job-store.mjs +51 -0
  121. package/tools/generation/providers/kling.mjs +76 -0
  122. package/tools/generation/providers/runway.mjs +101 -0
  123. package/tools/generation/providers/seedance.mjs +87 -0
  124. package/tools/generation/providers/shared.mjs +49 -0
  125. package/tools/generation/providers/veo.mjs +84 -0
  126. package/tools/metrics/archive-traffic.mjs +138 -0
  127. package/dist/assets/index-BW_S1YRy.js +0 -4395
  128. package/dist/assets/index-nRmmJgap.css +0 -1
  129. package/tools/ardy/__pycache__/cclay_sequence_generate.cpython-313.pyc +0 -0
  130. package/tools/ardy/out/gen-1786443326924-c6019e-generated.npz +0 -0
  131. package/tools/ardy/out/gen-1786443609325-0053db-generated.npz +0 -0
  132. package/tools/ardy/out/gen-1786443835628-749ed1-generated.npz +0 -0
  133. package/tools/ardy/out/gen-1786462605247-03cb19-generated.npz +0 -0
  134. package/tools/qa-browser.mjs +0 -84
  135. package/tools/qa-crop3.mjs +0 -22
  136. package/tools/qa-playview.mjs +0 -49
  137. package/tools/qa-screenshot.mjs +0 -24
  138. package/tools/qa-visual.mjs +0 -51
@@ -0,0 +1,462 @@
1
+ /**
2
+ * footage.mjs — the bridge's footage-fetch side: platform pages (YouTube,
3
+ * Vimeo, …) serve a player, not a file, and block cross-origin reads, so the
4
+ * BROWSER can never ingest them. This process can: yt-dlp downloads the
5
+ * video stream, ffmpeg locks it to a constant rate (platform video is often
6
+ * VFR — frame numbers would drift against seconds) no higher than extraction
7
+ * can use, and the result is served back over loopback where the ordinary
8
+ * ingest takes over.
9
+ *
10
+ * Same safety posture as motion delivery: served paths come only from an
11
+ * allowlist this process populated, a client disconnect kills the child
12
+ * process group, and every failure is a NAMED reason the UI can act on.
13
+ */
14
+
15
+ import { spawn } from "node:child_process";
16
+ import { randomBytes } from "node:crypto";
17
+ import { createReadStream, existsSync, mkdirSync, readdirSync, rmSync, statSync } from "node:fs";
18
+ import { dirname, join, sep } from "node:path";
19
+ import { fileURLToPath } from "node:url";
20
+ import { killGroup, track } from "./runners/proc.mjs";
21
+
22
+ const HERE = dirname(fileURLToPath(import.meta.url));
23
+ const FOOTAGE_DIR = join(HERE, "out", "footage");
24
+ // Normalisation locks the clip to a CONSTANT rate and keeps the source's own
25
+ // rate up to the extraction ceiling below: platform video is often VFR (frame
26
+ // numbers drift against seconds), but resampling 30 → 24 would drop every 5th
27
+ // frame and turn smooth motion into a 4-step-1-jump judder that extraction
28
+ // reads as real velocity. The take is retimed to the 24 fps timeline AFTER
29
+ // extraction, in rotation space.
30
+ const FOOTAGE_FPS_FALLBACK = 30;
31
+ const FOOTAGE_FPS_MIN = 10;
32
+ const FOOTAGE_FPS_MAX = 60;
33
+ // The ceiling extraction is allowed to see. SAM-3D-Body solves every frame
34
+ // INDEPENDENTLY and errs a little each time, so twice the rate feeds twice the
35
+ // estimation noise per second while the smoothing windows behind it are
36
+ // counted in FRAMES and therefore span half as much time. Measured on two
37
+ // takes of the same session: a 60 fps / 1632-frame extract trembled 3.36 and
38
+ // 3.54 cm at the feet, a 30 fps / 270-frame one 0.60 and 0.72 cm — five times
39
+ // the wobble, plus double the GPU cost, for frames a 24 fps timeline can
40
+ // never show. Both intake paths clamp here; this is the single edit that
41
+ // moves the ceiling.
42
+ export const EXTRACT_FPS_MAX = 30;
43
+ const FOOTAGE_MAX_S = 900; // 15 min: previs reference, not an archive mirror
44
+ const FOOTAGE_MAX_HEIGHT = 1080;
45
+ const FOOTAGE_ALLOWLIST_MAX = 16; // newest downloads only; evicted ids 404
46
+ const FOOTAGE_ID = /^[0-9]+-[0-9a-f]{6}$/;
47
+ // Prefer the Homebrew binaries over whatever shadows them on PATH: a stale
48
+ // pip-installed yt-dlp is the classic cause of "The page needs to be
49
+ // reloaded" refusals from YouTube, and it sorts FIRST in a default PATH.
50
+ function firstExisting(override, candidates, fallback) {
51
+ if (override) return override;
52
+ for (const candidate of candidates) if (existsSync(candidate)) return candidate;
53
+ return fallback;
54
+ }
55
+ const YTDLP = firstExisting(process.env.CCLAY_YTDLP, ["/opt/homebrew/bin/yt-dlp", "/usr/local/bin/yt-dlp"], "yt-dlp");
56
+ const FFMPEG = firstExisting(process.env.CCLAY_FFMPEG, ["/opt/homebrew/bin/ffmpeg", "/usr/local/bin/ffmpeg"], "ffmpeg");
57
+ const FFPROBE = firstExisting(process.env.CCLAY_FFPROBE, ["/opt/homebrew/bin/ffprobe", "/usr/local/bin/ffprobe"], "ffprobe");
58
+ // The default (web) client serves full resolution but hits YouTube's
59
+ // "confirm you're not a bot" wall on music/mix content; the android client
60
+ // always passes but is capped around 360p. So every yt-dlp call goes web
61
+ // first and falls back to android ONLY on that specific refusal — quality
62
+ // when possible, availability when not.
63
+ const YTDLP_BASE = ["--no-playlist", "--no-warnings"];
64
+ const YTDLP_BOTFIX = ["--extractor-args", "youtube:player_client=android"];
65
+ // Two ways YouTube refuses the web client: the bot wall up front, or a 403
66
+ // on the media URLs partway through a download. Both mean "switch client".
67
+ const WEB_CLIENT_REFUSED = /confirm you.re not a bot|HTTP Error 403/i;
68
+
69
+ const footageAllowlist = new Map(); // id -> absolute mp4 path under FOOTAGE_DIR
70
+
71
+ /** `[download] 42.3% of ...` -> 0.423; null for any other line. */
72
+ export function parseDownloadRatio(line) {
73
+ const match = /\[download\]\s+([\d.]+)%/.exec(line);
74
+ if (!match) return null;
75
+ const percent = Number(match[1]);
76
+ if (!Number.isFinite(percent)) return null;
77
+ return Math.min(1, Math.max(0, percent / 100));
78
+ }
79
+
80
+ /** ffmpeg's `... time=00:00:12.34 ...` against the clip length -> 0..1. */
81
+ export function parseEncodeRatio(line, durationS) {
82
+ if (!(durationS > 0)) return null;
83
+ const match = /time=(\d+):(\d\d):(\d\d(?:\.\d+)?)/.exec(line);
84
+ if (!match) return null;
85
+ const seconds = Number(match[1]) * 3600 + Number(match[2]) * 60 + Number(match[3]);
86
+ if (!Number.isFinite(seconds)) return null;
87
+ return Math.min(1, Math.max(0, seconds / durationS));
88
+ }
89
+
90
+ /** yt-dlp's probe line `"123.45\t29.97\tSome title"` -> { durationS, fps,
91
+ * title }. A missing/absurd rate falls back to 30 and is clamped to a sane
92
+ * band — the ffmpeg pass hard-locks whatever rate is returned here. */
93
+ export function parseProbeLine(line) {
94
+ const parts = line.split("\t");
95
+ if (parts.length < 3) return null;
96
+ const durationS = Number(parts[0]);
97
+ const rawFps = Number(parts[1]);
98
+ const title = parts.slice(2).join("\t").trim();
99
+ if (!Number.isFinite(durationS) || durationS <= 0) return null;
100
+ const fps = Number.isFinite(rawFps) && rawFps > 0
101
+ ? Math.min(FOOTAGE_FPS_MAX, Math.max(FOOTAGE_FPS_MIN, Math.round(rawFps)))
102
+ : FOOTAGE_FPS_FALLBACK;
103
+ return { durationS, fps, title };
104
+ }
105
+
106
+ /**
107
+ * The rate a source of `sourceFps` may be fed to extraction at. Never
108
+ * upsamples — 24 stays 24, 25 stays 25 — because inventing frames invents
109
+ * motion. Above the ceiling it prefers an INTEGER divisor of the source
110
+ * (60→30, 50→25, 48→24, 120→30): dropping every n-th frame keeps the
111
+ * surviving frames exactly as they were shot, while landing on a foreign
112
+ * rate has to duplicate frames at uneven intervals and reads as judder. A
113
+ * rate with no divisor that lands in range (35, 45, …) takes the ceiling
114
+ * itself, and an unreadable rate takes it too — the ceiling is the safe
115
+ * answer, never the source's word.
116
+ */
117
+ export function capExtractFps(sourceFps) {
118
+ const fps = Math.round(Number(sourceFps));
119
+ if (!Number.isFinite(fps) || fps <= 0) return EXTRACT_FPS_MAX;
120
+ if (fps <= EXTRACT_FPS_MAX) return fps;
121
+ const divisor = Math.ceil(fps / EXTRACT_FPS_MAX);
122
+ return fps % divisor === 0 ? fps / divisor : EXTRACT_FPS_MAX;
123
+ }
124
+
125
+ /**
126
+ * The one ffmpeg pass both intake paths use: a CONSTANT `fps`, no audio, a
127
+ * faststart mp4. Nothing here touches presentation timestamps as a whole —
128
+ * `fps=N` only drops or repeats frames, so the clip comes out the same
129
+ * length and the same speed it went in.
130
+ */
131
+ export function normalizeArgs(inputPath, outputPath, fps) {
132
+ return [
133
+ "-y", "-i", inputPath,
134
+ "-vf", `fps=${fps}`,
135
+ "-an",
136
+ "-c:v", "libx264", "-preset", "veryfast", "-crf", "20",
137
+ "-movflags", "+faststart",
138
+ outputPath,
139
+ ];
140
+ }
141
+
142
+ /** ffprobe's `key=num/den` block -> { fps, durationS }, either field null when
143
+ * the file would not say. avg_frame_rate is frames ÷ length over the whole
144
+ * stream, which is what extraction actually meets; r_frame_rate is only the
145
+ * container's base tick and reads as 90000 on some VFR muxes, so it is the
146
+ * fallback, not the answer. */
147
+ export function parseProbeStreamInfo(text) {
148
+ const source = typeof text === "string" ? text : "";
149
+ const ratio = (key) => {
150
+ const match = new RegExp(`^${key}=([\\d.]+)(?:/([\\d.]+))?$`, "m").exec(source);
151
+ if (!match) return null;
152
+ const value = match[2] === undefined ? Number(match[1]) : Number(match[1]) / Number(match[2]);
153
+ return Number.isFinite(value) && value > 0 ? value : null;
154
+ };
155
+ return { fps: ratio("avg_frame_rate") ?? ratio("r_frame_rate"), durationS: ratio("duration") };
156
+ }
157
+
158
+ function runCollect(command, args, { children, timeoutMs, onLine }) {
159
+ return new Promise((resolvePromise, reject) => {
160
+ const child = spawn(command, args, { detached: true, stdio: ["ignore", "pipe", "pipe"] });
161
+ children.add(child);
162
+ track(child);
163
+ let out = "";
164
+ let err = "";
165
+ let buffered = "";
166
+ const timer = timeoutMs
167
+ ? setTimeout(() => {
168
+ killGroup(child);
169
+ reject(new Error("footage-timeout"));
170
+ }, timeoutMs)
171
+ : null;
172
+ const feed = (text) => {
173
+ buffered += text;
174
+ // yt-dlp redraws progress with \r, ffmpeg logs stats to stderr the
175
+ // same way; both count as line breaks for the progress parsers.
176
+ const lines = buffered.split(/[\r\n]/);
177
+ buffered = lines.pop() ?? "";
178
+ for (const line of lines) if (line) onLine?.(line);
179
+ };
180
+ child.stdout.on("data", (chunk) => {
181
+ out += chunk;
182
+ feed(String(chunk));
183
+ });
184
+ child.stderr.on("data", (chunk) => {
185
+ err += chunk;
186
+ feed(String(chunk));
187
+ });
188
+ child.on("error", (error) => {
189
+ if (timer) clearTimeout(timer);
190
+ children.delete(child);
191
+ reject(new Error(`spawn ${command}: ${error.message}`));
192
+ });
193
+ child.on("close", (code) => {
194
+ if (timer) clearTimeout(timer);
195
+ children.delete(child);
196
+ if (buffered) onLine?.(buffered);
197
+ if (code === 0) resolvePromise({ out, err });
198
+ else reject(new Error(err.split("\n").filter(Boolean).pop() || `${command} exited ${code}`));
199
+ });
200
+ });
201
+ }
202
+
203
+ /** yt-dlp with the web client, retried via the android client only when
204
+ * YouTube's bot wall is the stated reason for the failure. */
205
+ async function runYtdlp(args, options) {
206
+ try {
207
+ return await runCollect(YTDLP, [...YTDLP_BASE, ...args], options);
208
+ } catch (error) {
209
+ if (!WEB_CLIENT_REFUSED.test(error?.message ?? "")) throw error;
210
+ console.error("[bridge] youtube refused the web client; retrying with the android client (≤360p)");
211
+ return runCollect(YTDLP, [...YTDLP_BASE, ...YTDLP_BOTFIX, ...args], options);
212
+ }
213
+ }
214
+
215
+ function cleanup(paths) {
216
+ for (const path of paths) {
217
+ try {
218
+ rmSync(path, { force: true });
219
+ } catch {
220
+ /* best effort */
221
+ }
222
+ }
223
+ }
224
+
225
+ /**
226
+ * Bring a local video down to EXTRACT_FPS_MAX, if it is over it. Resolves to
227
+ * `{ path, fps, capped }`: `path` is the INPUT untouched whenever the source
228
+ * is already at or under the ceiling, so a bridge download — normalized at
229
+ * this same ceiling on the way in — is never re-encoded a second time for
230
+ * nothing. `fps` is what the file at `path` now runs at, i.e. what extraction
231
+ * will see; it is null only when ffprobe would not name a rate, in which case
232
+ * the clip goes through as it is rather than be refused over a probe hiccup.
233
+ * The ffmpeg failure itself throws, for the caller to name.
234
+ */
235
+ export async function conformToExtractFps(inputPath, outputPath, { children = new Set(), timeoutMs = 600000, onCap, onProgress } = {}) {
236
+ let probed = { fps: null, durationS: null };
237
+ try {
238
+ const { out } = await runCollect(
239
+ FFPROBE,
240
+ [
241
+ "-v", "error", "-select_streams", "v:0",
242
+ "-show_entries", "stream=avg_frame_rate,r_frame_rate",
243
+ "-show_entries", "format=duration",
244
+ "-of", "default=noprint_wrappers=1",
245
+ inputPath,
246
+ ],
247
+ { children, timeoutMs: 60000 }
248
+ );
249
+ probed = parseProbeStreamInfo(out);
250
+ } catch (err) {
251
+ console.error(`[bridge] fps probe failed for ${inputPath}: ${err.message}`);
252
+ }
253
+ if (probed.fps === null) return { path: inputPath, fps: null, capped: false };
254
+ const targetFps = capExtractFps(probed.fps);
255
+ if (targetFps >= Math.round(probed.fps)) return { path: inputPath, fps: targetFps, capped: false };
256
+ onCap?.(targetFps);
257
+ await runCollect(FFMPEG, normalizeArgs(inputPath, outputPath, targetFps), {
258
+ children,
259
+ timeoutMs,
260
+ onLine: (line) => {
261
+ const ratio = parseEncodeRatio(line, probed.durationS ?? 0);
262
+ if (ratio !== null) onProgress?.(ratio);
263
+ },
264
+ });
265
+ return { path: outputPath, fps: targetFps, capped: true };
266
+ }
267
+
268
+ /**
269
+ * POST /ardy/footage — body {url}, answer ndjson:
270
+ * {event:"status", message} stage announcements
271
+ * {event:"progress", stage, ratio} download / normalize movement
272
+ * {event:"done", url, title, durationS, fps}
273
+ * {event:"error", message} message is a NAMED reason
274
+ */
275
+ export async function handleFootage(req, res, readBody) {
276
+ let body;
277
+ try {
278
+ body = JSON.parse(await readBody(req));
279
+ } catch (err) {
280
+ res.writeHead(400, { "Content-Type": "application/json" });
281
+ res.end(`${JSON.stringify({ ok: false, reason: err.message })}\n`);
282
+ return;
283
+ }
284
+ const url = typeof body?.url === "string" ? body.url.trim() : "";
285
+ if (!/^https?:\/\/\S{1,2000}$/.test(url)) {
286
+ res.writeHead(400, { "Content-Type": "application/json" });
287
+ res.end(`${JSON.stringify({ ok: false, reason: "footage-url-invalid" })}\n`);
288
+ return;
289
+ }
290
+
291
+ res.writeHead(200, { "Content-Type": "application/x-ndjson", "Cache-Control": "no-store" });
292
+ const send = (obj) => {
293
+ if (res.writableEnded) return;
294
+ try {
295
+ res.write(`${JSON.stringify(obj)}\n`);
296
+ } catch {
297
+ /* socket gone */
298
+ }
299
+ };
300
+ const fail = (message) => {
301
+ send({ event: "error", message });
302
+ res.end();
303
+ };
304
+
305
+ const children = new Set();
306
+ res.on("close", () => {
307
+ if (!res.writableEnded) {
308
+ console.error(`[bridge] client disconnected mid-footage; killing ${children.size} child group(s)`);
309
+ for (const child of children) killGroup(child);
310
+ children.clear();
311
+ }
312
+ });
313
+
314
+ mkdirSync(FOOTAGE_DIR, { recursive: true });
315
+ const stamp = `${Date.now()}-${randomBytes(3).toString("hex")}`;
316
+ const rawPrefix = `raw-${stamp}`;
317
+ const outPath = join(FOOTAGE_DIR, `${stamp}.mp4`);
318
+
319
+ // 1) probe: refuse lives (no duration) and long videos before any bytes.
320
+ send({ event: "status", message: "probing" });
321
+ let probe;
322
+ try {
323
+ const { out } = await runYtdlp(
324
+ ["--print", "%(duration)s\t%(fps)s\t%(title)s", url],
325
+ { children, timeoutMs: 60000 }
326
+ );
327
+ probe = parseProbeLine(out.split("\n").find((line) => line.includes("\t")) ?? "");
328
+ } catch (err) {
329
+ console.error(`[bridge] footage probe failed: ${err.message}`);
330
+ fail("footage-probe-failed");
331
+ return;
332
+ }
333
+ if (!probe) {
334
+ fail("footage-live-unsupported");
335
+ return;
336
+ }
337
+ if (probe.durationS > FOOTAGE_MAX_S) {
338
+ fail("footage-too-long");
339
+ return;
340
+ }
341
+
342
+ // 2) download the video stream only (audio never reaches the ingest).
343
+ send({ event: "status", message: "downloading" });
344
+ try {
345
+ await runYtdlp(
346
+ [
347
+ "--newline",
348
+ "-f", `bv*[height<=${FOOTAGE_MAX_HEIGHT}]/b[height<=${FOOTAGE_MAX_HEIGHT}]/b`,
349
+ "-o", join(FOOTAGE_DIR, `${rawPrefix}.%(ext)s`),
350
+ url,
351
+ ],
352
+ {
353
+ children,
354
+ timeoutMs: 10 * 60 * 1000,
355
+ onLine: (line) => {
356
+ const ratio = parseDownloadRatio(line);
357
+ if (ratio !== null) send({ event: "progress", stage: "download", ratio });
358
+ },
359
+ }
360
+ );
361
+ } catch (err) {
362
+ console.error(`[bridge] footage download failed: ${err.message}`);
363
+ cleanup(readdirSync(FOOTAGE_DIR).filter((f) => f.startsWith(rawPrefix)).map((f) => join(FOOTAGE_DIR, f)));
364
+ fail("footage-download-failed");
365
+ return;
366
+ }
367
+ const rawName = readdirSync(FOOTAGE_DIR).find((f) => f.startsWith(rawPrefix));
368
+ if (!rawName) {
369
+ fail("footage-download-failed");
370
+ return;
371
+ }
372
+ const rawPath = join(FOOTAGE_DIR, rawName);
373
+
374
+ // 3) lock to a CONSTANT rate at the source's own fps, capped at the rate
375
+ // extraction can use: VFR platform video would otherwise make "frame 385"
376
+ // a different instant than the timeline believes, and resampling to a
377
+ // foreign rate would judder the motion. Everything downstream — the done
378
+ // event below, the browser probe, the retime — is told THIS number, so the
379
+ // clip's declared rate is the rate SAM will actually meet.
380
+ const targetFps = capExtractFps(probe.fps);
381
+ send({ event: "status", message: "normalizing" });
382
+ try {
383
+ await runCollect(
384
+ FFMPEG,
385
+ normalizeArgs(rawPath, outPath, targetFps),
386
+ {
387
+ children,
388
+ timeoutMs: 10 * 60 * 1000,
389
+ onLine: (line) => {
390
+ const ratio = parseEncodeRatio(line, probe.durationS);
391
+ if (ratio !== null) send({ event: "progress", stage: "normalize", ratio });
392
+ },
393
+ }
394
+ );
395
+ } catch (err) {
396
+ console.error(`[bridge] footage normalize failed: ${err.message}`);
397
+ cleanup([rawPath, outPath]);
398
+ fail("footage-normalize-failed");
399
+ return;
400
+ }
401
+ cleanup([rawPath]);
402
+
403
+ footageAllowlist.set(stamp, outPath);
404
+ if (footageAllowlist.size > FOOTAGE_ALLOWLIST_MAX) {
405
+ const oldest = footageAllowlist.keys().next().value;
406
+ cleanup([footageAllowlist.get(oldest)]);
407
+ footageAllowlist.delete(oldest);
408
+ }
409
+ send({
410
+ event: "done",
411
+ url: `/ardy/footage/${stamp}.mp4`,
412
+ footage: stamp,
413
+ title: probe.title,
414
+ durationS: probe.durationS,
415
+ fps: targetFps,
416
+ });
417
+ res.end();
418
+ }
419
+
420
+ /** Absolute path for an allowlisted footage id, or null — the extract route
421
+ * reuses a bridge-downloaded clip without a second trip through the browser. */
422
+ export function footagePath(id) {
423
+ if (typeof id !== "string" || !FOOTAGE_ID.test(id)) return null;
424
+ return footageAllowlist.get(id) ?? null;
425
+ }
426
+
427
+ /** GET /ardy/footage/<id>.mp4 — allowlist only, exactly like motion delivery. */
428
+ export function serveFootage(req, res, pathname) {
429
+ const sendJson = (status, obj) => {
430
+ res.writeHead(status, { "Content-Type": "application/json", "Cache-Control": "no-store" });
431
+ res.end(`${JSON.stringify(obj)}\n`);
432
+ };
433
+ const match = /^\/ardy\/footage\/([^/]+)\.mp4$/.exec(pathname);
434
+ if (!match || !FOOTAGE_ID.test(match[1]) || !footageAllowlist.has(match[1])) {
435
+ sendJson(404, { ok: false, reason: `unknown or expired footage "${match?.[1] ?? pathname}"` });
436
+ return 404;
437
+ }
438
+ const absPath = footageAllowlist.get(match[1]);
439
+ if (!absPath.startsWith(`${FOOTAGE_DIR}${sep}`)) {
440
+ sendJson(404, { ok: false, reason: "footage path is outside the footage directory" });
441
+ return 404;
442
+ }
443
+ let size;
444
+ try {
445
+ size = statSync(absPath).size;
446
+ } catch {
447
+ sendJson(404, { ok: false, reason: "footage is no longer on disk" });
448
+ return 404;
449
+ }
450
+ res.writeHead(200, {
451
+ "Content-Type": "video/mp4",
452
+ "Content-Length": size,
453
+ "Cache-Control": "no-store",
454
+ });
455
+ createReadStream(absPath)
456
+ .on("error", (err) => {
457
+ console.error(`[bridge] error streaming ${absPath}: ${err.message}`);
458
+ res.destroy();
459
+ })
460
+ .pipe(res);
461
+ return 200;
462
+ }
@@ -0,0 +1,160 @@
1
+ #!/usr/bin/env python3
2
+ """merge-text-encoder.py - bake the MNTP LoRA into the base weights.
3
+
4
+ Run under the ARDY text-encoder venv (needs torch, peft, transformers, and
5
+ the `ardy` package importable — i.e. from the ARDY checkout):
6
+
7
+ cd ~/ardy && .venv/bin/python merge-text-encoder.py --dest ~/cclay-text-encoders
8
+
9
+ Why this exists: transformers v5 removed the automatic PEFT-adapter
10
+ resolution that `from_pretrained` did in v4 (`find_adapter_config_file` is
11
+ gone from modeling_utils). ARDY's vendored LLM2Vec loader relies on that
12
+ behavior — it points `from_pretrained` at the MNTP *adapter* repo and lets
13
+ transformers chase `adapter_config.json` to the (gated) base. Under v5 that
14
+ path simply fails with "no model.safetensors found".
15
+
16
+ So instead of an adapter directory, the local MNTP path becomes a full
17
+ model: this script loads the ungated base (NousResearch mirror, already
18
+ hash-verified by setup-text-encoder.py), applies the MNTP LoRA with peft,
19
+ `merge_and_unload()`s it — the exact math upstream LLM2Vec performs at load
20
+ time — and saves the merged weights to the path ARDY's TEXT_ENCODERS_DIR
21
+ override resolves. Plain full-model loading works identically under
22
+ transformers v4 and v5; the supervised LoRA keeps loading through peft
23
+ directly, which both versions support.
24
+
25
+ Layout before: <dest>/McGill-NLP/...-mntp (downloaded adapter files)
26
+ Layout after: <dest>/McGill-NLP/...-mntp-src (the downloads, kept for
27
+ provenance + re-merges)
28
+ <dest>/McGill-NLP/...-mntp (merged full model +
29
+ tokenizer + MERGE_MANIFEST)
30
+
31
+ The merged config.json keeps `_name_or_path` =
32
+ "meta-llama/Meta-Llama-3-8B-Instruct": it is a metadata string (no download
33
+ or gate involved) that LLM2Vec's `prepare_for_tokenization` matches to pick
34
+ the Llama-3 chat template — losing it would silently change every prompt
35
+ embedding.
36
+ """
37
+
38
+ import argparse
39
+ import hashlib
40
+ import json
41
+ import os
42
+ import shutil
43
+ import sys
44
+
45
+ BASE = "NousResearch/Meta-Llama-3-8B-Instruct"
46
+ MNTP = "McGill-NLP/LLM2Vec-Meta-Llama-3-8B-Instruct-mntp"
47
+ MNTP_SRC = MNTP + "-src"
48
+ NAME_OR_PATH = "meta-llama/Meta-Llama-3-8B-Instruct"
49
+ TOKENIZER_FILES = ["tokenizer.json", "tokenizer_config.json", "special_tokens_map.json"]
50
+
51
+
52
+ def log(msg):
53
+ print("merge-text-encoder: %s" % msg, flush=True)
54
+
55
+
56
+ def fail(msg):
57
+ print("merge-text-encoder: ERROR: %s" % msg, file=sys.stderr, flush=True)
58
+ sys.exit(1)
59
+
60
+
61
+ def sha256_file(path):
62
+ h = hashlib.sha256()
63
+ with open(path, "rb") as f:
64
+ for block in iter(lambda: f.read(1024 * 1024), b""):
65
+ h.update(block)
66
+ return h.hexdigest()
67
+
68
+
69
+ def main():
70
+ ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
71
+ ap.add_argument("--dest", default=os.environ.get(
72
+ "TEXT_ENCODERS_DIR", os.path.expanduser("~/cclay-text-encoders")))
73
+ args = ap.parse_args()
74
+ dest = os.path.abspath(os.path.expanduser(args.dest))
75
+ base_dir = os.path.join(dest, BASE)
76
+ mntp_dir = os.path.join(dest, MNTP)
77
+ src_dir = os.path.join(dest, MNTP_SRC)
78
+
79
+ if not os.path.isfile(os.path.join(base_dir, "model.safetensors.index.json")):
80
+ fail("base model missing under %s — run setup-text-encoder.py first" % base_dir)
81
+
82
+ # Migrate a pre-merge tree: the download step used to land the adapter
83
+ # files directly at the mntp path.
84
+ if os.path.isdir(mntp_dir) and not os.path.isdir(src_dir) \
85
+ and os.path.isfile(os.path.join(mntp_dir, "adapter_model.safetensors")) \
86
+ and not os.path.isfile(os.path.join(mntp_dir, "model.safetensors.index.json")):
87
+ os.rename(mntp_dir, src_dir)
88
+ log("moved downloaded adapter layout to %s" % MNTP_SRC)
89
+
90
+ if not os.path.isfile(os.path.join(src_dir, "adapter_model.safetensors")):
91
+ fail("MNTP adapter missing under %s — run setup-text-encoder.py first" % src_dir)
92
+
93
+ adapter_sha = sha256_file(os.path.join(src_dir, "adapter_model.safetensors"))
94
+ manifest_path = os.path.join(mntp_dir, "MERGE_MANIFEST.json")
95
+ if os.path.isfile(manifest_path):
96
+ with open(manifest_path, "r", encoding="utf-8") as f:
97
+ manifest = json.load(f)
98
+ if manifest.get("mntp_adapter_sha256") == adapter_sha \
99
+ and os.path.isfile(os.path.join(mntp_dir, "model.safetensors.index.json")):
100
+ log("merged model already present and matches the adapter hash — nothing to do")
101
+ return
102
+
103
+ import torch # deferred: keep --help fast and errors readable
104
+ import peft
105
+ import transformers
106
+ from peft import PeftModel
107
+ from ardy.model.llm2vec.llm2vec import LLM2Vec
108
+
109
+ log("loading base (bfloat16, cpu) from %s" % base_dir)
110
+ model_class = LLM2Vec._get_model_class("LlamaConfig", enable_bidirectional=True)
111
+ model = model_class.from_pretrained(base_dir, torch_dtype=torch.bfloat16)
112
+
113
+ log("applying + merging the MNTP LoRA from %s" % src_dir)
114
+ model = PeftModel.from_pretrained(model, src_dir)
115
+ model = model.merge_and_unload()
116
+
117
+ tmp_out = mntp_dir + ".merging"
118
+ if os.path.isdir(tmp_out):
119
+ shutil.rmtree(tmp_out)
120
+ log("saving merged model to %s" % mntp_dir)
121
+ model.save_pretrained(tmp_out, safe_serialization=True)
122
+ del model
123
+
124
+ for name in TOKENIZER_FILES:
125
+ src = os.path.join(src_dir, name)
126
+ if os.path.isfile(src):
127
+ shutil.copy2(src, os.path.join(tmp_out, name))
128
+
129
+ # Preserve the chat-template key and the source architecture label.
130
+ cfg_path = os.path.join(tmp_out, "config.json")
131
+ with open(cfg_path, "r", encoding="utf-8") as f:
132
+ cfg = json.load(f)
133
+ with open(os.path.join(src_dir, "config.json"), "r", encoding="utf-8") as f:
134
+ src_cfg = json.load(f)
135
+ cfg["_name_or_path"] = NAME_OR_PATH
136
+ if "architectures" in src_cfg:
137
+ cfg["architectures"] = src_cfg["architectures"]
138
+ with open(cfg_path, "w", encoding="utf-8") as f:
139
+ json.dump(cfg, f, indent=2)
140
+ f.write("\n")
141
+
142
+ with open(os.path.join(tmp_out, "MERGE_MANIFEST.json"), "w", encoding="utf-8") as f:
143
+ json.dump({
144
+ "base": BASE,
145
+ "mntp_adapter_sha256": adapter_sha,
146
+ "torch": torch.__version__,
147
+ "transformers": transformers.__version__,
148
+ "peft": peft.__version__,
149
+ "name_or_path": NAME_OR_PATH,
150
+ }, f, indent=2)
151
+ f.write("\n")
152
+
153
+ if os.path.isdir(mntp_dir):
154
+ shutil.rmtree(mntp_dir)
155
+ os.rename(tmp_out, mntp_dir)
156
+ log("done — %s is a full merged model; the supervised LoRA stays a peft adapter" % MNTP)
157
+
158
+
159
+ if __name__ == "__main__":
160
+ main()