cozyclay 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -24
- package/THIRD_PARTY_NOTICES.md +31 -0
- package/bin/cozyclay.mjs +105 -11
- package/dist/CNAME +1 -0
- package/dist/app/index.html +57 -0
- package/dist/assets/app-Cgpk2hwX.js +4803 -0
- package/dist/assets/app-DgZvaAE1.css +1 -0
- package/dist/favicon.ico +0 -0
- package/dist/icons/apple-touch-icon.png +0 -0
- package/dist/icons/icon-192.png +0 -0
- package/dist/icons/icon-32.png +0 -0
- package/dist/icons/icon-512.png +0 -0
- package/dist/icons/icon-maskable-512.png +0 -0
- package/dist/index.html +406 -5
- package/dist/manifest.webmanifest +42 -0
- package/dist/media/cozyclay-demo-poster.jpg +0 -0
- package/dist/media/cozyclay-logo.png +0 -0
- package/dist/robots.txt +4 -0
- package/dist/sitemap.xml +9 -0
- package/dist/sw.js +127 -0
- package/package.json +15 -3
- package/src/App.jsx +1393 -648
- package/src/ardy/client.js +5 -4
- package/src/ardy/mp4-duration.js +102 -0
- package/src/ardy/timeline-coordinates.js +11 -0
- package/src/ardy/timeline.jsx +528 -67
- package/src/ardy/waypoints.js +14 -9
- package/src/camera-block.js +60 -0
- package/src/camera-follow.js +49 -10
- package/src/camera-rail-schedule.js +176 -0
- package/src/controls.jsx +16 -2
- package/src/cuts.js +184 -0
- package/src/generation/client.js +56 -0
- package/src/generation/session.js +47 -0
- package/src/generation/shot-spec.js +40 -0
- package/src/generation/use-generation.js +55 -0
- package/src/hierarchy-model.js +9 -17
- package/src/hierarchy-panel.jsx +170 -23
- package/src/locale-toggle.jsx +18 -0
- package/src/locale.js +34 -0
- package/src/main.jsx +5 -0
- package/src/object-catalog.jsx +32 -3
- package/src/planview.jsx +174 -66
- package/src/pose-extract/browser.js +42 -0
- package/src/pose-extract/fit.js +194 -0
- package/src/pose-extract/index.js +53 -0
- package/src/pose-extract/keyframes.js +74 -0
- package/src/pose-extract/landmarks.js +162 -0
- package/src/poses.js +114 -0
- package/src/posestudio.jsx +75 -13
- package/src/pwa.js +42 -0
- package/src/result-modal.jsx +124 -0
- package/src/scenes.js +223 -0
- package/src/shot-authoring.js +204 -77
- package/src/shot.js +8 -4
- package/src/styles.css +839 -38
- package/src/ui.jsx +54 -5
- package/tools/ardy/BRIDGE.md +70 -4
- package/tools/ardy/README.md +55 -0
- package/tools/ardy/bridge.mjs +89 -389
- package/tools/ardy/cclay_constrained_generate.py +449 -0
- package/tools/ardy/cclay_motion_edit.py +74 -9
- package/tools/ardy/cclay_sequence_generate.py +68 -7
- package/tools/ardy/cclay_worker.py +304 -0
- package/tools/ardy/merge-text-encoder.py +160 -0
- package/tools/ardy/run-local.mjs +233 -0
- package/tools/ardy/runners/index.mjs +23 -0
- package/tools/ardy/runners/local.mjs +452 -0
- package/tools/ardy/runners/proc.mjs +125 -0
- package/tools/ardy/runners/remote.mjs +246 -0
- package/tools/ardy/setup-local.mjs +189 -0
- package/tools/ardy/setup-text-encoder-on-box.sh +93 -0
- package/tools/ardy/setup-text-encoder.py +296 -0
- package/tools/generation/README.md +29 -0
- package/tools/generation/bridge.mjs +149 -0
- package/tools/generation/job-store.mjs +51 -0
- package/tools/generation/providers/kling.mjs +76 -0
- package/tools/generation/providers/runway.mjs +101 -0
- package/tools/generation/providers/seedance.mjs +87 -0
- package/tools/generation/providers/shared.mjs +49 -0
- package/tools/generation/providers/veo.mjs +84 -0
- package/tools/metrics/archive-traffic.mjs +138 -0
- package/dist/assets/index-BW_S1YRy.js +0 -4395
- package/dist/assets/index-nRmmJgap.css +0 -1
- package/tools/ardy/__pycache__/cclay_sequence_generate.cpython-313.pyc +0 -0
- package/tools/ardy/out/gen-1786443326924-c6019e-generated.npz +0 -0
- package/tools/ardy/out/gen-1786443609325-0053db-generated.npz +0 -0
- package/tools/ardy/out/gen-1786443835628-749ed1-generated.npz +0 -0
- package/tools/ardy/out/gen-1786462605247-03cb19-generated.npz +0 -0
- package/tools/qa-browser.mjs +0 -84
- package/tools/qa-crop3.mjs +0 -22
- package/tools/qa-playview.mjs +0 -49
- package/tools/qa-screenshot.mjs +0 -24
- package/tools/qa-visual.mjs +0 -51
|
@@ -0,0 +1,304 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
3
|
+
# SPDX-FileCopyrightText: CozyClay contributors
|
|
4
|
+
#
|
|
5
|
+
# cclay-owned script. Source of truth lives in the CozyClay repo at
|
|
6
|
+
# tools/ardy/cclay_worker.py.
|
|
7
|
+
"""Persistent local generation worker for the CozyClay ARDY bridge.
|
|
8
|
+
|
|
9
|
+
Loading the motion model and compiling its GPU kernels costs minutes on MPS,
|
|
10
|
+
while the sampling itself takes seconds. Spawning a fresh python per request
|
|
11
|
+
(the remote-box contract run-local.mjs mirrors) pays that cost every time, so
|
|
12
|
+
this worker pays it ONCE: it loads the model, runs a tiny no-text warmup to
|
|
13
|
+
force kernel compilation, and then serves generation jobs from the warm
|
|
14
|
+
process. run-local.mjs prefers the worker and silently falls back to a direct
|
|
15
|
+
spawn when it is not running, so the worker is an accelerator, never a
|
|
16
|
+
dependency.
|
|
17
|
+
|
|
18
|
+
Protocol (loopback TCP, one connection per job, one job at a time):
|
|
19
|
+
|
|
20
|
+
client -> worker one JSON line
|
|
21
|
+
{"mode": "ping"}
|
|
22
|
+
{"mode": "single"|"sequence"|"edit", "argv": [...]}
|
|
23
|
+
worker -> client the generator's stdout lines, verbatim, then
|
|
24
|
+
"worker: pong" (ping) or "worker: exit <code>"
|
|
25
|
+
|
|
26
|
+
`argv` is exactly the CLI argument vector of the corresponding
|
|
27
|
+
cclay_*_generate/edit script; the worker calls that script's main() in
|
|
28
|
+
process with the cached model injected, so worker and direct-spawn runs share
|
|
29
|
+
one code path and cannot drift.
|
|
30
|
+
|
|
31
|
+
env:
|
|
32
|
+
CCLAY_ARDY_WORKER_PORT listen port (default 9552, loopback only)
|
|
33
|
+
CHECKPOINTS_DIR / TEXT_ENCODER_URL / TEXT_ENCODERS_DIR as usual
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
import importlib
|
|
37
|
+
import json
|
|
38
|
+
import os
|
|
39
|
+
import socket
|
|
40
|
+
import sys
|
|
41
|
+
import traceback
|
|
42
|
+
|
|
43
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
44
|
+
sys.path.insert(0, HERE)
|
|
45
|
+
|
|
46
|
+
MODULES = {
|
|
47
|
+
"single": "cclay_constrained_generate",
|
|
48
|
+
"sequence": "cclay_sequence_generate",
|
|
49
|
+
"edit": "cclay_motion_edit",
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class _Tee:
|
|
54
|
+
"""Mirror generator stdout to the job's socket AND the worker console.
|
|
55
|
+
|
|
56
|
+
A vanished client must not kill the job: the generation keeps running,
|
|
57
|
+
the npz still lands on disk, and only the socket copy is dropped.
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
def __init__(self, sock_file):
|
|
61
|
+
self._sock_file = sock_file
|
|
62
|
+
|
|
63
|
+
def write(self, data):
|
|
64
|
+
sys.__stdout__.write(data)
|
|
65
|
+
try:
|
|
66
|
+
self._sock_file.write(data)
|
|
67
|
+
self._sock_file.flush()
|
|
68
|
+
except OSError:
|
|
69
|
+
pass
|
|
70
|
+
|
|
71
|
+
def flush(self):
|
|
72
|
+
sys.__stdout__.flush()
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def requested_model_name(mode, argv, resolve_model_name, default_model, checkpoints_dir):
|
|
76
|
+
"""The model a job's argv asks for (edit has no --model flag)."""
|
|
77
|
+
name = default_model
|
|
78
|
+
if mode != "edit":
|
|
79
|
+
for index, arg in enumerate(argv):
|
|
80
|
+
if arg == "--model" and index + 1 < len(argv):
|
|
81
|
+
name = argv[index + 1]
|
|
82
|
+
return resolve_model_name(name, checkpoints_dir=checkpoints_dir)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def fix_text_encoder_device(model, device):
|
|
86
|
+
"""Move an in-process LLM2Vec encoder onto the generation device, fast.
|
|
87
|
+
|
|
88
|
+
ARDY's load_text_encoder ends with `.to(device or "cuda"/"cpu")`, so on a
|
|
89
|
+
Mac the fallback encoder lands on CPU even when TEXT_ENCODER_DEVICE says
|
|
90
|
+
mps. On MPS the dtype matters as much as the device: bf16 matmuls are
|
|
91
|
+
emulated (a single prompt costs ~30-50 s) while fp16 is native, so fp16
|
|
92
|
+
is tried first and a canary encode guards against fp16 overflow — NaNs
|
|
93
|
+
demote back to bf16. Only the real in-process encoder has a `.model`;
|
|
94
|
+
the API client stub is left alone.
|
|
95
|
+
"""
|
|
96
|
+
encoder = getattr(model, "text_encoder", None)
|
|
97
|
+
if encoder is None or not hasattr(encoder, "model") or device == "cpu":
|
|
98
|
+
return
|
|
99
|
+
import torch
|
|
100
|
+
|
|
101
|
+
for dtype in (torch.float16, torch.bfloat16):
|
|
102
|
+
try:
|
|
103
|
+
encoder.to(device=device, dtype=dtype)
|
|
104
|
+
canary, _ = encoder(["a person walks forward"])
|
|
105
|
+
if bool(torch.isfinite(canary.float()).all()):
|
|
106
|
+
print(f"worker: text encoder on {device} as {dtype}", flush=True)
|
|
107
|
+
return
|
|
108
|
+
print(f"worker: text encoder {dtype} produced non-finite values; trying next dtype", flush=True)
|
|
109
|
+
except Exception as exc:
|
|
110
|
+
print(f"worker: text encoder {dtype} on {device} failed ({exc}); trying next dtype", flush=True)
|
|
111
|
+
try: # both GPU dtypes failed: back to the slow-but-correct CPU encoder
|
|
112
|
+
encoder.to(device="cpu", dtype=torch.bfloat16)
|
|
113
|
+
print("worker: text encoder stays on cpu", flush=True)
|
|
114
|
+
except Exception as exc:
|
|
115
|
+
print(f"worker: text encoder left as loaded ({exc})", flush=True)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
class _CachingEncoder:
|
|
119
|
+
"""Per-prompt embedding cache in front of the in-process encoder.
|
|
120
|
+
|
|
121
|
+
Authoring iterates on ONE prompt (new seed, new pose pin, new waypoints),
|
|
122
|
+
so most requests re-encode text the worker has already seen. Embeddings
|
|
123
|
+
are deterministic per (model, dtype) — exactly the cache key's tag — and
|
|
124
|
+
a few KB each, so they live in memory and under ~/.cozyclay/embed-cache
|
|
125
|
+
across worker restarts. Misses are timed on the console; hits are free.
|
|
126
|
+
"""
|
|
127
|
+
|
|
128
|
+
def __init__(self, inner, cache_dir, tag):
|
|
129
|
+
import os
|
|
130
|
+
|
|
131
|
+
self._inner = inner
|
|
132
|
+
self._dir = cache_dir
|
|
133
|
+
self._tag = tag
|
|
134
|
+
self._mem = {}
|
|
135
|
+
os.makedirs(cache_dir, exist_ok=True)
|
|
136
|
+
|
|
137
|
+
def _key(self, text):
|
|
138
|
+
import hashlib
|
|
139
|
+
|
|
140
|
+
return hashlib.sha256(f"{self._tag}\x00{text}".encode("utf-8")).hexdigest()
|
|
141
|
+
|
|
142
|
+
def _encode_one(self, text):
|
|
143
|
+
import os
|
|
144
|
+
import time
|
|
145
|
+
|
|
146
|
+
import numpy as np
|
|
147
|
+
|
|
148
|
+
key = self._key(text)
|
|
149
|
+
if key in self._mem:
|
|
150
|
+
return self._mem[key]
|
|
151
|
+
path = os.path.join(self._dir, f"{key}.npy")
|
|
152
|
+
if os.path.isfile(path):
|
|
153
|
+
try:
|
|
154
|
+
embedding = np.load(path)
|
|
155
|
+
self._mem[key] = embedding
|
|
156
|
+
return embedding
|
|
157
|
+
except Exception:
|
|
158
|
+
pass # unreadable cache entry: re-encode below
|
|
159
|
+
started = time.time()
|
|
160
|
+
tensor, _ = self._inner([text])
|
|
161
|
+
embedding = tensor[0].detach().float().cpu().numpy()
|
|
162
|
+
print(f"worker: text encode took {time.time() - started:.1f}s (cache miss)", flush=True)
|
|
163
|
+
self._mem[key] = embedding
|
|
164
|
+
try:
|
|
165
|
+
np.save(path, embedding)
|
|
166
|
+
except Exception:
|
|
167
|
+
pass # cache write failure must never fail the job
|
|
168
|
+
return embedding
|
|
169
|
+
|
|
170
|
+
def __call__(self, texts):
|
|
171
|
+
import numpy as np
|
|
172
|
+
import torch
|
|
173
|
+
|
|
174
|
+
is_string = isinstance(texts, str)
|
|
175
|
+
text_list = [texts] if is_string else list(texts)
|
|
176
|
+
stacked = np.stack([self._encode_one(text) for text in text_list])
|
|
177
|
+
device = getattr(self._inner, "_device", "cpu")
|
|
178
|
+
encoded = torch.from_numpy(stacked).to(device)
|
|
179
|
+
lengths = [1] * len(text_list)
|
|
180
|
+
if is_string:
|
|
181
|
+
return encoded[0], 1
|
|
182
|
+
return encoded, lengths
|
|
183
|
+
|
|
184
|
+
def __getattr__(self, name):
|
|
185
|
+
return getattr(self._inner, name)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def warmup(model, device):
|
|
189
|
+
"""One throwaway sampling call so kernel compilation happens off the clock.
|
|
190
|
+
|
|
191
|
+
Zero text features skip the text encoder entirely — the warmup must not
|
|
192
|
+
drag the 16 GB Llama encoder into RAM just to compile Metal shaders.
|
|
193
|
+
"""
|
|
194
|
+
import torch
|
|
195
|
+
from ardy.motion_rep.tools import length_to_mask
|
|
196
|
+
|
|
197
|
+
try:
|
|
198
|
+
frames = int(model.gen_horizon_len)
|
|
199
|
+
lengths = torch.tensor([frames], device=device)
|
|
200
|
+
with torch.no_grad():
|
|
201
|
+
model(
|
|
202
|
+
[""],
|
|
203
|
+
frames,
|
|
204
|
+
num_denoising_steps=1,
|
|
205
|
+
pad_mask=length_to_mask(lengths),
|
|
206
|
+
first_heading_angle=torch.zeros(1, device=device),
|
|
207
|
+
motion_mask=None,
|
|
208
|
+
observed_motion=None,
|
|
209
|
+
cfg_weight=2.0,
|
|
210
|
+
text_feat=torch.zeros(1, 1, 4096, device=device),
|
|
211
|
+
text_pad_mask=torch.ones(1, 1, device=device, dtype=torch.bool),
|
|
212
|
+
progress_bar=lambda iterable: iterable,
|
|
213
|
+
)
|
|
214
|
+
print("worker: warmup complete", flush=True)
|
|
215
|
+
except Exception as exc: # non-fatal: first real job just pays the compile
|
|
216
|
+
print(f"worker: warmup skipped ({exc})", flush=True)
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def main():
|
|
220
|
+
port = int(os.environ.get("CCLAY_ARDY_WORKER_PORT", "9552"))
|
|
221
|
+
|
|
222
|
+
single = importlib.import_module(MODULES["single"])
|
|
223
|
+
device = single.cclay_pick_device()
|
|
224
|
+
single.cclay_mps_compat(device)
|
|
225
|
+
|
|
226
|
+
from ardy.model import DEFAULT_MODEL, load_model
|
|
227
|
+
from ardy.model.loading import get_env_var
|
|
228
|
+
from ardy.model.registry import resolve_model_name
|
|
229
|
+
|
|
230
|
+
checkpoints_dir = get_env_var("CHECKPOINTS_DIR", None)
|
|
231
|
+
cached_name = resolve_model_name(DEFAULT_MODEL, checkpoints_dir=checkpoints_dir)
|
|
232
|
+
print(f"worker: loading {cached_name} on {device} ...", flush=True)
|
|
233
|
+
cached_model = load_model(cached_name, device=device, checkpoints_dir=checkpoints_dir)
|
|
234
|
+
cached_model.eval()
|
|
235
|
+
fix_text_encoder_device(cached_model, device)
|
|
236
|
+
encoder = getattr(cached_model, "text_encoder", None)
|
|
237
|
+
if encoder is not None and hasattr(encoder, "model"):
|
|
238
|
+
try:
|
|
239
|
+
param = next(encoder.model.parameters())
|
|
240
|
+
tag = f"llm2vec:{param.dtype}:{param.device.type}"
|
|
241
|
+
except Exception:
|
|
242
|
+
tag = "llm2vec:unknown"
|
|
243
|
+
cache_dir = os.path.join(os.path.expanduser("~"), ".cozyclay", "embed-cache")
|
|
244
|
+
cached_model.text_encoder = _CachingEncoder(encoder, cache_dir, tag)
|
|
245
|
+
print(f"worker: embedding cache at {cache_dir} ({tag})", flush=True)
|
|
246
|
+
warmup(cached_model, device)
|
|
247
|
+
|
|
248
|
+
server = socket.create_server(("127.0.0.1", port))
|
|
249
|
+
print(f"worker: ready on 127.0.0.1:{port} (model {cached_name}, device {device})", flush=True)
|
|
250
|
+
|
|
251
|
+
while True:
|
|
252
|
+
conn, _ = server.accept()
|
|
253
|
+
with conn:
|
|
254
|
+
rfile = conn.makefile("r", encoding="utf-8")
|
|
255
|
+
wfile = conn.makefile("w", encoding="utf-8")
|
|
256
|
+
try:
|
|
257
|
+
job = json.loads(rfile.readline())
|
|
258
|
+
except (ValueError, OSError):
|
|
259
|
+
continue
|
|
260
|
+
mode = job.get("mode") if isinstance(job, dict) else None
|
|
261
|
+
if mode == "ping":
|
|
262
|
+
try:
|
|
263
|
+
wfile.write("worker: pong\n")
|
|
264
|
+
wfile.flush()
|
|
265
|
+
except OSError:
|
|
266
|
+
pass
|
|
267
|
+
continue
|
|
268
|
+
argv = job.get("argv") if isinstance(job, dict) else None
|
|
269
|
+
if mode not in MODULES or not isinstance(argv, list) or not all(isinstance(a, str) for a in argv):
|
|
270
|
+
try:
|
|
271
|
+
wfile.write("worker: exit 2\n")
|
|
272
|
+
wfile.flush()
|
|
273
|
+
except OSError:
|
|
274
|
+
pass
|
|
275
|
+
continue
|
|
276
|
+
|
|
277
|
+
module = importlib.import_module(MODULES[mode])
|
|
278
|
+
code = 0
|
|
279
|
+
old_stdout = sys.stdout
|
|
280
|
+
sys.stdout = _Tee(wfile)
|
|
281
|
+
try:
|
|
282
|
+
wanted = requested_model_name(mode, argv, resolve_model_name, DEFAULT_MODEL, checkpoints_dir)
|
|
283
|
+
if wanted != cached_name:
|
|
284
|
+
print(f"worker: switching model {cached_name} -> {wanted}", flush=True)
|
|
285
|
+
cached_model = load_model(wanted, device=device, checkpoints_dir=checkpoints_dir)
|
|
286
|
+
cached_model.eval()
|
|
287
|
+
cached_name = wanted
|
|
288
|
+
module.main(argv, preloaded_model=cached_model)
|
|
289
|
+
except SystemExit as exit_exc:
|
|
290
|
+
code = exit_exc.code if isinstance(exit_exc.code, int) else 1
|
|
291
|
+
except Exception:
|
|
292
|
+
traceback.print_exc(file=sys.stdout)
|
|
293
|
+
code = 1
|
|
294
|
+
finally:
|
|
295
|
+
sys.stdout = old_stdout
|
|
296
|
+
try:
|
|
297
|
+
wfile.write(f"worker: exit {code}\n")
|
|
298
|
+
wfile.flush()
|
|
299
|
+
except OSError:
|
|
300
|
+
pass
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
if __name__ == "__main__":
|
|
304
|
+
main()
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""merge-text-encoder.py - bake the MNTP LoRA into the base weights.
|
|
3
|
+
|
|
4
|
+
Run under the ARDY text-encoder venv (needs torch, peft, transformers, and
|
|
5
|
+
the `ardy` package importable — i.e. from the ARDY checkout):
|
|
6
|
+
|
|
7
|
+
cd ~/ardy && .venv/bin/python merge-text-encoder.py --dest ~/cclay-text-encoders
|
|
8
|
+
|
|
9
|
+
Why this exists: transformers v5 removed the automatic PEFT-adapter
|
|
10
|
+
resolution that `from_pretrained` did in v4 (`find_adapter_config_file` is
|
|
11
|
+
gone from modeling_utils). ARDY's vendored LLM2Vec loader relies on that
|
|
12
|
+
behavior — it points `from_pretrained` at the MNTP *adapter* repo and lets
|
|
13
|
+
transformers chase `adapter_config.json` to the (gated) base. Under v5 that
|
|
14
|
+
path simply fails with "no model.safetensors found".
|
|
15
|
+
|
|
16
|
+
So instead of an adapter directory, the local MNTP path becomes a full
|
|
17
|
+
model: this script loads the ungated base (NousResearch mirror, already
|
|
18
|
+
hash-verified by setup-text-encoder.py), applies the MNTP LoRA with peft,
|
|
19
|
+
`merge_and_unload()`s it — the exact math upstream LLM2Vec performs at load
|
|
20
|
+
time — and saves the merged weights to the path ARDY's TEXT_ENCODERS_DIR
|
|
21
|
+
override resolves. Plain full-model loading works identically under
|
|
22
|
+
transformers v4 and v5; the supervised LoRA keeps loading through peft
|
|
23
|
+
directly, which both versions support.
|
|
24
|
+
|
|
25
|
+
Layout before: <dest>/McGill-NLP/...-mntp (downloaded adapter files)
|
|
26
|
+
Layout after: <dest>/McGill-NLP/...-mntp-src (the downloads, kept for
|
|
27
|
+
provenance + re-merges)
|
|
28
|
+
<dest>/McGill-NLP/...-mntp (merged full model +
|
|
29
|
+
tokenizer + MERGE_MANIFEST)
|
|
30
|
+
|
|
31
|
+
The merged config.json keeps `_name_or_path` =
|
|
32
|
+
"meta-llama/Meta-Llama-3-8B-Instruct": it is a metadata string (no download
|
|
33
|
+
or gate involved) that LLM2Vec's `prepare_for_tokenization` matches to pick
|
|
34
|
+
the Llama-3 chat template — losing it would silently change every prompt
|
|
35
|
+
embedding.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
import argparse
|
|
39
|
+
import hashlib
|
|
40
|
+
import json
|
|
41
|
+
import os
|
|
42
|
+
import shutil
|
|
43
|
+
import sys
|
|
44
|
+
|
|
45
|
+
BASE = "NousResearch/Meta-Llama-3-8B-Instruct"
|
|
46
|
+
MNTP = "McGill-NLP/LLM2Vec-Meta-Llama-3-8B-Instruct-mntp"
|
|
47
|
+
MNTP_SRC = MNTP + "-src"
|
|
48
|
+
NAME_OR_PATH = "meta-llama/Meta-Llama-3-8B-Instruct"
|
|
49
|
+
TOKENIZER_FILES = ["tokenizer.json", "tokenizer_config.json", "special_tokens_map.json"]
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def log(msg):
|
|
53
|
+
print("merge-text-encoder: %s" % msg, flush=True)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def fail(msg):
|
|
57
|
+
print("merge-text-encoder: ERROR: %s" % msg, file=sys.stderr, flush=True)
|
|
58
|
+
sys.exit(1)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def sha256_file(path):
|
|
62
|
+
h = hashlib.sha256()
|
|
63
|
+
with open(path, "rb") as f:
|
|
64
|
+
for block in iter(lambda: f.read(1024 * 1024), b""):
|
|
65
|
+
h.update(block)
|
|
66
|
+
return h.hexdigest()
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def main():
|
|
70
|
+
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
71
|
+
ap.add_argument("--dest", default=os.environ.get(
|
|
72
|
+
"TEXT_ENCODERS_DIR", os.path.expanduser("~/cclay-text-encoders")))
|
|
73
|
+
args = ap.parse_args()
|
|
74
|
+
dest = os.path.abspath(os.path.expanduser(args.dest))
|
|
75
|
+
base_dir = os.path.join(dest, BASE)
|
|
76
|
+
mntp_dir = os.path.join(dest, MNTP)
|
|
77
|
+
src_dir = os.path.join(dest, MNTP_SRC)
|
|
78
|
+
|
|
79
|
+
if not os.path.isfile(os.path.join(base_dir, "model.safetensors.index.json")):
|
|
80
|
+
fail("base model missing under %s — run setup-text-encoder.py first" % base_dir)
|
|
81
|
+
|
|
82
|
+
# Migrate a pre-merge tree: the download step used to land the adapter
|
|
83
|
+
# files directly at the mntp path.
|
|
84
|
+
if os.path.isdir(mntp_dir) and not os.path.isdir(src_dir) \
|
|
85
|
+
and os.path.isfile(os.path.join(mntp_dir, "adapter_model.safetensors")) \
|
|
86
|
+
and not os.path.isfile(os.path.join(mntp_dir, "model.safetensors.index.json")):
|
|
87
|
+
os.rename(mntp_dir, src_dir)
|
|
88
|
+
log("moved downloaded adapter layout to %s" % MNTP_SRC)
|
|
89
|
+
|
|
90
|
+
if not os.path.isfile(os.path.join(src_dir, "adapter_model.safetensors")):
|
|
91
|
+
fail("MNTP adapter missing under %s — run setup-text-encoder.py first" % src_dir)
|
|
92
|
+
|
|
93
|
+
adapter_sha = sha256_file(os.path.join(src_dir, "adapter_model.safetensors"))
|
|
94
|
+
manifest_path = os.path.join(mntp_dir, "MERGE_MANIFEST.json")
|
|
95
|
+
if os.path.isfile(manifest_path):
|
|
96
|
+
with open(manifest_path, "r", encoding="utf-8") as f:
|
|
97
|
+
manifest = json.load(f)
|
|
98
|
+
if manifest.get("mntp_adapter_sha256") == adapter_sha \
|
|
99
|
+
and os.path.isfile(os.path.join(mntp_dir, "model.safetensors.index.json")):
|
|
100
|
+
log("merged model already present and matches the adapter hash — nothing to do")
|
|
101
|
+
return
|
|
102
|
+
|
|
103
|
+
import torch # deferred: keep --help fast and errors readable
|
|
104
|
+
import peft
|
|
105
|
+
import transformers
|
|
106
|
+
from peft import PeftModel
|
|
107
|
+
from ardy.model.llm2vec.llm2vec import LLM2Vec
|
|
108
|
+
|
|
109
|
+
log("loading base (bfloat16, cpu) from %s" % base_dir)
|
|
110
|
+
model_class = LLM2Vec._get_model_class("LlamaConfig", enable_bidirectional=True)
|
|
111
|
+
model = model_class.from_pretrained(base_dir, torch_dtype=torch.bfloat16)
|
|
112
|
+
|
|
113
|
+
log("applying + merging the MNTP LoRA from %s" % src_dir)
|
|
114
|
+
model = PeftModel.from_pretrained(model, src_dir)
|
|
115
|
+
model = model.merge_and_unload()
|
|
116
|
+
|
|
117
|
+
tmp_out = mntp_dir + ".merging"
|
|
118
|
+
if os.path.isdir(tmp_out):
|
|
119
|
+
shutil.rmtree(tmp_out)
|
|
120
|
+
log("saving merged model to %s" % mntp_dir)
|
|
121
|
+
model.save_pretrained(tmp_out, safe_serialization=True)
|
|
122
|
+
del model
|
|
123
|
+
|
|
124
|
+
for name in TOKENIZER_FILES:
|
|
125
|
+
src = os.path.join(src_dir, name)
|
|
126
|
+
if os.path.isfile(src):
|
|
127
|
+
shutil.copy2(src, os.path.join(tmp_out, name))
|
|
128
|
+
|
|
129
|
+
# Preserve the chat-template key and the source architecture label.
|
|
130
|
+
cfg_path = os.path.join(tmp_out, "config.json")
|
|
131
|
+
with open(cfg_path, "r", encoding="utf-8") as f:
|
|
132
|
+
cfg = json.load(f)
|
|
133
|
+
with open(os.path.join(src_dir, "config.json"), "r", encoding="utf-8") as f:
|
|
134
|
+
src_cfg = json.load(f)
|
|
135
|
+
cfg["_name_or_path"] = NAME_OR_PATH
|
|
136
|
+
if "architectures" in src_cfg:
|
|
137
|
+
cfg["architectures"] = src_cfg["architectures"]
|
|
138
|
+
with open(cfg_path, "w", encoding="utf-8") as f:
|
|
139
|
+
json.dump(cfg, f, indent=2)
|
|
140
|
+
f.write("\n")
|
|
141
|
+
|
|
142
|
+
with open(os.path.join(tmp_out, "MERGE_MANIFEST.json"), "w", encoding="utf-8") as f:
|
|
143
|
+
json.dump({
|
|
144
|
+
"base": BASE,
|
|
145
|
+
"mntp_adapter_sha256": adapter_sha,
|
|
146
|
+
"torch": torch.__version__,
|
|
147
|
+
"transformers": transformers.__version__,
|
|
148
|
+
"peft": peft.__version__,
|
|
149
|
+
"name_or_path": NAME_OR_PATH,
|
|
150
|
+
}, f, indent=2)
|
|
151
|
+
f.write("\n")
|
|
152
|
+
|
|
153
|
+
if os.path.isdir(mntp_dir):
|
|
154
|
+
shutil.rmtree(mntp_dir)
|
|
155
|
+
os.rename(tmp_out, mntp_dir)
|
|
156
|
+
log("done — %s is a full merged model; the supervised LoRA stays a peft adapter" % MNTP)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
if __name__ == "__main__":
|
|
160
|
+
main()
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* run-local.mjs - generate an ARDY clip on THIS machine.
|
|
4
|
+
*
|
|
5
|
+
* run-local.mjs single [--pose-from NPZ SRC DST]... --prompt P --duration S
|
|
6
|
+
* [--seed N] [--root-2d F X Z H]... --output OUT.npz
|
|
7
|
+
* run-local.mjs sequence --segment PROMPT SECONDS [--segment ...]...
|
|
8
|
+
* [--root-2d F X Z H]... [--seed N] --output OUT.npz
|
|
9
|
+
* run-local.mjs edit --source SRC.npz --manifest M.json --prompt P
|
|
10
|
+
* --context-before N --context-after N [--seed N]
|
|
11
|
+
* [--pose POSE.npz]... --output OUT.npz
|
|
12
|
+
*
|
|
13
|
+
* The local twin of run-on-box.sh / run-sequence-on-box.sh /
|
|
14
|
+
* run-edit-on-box.sh: same modes, same success marker
|
|
15
|
+
*
|
|
16
|
+
* run-local: done - <output> (<bytes> bytes)
|
|
17
|
+
*
|
|
18
|
+
* minus every ssh/scp/temp-dir round trip — the generator scripts run in
|
|
19
|
+
* place from this repo against the ARDY checkout's venv. Device selection
|
|
20
|
+
* lives in the python side (cclay_pick_device: CCLAY_ARDY_DEVICE override,
|
|
21
|
+
* else cuda > mps > cpu). The text encoder is the bridge's lazy sidecar; a
|
|
22
|
+
* dead encoder degrades to ARDY's in-process fallback rather than failing.
|
|
23
|
+
*
|
|
24
|
+
* env (set by runners/local.mjs; usable standalone too):
|
|
25
|
+
* CCLAY_ARDY_LOCAL_DIR ARDY checkout (default ~/.cozyclay/ardy)
|
|
26
|
+
* CCLAY_ARDY_LOCAL_VENV venv python (default <dir>/.venv/bin/python)
|
|
27
|
+
* TEXT_ENCODER_URL encoder service (default http://127.0.0.1:9550/)
|
|
28
|
+
* TEXT_ENCODERS_DIR encoder weights dir (default ~/.cozyclay/text-encoders)
|
|
29
|
+
* CCLAY_ARDY_DEVICE force a torch device (cpu|mps|cuda[:N])
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
import { spawn } from "node:child_process";
|
|
33
|
+
import { cpSync, existsSync, mkdtempSync, rmSync, statSync } from "node:fs";
|
|
34
|
+
import { createConnection } from "node:net";
|
|
35
|
+
import { homedir, tmpdir } from "node:os";
|
|
36
|
+
import { basename, dirname, join } from "node:path";
|
|
37
|
+
import { fileURLToPath } from "node:url";
|
|
38
|
+
|
|
39
|
+
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
40
|
+
const LOCAL_DIR = process.env.CCLAY_ARDY_LOCAL_DIR || join(homedir(), ".cozyclay", "ardy");
|
|
41
|
+
const VENV_PY =
|
|
42
|
+
process.env.CCLAY_ARDY_LOCAL_VENV ||
|
|
43
|
+
join(LOCAL_DIR, ".venv", process.platform === "win32" ? "Scripts\\python.exe" : "bin/python");
|
|
44
|
+
const WORKER_PORT = Number(process.env.CCLAY_ARDY_WORKER_PORT) || 9552;
|
|
45
|
+
// A sequence job can legitimately run a long time; the watchdog only guards
|
|
46
|
+
// against a worker that stopped talking entirely.
|
|
47
|
+
const WORKER_SILENCE_TIMEOUT_MS = 60 * 60_000;
|
|
48
|
+
|
|
49
|
+
const GENERATORS = {
|
|
50
|
+
single: join(HERE, "cclay_constrained_generate.py"),
|
|
51
|
+
sequence: join(HERE, "cclay_sequence_generate.py"),
|
|
52
|
+
edit: join(HERE, "cclay_motion_edit.py"),
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
function die(message) {
|
|
56
|
+
console.error(`run-local: ${message}`);
|
|
57
|
+
process.exit(2);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function usage() {
|
|
61
|
+
console.error("usage: run-local.mjs <single|sequence|edit> [flags] --output OUT.npz");
|
|
62
|
+
console.error("see the header of tools/ardy/run-local.mjs for the full grammar");
|
|
63
|
+
process.exit(2);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// The bridge builds these argv arrays itself, so parsing here is deliberately
|
|
67
|
+
// shallow: pull out the handful of values run-local needs (--output always;
|
|
68
|
+
// --cpu, --pose/--manifest for staging) and pass everything else through to
|
|
69
|
+
// the python generator untouched.
|
|
70
|
+
function parseArgs(argv) {
|
|
71
|
+
const passthrough = [];
|
|
72
|
+
let output = "";
|
|
73
|
+
let cpu = false;
|
|
74
|
+
const poses = [];
|
|
75
|
+
let manifest = "";
|
|
76
|
+
for (let i = 0; i < argv.length; i += 1) {
|
|
77
|
+
const arg = argv[i];
|
|
78
|
+
if (arg === "--output") {
|
|
79
|
+
output = argv[++i] ?? "";
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
if (arg === "--cpu") {
|
|
83
|
+
cpu = true;
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
if (arg === "--pose") {
|
|
87
|
+
poses.push(argv[++i] ?? "");
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
if (arg === "--manifest") {
|
|
91
|
+
manifest = argv[++i] ?? "";
|
|
92
|
+
continue;
|
|
93
|
+
}
|
|
94
|
+
passthrough.push(arg);
|
|
95
|
+
}
|
|
96
|
+
if (!output) die("--output is required");
|
|
97
|
+
return { passthrough, output, cpu, poses, manifest };
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Try to run the job on the persistent worker (tools/ardy/cclay_worker.py).
|
|
102
|
+
*
|
|
103
|
+
* Resolves {code} when the worker accepted and finished the job (its stdout
|
|
104
|
+
* lines have already been relayed), or null when the worker is unavailable
|
|
105
|
+
* or died mid-job — the caller falls back to a direct spawn, which is always
|
|
106
|
+
* correct because the job is idempotent (same --output, overwritten).
|
|
107
|
+
*/
|
|
108
|
+
function runViaWorker(mode, flagArgs) {
|
|
109
|
+
return new Promise((resolvePromise) => {
|
|
110
|
+
const sock = createConnection({ host: "127.0.0.1", port: WORKER_PORT });
|
|
111
|
+
let buffer = "";
|
|
112
|
+
let settled = false;
|
|
113
|
+
const finish = (value) => {
|
|
114
|
+
if (!settled) {
|
|
115
|
+
settled = true;
|
|
116
|
+
sock.destroy();
|
|
117
|
+
resolvePromise(value);
|
|
118
|
+
}
|
|
119
|
+
};
|
|
120
|
+
sock.setTimeout(WORKER_SILENCE_TIMEOUT_MS, () => finish(null));
|
|
121
|
+
sock.on("error", () => finish(null));
|
|
122
|
+
sock.on("close", () => finish(null));
|
|
123
|
+
sock.on("connect", () => {
|
|
124
|
+
sock.write(`${JSON.stringify({ mode, argv: flagArgs })}\n`);
|
|
125
|
+
});
|
|
126
|
+
sock.on("data", (chunk) => {
|
|
127
|
+
buffer += chunk.toString("utf8");
|
|
128
|
+
let nl;
|
|
129
|
+
while ((nl = buffer.indexOf("\n")) !== -1) {
|
|
130
|
+
const line = buffer.slice(0, nl);
|
|
131
|
+
buffer = buffer.slice(nl + 1);
|
|
132
|
+
const marker = /^worker: exit (\d+)$/.exec(line);
|
|
133
|
+
if (marker) {
|
|
134
|
+
finish({ code: Number(marker[1]) });
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
if (line) console.log(line);
|
|
138
|
+
}
|
|
139
|
+
});
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
async function main() {
|
|
144
|
+
const [mode, ...rest] = process.argv.slice(2);
|
|
145
|
+
if (!mode || mode === "-h" || mode === "--help") usage();
|
|
146
|
+
const generator = GENERATORS[mode];
|
|
147
|
+
if (!generator) die(`unknown mode '${mode}' (expected single, sequence, or edit)`);
|
|
148
|
+
if (!existsSync(VENV_PY)) {
|
|
149
|
+
die(`venv python not found at ${VENV_PY}; run \`node tools/ardy/setup-local.mjs\` once`);
|
|
150
|
+
}
|
|
151
|
+
if (!existsSync(LOCAL_DIR)) {
|
|
152
|
+
die(`ARDY checkout not found at ${LOCAL_DIR}; run \`node tools/ardy/setup-local.mjs\` once`);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
const { passthrough, output, cpu, poses, manifest } = parseArgs(rest);
|
|
156
|
+
|
|
157
|
+
const flagArgs = [...passthrough];
|
|
158
|
+
let stagingDir = null;
|
|
159
|
+
if (mode === "edit") {
|
|
160
|
+
if (!manifest) die("--manifest is required in edit mode");
|
|
161
|
+
if (poses.length === 0) die("edit mode needs at least one --pose");
|
|
162
|
+
// cclay_motion_edit.py resolves the manifest's relative pose_path
|
|
163
|
+
// entries against the manifest's own directory, expecting pose-<i>.npz
|
|
164
|
+
// siblings (the layout run-edit-on-box.sh created with scp). Stage the
|
|
165
|
+
// same layout in a fresh temp dir.
|
|
166
|
+
stagingDir = mkdtempSync(join(tmpdir(), "cozyclay-edit-"));
|
|
167
|
+
cpSync(manifest, join(stagingDir, "manifest.json"));
|
|
168
|
+
poses.forEach((posePath, index) => {
|
|
169
|
+
cpSync(posePath, join(stagingDir, `pose-${index}.npz`));
|
|
170
|
+
});
|
|
171
|
+
flagArgs.push("--manifest", join(stagingDir, "manifest.json"));
|
|
172
|
+
} else if (manifest) {
|
|
173
|
+
flagArgs.push("--manifest", manifest);
|
|
174
|
+
}
|
|
175
|
+
flagArgs.push("--output", output);
|
|
176
|
+
|
|
177
|
+
const env = { ...process.env };
|
|
178
|
+
if (cpu) env.CCLAY_ARDY_DEVICE = "cpu";
|
|
179
|
+
|
|
180
|
+
// Prefer the warm worker; a forced device (--cpu / CCLAY_ARDY_DEVICE)
|
|
181
|
+
// bypasses it because the worker's model already lives on the auto-picked
|
|
182
|
+
// device. null = worker unavailable, fall through to a direct spawn.
|
|
183
|
+
let code = null;
|
|
184
|
+
const forcedDevice = cpu || Boolean(process.env.CCLAY_ARDY_DEVICE);
|
|
185
|
+
if (!forcedDevice) {
|
|
186
|
+
const workerResult = await runViaWorker(mode, flagArgs);
|
|
187
|
+
if (workerResult) {
|
|
188
|
+
console.log("run-local: served by warm worker");
|
|
189
|
+
code = workerResult.code;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
if (code === null) {
|
|
193
|
+
console.log(`run-local: ${mode} generation with ${basename(VENV_PY)} (${LOCAL_DIR})`);
|
|
194
|
+
const child = spawn(VENV_PY, [generator, ...flagArgs], {
|
|
195
|
+
cwd: LOCAL_DIR, // keeps any generator-relative paths (outputs/, HF cache) out of this repo
|
|
196
|
+
stdio: ["ignore", "inherit", "inherit"],
|
|
197
|
+
env,
|
|
198
|
+
});
|
|
199
|
+
code = await new Promise((resolvePromise, reject) => {
|
|
200
|
+
child.on("error", reject);
|
|
201
|
+
child.on("close", resolvePromise);
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
if (stagingDir) {
|
|
205
|
+
// Cleanup failure must never mask the real result.
|
|
206
|
+
try {
|
|
207
|
+
rmSync(stagingDir, { recursive: true, force: true });
|
|
208
|
+
} catch (err) {
|
|
209
|
+
console.error(`run-local: could not remove ${stagingDir}: ${err.message}`);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
if (code !== 0) {
|
|
213
|
+
console.error(`run-local: generator exited with code ${code}`);
|
|
214
|
+
process.exit(code || 1);
|
|
215
|
+
}
|
|
216
|
+
let size;
|
|
217
|
+
try {
|
|
218
|
+
size = statSync(output).size;
|
|
219
|
+
} catch {
|
|
220
|
+
console.error(`run-local: generator exited 0 but ${output} does not exist`);
|
|
221
|
+
process.exit(1);
|
|
222
|
+
}
|
|
223
|
+
if (size === 0) {
|
|
224
|
+
console.error(`run-local: generator produced an empty file at ${output}`);
|
|
225
|
+
process.exit(1);
|
|
226
|
+
}
|
|
227
|
+
console.log(`run-local: done - ${output} (${size} bytes)`);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
main().catch((err) => {
|
|
231
|
+
console.error(`run-local: ${err.stack || err}`);
|
|
232
|
+
process.exit(1);
|
|
233
|
+
});
|