cozyclay 1.0.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/README.md +59 -24
  2. package/THIRD_PARTY_NOTICES.md +31 -0
  3. package/bin/cozyclay.mjs +105 -11
  4. package/dist/CNAME +1 -0
  5. package/dist/app/index.html +57 -0
  6. package/dist/assets/app-Cgpk2hwX.js +4803 -0
  7. package/dist/assets/app-DgZvaAE1.css +1 -0
  8. package/dist/favicon.ico +0 -0
  9. package/dist/icons/apple-touch-icon.png +0 -0
  10. package/dist/icons/icon-192.png +0 -0
  11. package/dist/icons/icon-32.png +0 -0
  12. package/dist/icons/icon-512.png +0 -0
  13. package/dist/icons/icon-maskable-512.png +0 -0
  14. package/dist/index.html +406 -5
  15. package/dist/manifest.webmanifest +42 -0
  16. package/dist/media/cozyclay-demo-poster.jpg +0 -0
  17. package/dist/media/cozyclay-logo.png +0 -0
  18. package/dist/robots.txt +4 -0
  19. package/dist/sitemap.xml +9 -0
  20. package/dist/sw.js +127 -0
  21. package/package.json +15 -3
  22. package/src/App.jsx +1393 -648
  23. package/src/ardy/client.js +5 -4
  24. package/src/ardy/mp4-duration.js +102 -0
  25. package/src/ardy/timeline-coordinates.js +11 -0
  26. package/src/ardy/timeline.jsx +528 -67
  27. package/src/ardy/waypoints.js +14 -9
  28. package/src/camera-block.js +60 -0
  29. package/src/camera-follow.js +49 -10
  30. package/src/camera-rail-schedule.js +176 -0
  31. package/src/controls.jsx +16 -2
  32. package/src/cuts.js +184 -0
  33. package/src/generation/client.js +56 -0
  34. package/src/generation/session.js +47 -0
  35. package/src/generation/shot-spec.js +40 -0
  36. package/src/generation/use-generation.js +55 -0
  37. package/src/hierarchy-model.js +9 -17
  38. package/src/hierarchy-panel.jsx +170 -23
  39. package/src/locale-toggle.jsx +18 -0
  40. package/src/locale.js +34 -0
  41. package/src/main.jsx +5 -0
  42. package/src/object-catalog.jsx +32 -3
  43. package/src/planview.jsx +174 -66
  44. package/src/pose-extract/browser.js +42 -0
  45. package/src/pose-extract/fit.js +194 -0
  46. package/src/pose-extract/index.js +53 -0
  47. package/src/pose-extract/keyframes.js +74 -0
  48. package/src/pose-extract/landmarks.js +162 -0
  49. package/src/poses.js +114 -0
  50. package/src/posestudio.jsx +75 -13
  51. package/src/pwa.js +42 -0
  52. package/src/result-modal.jsx +124 -0
  53. package/src/scenes.js +223 -0
  54. package/src/shot-authoring.js +204 -77
  55. package/src/shot.js +8 -4
  56. package/src/styles.css +839 -38
  57. package/src/ui.jsx +54 -5
  58. package/tools/ardy/BRIDGE.md +70 -4
  59. package/tools/ardy/README.md +55 -0
  60. package/tools/ardy/bridge.mjs +89 -389
  61. package/tools/ardy/cclay_constrained_generate.py +449 -0
  62. package/tools/ardy/cclay_motion_edit.py +74 -9
  63. package/tools/ardy/cclay_sequence_generate.py +68 -7
  64. package/tools/ardy/cclay_worker.py +304 -0
  65. package/tools/ardy/merge-text-encoder.py +160 -0
  66. package/tools/ardy/run-local.mjs +233 -0
  67. package/tools/ardy/runners/index.mjs +23 -0
  68. package/tools/ardy/runners/local.mjs +452 -0
  69. package/tools/ardy/runners/proc.mjs +125 -0
  70. package/tools/ardy/runners/remote.mjs +246 -0
  71. package/tools/ardy/setup-local.mjs +189 -0
  72. package/tools/ardy/setup-text-encoder-on-box.sh +93 -0
  73. package/tools/ardy/setup-text-encoder.py +296 -0
  74. package/tools/generation/README.md +29 -0
  75. package/tools/generation/bridge.mjs +149 -0
  76. package/tools/generation/job-store.mjs +51 -0
  77. package/tools/generation/providers/kling.mjs +76 -0
  78. package/tools/generation/providers/runway.mjs +101 -0
  79. package/tools/generation/providers/seedance.mjs +87 -0
  80. package/tools/generation/providers/shared.mjs +49 -0
  81. package/tools/generation/providers/veo.mjs +84 -0
  82. package/tools/metrics/archive-traffic.mjs +138 -0
  83. package/dist/assets/index-BW_S1YRy.js +0 -4395
  84. package/dist/assets/index-nRmmJgap.css +0 -1
  85. package/tools/ardy/__pycache__/cclay_sequence_generate.cpython-313.pyc +0 -0
  86. package/tools/ardy/out/gen-1786443326924-c6019e-generated.npz +0 -0
  87. package/tools/ardy/out/gen-1786443609325-0053db-generated.npz +0 -0
  88. package/tools/ardy/out/gen-1786443835628-749ed1-generated.npz +0 -0
  89. package/tools/ardy/out/gen-1786462605247-03cb19-generated.npz +0 -0
  90. package/tools/qa-browser.mjs +0 -84
  91. package/tools/qa-crop3.mjs +0 -22
  92. package/tools/qa-playview.mjs +0 -49
  93. package/tools/qa-screenshot.mjs +0 -24
  94. package/tools/qa-visual.mjs +0 -51
@@ -0,0 +1,304 @@
1
+ #!/usr/bin/env python3
2
+ # SPDX-License-Identifier: GPL-3.0-or-later
3
+ # SPDX-FileCopyrightText: CozyClay contributors
4
+ #
5
+ # cclay-owned script. Source of truth lives in the CozyClay repo at
6
+ # tools/ardy/cclay_worker.py.
7
+ """Persistent local generation worker for the CozyClay ARDY bridge.
8
+
9
+ Loading the motion model and compiling its GPU kernels costs minutes on MPS,
10
+ while the sampling itself takes seconds. Spawning a fresh python per request
11
+ (the remote-box contract run-local.mjs mirrors) pays that cost every time, so
12
+ this worker pays it ONCE: it loads the model, runs a tiny no-text warmup to
13
+ force kernel compilation, and then serves generation jobs from the warm
14
+ process. run-local.mjs prefers the worker and silently falls back to a direct
15
+ spawn when it is not running, so the worker is an accelerator, never a
16
+ dependency.
17
+
18
+ Protocol (loopback TCP, one connection per job, one job at a time):
19
+
20
+ client -> worker one JSON line
21
+ {"mode": "ping"}
22
+ {"mode": "single"|"sequence"|"edit", "argv": [...]}
23
+ worker -> client the generator's stdout lines, verbatim, then
24
+ "worker: pong" (ping) or "worker: exit <code>"
25
+
26
+ `argv` is exactly the CLI argument vector of the corresponding
27
+ cclay_*_generate/edit script; the worker calls that script's main() in
28
+ process with the cached model injected, so worker and direct-spawn runs share
29
+ one code path and cannot drift.
30
+
31
+ env:
32
+ CCLAY_ARDY_WORKER_PORT listen port (default 9552, loopback only)
33
+ CHECKPOINTS_DIR / TEXT_ENCODER_URL / TEXT_ENCODERS_DIR as usual
34
+ """
35
+
36
+ import importlib
37
+ import json
38
+ import os
39
+ import socket
40
+ import sys
41
+ import traceback
42
+
43
+ HERE = os.path.dirname(os.path.abspath(__file__))
44
+ sys.path.insert(0, HERE)
45
+
46
+ MODULES = {
47
+ "single": "cclay_constrained_generate",
48
+ "sequence": "cclay_sequence_generate",
49
+ "edit": "cclay_motion_edit",
50
+ }
51
+
52
+
53
+ class _Tee:
54
+ """Mirror generator stdout to the job's socket AND the worker console.
55
+
56
+ A vanished client must not kill the job: the generation keeps running,
57
+ the npz still lands on disk, and only the socket copy is dropped.
58
+ """
59
+
60
+ def __init__(self, sock_file):
61
+ self._sock_file = sock_file
62
+
63
+ def write(self, data):
64
+ sys.__stdout__.write(data)
65
+ try:
66
+ self._sock_file.write(data)
67
+ self._sock_file.flush()
68
+ except OSError:
69
+ pass
70
+
71
+ def flush(self):
72
+ sys.__stdout__.flush()
73
+
74
+
75
+ def requested_model_name(mode, argv, resolve_model_name, default_model, checkpoints_dir):
76
+ """The model a job's argv asks for (edit has no --model flag)."""
77
+ name = default_model
78
+ if mode != "edit":
79
+ for index, arg in enumerate(argv):
80
+ if arg == "--model" and index + 1 < len(argv):
81
+ name = argv[index + 1]
82
+ return resolve_model_name(name, checkpoints_dir=checkpoints_dir)
83
+
84
+
85
+ def fix_text_encoder_device(model, device):
86
+ """Move an in-process LLM2Vec encoder onto the generation device, fast.
87
+
88
+ ARDY's load_text_encoder ends with `.to(device or "cuda"/"cpu")`, so on a
89
+ Mac the fallback encoder lands on CPU even when TEXT_ENCODER_DEVICE says
90
+ mps. On MPS the dtype matters as much as the device: bf16 matmuls are
91
+ emulated (a single prompt costs ~30-50 s) while fp16 is native, so fp16
92
+ is tried first and a canary encode guards against fp16 overflow — NaNs
93
+ demote back to bf16. Only the real in-process encoder has a `.model`;
94
+ the API client stub is left alone.
95
+ """
96
+ encoder = getattr(model, "text_encoder", None)
97
+ if encoder is None or not hasattr(encoder, "model") or device == "cpu":
98
+ return
99
+ import torch
100
+
101
+ for dtype in (torch.float16, torch.bfloat16):
102
+ try:
103
+ encoder.to(device=device, dtype=dtype)
104
+ canary, _ = encoder(["a person walks forward"])
105
+ if bool(torch.isfinite(canary.float()).all()):
106
+ print(f"worker: text encoder on {device} as {dtype}", flush=True)
107
+ return
108
+ print(f"worker: text encoder {dtype} produced non-finite values; trying next dtype", flush=True)
109
+ except Exception as exc:
110
+ print(f"worker: text encoder {dtype} on {device} failed ({exc}); trying next dtype", flush=True)
111
+ try: # both GPU dtypes failed: back to the slow-but-correct CPU encoder
112
+ encoder.to(device="cpu", dtype=torch.bfloat16)
113
+ print("worker: text encoder stays on cpu", flush=True)
114
+ except Exception as exc:
115
+ print(f"worker: text encoder left as loaded ({exc})", flush=True)
116
+
117
+
118
+ class _CachingEncoder:
119
+ """Per-prompt embedding cache in front of the in-process encoder.
120
+
121
+ Authoring iterates on ONE prompt (new seed, new pose pin, new waypoints),
122
+ so most requests re-encode text the worker has already seen. Embeddings
123
+ are deterministic per (model, dtype) — exactly the cache key's tag — and
124
+ a few KB each, so they live in memory and under ~/.cozyclay/embed-cache
125
+ across worker restarts. Misses are timed on the console; hits are free.
126
+ """
127
+
128
+ def __init__(self, inner, cache_dir, tag):
129
+ import os
130
+
131
+ self._inner = inner
132
+ self._dir = cache_dir
133
+ self._tag = tag
134
+ self._mem = {}
135
+ os.makedirs(cache_dir, exist_ok=True)
136
+
137
+ def _key(self, text):
138
+ import hashlib
139
+
140
+ return hashlib.sha256(f"{self._tag}\x00{text}".encode("utf-8")).hexdigest()
141
+
142
+ def _encode_one(self, text):
143
+ import os
144
+ import time
145
+
146
+ import numpy as np
147
+
148
+ key = self._key(text)
149
+ if key in self._mem:
150
+ return self._mem[key]
151
+ path = os.path.join(self._dir, f"{key}.npy")
152
+ if os.path.isfile(path):
153
+ try:
154
+ embedding = np.load(path)
155
+ self._mem[key] = embedding
156
+ return embedding
157
+ except Exception:
158
+ pass # unreadable cache entry: re-encode below
159
+ started = time.time()
160
+ tensor, _ = self._inner([text])
161
+ embedding = tensor[0].detach().float().cpu().numpy()
162
+ print(f"worker: text encode took {time.time() - started:.1f}s (cache miss)", flush=True)
163
+ self._mem[key] = embedding
164
+ try:
165
+ np.save(path, embedding)
166
+ except Exception:
167
+ pass # cache write failure must never fail the job
168
+ return embedding
169
+
170
+ def __call__(self, texts):
171
+ import numpy as np
172
+ import torch
173
+
174
+ is_string = isinstance(texts, str)
175
+ text_list = [texts] if is_string else list(texts)
176
+ stacked = np.stack([self._encode_one(text) for text in text_list])
177
+ device = getattr(self._inner, "_device", "cpu")
178
+ encoded = torch.from_numpy(stacked).to(device)
179
+ lengths = [1] * len(text_list)
180
+ if is_string:
181
+ return encoded[0], 1
182
+ return encoded, lengths
183
+
184
+ def __getattr__(self, name):
185
+ return getattr(self._inner, name)
186
+
187
+
188
+ def warmup(model, device):
189
+ """One throwaway sampling call so kernel compilation happens off the clock.
190
+
191
+ Zero text features skip the text encoder entirely — the warmup must not
192
+ drag the 16 GB Llama encoder into RAM just to compile Metal shaders.
193
+ """
194
+ import torch
195
+ from ardy.motion_rep.tools import length_to_mask
196
+
197
+ try:
198
+ frames = int(model.gen_horizon_len)
199
+ lengths = torch.tensor([frames], device=device)
200
+ with torch.no_grad():
201
+ model(
202
+ [""],
203
+ frames,
204
+ num_denoising_steps=1,
205
+ pad_mask=length_to_mask(lengths),
206
+ first_heading_angle=torch.zeros(1, device=device),
207
+ motion_mask=None,
208
+ observed_motion=None,
209
+ cfg_weight=2.0,
210
+ text_feat=torch.zeros(1, 1, 4096, device=device),
211
+ text_pad_mask=torch.ones(1, 1, device=device, dtype=torch.bool),
212
+ progress_bar=lambda iterable: iterable,
213
+ )
214
+ print("worker: warmup complete", flush=True)
215
+ except Exception as exc: # non-fatal: first real job just pays the compile
216
+ print(f"worker: warmup skipped ({exc})", flush=True)
217
+
218
+
219
+ def main():
220
+ port = int(os.environ.get("CCLAY_ARDY_WORKER_PORT", "9552"))
221
+
222
+ single = importlib.import_module(MODULES["single"])
223
+ device = single.cclay_pick_device()
224
+ single.cclay_mps_compat(device)
225
+
226
+ from ardy.model import DEFAULT_MODEL, load_model
227
+ from ardy.model.loading import get_env_var
228
+ from ardy.model.registry import resolve_model_name
229
+
230
+ checkpoints_dir = get_env_var("CHECKPOINTS_DIR", None)
231
+ cached_name = resolve_model_name(DEFAULT_MODEL, checkpoints_dir=checkpoints_dir)
232
+ print(f"worker: loading {cached_name} on {device} ...", flush=True)
233
+ cached_model = load_model(cached_name, device=device, checkpoints_dir=checkpoints_dir)
234
+ cached_model.eval()
235
+ fix_text_encoder_device(cached_model, device)
236
+ encoder = getattr(cached_model, "text_encoder", None)
237
+ if encoder is not None and hasattr(encoder, "model"):
238
+ try:
239
+ param = next(encoder.model.parameters())
240
+ tag = f"llm2vec:{param.dtype}:{param.device.type}"
241
+ except Exception:
242
+ tag = "llm2vec:unknown"
243
+ cache_dir = os.path.join(os.path.expanduser("~"), ".cozyclay", "embed-cache")
244
+ cached_model.text_encoder = _CachingEncoder(encoder, cache_dir, tag)
245
+ print(f"worker: embedding cache at {cache_dir} ({tag})", flush=True)
246
+ warmup(cached_model, device)
247
+
248
+ server = socket.create_server(("127.0.0.1", port))
249
+ print(f"worker: ready on 127.0.0.1:{port} (model {cached_name}, device {device})", flush=True)
250
+
251
+ while True:
252
+ conn, _ = server.accept()
253
+ with conn:
254
+ rfile = conn.makefile("r", encoding="utf-8")
255
+ wfile = conn.makefile("w", encoding="utf-8")
256
+ try:
257
+ job = json.loads(rfile.readline())
258
+ except (ValueError, OSError):
259
+ continue
260
+ mode = job.get("mode") if isinstance(job, dict) else None
261
+ if mode == "ping":
262
+ try:
263
+ wfile.write("worker: pong\n")
264
+ wfile.flush()
265
+ except OSError:
266
+ pass
267
+ continue
268
+ argv = job.get("argv") if isinstance(job, dict) else None
269
+ if mode not in MODULES or not isinstance(argv, list) or not all(isinstance(a, str) for a in argv):
270
+ try:
271
+ wfile.write("worker: exit 2\n")
272
+ wfile.flush()
273
+ except OSError:
274
+ pass
275
+ continue
276
+
277
+ module = importlib.import_module(MODULES[mode])
278
+ code = 0
279
+ old_stdout = sys.stdout
280
+ sys.stdout = _Tee(wfile)
281
+ try:
282
+ wanted = requested_model_name(mode, argv, resolve_model_name, DEFAULT_MODEL, checkpoints_dir)
283
+ if wanted != cached_name:
284
+ print(f"worker: switching model {cached_name} -> {wanted}", flush=True)
285
+ cached_model = load_model(wanted, device=device, checkpoints_dir=checkpoints_dir)
286
+ cached_model.eval()
287
+ cached_name = wanted
288
+ module.main(argv, preloaded_model=cached_model)
289
+ except SystemExit as exit_exc:
290
+ code = exit_exc.code if isinstance(exit_exc.code, int) else 1
291
+ except Exception:
292
+ traceback.print_exc(file=sys.stdout)
293
+ code = 1
294
+ finally:
295
+ sys.stdout = old_stdout
296
+ try:
297
+ wfile.write(f"worker: exit {code}\n")
298
+ wfile.flush()
299
+ except OSError:
300
+ pass
301
+
302
+
303
+ if __name__ == "__main__":
304
+ main()
@@ -0,0 +1,160 @@
1
+ #!/usr/bin/env python3
2
+ """merge-text-encoder.py - bake the MNTP LoRA into the base weights.
3
+
4
+ Run under the ARDY text-encoder venv (needs torch, peft, transformers, and
5
+ the `ardy` package importable — i.e. from the ARDY checkout):
6
+
7
+ cd ~/ardy && .venv/bin/python merge-text-encoder.py --dest ~/cclay-text-encoders
8
+
9
+ Why this exists: transformers v5 removed the automatic PEFT-adapter
10
+ resolution that `from_pretrained` did in v4 (`find_adapter_config_file` is
11
+ gone from modeling_utils). ARDY's vendored LLM2Vec loader relies on that
12
+ behavior — it points `from_pretrained` at the MNTP *adapter* repo and lets
13
+ transformers chase `adapter_config.json` to the (gated) base. Under v5 that
14
+ path simply fails with "no model.safetensors found".
15
+
16
+ So instead of an adapter directory, the local MNTP path becomes a full
17
+ model: this script loads the ungated base (NousResearch mirror, already
18
+ hash-verified by setup-text-encoder.py), applies the MNTP LoRA with peft,
19
+ `merge_and_unload()`s it — the exact math upstream LLM2Vec performs at load
20
+ time — and saves the merged weights to the path ARDY's TEXT_ENCODERS_DIR
21
+ override resolves. Plain full-model loading works identically under
22
+ transformers v4 and v5; the supervised LoRA keeps loading through peft
23
+ directly, which both versions support.
24
+
25
+ Layout before: <dest>/McGill-NLP/...-mntp (downloaded adapter files)
26
+ Layout after: <dest>/McGill-NLP/...-mntp-src (the downloads, kept for
27
+ provenance + re-merges)
28
+ <dest>/McGill-NLP/...-mntp (merged full model +
29
+ tokenizer + MERGE_MANIFEST)
30
+
31
+ The merged config.json keeps `_name_or_path` =
32
+ "meta-llama/Meta-Llama-3-8B-Instruct": it is a metadata string (no download
33
+ or gate involved) that LLM2Vec's `prepare_for_tokenization` matches to pick
34
+ the Llama-3 chat template — losing it would silently change every prompt
35
+ embedding.
36
+ """
37
+
38
+ import argparse
39
+ import hashlib
40
+ import json
41
+ import os
42
+ import shutil
43
+ import sys
44
+
45
+ BASE = "NousResearch/Meta-Llama-3-8B-Instruct"
46
+ MNTP = "McGill-NLP/LLM2Vec-Meta-Llama-3-8B-Instruct-mntp"
47
+ MNTP_SRC = MNTP + "-src"
48
+ NAME_OR_PATH = "meta-llama/Meta-Llama-3-8B-Instruct"
49
+ TOKENIZER_FILES = ["tokenizer.json", "tokenizer_config.json", "special_tokens_map.json"]
50
+
51
+
52
+ def log(msg):
53
+ print("merge-text-encoder: %s" % msg, flush=True)
54
+
55
+
56
+ def fail(msg):
57
+ print("merge-text-encoder: ERROR: %s" % msg, file=sys.stderr, flush=True)
58
+ sys.exit(1)
59
+
60
+
61
+ def sha256_file(path):
62
+ h = hashlib.sha256()
63
+ with open(path, "rb") as f:
64
+ for block in iter(lambda: f.read(1024 * 1024), b""):
65
+ h.update(block)
66
+ return h.hexdigest()
67
+
68
+
69
+ def main():
70
+ ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
71
+ ap.add_argument("--dest", default=os.environ.get(
72
+ "TEXT_ENCODERS_DIR", os.path.expanduser("~/cclay-text-encoders")))
73
+ args = ap.parse_args()
74
+ dest = os.path.abspath(os.path.expanduser(args.dest))
75
+ base_dir = os.path.join(dest, BASE)
76
+ mntp_dir = os.path.join(dest, MNTP)
77
+ src_dir = os.path.join(dest, MNTP_SRC)
78
+
79
+ if not os.path.isfile(os.path.join(base_dir, "model.safetensors.index.json")):
80
+ fail("base model missing under %s — run setup-text-encoder.py first" % base_dir)
81
+
82
+ # Migrate a pre-merge tree: the download step used to land the adapter
83
+ # files directly at the mntp path.
84
+ if os.path.isdir(mntp_dir) and not os.path.isdir(src_dir) \
85
+ and os.path.isfile(os.path.join(mntp_dir, "adapter_model.safetensors")) \
86
+ and not os.path.isfile(os.path.join(mntp_dir, "model.safetensors.index.json")):
87
+ os.rename(mntp_dir, src_dir)
88
+ log("moved downloaded adapter layout to %s" % MNTP_SRC)
89
+
90
+ if not os.path.isfile(os.path.join(src_dir, "adapter_model.safetensors")):
91
+ fail("MNTP adapter missing under %s — run setup-text-encoder.py first" % src_dir)
92
+
93
+ adapter_sha = sha256_file(os.path.join(src_dir, "adapter_model.safetensors"))
94
+ manifest_path = os.path.join(mntp_dir, "MERGE_MANIFEST.json")
95
+ if os.path.isfile(manifest_path):
96
+ with open(manifest_path, "r", encoding="utf-8") as f:
97
+ manifest = json.load(f)
98
+ if manifest.get("mntp_adapter_sha256") == adapter_sha \
99
+ and os.path.isfile(os.path.join(mntp_dir, "model.safetensors.index.json")):
100
+ log("merged model already present and matches the adapter hash — nothing to do")
101
+ return
102
+
103
+ import torch # deferred: keep --help fast and errors readable
104
+ import peft
105
+ import transformers
106
+ from peft import PeftModel
107
+ from ardy.model.llm2vec.llm2vec import LLM2Vec
108
+
109
+ log("loading base (bfloat16, cpu) from %s" % base_dir)
110
+ model_class = LLM2Vec._get_model_class("LlamaConfig", enable_bidirectional=True)
111
+ model = model_class.from_pretrained(base_dir, torch_dtype=torch.bfloat16)
112
+
113
+ log("applying + merging the MNTP LoRA from %s" % src_dir)
114
+ model = PeftModel.from_pretrained(model, src_dir)
115
+ model = model.merge_and_unload()
116
+
117
+ tmp_out = mntp_dir + ".merging"
118
+ if os.path.isdir(tmp_out):
119
+ shutil.rmtree(tmp_out)
120
+ log("saving merged model to %s" % mntp_dir)
121
+ model.save_pretrained(tmp_out, safe_serialization=True)
122
+ del model
123
+
124
+ for name in TOKENIZER_FILES:
125
+ src = os.path.join(src_dir, name)
126
+ if os.path.isfile(src):
127
+ shutil.copy2(src, os.path.join(tmp_out, name))
128
+
129
+ # Preserve the chat-template key and the source architecture label.
130
+ cfg_path = os.path.join(tmp_out, "config.json")
131
+ with open(cfg_path, "r", encoding="utf-8") as f:
132
+ cfg = json.load(f)
133
+ with open(os.path.join(src_dir, "config.json"), "r", encoding="utf-8") as f:
134
+ src_cfg = json.load(f)
135
+ cfg["_name_or_path"] = NAME_OR_PATH
136
+ if "architectures" in src_cfg:
137
+ cfg["architectures"] = src_cfg["architectures"]
138
+ with open(cfg_path, "w", encoding="utf-8") as f:
139
+ json.dump(cfg, f, indent=2)
140
+ f.write("\n")
141
+
142
+ with open(os.path.join(tmp_out, "MERGE_MANIFEST.json"), "w", encoding="utf-8") as f:
143
+ json.dump({
144
+ "base": BASE,
145
+ "mntp_adapter_sha256": adapter_sha,
146
+ "torch": torch.__version__,
147
+ "transformers": transformers.__version__,
148
+ "peft": peft.__version__,
149
+ "name_or_path": NAME_OR_PATH,
150
+ }, f, indent=2)
151
+ f.write("\n")
152
+
153
+ if os.path.isdir(mntp_dir):
154
+ shutil.rmtree(mntp_dir)
155
+ os.rename(tmp_out, mntp_dir)
156
+ log("done — %s is a full merged model; the supervised LoRA stays a peft adapter" % MNTP)
157
+
158
+
159
+ if __name__ == "__main__":
160
+ main()
@@ -0,0 +1,233 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * run-local.mjs - generate an ARDY clip on THIS machine.
4
+ *
5
+ * run-local.mjs single [--pose-from NPZ SRC DST]... --prompt P --duration S
6
+ * [--seed N] [--root-2d F X Z H]... --output OUT.npz
7
+ * run-local.mjs sequence --segment PROMPT SECONDS [--segment ...]...
8
+ * [--root-2d F X Z H]... [--seed N] --output OUT.npz
9
+ * run-local.mjs edit --source SRC.npz --manifest M.json --prompt P
10
+ * --context-before N --context-after N [--seed N]
11
+ * [--pose POSE.npz]... --output OUT.npz
12
+ *
13
+ * The local twin of run-on-box.sh / run-sequence-on-box.sh /
14
+ * run-edit-on-box.sh: same modes, same success marker
15
+ *
16
+ * run-local: done - <output> (<bytes> bytes)
17
+ *
18
+ * minus every ssh/scp/temp-dir round trip — the generator scripts run in
19
+ * place from this repo against the ARDY checkout's venv. Device selection
20
+ * lives in the python side (cclay_pick_device: CCLAY_ARDY_DEVICE override,
21
+ * else cuda > mps > cpu). The text encoder is the bridge's lazy sidecar; a
22
+ * dead encoder degrades to ARDY's in-process fallback rather than failing.
23
+ *
24
+ * env (set by runners/local.mjs; usable standalone too):
25
+ * CCLAY_ARDY_LOCAL_DIR ARDY checkout (default ~/.cozyclay/ardy)
26
+ * CCLAY_ARDY_LOCAL_VENV venv python (default <dir>/.venv/bin/python)
27
+ * TEXT_ENCODER_URL encoder service (default http://127.0.0.1:9550/)
28
+ * TEXT_ENCODERS_DIR encoder weights dir (default ~/.cozyclay/text-encoders)
29
+ * CCLAY_ARDY_DEVICE force a torch device (cpu|mps|cuda[:N])
30
+ */
31
+
32
+ import { spawn } from "node:child_process";
33
+ import { cpSync, existsSync, mkdtempSync, rmSync, statSync } from "node:fs";
34
+ import { createConnection } from "node:net";
35
+ import { homedir, tmpdir } from "node:os";
36
+ import { basename, dirname, join } from "node:path";
37
+ import { fileURLToPath } from "node:url";
38
+
39
+ const HERE = dirname(fileURLToPath(import.meta.url));
40
+ const LOCAL_DIR = process.env.CCLAY_ARDY_LOCAL_DIR || join(homedir(), ".cozyclay", "ardy");
41
+ const VENV_PY =
42
+ process.env.CCLAY_ARDY_LOCAL_VENV ||
43
+ join(LOCAL_DIR, ".venv", process.platform === "win32" ? "Scripts\\python.exe" : "bin/python");
44
+ const WORKER_PORT = Number(process.env.CCLAY_ARDY_WORKER_PORT) || 9552;
45
+ // A sequence job can legitimately run a long time; the watchdog only guards
46
+ // against a worker that stopped talking entirely.
47
+ const WORKER_SILENCE_TIMEOUT_MS = 60 * 60_000;
48
+
49
+ const GENERATORS = {
50
+ single: join(HERE, "cclay_constrained_generate.py"),
51
+ sequence: join(HERE, "cclay_sequence_generate.py"),
52
+ edit: join(HERE, "cclay_motion_edit.py"),
53
+ };
54
+
55
+ function die(message) {
56
+ console.error(`run-local: ${message}`);
57
+ process.exit(2);
58
+ }
59
+
60
+ function usage() {
61
+ console.error("usage: run-local.mjs <single|sequence|edit> [flags] --output OUT.npz");
62
+ console.error("see the header of tools/ardy/run-local.mjs for the full grammar");
63
+ process.exit(2);
64
+ }
65
+
66
+ // The bridge builds these argv arrays itself, so parsing here is deliberately
67
+ // shallow: pull out the handful of values run-local needs (--output always;
68
+ // --cpu, --pose/--manifest for staging) and pass everything else through to
69
+ // the python generator untouched.
70
+ function parseArgs(argv) {
71
+ const passthrough = [];
72
+ let output = "";
73
+ let cpu = false;
74
+ const poses = [];
75
+ let manifest = "";
76
+ for (let i = 0; i < argv.length; i += 1) {
77
+ const arg = argv[i];
78
+ if (arg === "--output") {
79
+ output = argv[++i] ?? "";
80
+ continue;
81
+ }
82
+ if (arg === "--cpu") {
83
+ cpu = true;
84
+ continue;
85
+ }
86
+ if (arg === "--pose") {
87
+ poses.push(argv[++i] ?? "");
88
+ continue;
89
+ }
90
+ if (arg === "--manifest") {
91
+ manifest = argv[++i] ?? "";
92
+ continue;
93
+ }
94
+ passthrough.push(arg);
95
+ }
96
+ if (!output) die("--output is required");
97
+ return { passthrough, output, cpu, poses, manifest };
98
+ }
99
+
100
+ /**
101
+ * Try to run the job on the persistent worker (tools/ardy/cclay_worker.py).
102
+ *
103
+ * Resolves {code} when the worker accepted and finished the job (its stdout
104
+ * lines have already been relayed), or null when the worker is unavailable
105
+ * or died mid-job — the caller falls back to a direct spawn, which is always
106
+ * correct because the job is idempotent (same --output, overwritten).
107
+ */
108
+ function runViaWorker(mode, flagArgs) {
109
+ return new Promise((resolvePromise) => {
110
+ const sock = createConnection({ host: "127.0.0.1", port: WORKER_PORT });
111
+ let buffer = "";
112
+ let settled = false;
113
+ const finish = (value) => {
114
+ if (!settled) {
115
+ settled = true;
116
+ sock.destroy();
117
+ resolvePromise(value);
118
+ }
119
+ };
120
+ sock.setTimeout(WORKER_SILENCE_TIMEOUT_MS, () => finish(null));
121
+ sock.on("error", () => finish(null));
122
+ sock.on("close", () => finish(null));
123
+ sock.on("connect", () => {
124
+ sock.write(`${JSON.stringify({ mode, argv: flagArgs })}\n`);
125
+ });
126
+ sock.on("data", (chunk) => {
127
+ buffer += chunk.toString("utf8");
128
+ let nl;
129
+ while ((nl = buffer.indexOf("\n")) !== -1) {
130
+ const line = buffer.slice(0, nl);
131
+ buffer = buffer.slice(nl + 1);
132
+ const marker = /^worker: exit (\d+)$/.exec(line);
133
+ if (marker) {
134
+ finish({ code: Number(marker[1]) });
135
+ return;
136
+ }
137
+ if (line) console.log(line);
138
+ }
139
+ });
140
+ });
141
+ }
142
+
143
+ async function main() {
144
+ const [mode, ...rest] = process.argv.slice(2);
145
+ if (!mode || mode === "-h" || mode === "--help") usage();
146
+ const generator = GENERATORS[mode];
147
+ if (!generator) die(`unknown mode '${mode}' (expected single, sequence, or edit)`);
148
+ if (!existsSync(VENV_PY)) {
149
+ die(`venv python not found at ${VENV_PY}; run \`node tools/ardy/setup-local.mjs\` once`);
150
+ }
151
+ if (!existsSync(LOCAL_DIR)) {
152
+ die(`ARDY checkout not found at ${LOCAL_DIR}; run \`node tools/ardy/setup-local.mjs\` once`);
153
+ }
154
+
155
+ const { passthrough, output, cpu, poses, manifest } = parseArgs(rest);
156
+
157
+ const flagArgs = [...passthrough];
158
+ let stagingDir = null;
159
+ if (mode === "edit") {
160
+ if (!manifest) die("--manifest is required in edit mode");
161
+ if (poses.length === 0) die("edit mode needs at least one --pose");
162
+ // cclay_motion_edit.py resolves the manifest's relative pose_path
163
+ // entries against the manifest's own directory, expecting pose-<i>.npz
164
+ // siblings (the layout run-edit-on-box.sh created with scp). Stage the
165
+ // same layout in a fresh temp dir.
166
+ stagingDir = mkdtempSync(join(tmpdir(), "cozyclay-edit-"));
167
+ cpSync(manifest, join(stagingDir, "manifest.json"));
168
+ poses.forEach((posePath, index) => {
169
+ cpSync(posePath, join(stagingDir, `pose-${index}.npz`));
170
+ });
171
+ flagArgs.push("--manifest", join(stagingDir, "manifest.json"));
172
+ } else if (manifest) {
173
+ flagArgs.push("--manifest", manifest);
174
+ }
175
+ flagArgs.push("--output", output);
176
+
177
+ const env = { ...process.env };
178
+ if (cpu) env.CCLAY_ARDY_DEVICE = "cpu";
179
+
180
+ // Prefer the warm worker; a forced device (--cpu / CCLAY_ARDY_DEVICE)
181
+ // bypasses it because the worker's model already lives on the auto-picked
182
+ // device. null = worker unavailable, fall through to a direct spawn.
183
+ let code = null;
184
+ const forcedDevice = cpu || Boolean(process.env.CCLAY_ARDY_DEVICE);
185
+ if (!forcedDevice) {
186
+ const workerResult = await runViaWorker(mode, flagArgs);
187
+ if (workerResult) {
188
+ console.log("run-local: served by warm worker");
189
+ code = workerResult.code;
190
+ }
191
+ }
192
+ if (code === null) {
193
+ console.log(`run-local: ${mode} generation with ${basename(VENV_PY)} (${LOCAL_DIR})`);
194
+ const child = spawn(VENV_PY, [generator, ...flagArgs], {
195
+ cwd: LOCAL_DIR, // keeps any generator-relative paths (outputs/, HF cache) out of this repo
196
+ stdio: ["ignore", "inherit", "inherit"],
197
+ env,
198
+ });
199
+ code = await new Promise((resolvePromise, reject) => {
200
+ child.on("error", reject);
201
+ child.on("close", resolvePromise);
202
+ });
203
+ }
204
+ if (stagingDir) {
205
+ // Cleanup failure must never mask the real result.
206
+ try {
207
+ rmSync(stagingDir, { recursive: true, force: true });
208
+ } catch (err) {
209
+ console.error(`run-local: could not remove ${stagingDir}: ${err.message}`);
210
+ }
211
+ }
212
+ if (code !== 0) {
213
+ console.error(`run-local: generator exited with code ${code}`);
214
+ process.exit(code || 1);
215
+ }
216
+ let size;
217
+ try {
218
+ size = statSync(output).size;
219
+ } catch {
220
+ console.error(`run-local: generator exited 0 but ${output} does not exist`);
221
+ process.exit(1);
222
+ }
223
+ if (size === 0) {
224
+ console.error(`run-local: generator produced an empty file at ${output}`);
225
+ process.exit(1);
226
+ }
227
+ console.log(`run-local: done - ${output} (${size} bytes)`);
228
+ }
229
+
230
+ main().catch((err) => {
231
+ console.error(`run-local: ${err.stack || err}`);
232
+ process.exit(1);
233
+ });