cozyclay 1.9.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (286) hide show
  1. package/CHANGELOG.md +171 -0
  2. package/bin/agent/agent-routes.mjs +74 -70
  3. package/bin/agent/agent-runner.mjs +10 -3
  4. package/bin/agent/motion-runtime.mjs +32 -260
  5. package/bin/agent/providers.mjs +46 -3
  6. package/bin/agent/session-store.mjs +41 -5
  7. package/bin/agent/studio-prompt.mjs +1 -1
  8. package/bin/agent/studio-tools.mjs +48 -11
  9. package/bin/cozyclay.mjs +26 -8
  10. package/bin/live/cli.mjs +22 -3
  11. package/bin/mcp-runtime.mjs +35 -20
  12. package/bin/telemetry-state.mjs +5 -2
  13. package/bin/update-check.mjs +15 -0
  14. package/dist/app/index.html +6 -6
  15. package/dist/assets/analytics-MVZOEW8O.js +1 -0
  16. package/dist/assets/app-C1sdVbg4.css +1 -0
  17. package/dist/assets/app-CA55tcI3.js +4876 -0
  18. package/dist/assets/{demo-B-eg1CT8.js → demo-ByeYp-dY.js} +1 -1
  19. package/dist/assets/{first-shot-handoff-CGgnvNu8.js → first-shot-handoff-DgT28ZiV.js} +1 -1
  20. package/dist/assets/{landing-IbRp3FnQ.js → landing-BaXkpFip.js} +1 -1
  21. package/dist/assets/shot-prompt-CbqWcWYi.css +1 -0
  22. package/dist/assets/shot-prompt-Yrt99wZx.js +17 -0
  23. package/dist/assets/{ticket-BgIzOBOf.js → ticket-BcmsnHn5.js} +1 -1
  24. package/dist/assets/workflow-DRdwoDBT.css +1 -0
  25. package/dist/assets/workflow-dNPGY9xz.js +179 -0
  26. package/dist/cozyclay-package.json +1 -1
  27. package/dist/fonts/IBM-Plex-Sans-OFL.txt +92 -0
  28. package/dist/fonts/JetBrains-Mono-OFL.txt +93 -0
  29. package/dist/fonts/README.md +13 -9
  30. package/dist/fonts/ibm-plex-sans-400-latin.woff2 +0 -0
  31. package/dist/fonts/ibm-plex-sans-500-latin.woff2 +0 -0
  32. package/dist/fonts/ibm-plex-sans-600-latin.woff2 +0 -0
  33. package/dist/fonts/jetbrains-mono-400-latin.woff2 +0 -0
  34. package/dist/fonts/jetbrains-mono-500-latin.woff2 +0 -0
  35. package/dist/index.html +7 -7
  36. package/dist/privacy/index.html +13 -8
  37. package/dist/sitemap.xml +6 -6
  38. package/dist/workflow/index.html +5 -5
  39. package/mcp/LIVE-PROTOCOL.md +1 -1
  40. package/mcp/live-hub.mjs +35 -93
  41. package/mcp/mesh-file.mjs +48 -0
  42. package/mcp/server.mjs +6 -77
  43. package/mcp/tool-handlers.mjs +224 -377
  44. package/package.json +2 -1
  45. package/src/App.jsx +1273 -9075
  46. package/src/analytics.js +393 -9
  47. package/src/app-context.js +211 -0
  48. package/src/app-stage.jsx +25 -24
  49. package/src/ardy/auto-fix-panel.css +108 -0
  50. package/src/ardy/collision-blockers.js +9 -3
  51. package/src/ardy/fix-collisions.js +10 -66
  52. package/src/ardy/ground.js +4 -4
  53. package/src/ardy/ik-drag.js +101 -0
  54. package/src/ardy/ik-key-json.js +43 -0
  55. package/src/ardy/ik.js +122 -76
  56. package/src/ardy/physics-panel.css +2 -0
  57. package/src/ardy/physics-panel.jsx +17 -5
  58. package/src/ardy/platform-fit-panel.css +64 -0
  59. package/src/ardy/platform-fit-panel.jsx +40 -0
  60. package/src/ardy/platform-fit.js +326 -0
  61. package/src/ardy/playback.js +166 -20
  62. package/src/ardy/range-pin.js +299 -0
  63. package/src/ardy/timeline-coordinates.js +26 -0
  64. package/src/ardy/timeline.css +1199 -0
  65. package/src/ardy/timeline.jsx +149 -41
  66. package/src/ardy/waypoints.js +2 -2
  67. package/src/asset-pane.css +462 -0
  68. package/src/asset-pane.jsx +461 -232
  69. package/src/command-bus.js +349 -0
  70. package/src/commands/ai.js +31 -0
  71. package/src/commands/cast.js +140 -0
  72. package/src/commands/elements/character.js +13 -0
  73. package/src/commands/elements/motion.js +14 -0
  74. package/src/commands/elements/object.js +22 -0
  75. package/src/commands/elements/scene.js +16 -0
  76. package/src/commands/elements/shot.js +9 -0
  77. package/src/commands/elements/stage.js +11 -0
  78. package/src/commands/elements.js +155 -0
  79. package/src/commands/export.js +20 -0
  80. package/src/commands/index.js +40 -0
  81. package/src/commands/motion.js +231 -0
  82. package/src/commands/objects.js +174 -0
  83. package/src/commands/project.js +53 -0
  84. package/src/commands/scene.js +81 -0
  85. package/src/commands/shared.js +19 -0
  86. package/src/commands/shot.js +103 -0
  87. package/src/commands/stage.js +17 -0
  88. package/src/commands/view.js +42 -0
  89. package/src/document-store.js +144 -0
  90. package/src/domains/cast.js +1007 -0
  91. package/src/domains/motion.js +3219 -0
  92. package/src/domains/objects.js +932 -0
  93. package/src/domains/scenes.js +817 -0
  94. package/src/domains/shots.js +404 -0
  95. package/src/domains/stage.js +93 -0
  96. package/src/dualview.jsx +113 -57
  97. package/src/facing-marks.js +3 -0
  98. package/src/first-success-guide.jsx +14 -14
  99. package/src/grid-view.js +18 -5
  100. package/src/hierarchy-model.js +24 -16
  101. package/src/hierarchy-panel.css +386 -0
  102. package/src/hierarchy-panel.jsx +150 -21
  103. package/src/{fal-motion-client.js → i2v-motion-client.js} +13 -13
  104. package/src/i2v-motion-studio.jsx +180 -0
  105. package/src/ik-camera.js +3 -0
  106. package/src/main.jsx +6 -0
  107. package/src/motion/generation.js +105 -0
  108. package/src/motion-readiness-ui.jsx +4 -4
  109. package/src/motion-readiness.js +11 -0
  110. package/src/motion-trail.js +366 -9
  111. package/src/object-gizmo.jsx +7 -1
  112. package/src/otio.js +2 -1
  113. package/src/panels/CameraPanel.jsx +60 -0
  114. package/src/panels/CharacterTransformPanel.jsx +48 -0
  115. package/src/panels/EnvironmentPanel.jsx +40 -0
  116. package/src/panels/Foldout.jsx +32 -0
  117. package/src/panels/LightPanel.jsx +24 -0
  118. package/src/panels/ObjectTransformPanel.jsx +554 -0
  119. package/src/panels/PosePanel.jsx +97 -0
  120. package/src/panels/ProjectPanel.jsx +33 -0
  121. package/src/panels/PromptBlocksPanel.jsx +442 -0
  122. package/src/panels/PropsPanel.jsx +70 -0
  123. package/src/panels/ReferenceImageField.jsx +91 -0
  124. package/src/panels/RigControlPanel.jsx +78 -0
  125. package/src/panels/RigPanel.jsx +37 -0
  126. package/src/panels/SubjectBox.jsx +47 -0
  127. package/src/panels/SubjectsPanel.jsx +39 -0
  128. package/src/panels/VideoCapturePanel.jsx +180 -0
  129. package/src/panels/details.css +1171 -0
  130. package/src/panels/motion.css +197 -0
  131. package/src/panels/pose.css +39 -0
  132. package/src/planview.jsx +60 -31
  133. package/src/posestudio.jsx +59 -4
  134. package/src/project-browser.css +1073 -0
  135. package/src/project-browser.jsx +284 -123
  136. package/src/range-pin-object-transform.js +19 -0
  137. package/src/range-pin-panel.css +518 -0
  138. package/src/range-pin-panel.jsx +344 -0
  139. package/src/result-modal.jsx +5 -5
  140. package/src/room.jsx +79 -51
  141. package/src/scene-objects.js +37 -1
  142. package/src/scenes.js +5 -0
  143. package/src/semantic-edit.js +2 -0
  144. package/src/settings-menu.jsx +39 -105
  145. package/src/shell/BottomDock.jsx +450 -0
  146. package/src/shell/DetailsSlot.jsx +513 -0
  147. package/src/shell/LibrarySlot.jsx +86 -0
  148. package/src/shell/MenuBar.jsx +601 -0
  149. package/src/shell/OutlinerSlot.jsx +50 -0
  150. package/src/shell/PreferencesDialog.jsx +538 -0
  151. package/src/shell/PreferencesSlot.jsx +40 -0
  152. package/src/shell/StatusBar.jsx +125 -0
  153. package/src/shell/StudioShell.jsx +92 -0
  154. package/src/shell/TopBar.jsx +129 -0
  155. package/src/shell/ViewportToolbar.jsx +656 -0
  156. package/src/shell/agent-glass.css +296 -0
  157. package/src/shell/dock.css +93 -0
  158. package/src/shell/glass-regions.css +985 -0
  159. package/src/shell/glass.css +687 -0
  160. package/src/shell/log-store.js +52 -0
  161. package/src/shell/mode.css +9 -0
  162. package/src/shell/preferences.css +651 -0
  163. package/src/shell/shell.css +282 -0
  164. package/src/shell/studio-shell-context.js +19 -0
  165. package/src/shell/topbar.css +435 -0
  166. package/src/shell/viewport.css +451 -0
  167. package/src/store/authored-intent.js +22 -0
  168. package/src/store/runtime-adapters.js +92 -0
  169. package/src/store/scene-stage.js +21 -0
  170. package/src/store/use-document-store.js +14 -0
  171. package/src/studio-actions.js +210 -0
  172. package/src/studio-agent-commands.js +41 -271
  173. package/src/studio-agent-context.js +39 -10
  174. package/src/studio-agent-motion.js +122 -373
  175. package/src/studio-agent-protocol.js +67 -31
  176. package/src/studio-app-binding.js +379 -0
  177. package/src/studio-contact-sheet.js +73 -0
  178. package/src/studio-elements.js +58 -51
  179. package/src/styles/themes.css +200 -0
  180. package/src/styles/tokens.css +69 -0
  181. package/src/styles.css +460 -1742
  182. package/src/theme.js +44 -0
  183. package/src/timeline-extent.js +16 -0
  184. package/src/trail-key-conflicts.js +39 -0
  185. package/src/trail-pick.js +59 -0
  186. package/src/ui.jsx +8 -8
  187. package/src/use-case-question.jsx +66 -0
  188. package/src/workflow/AgentPanel.jsx +82 -69
  189. package/src/workflow/agent-client.js +12 -2
  190. package/src/workflow/agent-panel.css +34 -32
  191. package/src/workflow/cozy-scene-node.css +2 -0
  192. package/src/workflow/workflow.css +2 -0
  193. package/tools/ardy/__pycache__/cclay_gvhmr_worker.cpython-313.pyc +0 -0
  194. package/tools/ardy/bridge.mjs +164 -146
  195. package/tools/ardy/visual-qa.mjs +5 -5
  196. package/tools/bench/EXP3.md +61 -0
  197. package/tools/bench/cclay_bench_extract_incam.py +125 -0
  198. package/tools/bench/cclay_bench_extract_obs.py +204 -0
  199. package/tools/bench/cclay_bench_runner.py +42 -0
  200. package/tools/bench/cube-contact.mjs +162 -0
  201. package/tools/bench/exp3.mjs +152 -0
  202. package/tools/bench/extract-bench-lib.mjs +216 -0
  203. package/tools/bench/extract-bench.mjs +302 -0
  204. package/tools/bench/fal-generate.mjs +88 -0
  205. package/tools/bench/fit/README.md +189 -0
  206. package/tools/bench/fit/camera.mjs +59 -0
  207. package/tools/bench/fit/contact.mjs +419 -0
  208. package/tools/bench/fit/footlock.mjs +234 -0
  209. package/tools/bench/fit/motion.mjs +85 -0
  210. package/tools/bench/fit/pin.mjs +27 -0
  211. package/tools/bench/fit/remote.mjs +105 -0
  212. package/tools/bench/fit-bench.mjs +126 -0
  213. package/tools/bench/fit-sanity.mjs +45 -0
  214. package/tools/bench/metrics.mjs +313 -0
  215. package/tools/bench/obs/depth.mjs +258 -0
  216. package/tools/bench/obs/extrinsics.mjs +175 -0
  217. package/tools/bench/obs/fit_mannequin_betas.py +340 -0
  218. package/tools/bench/obs/ground.mjs +294 -0
  219. package/tools/bench/obs/heading.mjs +151 -0
  220. package/tools/bench/obs/ladder.mjs +555 -0
  221. package/tools/bench/obs/mannequin-betas.json +124 -0
  222. package/tools/bench/obs/remote.mjs +83 -0
  223. package/tools/bench/obs/rest_joints.py +52 -0
  224. package/tools/bench/obs/ybot-targets.mjs +49 -0
  225. package/tools/bench/obs-bench.mjs +520 -0
  226. package/tools/bench/score.mjs +343 -0
  227. package/tools/bench/summarize.mjs +86 -0
  228. package/tools/dev/pages/privacy.html +13 -8
  229. package/tools/gt-render/browser.mjs +159 -0
  230. package/tools/gt-render/camera-math.mjs +276 -0
  231. package/tools/gt-render/page.mjs +267 -0
  232. package/tools/gt-render/render.mjs +514 -0
  233. package/tools/gt-render/scene-box.mjs +31 -0
  234. package/tools/gt-render/take-transform.mjs +87 -0
  235. package/tools/morphgs/assets/gen_truth.py +39 -0
  236. package/tools/morphgs/assets/mesh_ori_rig.txt +27 -0
  237. package/tools/morphgs/assets/playback-check.mjs +30 -0
  238. package/tools/morphgs/demo-gate.sh +26 -0
  239. package/tools/morphgs/fbx2morphgs.mjs +68 -0
  240. package/tools/morphgs/morphgs-to-cskel27.mjs +105 -0
  241. package/tools/morphgs/patches/preprocess_src-none-mode.patch +76 -0
  242. package/tools/morphgs/setup-on-cluster.sh +126 -0
  243. package/tools/qa/css-rule-usage.mjs +283 -0
  244. package/tools/qa/studio-control-count.mjs +70 -32
  245. package/tools/run-tests.mjs +134 -12
  246. package/tools/track/DESIGN.md +513 -0
  247. package/tools/track/backfill-provenance.mjs +116 -0
  248. package/tools/track/budget.mjs +8 -0
  249. package/tools/track/check-rig.mjs +209 -0
  250. package/tools/track/diagnostics.schema.json +60 -0
  251. package/tools/track/export-rig.mjs +256 -0
  252. package/tools/track/fallback.mjs +7 -0
  253. package/tools/track/fk-parity-fixture.mjs +161 -0
  254. package/tools/track/gate.mjs +349 -0
  255. package/tools/track/masks.mjs +318 -0
  256. package/tools/track/metrics.mjs +201 -0
  257. package/tools/track/publish-obs.mjs +108 -0
  258. package/tools/track/py/check_env.py +61 -0
  259. package/tools/track/py/eval_lr.py +229 -0
  260. package/tools/track/py/lr_viterbi.py +294 -0
  261. package/tools/track/py/masks.py +405 -0
  262. package/tools/track/py/objective.py +382 -0
  263. package/tools/track/py/rig.py +508 -0
  264. package/tools/track/py/scene.py +326 -0
  265. package/tools/track/py/test_joint_indices.py +123 -0
  266. package/tools/track/py/test_lr_viterbi.py +195 -0
  267. package/tools/track/py/test_masks.py +229 -0
  268. package/tools/track/py/test_rig.py +288 -0
  269. package/tools/track/py/test_scene.py +354 -0
  270. package/tools/track/py/test_track.py +328 -0
  271. package/tools/track/py/track.py +411 -0
  272. package/tools/track/remote.mjs +140 -0
  273. package/tools/track/rig-dump.mjs +227 -0
  274. package/tools/track/run-box-tests.mjs +14 -0
  275. package/tools/track/run-box.mjs +168 -0
  276. package/tools/track/setup-box.sh +59 -0
  277. package/tools/track/study-2d.mjs +728 -0
  278. package/dist/assets/analytics-B1hnH66c.js +0 -1
  279. package/dist/assets/app-BWusbqgO.js +0 -4861
  280. package/dist/assets/app-qKDo4PBX.css +0 -1
  281. package/dist/assets/shot-prompt-CHUtw6af.js +0 -17
  282. package/dist/assets/shot-prompt-C_g2BHVc.css +0 -1
  283. package/dist/assets/workflow-CVwzMhz2.js +0 -179
  284. package/dist/assets/workflow-DsudxKHF.css +0 -1
  285. package/src/fal-motion-studio.jsx +0 -180
  286. package/src/scene-history.js +0 -129
@@ -0,0 +1,229 @@
1
+ from __future__ import annotations
2
+
3
+ import contextlib
4
+ import os
5
+
6
+ import numpy as np
7
+ import pytest
8
+
9
+ from masks import (
10
+ NoPersonPrompt,
11
+ box_from_xys,
12
+ choose_anchor,
13
+ main,
14
+ plan_chunks,
15
+ prompt_boxes,
16
+ prompt_points,
17
+ reprompt_frames,
18
+ segment,
19
+ should_reprompt,
20
+ track_clip,
21
+ )
22
+
23
+
24
+ def test_reprompt_rule_on_synthetic_area_sequence():
25
+ areas = [1000, 1100, 1200, 600, 620, 1000, 1000, 0, 0, 500, 700]
26
+ # 1200->600 (-50 %), 620->1000 (+61 %), 1000->0 (-100 %); 0->500 is a recovery, 500->700 is +40 % (not > 40 %).
27
+ assert reprompt_frames(areas) == [3, 5, 7]
28
+ assert reprompt_frames(areas, eligible=[True] * 5 + [False] * 6) == [3]
29
+ assert not should_reprompt(1000, 1400)
30
+ assert should_reprompt(1000, 1401)
31
+ assert not should_reprompt(0, 5000)
32
+
33
+
34
+ def test_box_conversion_clips_and_rejects():
35
+ assert box_from_xys([100, 50, 40], 832, 480).tolist() == [80, 30, 120, 70]
36
+ assert box_from_xys([441.5, 49.0, 249.4], 832, 480)[1] == 0.0 # clipped at the top edge
37
+ assert box_from_xys([10, 10, 0], 832, 480) is None
38
+ assert box_from_xys([np.nan, 10, 50], 832, 480) is None
39
+ assert box_from_xys([-500, 10, 50], 832, 480) is None # entirely outside the image
40
+
41
+
42
+ def test_empty_bbx_raises_no_person_prompt():
43
+ for bbx in (None, np.zeros((0, 3), np.float32), np.zeros((5, 3), np.float32)):
44
+ with pytest.raises(NoPersonPrompt, match="no-person-prompt"):
45
+ prompt_boxes(bbx, 832, 480)
46
+
47
+
48
+ def test_cli_empty_bbx_exits_3_without_output(tmp_path, capsys):
49
+ obs = tmp_path / "obs.npz"
50
+ np.savez(obs, bbx_xys=np.zeros((0, 3), np.float32), kp2d=np.zeros((0, 17, 3), np.float32))
51
+ out = tmp_path / "out"
52
+ assert main(["--video", str(tmp_path / "missing.mp4"), "--obs", str(obs), "--out", str(out)]) == 3
53
+ assert "no-person-prompt" in capsys.readouterr().err
54
+ assert not out.exists()
55
+
56
+
57
+ def test_prompt_points_use_confident_joints_inside_the_box():
58
+ box = np.array([100, 100, 300, 400], np.float32)
59
+ kp = np.zeros((17, 3))
60
+ kp[:, :2] = 200
61
+ kp[:, 2] = 0.9
62
+ kp[5, :2], kp[6, :2] = (180, 150), (220, 150) # shoulders -> midpoint (200, 150)
63
+ kp[0, :2], kp[0, 2] = (200, 120), 0.9 # nose
64
+ kp[9, 2] = 0.1 # unconfident wrist is dropped
65
+ kp[16, :2] = (500, 390) # ankle outside the box is dropped
66
+ points = prompt_points("joints", box, kp)
67
+ assert [tuple(p) for p in points.tolist()] == [(200, 150), (200, 200), (200, 120), (200, 200), (200, 200)]
68
+ kp[15, 2] = 0.6 # an ankle needs ANKLE_TRUST (hidden feet behind props score 0.46-0.71)
69
+ assert len(prompt_points("joints", box, kp)) == 4
70
+ kp[9, 2] = 0.6 # a wrist at the same confidence is kept
71
+ assert len(prompt_points("joints", box, kp)) == 5
72
+ assert prompt_points("box", box, kp) is None
73
+ assert prompt_points("centre", box, kp).tolist() == [[200, 250]]
74
+ assert prompt_points("joints", box, np.zeros((17, 3))) is None # nothing confident: box only
75
+ assert prompt_points("joints", None, kp) is None
76
+ with pytest.raises(ValueError):
77
+ prompt_points("palm", box, kp)
78
+
79
+
80
+ def test_anchor_skips_untrusted_leading_frames():
81
+ boxes = [np.array([0, 0, 10, 10], np.float32)] * 6
82
+ assert choose_anchor(boxes, [0.02, 0.1, 0.6, 0.9, 0.9, 0.9]) == 2
83
+ assert choose_anchor(boxes, [0.02, 0.3, 0.1, 0.2, 0.4, 0.1]) == 4 # nothing trusted: most confident
84
+ assert choose_anchor([None, None] + boxes[:4], [0.9] * 6) == 2
85
+
86
+
87
+ def test_chunk_plan_covers_every_frame_with_shared_boundaries():
88
+ assert plan_chunks(124, 0, 180) == [(1, 0, 123)]
89
+ assert plan_chunks(362, 0, 180) == [(1, 0, 179), (1, 179, 358), (1, 358, 361)]
90
+ assert plan_chunks(20, 5, 8) == [(1, 5, 12), (1, 12, 19), (-1, 0, 5)]
91
+ for frames, anchor, size in ((362, 0, 180), (20, 5, 8), (20, 19, 4), (1, 0, 2)):
92
+ chunks = plan_chunks(frames, anchor, size)
93
+ covered = set()
94
+ for _, start, end in chunks:
95
+ assert end - start + 1 <= size
96
+ covered.update(range(start, end + 1))
97
+ assert covered == set(range(frames))
98
+
99
+
100
+ class FakePredictor:
101
+ """SAM2 video-predictor stand-in: frame t tracks to TRUE area, except in the
102
+ drift window where tracking alone shrinks the mask to 30 % until a box prompt
103
+ is placed inside the window. Mask seeds reproduce their own area."""
104
+
105
+ def __init__(self, count, drift, hw=(20, 50)):
106
+ self.count, self.drift, self.hw = count, drift, hw
107
+ self.true = 600
108
+ self.box_prompts = []
109
+ self.recomputed = []
110
+
111
+ def open_chunk(self, start, end):
112
+ @contextlib.contextmanager
113
+ def chunk():
114
+ yield {"start": start, "n": end - start + 1, "prompts": {},
115
+ "output_dict_per_obj": {0: {"non_cond_frame_outputs": {}}}, "frames_tracked_per_obj": {0: {}}}
116
+ return chunk()
117
+
118
+ def logits(self, area):
119
+ import torch
120
+
121
+ flat = torch.full((self.hw[0] * self.hw[1],), -10.0)
122
+ flat[:area] = 10.0
123
+ return flat.reshape(1, 1, *self.hw)
124
+
125
+ def add_new_points_or_box(self, state, frame_idx, obj_id, box=None, points=None, labels=None):
126
+ assert frame_idx not in state["frames_tracked_per_obj"][0], "re-prompt must be a fresh conditioning frame"
127
+ state["prompts"][frame_idx] = ("box", self.true)
128
+ self.box_prompts.append(state["start"] + frame_idx)
129
+
130
+ def add_new_mask(self, state, frame_idx, obj_id, mask):
131
+ state["prompts"][frame_idx] = ("mask", int(mask.sum()))
132
+
133
+ def propagate_in_video(self, state, start_frame_idx, reverse=False):
134
+ order = range(start_frame_idx, -1, -1) if reverse else range(start_frame_idx, state["n"])
135
+ for local in order:
136
+ t = state["start"] + local
137
+ if local in state["prompts"]:
138
+ area = state["prompts"][local][1]
139
+ elif t in self.drift and not any(p in self.drift and (p <= t if not reverse else p >= t) for p in self.box_prompts):
140
+ area = int(self.true * 0.3)
141
+ else:
142
+ area = self.true
143
+ if local in state["output_dict_per_obj"][0]["non_cond_frame_outputs"]:
144
+ self.recomputed.append(t)
145
+ state["output_dict_per_obj"][0]["non_cond_frame_outputs"][local] = area
146
+ state["frames_tracked_per_obj"][0][local] = {"reverse": reverse}
147
+ yield local, [1], self.logits(area)
148
+
149
+
150
+ def run_fake(count, anchor, size, drift, trusted):
151
+ fake = FakePredictor(count, set(drift))
152
+ boxes = [np.array([0, 0, 10, 10], np.float32)] * count
153
+ prob, area, reprompted = track_clip(fake, fake.open_chunk, count, (10, 25), boxes, np.array(trusted),
154
+ anchor, plan_chunks(count, anchor, size))
155
+ return fake, prob, area, reprompted
156
+
157
+
158
+ def test_track_clip_reprompts_drift_once_and_resumes():
159
+ fake, prob, area, reprompted = run_fake(20, 5, 8, range(14, 18), [False] * 5 + [True] * 15)
160
+ assert reprompted == [14]
161
+ assert fake.box_prompts == [5, 14]
162
+ assert area.tolist() == [600] * 20
163
+ assert prob.dtype == np.float16 and prob.shape == (20, 10, 25)
164
+ assert float(prob.min()) >= 0.0 and float(prob.max()) <= 1.0
165
+ assert fake.recomputed == [] # no frame is tracked twice: the restart starts at the re-prompted frame
166
+
167
+
168
+ def test_track_clip_never_reprompts_untrusted_frames():
169
+ fake, _, area, reprompted = run_fake(20, 5, 8, range(1, 3), [False] * 5 + [True] * 15)
170
+ assert reprompted == []
171
+ assert fake.box_prompts == [5]
172
+ assert area[1:3].tolist() == [180, 180] and area[0] == 600
173
+
174
+
175
+ def draw_person(frame, cx, cy, scale):
176
+ import cv2
177
+
178
+ mask = np.zeros(frame.shape[:2], np.uint8)
179
+ cv2.circle(mask, (cx, cy - int(70 * scale)), int(14 * scale), 255, -1)
180
+ cv2.ellipse(mask, (cx, cy - int(25 * scale)), (int(18 * scale), int(34 * scale)), 0, 0, 360, 255, -1)
181
+ for dx in (-10, 10):
182
+ cv2.line(mask, (cx + dx, cy), (cx + dx * 2, cy + int(60 * scale)), 255, int(10 * scale))
183
+ cv2.line(mask, (cx + dx * 2, cy - int(50 * scale)), (cx + dx * 4, cy - int(10 * scale)), 255, int(8 * scale))
184
+ frame[mask > 0] = (60, 90, 200)
185
+ return mask > 0
186
+
187
+
188
+ def test_sam2_synthetic_clip_backward_and_forward(tmp_path):
189
+ cv2 = pytest.importorskip("cv2")
190
+ count, width, height = 20, 832, 480
191
+ rng = np.random.default_rng(3)
192
+ background = np.full((height, width, 3), (200, 205, 210), np.uint8)
193
+ background[height // 2:] = (170, 180, 185)
194
+ for x in range(0, width, 64):
195
+ cv2.line(background, (x, height // 2), (x - 200, height), (150, 150, 150), 2)
196
+ background = np.clip(background.astype(int) + rng.integers(-4, 5, background.shape), 0, 255).astype(np.uint8)
197
+ video = str(tmp_path / "clip.avi")
198
+ writer = cv2.VideoWriter(video, cv2.VideoWriter_fourcc(*"MJPG"), 24, (width, height))
199
+ truth, bbx = [], []
200
+ for t in range(count):
201
+ frame = background.copy()
202
+ cx, cy = 250 + 12 * t, 300
203
+ mask = draw_person(frame, cx, cy, 1.4)
204
+ truth.append(mask)
205
+ ys, xs = np.nonzero(mask)
206
+ size = 1.2 * max(np.ptp(xs), np.ptp(ys))
207
+ bbx.append([(xs.min() + xs.max()) / 2, (ys.min() + ys.max()) / 2, size])
208
+ writer.write(frame)
209
+ writer.release()
210
+ bbx = np.array(bbx, np.float32)
211
+ bbx[:5] = [441.5, 49.0, 249.4] # garbage leading boxes, as GVHMR emits before its first detection
212
+ kp2d = np.zeros((count, 17, 3), np.float32)
213
+ kp2d[:, :, 2] = 0.9
214
+ kp2d[:5, :, 2] = 0.02
215
+ obs = str(tmp_path / "obs.npz")
216
+ np.savez(obs, bbx_xys=bbx, kp2d=kp2d)
217
+
218
+ arrays, summary = segment(video, obs, chunk_frames=8)
219
+ assert summary["anchor"] == 5
220
+ assert summary["chunks"] == [[1, 5, 12], [1, 12, 19], [-1, 0, 5]]
221
+ assert arrays["prob"].shape == (count, height // 2, width // 2) and arrays["prob"].dtype == np.float16
222
+ assert summary["peakReservedMiB"] <= 5632
223
+ ious = []
224
+ for t in range(count):
225
+ prob = cv2.resize(arrays["prob"][t].astype(np.float32), (width, height), interpolation=cv2.INTER_LINEAR)
226
+ pred = prob > 0.5
227
+ ious.append((pred & truth[t]).sum() / max(1, (pred | truth[t]).sum()))
228
+ assert min(ious) >= 0.8, [round(v, 3) for v in ious]
229
+ assert int(np.median(ious) * 100) >= 90, [round(v, 3) for v in ious]
@@ -0,0 +1,288 @@
1
+ """Parity tests for rig.py (plan mocap-rearch todo 6, tests (a)-(e) + malformed input).
2
+
3
+ Data (resolved in this order; a missing file FAILS, never skips):
4
+ - parity fixture: <repo>/test/fixtures/track-fk-parity.json
5
+ (node tools/track/fk-parity-fixture.mjs)
6
+ - rig npz: $COZYFIT_RIG, <repo>/node_modules/.cache/cozyfit/rig-y-bot-tpose.npz
7
+ (node tools/track/export-rig.mjs --model y-bot-tpose)
8
+ - truth motion: $COZYFIT_WALK, <repo>/../../evidence/prep/gt-motions/walk.npz
9
+ On the box (tools/track/run-box-tests.mjs uploads only tools/track/py/*.py) the three
10
+ files are read from ~/cclay-ingest/cozyfit/testdata/ - stage them with
11
+ scp test/fixtures/track-fk-parity.json node_modules/.cache/cozyfit/rig-y-bot-tpose.npz \
12
+ <evidence>/prep/gt-motions/walk.npz ubuntu-baremetal:cclay-ingest/cozyfit/testdata/
13
+
14
+ Measured errors are printed as one `[rig-parity] {json}` line when pytest exits, so a
15
+ passing log carries the numbers, not just dots.
16
+ """
17
+ import atexit
18
+ import json
19
+ import os
20
+ from pathlib import Path
21
+
22
+ import numpy as np
23
+ import pytest
24
+ import torch
25
+
26
+ import rig as rigmod
27
+ from rig import RigError, State
28
+
29
+ HERE = Path(__file__).resolve().parent
30
+ # <repo>/tools/track/py locally; a flat /tmp/cozyfit-* upload dir on the box (no repo around it)
31
+ REPO = HERE.parents[2] if len(HERE.parents) > 2 else HERE
32
+ EVIDENCE = REPO.parents[1] / "evidence" if len(REPO.parents) > 1 else REPO / "evidence"
33
+ BOX_DATA = Path.home() / "cclay-ingest" / "cozyfit" / "testdata"
34
+ PARITY: dict = {}
35
+
36
+
37
+ @atexit.register
38
+ def _report():
39
+ if PARITY:
40
+ print("[rig-parity] " + json.dumps(PARITY, sort_keys=True))
41
+
42
+
43
+ def _find(label, env, *candidates):
44
+ paths = ([Path(os.environ[env])] if env and os.environ.get(env) else []) + [Path(c) for c in candidates]
45
+ for p in paths:
46
+ if p.is_file():
47
+ return p
48
+ pytest.fail(f"missing {label}; looked in {[str(p) for p in paths]} (see test_rig.py docstring for staging)")
49
+
50
+
51
+ @pytest.fixture(scope="module")
52
+ def fixture():
53
+ path = _find("fk parity fixture", None, REPO / "test/fixtures/track-fk-parity.json", BOX_DATA / "track-fk-parity.json")
54
+ return json.loads(path.read_text())
55
+
56
+
57
+ @pytest.fixture(scope="module")
58
+ def rig_path():
59
+ return _find("rig npz", "COZYFIT_RIG", REPO / "node_modules/.cache/cozyfit/rig-y-bot-tpose.npz", BOX_DATA / "rig-y-bot-tpose.npz")
60
+
61
+
62
+ @pytest.fixture(scope="module")
63
+ def rig(rig_path, fixture):
64
+ torch.manual_seed(0)
65
+ r = rigmod.load_rig(rig_path)
66
+ source = (r.meta.get("source") or {}).get("sha256")
67
+ assert source == fixture["fbxSha256"], "rig npz and parity fixture come from different FBX files; regenerate one"
68
+ assert r.bone_names == fixture["bones"], "rig npz bone order differs from the fixture"
69
+ return r
70
+
71
+
72
+ def _err(a, b):
73
+ return float(torch.linalg.norm(torch.as_tensor(a, dtype=torch.float64) - torch.as_tensor(b, dtype=torch.float64), dim=-1).max())
74
+
75
+
76
+ # (a) ---------------------------------------------------------------------------------
77
+
78
+ def test_a_identity_reproduces_neutral(rig, fixture):
79
+ canonical = torch.tensor(fixture["canonicalPosedJoints"]) # JS canonicalCskel27Reference().posed_joints
80
+ state = State.identity(1, root=(0.0, -rig.min_y, 0.0))
81
+ joints, globals_ = rigmod.cskel27_fk(state, None, rig)
82
+ err = _err(joints[0], canonical)
83
+ ones, _ = rigmod.cskel27_fk(state, np.ones(27, dtype=np.float32), rig)
84
+ PARITY["a_identity_max_m"] = err
85
+ assert err < 1e-5, f"identity pose vs neutral posedJoints: {err:.3e} m"
86
+ assert _err(ones, joints) < 1e-7
87
+ assert torch.allclose(globals_, torch.eye(3).expand_as(globals_))
88
+
89
+
90
+ # (b) ---------------------------------------------------------------------------------
91
+
92
+ def test_b_layer1_matches_truth_posed_joints(rig):
93
+ path = _find("truth motion walk.npz", "COZYFIT_WALK", EVIDENCE / "prep/gt-motions/walk.npz", BOX_DATA / "walk.npz")
94
+ with np.load(path) as z:
95
+ rot, root, posed = z["local_rot_mats"], z["root_positions"], z["posed_joints"]
96
+ bone_scale = z["bone_scale"] if "bone_scale" in z else None
97
+ state = State.from_motion(rot, root)
98
+ joints, _ = rigmod.cskel27_fk(state, bone_scale, rig)
99
+ per_frame = torch.linalg.norm(joints.double() - torch.as_tensor(posed, dtype=torch.float64), dim=-1).amax(dim=1)
100
+ PARITY["b_walk_frames"] = int(len(per_frame))
101
+ PARITY["b_walk_max_m"] = float(per_frame.max())
102
+ PARITY["b_walk_bone_scale"] = "npz" if bone_scale is not None else "none (canonical)"
103
+ assert len(per_frame) == len(posed) > 100
104
+ assert float(per_frame.max()) < 1e-4, f"worst frame {int(per_frame.argmax())}: {float(per_frame.max()):.3e} m"
105
+
106
+
107
+ def test_b_layer1_matches_js_fk_with_bone_scale(rig, fixture):
108
+ worst = 0.0
109
+ for case in fixture["cases"]:
110
+ rot = np.asarray(case["rotMats"], dtype=np.float32).reshape(-1, 27, 3, 3)
111
+ root = np.asarray(case["rootPos"], dtype=np.float32).reshape(-1, 3)
112
+ posed = np.asarray(case["posedJoints"], dtype=np.float32).reshape(-1, 27, 3)
113
+ joints, _ = rigmod.cskel27_fk(State.from_motion(rot, root), case["boneScale"], rig)
114
+ worst = max(worst, _err(joints, posed))
115
+ PARITY["b_js_fk_max_m"] = worst
116
+ assert worst < 1e-4
117
+
118
+
119
+ # (c) ---------------------------------------------------------------------------------
120
+
121
+ def test_c_rot6d_round_trip():
122
+ g = torch.Generator().manual_seed(6)
123
+ q = torch.randn(4096, 4, generator=g)
124
+ R = rigmod.quat_to_matrix(q)
125
+ back = rigmod.rot6d_to_matrix(rigmod.matrix_to_rot6d(R))
126
+ err = float((back - R).abs().max())
127
+ # arbitrary (non-orthonormal) 6D vectors land on SO(3) and are a fixed point after one pass.
128
+ # The property is checked in float64; float32 Gram-Schmidt on near-parallel random column
129
+ # pairs loses ~1e-5 to cancellation (measured 1.02e-5 on the box), so its bound is looser.
130
+ a = torch.randn(4096, 6, generator=g, dtype=torch.float64)
131
+ Ra = rigmod.rot6d_to_matrix(a)
132
+ ortho = float((Ra.transpose(-1, -2) @ Ra - torch.eye(3, dtype=torch.float64)).abs().max())
133
+ det = float((torch.linalg.det(Ra) - 1).abs().max())
134
+ fixed = float((rigmod.rot6d_to_matrix(rigmod.matrix_to_rot6d(Ra)) - Ra).abs().max())
135
+ Ra32 = rigmod.rot6d_to_matrix(a.float())
136
+ ortho32 = float((Ra32.transpose(-1, -2) @ Ra32 - torch.eye(3)).abs().max())
137
+ PARITY["c_rot6d_round_trip_max"] = err
138
+ PARITY["c_rot6d_orthonormal_f64_max"] = ortho
139
+ PARITY["c_rot6d_orthonormal_f32_max"] = ortho32
140
+ assert err < 1e-5, f"6D round trip {err:.3e}"
141
+ assert ortho < 1e-12 and det < 1e-12 and fixed < 1e-12, (ortho, det, fixed)
142
+ assert ortho32 < 1e-4, f"float32 orthonormality {ortho32:.3e}"
143
+ eye = rigmod.rot6d_to_matrix(torch.tensor([1.0, 0, 0, 0, 1, 0]))
144
+ assert torch.equal(eye, torch.eye(3))
145
+
146
+
147
+ # (d) ---------------------------------------------------------------------------------
148
+
149
+ def test_d_gradients_finite(rig, fixture):
150
+ case = fixture["cases"][0]
151
+ rot = np.asarray(case["rotMats"], dtype=np.float32).reshape(-1, 27, 3, 3)
152
+ root = np.asarray(case["rootPos"], dtype=np.float32).reshape(-1, 3)
153
+ state = State.from_motion(rot, root)
154
+ g = torch.Generator().manual_seed(7)
155
+ for name in ("transl", "root6d", "local6d"):
156
+ t = getattr(state, name)
157
+ setattr(state, name, (t + 0.01 * torch.randn(t.shape, generator=g)).requires_grad_(True))
158
+ joints, globals_ = rigmod.cskel27_fk(state, case["boneScale"], rig)
159
+ _, verts = rigmod.studio_skin(globals_, joints, rig, bone_scale=case["boneScale"])
160
+ loss = joints.square().sum() + verts.square().mean()
161
+ loss.backward()
162
+ norms = {name: float(getattr(state, name).grad.norm()) for name in ("transl", "root6d", "local6d")}
163
+ PARITY["d_grad_norms"] = norms
164
+ for name in norms:
165
+ grad = getattr(state, name).grad
166
+ assert torch.isfinite(grad).all(), name
167
+ assert norms[name] > 0, name
168
+ # every limb rotation reaches the skinned surface (no detached joint)
169
+ driven = [j for j in range(1, 27) if rig.prep_bone[j] >= 0]
170
+ assert (state.local6d.grad[:, [j - 1 for j in driven]].abs().sum(dim=(0, 2)) > 0).all()
171
+
172
+
173
+ # (e) ---------------------------------------------------------------------------------
174
+
175
+ def test_e_layer2_matches_studio_playback(rig, fixture):
176
+ idx = fixture["vertexIndices"]
177
+ bone_err, rot_err, vert_err = 0.0, 0.0, 0.0
178
+ for case in fixture["cases"]:
179
+ rot = torch.tensor(case["rotMats"]).reshape(-1, 27, 3, 3)
180
+ posed = torch.tensor(case["posedJoints"]).reshape(-1, 27, 3)
181
+ f, a = case["frame"], case["anchorFrame"]
182
+ globals_ = rigmod.global_rotations(rot[f:f + 1], rig.parents)
183
+ anchor = posed[a, 0, [0, 2]]
184
+ W, verts = rigmod.studio_skin(globals_, posed[f:f + 1], rig, bone_scale=case["boneScale"],
185
+ anchor_xz=anchor, vertex_indices=idx)
186
+ ref_W = torch.tensor(case["boneWorld"]).reshape(-1, 4, 4)
187
+ ref_v = torch.tensor(case["vertices"]).reshape(-1, 3)
188
+ bone_err = max(bone_err, _err(W[0, :, :3, 3], ref_W[:, :3, 3]))
189
+ # linear part carries the 0.01 rig root scale; compare the rotation itself
190
+ rot_err = max(rot_err, float((W[0, :, :3, :3] - ref_W[:, :3, :3]).abs().max()) / rig.root_scale)
191
+ vert_err = max(vert_err, _err(verts[0], ref_v))
192
+ PARITY.update(e_cases=len(fixture["cases"]), e_bones=len(fixture["bones"]), e_vertices=len(idx),
193
+ e_bone_pos_max_m=bone_err, e_bone_mat_max=rot_err, e_vertex_max_m=vert_err)
194
+ assert bone_err < 1e-4, f"bone world position {bone_err:.3e} m"
195
+ assert rot_err < 1e-4, f"bone world rotation entries {rot_err:.3e}"
196
+ assert vert_err < 1e-4, f"skinned vertex {vert_err:.3e} m"
197
+
198
+
199
+ def test_e_parity_detects_a_wrong_mode(rig, fixture):
200
+ """Guard against a vacuous parity: dropping boneScale must move the arms by > 1 mm."""
201
+ case = fixture["cases"][0]
202
+ rot = torch.tensor(case["rotMats"]).reshape(-1, 27, 3, 3)
203
+ posed = torch.tensor(case["posedJoints"]).reshape(-1, 27, 3)
204
+ globals_ = rigmod.global_rotations(rot[1:2], rig.parents)
205
+ _, verts = rigmod.studio_skin(globals_, posed[1:2], rig, bone_scale=None, anchor_xz=posed[0, 0, [0, 2]],
206
+ vertex_indices=fixture["vertexIndices"])
207
+ wrong = _err(verts[0], torch.tensor(case["vertices"]).reshape(-1, 3))
208
+ PARITY["e_wrong_mode_vertex_max_m"] = wrong
209
+ assert wrong > 1e-3
210
+
211
+
212
+ # decimated tracker mesh ----------------------------------------------------------------
213
+
214
+ def test_decimated_mesh(rig):
215
+ faces, verts = rig.dec_faces, rig.dec_vertices
216
+ w = rig.dec_skin_weight.double()
217
+ PARITY.update(dec_faces=int(faces.shape[0]), dec_vertices=int(verts.shape[0]),
218
+ full_vertices=int(rig.vertices.shape[0]))
219
+ assert 1000 < faces.shape[0] <= rigmod.MAX_FACES
220
+ assert int(faces.max()) < verts.shape[0] and int(faces.min()) >= 0
221
+ assert (w >= 0).all() and float((w.sum(dim=1) - 1).abs().max()) < 1e-6
222
+ assert torch.allclose(rig.dec_weights.sum(dim=1), torch.ones(len(verts)), atol=1e-6)
223
+ # weights come from the nearest original vertex: that vertex is close by
224
+ gap = torch.linalg.norm(verts - rig.vertices[torch.as_tensor(rig.dec_source)], dim=-1)
225
+ PARITY["dec_source_gap_max_m"] = float(gap.max())
226
+ assert float(gap.max()) < 0.05
227
+ # rest pose: identity-ish skinning leaves the rest mesh where it is
228
+ B = len(rig.bone_names)
229
+ rest_worlds = torch.linalg.inv(rig.bind_inverse).unsqueeze(0)
230
+ moved = rigmod.lbs(rest_worlds, rig, verts, rig.dec_weights)
231
+ assert rest_worlds.shape == (1, B, 4, 4)
232
+ assert float((moved[0] - verts).abs().max()) < 1e-4
233
+
234
+
235
+ # malformed input -------------------------------------------------------------------------
236
+
237
+ def _rewrite(rig_path, tmp_path, **changes):
238
+ with np.load(rig_path) as z:
239
+ members = {k: z[k] for k in z.files}
240
+ for key, value in changes.items():
241
+ if value is None:
242
+ members.pop(key)
243
+ else:
244
+ members[key] = value
245
+ out = tmp_path / "rig.npz"
246
+ np.savez(out, **members)
247
+ return out
248
+
249
+
250
+ def test_bad_skin_weights_rejected(rig_path, tmp_path):
251
+ with np.load(rig_path) as z:
252
+ w = z["skin_weight"].copy()
253
+ w[123] *= 0.9
254
+ with pytest.raises(RigError) as err:
255
+ rigmod.load_rig(_rewrite(rig_path, tmp_path, skin_weight=w))
256
+ assert err.value.code == "bad-skin-weights" and "vertex 123" in str(err.value)
257
+ w = np.load(rig_path)["skin_weight"].copy()
258
+ w[7, 0], w[7, 1] = -0.25, w[7, 1] + 0.25 + w[7, 0]
259
+ with pytest.raises(RigError) as err:
260
+ rigmod.load_rig(_rewrite(rig_path, tmp_path, skin_weight=w))
261
+ assert err.value.code == "bad-skin-weights"
262
+
263
+
264
+ def test_missing_and_malformed_members_rejected(rig_path, tmp_path):
265
+ with pytest.raises(RigError) as err:
266
+ rigmod.load_rig(_rewrite(rig_path, tmp_path, prep_chain_rel=None))
267
+ assert err.value.code == "bad-rig" and "prep_chain_rel" in str(err.value)
268
+ with pytest.raises(RigError) as err:
269
+ rigmod.load_rig(_rewrite(rig_path, tmp_path, prep_offsets=np.zeros((26, 3), np.float32)))
270
+ assert err.value.code == "bad-rig" and "prep_offsets" in str(err.value)
271
+ with pytest.raises(RigError) as err:
272
+ rigmod.load_rig(tmp_path / "no-such-rig.npz")
273
+ assert err.value.code == "bad-rig"
274
+
275
+
276
+ def test_bad_bone_scale_and_motion_rejected(rig):
277
+ state = State.identity(2)
278
+ with pytest.raises(RigError) as err:
279
+ rigmod.cskel27_fk(state, np.ones(26), rig)
280
+ assert err.value.code == "bad-bone-scale"
281
+ bad = np.ones(27)
282
+ bad[3] = 0
283
+ with pytest.raises(RigError) as err:
284
+ rigmod.cskel27_fk(state, bad, rig)
285
+ assert err.value.code == "bad-bone-scale"
286
+ with pytest.raises(RigError) as err:
287
+ State.from_motion(np.zeros((4, 26, 3, 3)), np.zeros((4, 3)))
288
+ assert err.value.code == "bad-motion"