cozyclay 2.1.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (583) hide show
  1. package/CHANGELOG.md +65 -0
  2. package/README.md +12 -30
  3. package/bin/agent/agent-routes.mjs +40 -113
  4. package/bin/agent/agent-runner.mjs +9 -10
  5. package/bin/agent/cliproxy-provider.mjs +135 -0
  6. package/bin/agent/credential-store.mjs +30 -7
  7. package/bin/agent/pi-tools.mjs +4 -1
  8. package/bin/agent/provider-keys.mjs +32 -0
  9. package/bin/agent/provider-login.mjs +110 -0
  10. package/bin/agent/provider-routes.mjs +99 -0
  11. package/bin/agent/providers.mjs +19 -97
  12. package/bin/agent/script-tool.mjs +17 -4
  13. package/bin/agent/studio-motion-reference.mjs +23 -0
  14. package/bin/agent/studio-prompt.mjs +1 -1
  15. package/bin/agent/studio-rebase-text.mjs +32 -0
  16. package/bin/agent/studio-rebase.mjs +26 -39
  17. package/bin/agent/studio-tools.mjs +9 -28
  18. package/bin/agent/tool-arguments.mjs +49 -0
  19. package/bin/agent/turn-frames.mjs +66 -0
  20. package/bin/codex-auth.mjs +9 -2
  21. package/bin/cozyclay.mjs +10 -116
  22. package/bin/live/cli.mjs +22 -6
  23. package/bin/mcp-runtime.mjs +2 -12
  24. package/bin/package-signature.mjs +0 -4
  25. package/dist/app/index.html +7 -6
  26. package/dist/assets/EXRLoader-CvVIvGVE.js +1 -0
  27. package/dist/assets/analytics-C9sUnndd.js +1 -0
  28. package/dist/assets/app-CCh--1Hc.js +5087 -0
  29. package/dist/assets/app-f-fkV5uP.css +1 -0
  30. package/dist/assets/compile-worker-CkxiD6lH.js +5 -0
  31. package/dist/assets/fflate.module-KbGj5-H6.js +5 -0
  32. package/dist/assets/{first-shot-handoff-BlT33CGf.js → first-shot-handoff-CC4xe4HM.js} +1 -1
  33. package/dist/assets/{landing-BSesrCpL.js → landing-DjHD8u7k.js} +1 -1
  34. package/dist/assets/scene-asset-sync-Hx_FzdOH.js +20 -0
  35. package/dist/assets/scene-asset-sync-_S8NQl5B.css +1 -0
  36. package/dist/assets/workflow-C4yKH1p4.js +179 -0
  37. package/dist/cozyclay-package.json +1 -1
  38. package/dist/index.html +3 -3
  39. package/dist/llms.txt +14 -0
  40. package/dist/sw.js +3 -6
  41. package/dist/workflow/index.html +5 -4
  42. package/mcp/LIVE-PROTOCOL.md +7 -47
  43. package/mcp/README.md +16 -26
  44. package/mcp/live-hub.mjs +6 -10
  45. package/mcp/package.json +1 -3
  46. package/mcp/tool-handlers.mjs +4 -168
  47. package/package.json +3 -22
  48. package/src/App.jsx +174 -2639
  49. package/src/agent-progress.js +79 -0
  50. package/src/agent-turn-mark.js +30 -0
  51. package/src/analytics.js +1 -19
  52. package/src/app-context.js +5 -0
  53. package/src/app-stage.jsx +2 -34
  54. package/src/asset-pane.jsx +6 -5
  55. package/src/camera-block.js +21 -0
  56. package/src/camera-direction.js +120 -0
  57. package/src/character-kind.js +3 -9
  58. package/src/command-bus.js +8 -6
  59. package/src/commands/cast.js +5 -34
  60. package/src/commands/elements/character.js +1 -1
  61. package/src/commands/elements/motion.js +1 -6
  62. package/src/commands/elements/object.js +6 -2
  63. package/src/commands/elements/shot.js +4 -1
  64. package/src/commands/elements.js +38 -14
  65. package/src/commands/index.js +3 -1
  66. package/src/commands/kits.js +37 -0
  67. package/src/commands/motion-timing.js +57 -0
  68. package/src/commands/motion.js +9 -152
  69. package/src/commands/motion3-compile.js +65 -0
  70. package/src/commands/motion3.js +207 -0
  71. package/src/commands/objects.js +6 -4
  72. package/src/commands/shot.js +49 -11
  73. package/src/commands/stage.js +14 -1
  74. package/src/commands/view.js +4 -1
  75. package/src/document-mirrors.js +42 -0
  76. package/src/document-store.js +4 -0
  77. package/src/domains/cast.js +19 -194
  78. package/src/domains/motion/motion-range-pin-rules.js +1 -1
  79. package/src/domains/motion/take-source.js +51 -0
  80. package/src/domains/motion.js +60 -1838
  81. package/src/domains/scenes.js +12 -55
  82. package/src/domains/shots.js +2 -2
  83. package/src/domains/stage.js +4 -2
  84. package/src/draft-ghost.js +46 -0
  85. package/src/draft-ghost.jsx +129 -0
  86. package/src/dualview.jsx +4 -3
  87. package/src/effects/EffectsLayer.jsx +112 -0
  88. package/src/effects/effect-meshes.js +232 -0
  89. package/src/effects/effect-record.js +103 -0
  90. package/src/effects/emitter.js +59 -0
  91. package/src/effects/simulate.js +261 -0
  92. package/src/effects/state.js +59 -0
  93. package/src/execution-telemetry.js +2 -3
  94. package/src/first-success-guide.jsx +5 -5
  95. package/src/ghost-set.js +257 -0
  96. package/src/held-object-edit.jsx +113 -0
  97. package/src/held-object-pose.js +175 -0
  98. package/src/hierarchy-model.js +1 -1
  99. package/src/hierarchy-panel.jsx +1 -1
  100. package/src/imported-mesh.jsx +103 -0
  101. package/src/kits/held.js +84 -0
  102. package/src/kits/index.js +70 -0
  103. package/src/kits/smoking.js +57 -0
  104. package/src/light-blockers.js +93 -0
  105. package/src/motion/beat-board-model.js +84 -0
  106. package/src/motion/beat-board.css +124 -0
  107. package/src/motion/beat-board.jsx +59 -0
  108. package/src/motion/center-of-mass.js +66 -0
  109. package/src/motion/fix-collisions.js +850 -0
  110. package/src/motion/physics-review.js +118 -0
  111. package/src/{ardy → motion}/physics-support.js +1 -76
  112. package/src/motion/rig-snapshot.js +22 -0
  113. package/src/motion/take-segments.jsx +140 -0
  114. package/src/{ardy → motion}/timeline-coordinates.js +0 -21
  115. package/src/{ardy → motion}/timeline.css +1 -2
  116. package/src/{ardy → motion}/timeline.jsx +48 -419
  117. package/src/motion/waypoints.js +76 -0
  118. package/src/motion3/beats.js +83 -0
  119. package/src/motion3/body.js +158 -0
  120. package/src/motion3/bones.js +47 -0
  121. package/src/motion3/carried.js +47 -0
  122. package/src/motion3/compile-job.js +75 -0
  123. package/src/motion3/compile-worker.js +10 -0
  124. package/src/motion3/compiler/anchors.js +105 -0
  125. package/src/motion3/compiler/arm-orient.js +128 -0
  126. package/src/motion3/compiler/channels.js +125 -0
  127. package/src/motion3/compiler/compile.js +205 -0
  128. package/src/motion3/compiler/gait-rhythm.js +97 -0
  129. package/src/motion3/compiler/grip.js +171 -0
  130. package/src/motion3/compiler/head-steady.js +48 -0
  131. package/src/motion3/compiler/hold.js +134 -0
  132. package/src/motion3/compiler/idle.js +113 -0
  133. package/src/motion3/compiler/limbs.js +170 -0
  134. package/src/motion3/compiler/math.js +72 -0
  135. package/src/motion3/compiler/pose.js +162 -0
  136. package/src/motion3/compiler/quat.js +49 -0
  137. package/src/motion3/compiler/skeleton.js +74 -0
  138. package/src/motion3/compiler/validate-common.js +32 -0
  139. package/src/motion3/compiler/validate-goals.js +118 -0
  140. package/src/motion3/compiler/validate-objects.js +143 -0
  141. package/src/motion3/compiler/validate-parts.js +116 -0
  142. package/src/motion3/compiler/validate.js +122 -0
  143. package/src/motion3/compiler/vocab.js +73 -0
  144. package/src/motion3/compiler/walk-pelvis.js +275 -0
  145. package/src/motion3/compiler/walk.js +207 -0
  146. package/src/motion3/compiler/wrist-clamp.js +32 -0
  147. package/src/motion3/delta.js +47 -0
  148. package/src/motion3/describe.js +174 -0
  149. package/src/motion3/docs.js +27 -0
  150. package/src/motion3/edit-carry.js +21 -0
  151. package/src/motion3/fix/ease.js +16 -0
  152. package/src/motion3/fix/ground.js +37 -0
  153. package/src/motion3/fix/index.js +52 -0
  154. package/src/motion3/fix/push.js +71 -0
  155. package/src/motion3/fix/reach.js +46 -0
  156. package/src/motion3/fix/retime.js +79 -0
  157. package/src/motion3/fix/self.js +211 -0
  158. package/src/motion3/fix/slide.js +70 -0
  159. package/src/motion3/fix/turn.js +51 -0
  160. package/src/motion3/follow.js +39 -0
  161. package/src/motion3/free-motion.js +173 -0
  162. package/src/motion3/grip-choice.js +130 -0
  163. package/src/motion3/hold-plan.js +157 -0
  164. package/src/motion3/json-patch.js +77 -0
  165. package/src/motion3/key-aim.js +105 -0
  166. package/src/motion3/key-edit.js +82 -0
  167. package/src/motion3/lenient.js +127 -0
  168. package/src/motion3/object-keys.js +236 -0
  169. package/src/motion3/object-pose.js +182 -0
  170. package/src/motion3/object-tracks.js +12 -0
  171. package/src/motion3/pipeline.js +102 -0
  172. package/src/motion3/playback.js +51 -0
  173. package/src/motion3/probes.js +91 -0
  174. package/src/motion3/qa/body-checks.js +34 -0
  175. package/src/motion3/qa/check-names.js +14 -0
  176. package/src/motion3/qa/checks.js +121 -0
  177. package/src/motion3/qa/contact-checks.js +51 -0
  178. package/src/motion3/qa/explain.js +80 -0
  179. package/src/motion3/qa/fmt.js +5 -0
  180. package/src/motion3/qa/foot-checks.js +72 -0
  181. package/src/motion3/qa/geometry.js +77 -0
  182. package/src/motion3/qa/goals.js +191 -0
  183. package/src/motion3/qa/ground.js +60 -0
  184. package/src/motion3/qa/hints-contact.js +179 -0
  185. package/src/motion3/qa/hints-goal.js +160 -0
  186. package/src/motion3/qa/hints-knee.js +24 -0
  187. package/src/motion3/qa/hints-motion.js +209 -0
  188. package/src/motion3/qa/hints-object.js +91 -0
  189. package/src/motion3/qa/index.js +42 -0
  190. package/src/motion3/qa/key-checks.js +82 -0
  191. package/src/motion3/qa/near.js +96 -0
  192. package/src/motion3/qa/object-checks.js +135 -0
  193. package/src/motion3/qa/patch.js +161 -0
  194. package/src/motion3/qa/severity.js +38 -0
  195. package/src/motion3/qa/spec-checks.js +65 -0
  196. package/src/motion3/qa/style.js +218 -0
  197. package/src/motion3/qa/wrist-checks.js +69 -0
  198. package/src/motion3/qa/wrist-obs.js +60 -0
  199. package/src/motion3/refs.js +120 -0
  200. package/src/motion3/repair-tune.js +121 -0
  201. package/src/motion3/repair.js +83 -0
  202. package/src/motion3/report.js +67 -0
  203. package/src/motion3/rigs/index.js +14 -0
  204. package/src/motion3/rigs/x-bot-tpose-body.json +1 -0
  205. package/src/motion3/rigs/x-bot-tpose.json +1 -0
  206. package/src/motion3/rigs/y-bot-tpose-body.json +1 -0
  207. package/src/motion3/rigs/y-bot-tpose.json +1 -0
  208. package/src/motion3/scene.js +60 -0
  209. package/src/motion3/schema/brief.js +89 -0
  210. package/src/motion3/schema/fields.js +44 -0
  211. package/src/motion3/schema/spec-examples.js +46 -0
  212. package/src/motion3/schema/topics.js +111 -0
  213. package/src/motion3/schema/types.js +147 -0
  214. package/src/motion3/search.js +117 -0
  215. package/src/motion3/second-hand.js +112 -0
  216. package/src/motion3/skills/basic.js +82 -0
  217. package/src/motion3/skills/book.js +76 -0
  218. package/src/motion3/skills/car.js +145 -0
  219. package/src/motion3/skills/frames-auto.js +29 -0
  220. package/src/motion3/skills/index.js +83 -0
  221. package/src/motion3/skills/info.js +65 -0
  222. package/src/motion3/skills/smoke-kit.js +109 -0
  223. package/src/motion3/skills/smoke.js +160 -0
  224. package/src/motion3/spec-store.js +59 -0
  225. package/src/motion3/take-edit.js +155 -0
  226. package/src/motion3/take.js +63 -0
  227. package/src/motion3/timeline.js +127 -0
  228. package/src/motion3/views.js +121 -0
  229. package/src/motion3/walk-plan.js +119 -0
  230. package/src/object-catalog.jsx +7 -1
  231. package/src/object-colors.js +98 -0
  232. package/src/object-gizmo-parts.js +115 -0
  233. package/src/object-gizmo.jsx +37 -122
  234. package/src/object-glow.js +139 -0
  235. package/src/object-hidden.js +33 -0
  236. package/src/object-look.js +196 -0
  237. package/src/object-size.js +10 -0
  238. package/src/object-travel.js +71 -6
  239. package/src/panels/CameraPanel.jsx +7 -7
  240. package/src/panels/HdrEnvironmentControls.jsx +59 -0
  241. package/src/panels/LightPanel.jsx +25 -4
  242. package/src/panels/ObjectLookPanel.jsx +88 -0
  243. package/src/panels/PoseImagePanel.jsx +1 -1
  244. package/src/panels/PosePanel.jsx +2 -17
  245. package/src/panels/RigControlPanel.jsx +5 -53
  246. package/src/panels/SubjectBox.jsx +4 -1
  247. package/src/panels/details.css +14 -0
  248. package/src/posestudio.jsx +4 -27
  249. package/src/project-browser.jsx +3 -4
  250. package/src/project-resources.js +18 -31
  251. package/src/project.js +13 -104
  252. package/src/props.jsx +25 -280
  253. package/src/rail-dot-drag.js +23 -0
  254. package/src/render-passes.js +10 -0
  255. package/src/resource-status.jsx +1 -0
  256. package/src/result-modal.jsx +4 -19
  257. package/src/room.jsx +20 -12
  258. package/src/scene-assets.js +8 -6
  259. package/src/scene-light-place.js +72 -0
  260. package/src/scene-lights.jsx +148 -0
  261. package/src/scene-objects.js +31 -139
  262. package/src/scenes.js +15 -105
  263. package/src/semantic-edit.js +2 -5
  264. package/src/set-pieces.jsx +183 -0
  265. package/src/settings-menu.jsx +3 -11
  266. package/src/shell/BottomDock.jsx +11 -53
  267. package/src/shell/CompositionGuideButton.jsx +20 -0
  268. package/src/shell/DetailsSlot.jsx +16 -200
  269. package/src/shell/GhostsMenuGroup.jsx +74 -0
  270. package/src/shell/LibrarySlot.jsx +1 -3
  271. package/src/shell/MenuBar.jsx +3 -3
  272. package/src/shell/PreferencesDialog.jsx +15 -264
  273. package/src/shell/PreferencesSlot.jsx +2 -8
  274. package/src/shell/ShotPreviewHeader.jsx +17 -0
  275. package/src/shell/TopBar.jsx +5 -83
  276. package/src/shell/ViewportToolbar.jsx +23 -93
  277. package/src/shell/agent-glass.css +19 -6
  278. package/src/shell/details-selection.js +42 -0
  279. package/src/shell/ghosts-menu.css +22 -0
  280. package/src/shell/glass-regions.css +29 -3
  281. package/src/shell/preferences.css +6 -1
  282. package/src/shell/viewport.css +12 -0
  283. package/src/shot-authoring.js +4 -7
  284. package/src/soft-shadows.js +37 -0
  285. package/src/stage-attach-frames.js +12 -12
  286. package/src/stage-canvas-rigs.jsx +3 -3
  287. package/src/stage-capture.jsx +42 -8
  288. package/src/stage-characters.jsx +22 -216
  289. package/src/stage-editor-gizmos.jsx +6 -6
  290. package/src/stage-environment.js +93 -0
  291. package/src/stage-hdr-builtins.js +107 -0
  292. package/src/stage-hdr-image.js +116 -0
  293. package/src/stage-hdr-library.js +70 -0
  294. package/src/stage-hdr-map.js +94 -0
  295. package/src/stage-hdr.js +130 -0
  296. package/src/stage-lighting.js +149 -0
  297. package/src/stage-occlusion.js +210 -0
  298. package/src/stage-sky.jsx +159 -0
  299. package/src/stage-startup-defaults.js +5 -9
  300. package/src/studio-actions.js +12 -14
  301. package/src/studio-agent-commands.js +46 -16
  302. package/src/studio-agent-motion.js +9 -9
  303. package/src/studio-agent-protocol.js +46 -73
  304. package/src/studio-app-binding.js +29 -45
  305. package/src/studio-asset-list.js +17 -0
  306. package/src/studio-contact-sheet.js +37 -5
  307. package/src/studio-elements.js +27 -7
  308. package/src/studio-entity-bounds.js +18 -0
  309. package/src/studio-error-codes.js +8 -0
  310. package/src/studio-export-framing.js +152 -0
  311. package/src/studio-lighting-check.js +103 -0
  312. package/src/studio-motion-visual.js +150 -0
  313. package/src/studio-schema-errors.js +80 -0
  314. package/src/studio-support-gap.js +74 -0
  315. package/src/styles.css +1 -19
  316. package/src/timeline-defaults.js +5 -0
  317. package/src/timeline-extent.js +8 -20
  318. package/src/tutorial-walk.js +17 -0
  319. package/src/ui-scrub.js +1 -1
  320. package/src/use-shot-camera-tracks.js +12 -0
  321. package/src/view-ghosts.js +97 -0
  322. package/src/view-ghosts.jsx +49 -0
  323. package/src/workflow/AgentPanel.jsx +47 -166
  324. package/src/workflow/AgentProgressCard.jsx +107 -0
  325. package/src/workflow/AgentReplyText.jsx +37 -0
  326. package/src/workflow/AgentTurnSteps.jsx +84 -0
  327. package/src/workflow/LoginPanel.jsx +140 -0
  328. package/src/workflow/WorkflowBuilder.jsx +7 -23
  329. package/src/workflow/agent-client.js +54 -51
  330. package/src/workflow/agent-panel.css +71 -1
  331. package/src/workflow/agent-progress-card.css +167 -0
  332. package/src/workflow/agent-progress-view.js +47 -0
  333. package/src/workflow/agent-turn-steps.css +113 -0
  334. package/src/workflow/agent-turn.js +191 -0
  335. package/src/workflow/login-client.js +34 -0
  336. package/src/workflow/motion-input.js +18 -31
  337. package/src/workflow/node-schema.js +1 -1
  338. package/src/workflow/reply-markdown.js +69 -0
  339. package/src/workflow/scene-asset-sync.js +8 -8
  340. package/tools/agent-session-stats.mjs +61 -0
  341. package/tools/dev-full.mjs +5 -56
  342. package/tools/eslint-baseline.json +12 -13
  343. package/tools/headless-studio.mjs +85 -0
  344. package/tools/knip-baseline.json +0 -179
  345. package/tools/loc-baseline.json +33 -83
  346. package/tools/motion3/extract-body.mjs +103 -0
  347. package/tools/motion3/extract-rig.mjs +85 -0
  348. package/tools/process-supervisor.mjs +0 -88
  349. package/tools/run-tests.mjs +9 -124
  350. package/bin/agent/motion-runtime.mjs +0 -76
  351. package/dist/ardy/cskel27-rest.json +0 -557
  352. package/dist/assets/analytics-B41FmPYI.js +0 -1
  353. package/dist/assets/app-Do-Og5Pu.css +0 -1
  354. package/dist/assets/app-DwP-tl2x.js +0 -4920
  355. package/dist/assets/config-Bx7pYsD9.js +0 -1
  356. package/dist/assets/config-DeoFTYzb.css +0 -1
  357. package/dist/assets/demo-Dt0qZfS7.js +0 -1
  358. package/dist/assets/scene-asset-sync-BCWDJB1d.js +0 -19
  359. package/dist/assets/scene-asset-sync-DjJ_cJ2R.css +0 -1
  360. package/dist/assets/ticket-jqMhqL88.js +0 -1
  361. package/dist/assets/workflow-Br-K7Srs.js +0 -179
  362. package/dist/demo/walk-then-stop.npz +0 -0
  363. package/mcp/ardy-prompts.mjs +0 -218
  364. package/src/ardy/auto-fix-panel.css +0 -108
  365. package/src/ardy/auto-physics.js +0 -1194
  366. package/src/ardy/client.js +0 -153
  367. package/src/ardy/convert.js +0 -313
  368. package/src/ardy/cskel27-neutral.js +0 -39
  369. package/src/ardy/cskel27.js +0 -77
  370. package/src/ardy/export.js +0 -165
  371. package/src/ardy/fix-collisions.js +0 -2257
  372. package/src/ardy/foot-lock.js +0 -230
  373. package/src/ardy/motion-calibration.js +0 -118
  374. package/src/ardy/motion-edit.js +0 -213
  375. package/src/ardy/motion-url.js +0 -55
  376. package/src/ardy/mp4-duration.js +0 -102
  377. package/src/ardy/npz.js +0 -595
  378. package/src/ardy/physics-panel.css +0 -21
  379. package/src/ardy/physics-panel.jsx +0 -89
  380. package/src/ardy/physics-refine.js +0 -111
  381. package/src/ardy/physics-review.js +0 -567
  382. package/src/ardy/physics-temporal.js +0 -54
  383. package/src/ardy/platform-fit-panel.css +0 -64
  384. package/src/ardy/platform-fit-panel.jsx +0 -41
  385. package/src/ardy/platform-fit.js +0 -326
  386. package/src/ardy/playback.js +0 -711
  387. package/src/ardy/pose-pin.js +0 -97
  388. package/src/ardy/prompt-clips.js +0 -30
  389. package/src/ardy/retime.js +0 -211
  390. package/src/ardy/root-drop.js +0 -369
  391. package/src/ardy/timeline-resize.js +0 -13
  392. package/src/ardy/to-cskel27.js +0 -225
  393. package/src/ardy/trim.js +0 -33
  394. package/src/ardy/waypoints.js +0 -369
  395. package/src/ardy-bridge-contract.js +0 -281
  396. package/src/domains/motion/motion-video-capture.js +0 -664
  397. package/src/i2v-motion-client.js +0 -77
  398. package/src/i2v-motion-studio.jsx +0 -180
  399. package/src/line-edit.js +0 -1348
  400. package/src/motion/generation.js +0 -105
  401. package/src/motion-readiness-ui.jsx +0 -67
  402. package/src/motion-readiness.js +0 -40
  403. package/src/motion-resources.js +0 -177
  404. package/src/motion-store.js +0 -65
  405. package/src/motion-trail.js +0 -601
  406. package/src/multimodel-ingest.js +0 -394
  407. package/src/panels/PromptBlocksPanel.jsx +0 -444
  408. package/src/panels/VideoCapturePanel.jsx +0 -181
  409. package/src/pose-extract/browser.js +0 -42
  410. package/src/pose-extract/detector.js +0 -100
  411. package/src/pose-extract/fit.js +0 -227
  412. package/src/pose-extract/image-frame.js +0 -51
  413. package/src/pose-extract/index.js +0 -58
  414. package/src/pose-extract/keyframes.js +0 -74
  415. package/src/pose-extract/landmarks.js +0 -162
  416. package/src/pose-extract/mirror.js +0 -153
  417. package/src/pose-extract/take.js +0 -131
  418. package/src/pose-extract/video-frames.js +0 -91
  419. package/src/take-recipe.js +0 -165
  420. package/src/trail-key-conflicts.js +0 -39
  421. package/src/trail-pick.js +0 -59
  422. package/src/use-i2v-capture-modal.js +0 -15
  423. package/tools/ardy/BRIDGE.md +0 -37
  424. package/tools/ardy/GVHMR-NPZ.md +0 -68
  425. package/tools/ardy/README.md +0 -124
  426. package/tools/ardy/__pycache__/cclay_gvhmr_worker.cpython-313.pyc +0 -0
  427. package/tools/ardy/artifacts.mjs +0 -34
  428. package/tools/ardy/bridge.mjs +0 -1732
  429. package/tools/ardy/bvh-cskel27.mjs +0 -1265
  430. package/tools/ardy/cclay_constrained_generate.py +0 -561
  431. package/tools/ardy/cclay_gvhmr_worker.py +0 -264
  432. package/tools/ardy/cclay_motion_edit.py +0 -510
  433. package/tools/ardy/cclay_sequence_generate.py +0 -705
  434. package/tools/ardy/cclay_worker.py +0 -304
  435. package/tools/ardy/dump-npz.py +0 -205
  436. package/tools/ardy/extract-rest.mjs +0 -299
  437. package/tools/ardy/extract.mjs +0 -507
  438. package/tools/ardy/footage.mjs +0 -467
  439. package/tools/ardy/gvhmr-floor.mjs +0 -102
  440. package/tools/ardy/gvhmr-to-cskel27.mjs +0 -21
  441. package/tools/ardy/gvhmr_fastpath.py +0 -168
  442. package/tools/ardy/gvhmr_trajectory.py +0 -240
  443. package/tools/ardy/merge-text-encoder.py +0 -160
  444. package/tools/ardy/mocap-metrics.mjs +0 -200
  445. package/tools/ardy/mocap-quality-gate.mjs +0 -110
  446. package/tools/ardy/motion-stabilize.mjs +0 -348
  447. package/tools/ardy/npz.mjs +0 -414
  448. package/tools/ardy/pose-to-npz.mjs +0 -106
  449. package/tools/ardy/prompt-limits.mjs +0 -5
  450. package/tools/ardy/run-edit-on-box.sh +0 -73
  451. package/tools/ardy/run-local.mjs +0 -233
  452. package/tools/ardy/run-on-box.sh +0 -593
  453. package/tools/ardy/run-sequence-on-box.sh +0 -177
  454. package/tools/ardy/runners/gvhmr-worker.mjs +0 -206
  455. package/tools/ardy/runners/index.mjs +0 -18
  456. package/tools/ardy/runners/local.mjs +0 -455
  457. package/tools/ardy/runners/proc.mjs +0 -125
  458. package/tools/ardy/runners/remote.mjs +0 -252
  459. package/tools/ardy/setup-local.mjs +0 -189
  460. package/tools/ardy/setup-text-encoder-on-box.sh +0 -93
  461. package/tools/ardy/setup-text-encoder.py +0 -306
  462. package/tools/ardy/smpl-cskel27.mjs +0 -178
  463. package/tools/ardy/visual-qa.mjs +0 -196
  464. package/tools/ardy/vq-car.mjs +0 -52
  465. package/tools/bench/EXP3.md +0 -61
  466. package/tools/bench/cclay_bench_extract_incam.py +0 -125
  467. package/tools/bench/cclay_bench_extract_obs.py +0 -204
  468. package/tools/bench/cclay_bench_runner.py +0 -42
  469. package/tools/bench/cube-contact.mjs +0 -162
  470. package/tools/bench/exp3.mjs +0 -152
  471. package/tools/bench/extract-bench-lib.mjs +0 -216
  472. package/tools/bench/extract-bench.mjs +0 -302
  473. package/tools/bench/fal-generate.mjs +0 -88
  474. package/tools/bench/fit/README.md +0 -189
  475. package/tools/bench/fit/camera.mjs +0 -59
  476. package/tools/bench/fit/contact.mjs +0 -419
  477. package/tools/bench/fit/footlock.mjs +0 -234
  478. package/tools/bench/fit/motion.mjs +0 -85
  479. package/tools/bench/fit/pin.mjs +0 -27
  480. package/tools/bench/fit/remote.mjs +0 -105
  481. package/tools/bench/fit-bench.mjs +0 -126
  482. package/tools/bench/fit-sanity.mjs +0 -45
  483. package/tools/bench/metrics.mjs +0 -313
  484. package/tools/bench/obs/depth.mjs +0 -258
  485. package/tools/bench/obs/extrinsics.mjs +0 -175
  486. package/tools/bench/obs/fit_mannequin_betas.py +0 -340
  487. package/tools/bench/obs/ground.mjs +0 -294
  488. package/tools/bench/obs/heading.mjs +0 -151
  489. package/tools/bench/obs/ladder.mjs +0 -555
  490. package/tools/bench/obs/mannequin-betas.json +0 -124
  491. package/tools/bench/obs/remote.mjs +0 -83
  492. package/tools/bench/obs/rest_joints.py +0 -52
  493. package/tools/bench/obs/ybot-targets.mjs +0 -49
  494. package/tools/bench/obs-bench.mjs +0 -520
  495. package/tools/bench/score.mjs +0 -343
  496. package/tools/bench/summarize.mjs +0 -86
  497. package/tools/demo-worker/README.md +0 -137
  498. package/tools/demo-worker/api-client.mjs +0 -233
  499. package/tools/demo-worker/generate.mjs +0 -301
  500. package/tools/demo-worker/index.mjs +0 -466
  501. package/tools/demo-worker/sign-core.mjs +0 -112
  502. package/tools/demo-worker/sign.mjs +0 -161
  503. package/tools/gt-render/browser.mjs +0 -159
  504. package/tools/gt-render/camera-math.mjs +0 -276
  505. package/tools/gt-render/page.mjs +0 -267
  506. package/tools/gt-render/render.mjs +0 -514
  507. package/tools/gt-render/scene-box.mjs +0 -31
  508. package/tools/gt-render/take-transform.mjs +0 -87
  509. package/tools/kimodo/compare-seam.mjs +0 -108
  510. package/tools/kimodo/constraints.mjs +0 -244
  511. package/tools/kimodo/edit.mjs +0 -259
  512. package/tools/kimodo/effector-constraints.mjs +0 -200
  513. package/tools/kimodo/generate.mjs +0 -646
  514. package/tools/kimodo/local-output.mjs +0 -265
  515. package/tools/kimodo/measure-pose.mjs +0 -65
  516. package/tools/kimodo/measure-preserve.mjs +0 -899
  517. package/tools/kimodo/measure-waypoints.mjs +0 -55
  518. package/tools/kimodo/mlx-generate.py +0 -111
  519. package/tools/kimodo/pose-constraints.mjs +0 -248
  520. package/tools/kimodo/preserve-mask.mjs +0 -543
  521. package/tools/kimodo/read-npz.mjs +0 -189
  522. package/tools/kimodo/run-edit-on-box.mjs +0 -155
  523. package/tools/kimodo/run-sequence-on-box.mjs +0 -178
  524. package/tools/kimodo/runner.mjs +0 -284
  525. package/tools/kimodo/seam-batch.mjs +0 -77
  526. package/tools/kimodo/setup-local.mjs +0 -30
  527. package/tools/kimodo/setup-on-box.sh +0 -339
  528. package/tools/kimodo/soma77-to-cskel27.mjs +0 -305
  529. package/tools/morphgs/assets/gen_truth.py +0 -39
  530. package/tools/morphgs/assets/mesh_ori_rig.txt +0 -27
  531. package/tools/morphgs/assets/playback-check.mjs +0 -30
  532. package/tools/morphgs/demo-gate.sh +0 -26
  533. package/tools/morphgs/fbx2morphgs.mjs +0 -68
  534. package/tools/morphgs/morphgs-to-cskel27.mjs +0 -105
  535. package/tools/morphgs/patches/preprocess_src-none-mode.patch +0 -76
  536. package/tools/morphgs/setup-on-cluster.sh +0 -126
  537. package/tools/projflow/driver.py +0 -1069
  538. package/tools/projflow/generate.mjs +0 -805
  539. package/tools/projflow/hml22-to-cskel27.mjs +0 -735
  540. package/tools/projflow/line-edit-job.mjs +0 -547
  541. package/tools/projflow/replay.mjs +0 -452
  542. package/tools/projflow/runner.mjs +0 -269
  543. package/tools/projflow/service.mjs +0 -742
  544. package/tools/track/DESIGN.md +0 -513
  545. package/tools/track/backfill-provenance.mjs +0 -116
  546. package/tools/track/budget.mjs +0 -8
  547. package/tools/track/check-rig.mjs +0 -209
  548. package/tools/track/diagnostics.schema.json +0 -60
  549. package/tools/track/export-rig.mjs +0 -256
  550. package/tools/track/fallback.mjs +0 -7
  551. package/tools/track/fk-parity-fixture.mjs +0 -161
  552. package/tools/track/gate.mjs +0 -349
  553. package/tools/track/masks.mjs +0 -318
  554. package/tools/track/metrics.mjs +0 -201
  555. package/tools/track/publish-obs.mjs +0 -108
  556. package/tools/track/py/check_env.py +0 -61
  557. package/tools/track/py/eval_lr.py +0 -229
  558. package/tools/track/py/lr_viterbi.py +0 -294
  559. package/tools/track/py/masks.py +0 -405
  560. package/tools/track/py/objective.py +0 -382
  561. package/tools/track/py/rig.py +0 -508
  562. package/tools/track/py/scene.py +0 -326
  563. package/tools/track/py/test_joint_indices.py +0 -123
  564. package/tools/track/py/test_lr_viterbi.py +0 -195
  565. package/tools/track/py/test_masks.py +0 -229
  566. package/tools/track/py/test_rig.py +0 -288
  567. package/tools/track/py/test_scene.py +0 -354
  568. package/tools/track/py/test_track.py +0 -328
  569. package/tools/track/py/track.py +0 -411
  570. package/tools/track/remote.mjs +0 -140
  571. package/tools/track/rig-dump.mjs +0 -227
  572. package/tools/track/run-box-tests.mjs +0 -14
  573. package/tools/track/run-box.mjs +0 -168
  574. package/tools/track/setup-box.sh +0 -59
  575. package/tools/track/study-2d.mjs +0 -728
  576. /package/src/{ardy → motion}/collision-blockers.js +0 -0
  577. /package/src/{ardy → motion}/ground.js +0 -0
  578. /package/src/{ardy → motion}/ik-drag.js +0 -0
  579. /package/src/{ardy → motion}/ik-key-json.js +0 -0
  580. /package/src/{ardy → motion}/ik.js +0 -0
  581. /package/src/{ardy → motion}/physics-surface.js +0 -0
  582. /package/src/{ardy → motion}/playback-clock.js +0 -0
  583. /package/src/{ardy → motion}/range-pin.js +0 -0
@@ -1,1069 +0,0 @@
1
- #!/usr/bin/env python3
2
- """driver.py — ProjFlow line-edit driver, shipped to the GPU box per run.
3
-
4
- CozyClay ships THIS file to the box the same way tools/kimodo/generate.mjs
5
- ships its constraint JSONs: nothing is installed on the box, the scout venv is
6
- the only python, and a run leaves a mkdtemp behind that the wrapper deletes.
7
-
8
- WHY A DRIVER AND NOT `demo/run.py`. Contract C7: the two ProjFlow samplers each
9
- carry exactly half of a CozyClay line edit and neither CLI exposes the other
10
- half.
11
-
12
- * `sample_projflow(A=(B,3,T,J) mask, y=values)` preserves source frames
13
- exactly (S1 measured 4.8e-7 m), but its targets are 3D WORLD points — it has
14
- no camera, so a drawn screen-space line cannot be expressed at all.
15
- * `sample_linear_constraints(operator A=(m, 3*T*J), measurement y=(m,))` takes
16
- a GENERAL dense operator, so a camera row is just another row — but
17
- `demo/run.py` only ever builds projection rows plus a frame-0 keyframe, so
18
- "keep the rest of the take" does not exist on that path.
19
-
20
- `sample_linear_constraints` is the general one and the cheaper one (S1: ~2.7x
21
- faster per step than the projflow inpainting sampler), and demo/run.py ALREADY
22
- stacks two kinds of row into one operator (projection rows for the lifted joint
23
- + identity-ish keyframe rows at frame 0). So preserved frames are nothing more
24
- exotic than more rows: one identity row per kept (frame, joint, axis) whose
25
- measurement is the source coordinate. One operator, one pass, both halves.
26
-
27
- WHY THE ROWS ARE EXACTLY LINEAR UNDER A PERSPECTIVE CAMERA (contract C6, as
28
- amended after Phase 0). The paper's demo only accepts an orthographic camera and
29
- the plan originally carried an orthographic approximation of CozyClay's shot
30
- camera as a known risk. It is not needed. With
31
-
32
- X_cam = R @ x + t (R world->camera, t in camera space)
33
- u = fx * X_cam[0]/X_cam[2] + cx
34
- v = fy * X_cam[1]/X_cam[2] + cy
35
-
36
- the depth division is the only nonlinearity, and multiplying it out cancels it:
37
-
38
- (u - cx) * (R2.x + t2) = fx * (R0.x + t0)
39
- => [fx*R0 - (u-cx)*R2] . x = -[fx*t0 - (u-cx)*t2]
40
- and [fy*R1 - (v-cy)*R2] . x = -[fy*t1 - (v-cy)*t2]
41
-
42
- Two rows per constrained frame, each with exactly THREE nonzero entries (the
43
- x/y/z columns of the target joint at that frame). u and v are KNOWN — they are
44
- what the artist drew — so nothing here depends on the unknown depth. This is
45
- exact at every focal length, which is why gate GP1's "<1 px" now applies to the
46
- close-up case too.
47
-
48
- UNITS. `points2d` and the intrinsics must share ONE unit system; the algebra
49
- above is unit-agnostic as long as they agree. CozyClay sends viewport-normalised
50
- 0..1 for both (fx/fy/cx/cy pre-divided by the viewport size), so the reprojection
51
- errors reported in the metadata are in normalised viewport units.
52
-
53
- THE COST OF A ROW, AND WHY PRESERVE ROWS ARE SUBSAMPLED. The caller precomputes
54
- a Cholesky of `A A^T + lambda I`, which is m x m — the ROW count, not the column
55
- count, is what has to stay bounded. Freezing a 196-frame take densely would be
56
- 196 * 22 * 3 = 12936 rows and a 12936^2 factorisation (670 MB fp32) for
57
- information that is 90% redundant: adjacent frames of a real take are nearly the
58
- same pose. The subsampling policy is documented at `build_preserve_rows`.
59
-
60
- OUTPUT. `<out>.npy` (T,22,3) float32 world metres @ 20 fps, HumanML3D 22-joint
61
- order, plus `<out>.meta.json`. POSITIONS ONLY — ACMDM's "Raw" family generates
62
- absolute coordinates and has no rotation channel. Converting to cskel27 (which
63
- must LIFT rotations) is a separate module and deliberately not done here; see
64
- the seam comment in generate.mjs.
65
-
66
- TWO RUN MODES, ONE SAMPLER (contract C11). `--source/--line/--out` is the
67
- ONE-SHOT mode above: load the model, edit once, exit. It is the contract of
68
- record and nothing below changes a number it produces. `--serve` is the
69
- RESIDENT mode: load the model ONCE and then answer requests forever on
70
- stdin/stdout, because the measured cost of an edit is 3.9 s of model load
71
- against 1.3 s of sampling and paying the load per drag is what makes the detail
72
- loop feel like a batch job. Both modes call exactly the same `Engine` and
73
- `run_line_edit`, so a warm edit and a cold edit differ in nothing but when the
74
- weights were read (`sample_with_linear_constraints` calls
75
- `torch.manual_seed(seed)` itself, so a seeded request is bit-identical either
76
- way).
77
-
78
- THE SERVE PROTOCOL. NDJSON, one request object per stdin line, one response
79
- object per stdout line, arrays as base64 little-endian float32 blobs beside an
80
- explicit shape (a 196-frame source is ~52 KB raw / 69 KB encoded — small enough
81
- that a pipe is the whole transport and there is no port, no tunnel, no file):
82
-
83
- ready (unsolicited, once, after the model is loaded):
84
- {"type":"ready","protocol":1,"device":"cuda:0","loadSeconds":3.9}
85
- request:
86
- {"id":"7","type":"lineEdit","line":{...C6...},
87
- "source":{"shape":[T,22,3],"data":"<base64 float32>"},
88
- "steps":100,"ridge":1e-6,"preserveStride":2,"preserveMargin":20,
89
- "seamPin":2,"seed":0,"cfg":3.0,"preview":false}
90
- {"id":"8","type":"ping"} {"id":"9","type":"shutdown"}
91
- response:
92
- {"id":"7","ok":true,"result":{"shape":[T,22,3],"data":"<base64>",
93
- "meta":{...the one-shot meta, verbatim...}}}
94
- {"id":"7","ok":false,"error":{"message":"...","type":"ValueError"}}
95
-
96
- STDOUT IS THE PROTOCOL AND NOTHING ELSE. torch, CLIP and the repo's own loaders
97
- all print to stdout, and one stray progress line would desynchronise the stream
98
- for the rest of the session. So `--serve` DUPLICATES fd 1 to a private handle
99
- and then points fd 1 at fd 2 before importing anything: every `print`, every
100
- tqdm bar, every C-level write from a native extension lands on stderr (where
101
- the wrapper forwards it as status), and the only writer to the real stdout is
102
- `_write_message`. A request whose JSON cannot even be parsed is answered with
103
- `"id": null`, which the wrapper reads as a protocol failure — it must, because
104
- an unparseable line means the two sides no longer agree about framing.
105
- """
106
-
107
- from __future__ import annotations
108
-
109
- import argparse
110
- import base64
111
- import json
112
- import os
113
- import sys
114
- import time
115
- import traceback
116
- from pathlib import Path
117
-
118
- import numpy as np
119
-
120
- # Bumped when a request or response field changes meaning. The wrapper ships
121
- # THIS file to the box per session, so the two sides cannot drift by more than
122
- # one deploy — the number exists so that a stale resident left over from an
123
- # earlier build is refused loudly instead of answering with the old shape.
124
- PROTOCOL_VERSION = 1
125
-
126
-
127
- # --- the hml22 skeleton ------------------------------------------------------
128
- #
129
- # HumanML3D / T2M 22-joint order, from `utils.motion_process.t2m_kinematic_chain`
130
- # and SMPL_JOINT_NAMES[0:22]. S2's mapping table is the citation; this is the
131
- # subset the driver actually needs.
132
- HML22_PARENT = (
133
- -1, # 0 pelvis
134
- 0, # 1 left_hip
135
- 0, # 2 right_hip
136
- 0, # 3 spine1
137
- 1, # 4 left_knee
138
- 2, # 5 right_knee
139
- 3, # 6 spine2
140
- 4, # 7 left_ankle
141
- 5, # 8 right_ankle
142
- 6, # 9 spine3
143
- 7, # 10 left_foot (TOE BASE, not the foot — S2's naming trap)
144
- 8, # 11 right_foot (toe base)
145
- 9, # 12 neck
146
- 9, # 13 left_collar (the CLAVICLE — cskel27 LeftShoulder)
147
- 9, # 14 right_collar
148
- 12, # 15 head
149
- 13, # 16 left_shoulder (the UPPER-ARM ROOT — cskel27 LeftArm)
150
- 14, # 17 right_shoulder
151
- 16, # 18 left_elbow
152
- 17, # 19 right_elbow
153
- 18, # 20 left_wrist
154
- 19, # 21 right_wrist
155
- )
156
-
157
- NUM_JOINTS = 22
158
-
159
- # The joints where independent limbs branch: the pelvis and the spine3 junction
160
- # that carries the neck and both clavicles. An edit frees its limb UP TO its
161
- # anchor and no further, so the body cannot drift while one arm is redrawn.
162
- LIMB_ANCHORS = frozenset({0, 9})
163
-
164
- # CozyClay IK track id -> hml22 joint index.
165
- #
166
- # THE JS SIDE IS THE AUTHORING SOURCE (generate.mjs exports the same table and
167
- # writes the resolved `jointId` into line.json); this copy is a GUARD, re-checked
168
- # below, so a stale wrapper shipping the wrong joint fails loudly on the box
169
- # instead of silently editing the wrong limb.
170
- #
171
- # Derived from S2's mapping table by composing the two hops it documents:
172
- # track -> cskel27 target (the pose studio's own table, ik.js) -> hml22 source.
173
- # Two of those hops look like typos and are not:
174
- # * `leftElbow` is cskel27 LeftForeArm, whose hml22 source is left_elbow (18).
175
- # A mid-joint track therefore constrains its OWN joint, not its chain's
176
- # effector: the artist dragged the elbow handle and drew the ELBOW's path,
177
- # and pinning joint 20 instead would move the wrist along the drawn line and
178
- # leave the elbow wherever the model liked.
179
- # * `spine` is cskel27 Spine1, whose hml22 source is spine2 (6) — cskel27 has
180
- # three spine links between the anchors where hml22 has two.
181
- TRACK_TO_JOINT = {
182
- # IK effectors
183
- "leftHand": 20, # cskel27 LeftHand <- left_wrist
184
- "rightHand": 21, # cskel27 RightHand <- right_wrist
185
- "leftFoot": 7, # cskel27 LeftFoot <- left_ANKLE (10 is the toe base)
186
- "rightFoot": 8, # cskel27 RightFoot <- right_ankle
187
- # mid-chain handles
188
- "leftElbow": 18, # cskel27 LeftForeArm <- left_elbow
189
- "rightElbow": 19,
190
- "leftKnee": 4, # cskel27 LeftLeg <- left_knee
191
- "rightKnee": 5,
192
- # FK handles
193
- "hips": 0, # cskel27 Hips <- pelvis
194
- "spine": 6, # cskel27 Spine1 <- spine2
195
- "neck": 12,
196
- "head": 15,
197
- "leftShoulder": 13, # cskel27 LeftShoulder is the CLAVICLE <- left_collar
198
- "rightShoulder": 14,
199
- }
200
-
201
- # `chest` is cskel27 Spine2, and Spine2 is one of S2's five FILLED joints: hml22
202
- # has no source for it at all (three cskel27 spine links between Hips and Spine3
203
- # against hml22's two). There is no joint to constrain, so a chest line edit is
204
- # refused by name rather than silently retargeted onto a neighbouring spine
205
- # joint, which would put the drawn line on a curve the artist did not draw.
206
- UNMAPPABLE_TRACKS = {
207
- "chest": "cskel27 Spine2 has no hml22 source joint (S2: a FILLED joint)",
208
- }
209
-
210
-
211
- def limb_chain(joint: int) -> frozenset:
212
- """Joints the edit is allowed to move: `joint`, its descendants, and its
213
- ancestors up to (not including) the nearest limb anchor.
214
-
215
- For joint 20 (left wrist) this is {13, 16, 18, 20} — the whole left arm from
216
- the clavicle out. For joint 0 (pelvis) it is the whole body, which is right:
217
- a hips line edit drags the figure.
218
- """
219
- out = {int(joint)}
220
- # descendants
221
- changed = True
222
- while changed:
223
- changed = False
224
- for child, parent in enumerate(HML22_PARENT):
225
- if parent in out and child not in out:
226
- out.add(child)
227
- changed = True
228
- # ancestors, up to the anchor
229
- node = HML22_PARENT[int(joint)]
230
- while node is not None and node >= 0 and node not in LIMB_ANCHORS:
231
- out.add(node)
232
- node = HML22_PARENT[node]
233
- return frozenset(out)
234
-
235
-
236
- def resample_polyline(points, count):
237
- """Resample an (P,2) polyline to `count` points, arc-index-linearly.
238
-
239
- Deliberately the SAME convention as the repo's own
240
- `demo/lift/trajectory_io._resample_xy`: uniform parameter over the point
241
- INDEX (not arc length), linear interpolation. Matching it means a polyline
242
- that the demo would place at frame t lands on frame t here too, so S1's
243
- measured 2.4e-7 reprojection number transfers.
244
- """
245
- pts = np.asarray(points, dtype=np.float32)
246
- if pts.ndim != 2 or pts.shape[1] != 2:
247
- raise ValueError(f"points2d must be (P,2), got {pts.shape}")
248
- if pts.shape[0] < 2:
249
- raise ValueError("points2d needs at least 2 points")
250
- if pts.shape[0] == int(count):
251
- return pts
252
- t_src = np.linspace(0.0, 1.0, num=pts.shape[0], dtype=np.float32)
253
- t_dst = np.linspace(0.0, 1.0, num=int(count), dtype=np.float32)
254
- out = np.empty((int(count), 2), dtype=np.float32)
255
- out[:, 0] = np.interp(t_dst, t_src, pts[:, 0])
256
- out[:, 1] = np.interp(t_dst, t_src, pts[:, 1])
257
- return out.astype(np.float32)
258
-
259
-
260
- def column_index(axis: int, frame: int, joint: int, *, num_frames: int, num_joints: int) -> int:
261
- """Column of x[axis, frame, joint] in the flattened (3, L, J) operand.
262
-
263
- Copied from `demo/lift/projection_operator.column_index` rather than
264
- imported so this file states the layout it depends on; the two are asserted
265
- equal at startup (see `_assert_column_layout`).
266
- """
267
- return (axis * num_frames + frame) * num_joints + joint
268
-
269
-
270
- def _assert_column_layout(num_frames: int) -> None:
271
- from demo.lift.projection_operator import column_index as repo_column_index
272
-
273
- for axis, frame, joint in ((0, 0, 0), (1, 3, 7), (2, num_frames - 1, NUM_JOINTS - 1)):
274
- mine = column_index(axis, frame, joint, num_frames=num_frames, num_joints=NUM_JOINTS)
275
- theirs = repo_column_index(axis, frame, joint, num_frames=num_frames, num_joints=NUM_JOINTS)
276
- if mine != theirs:
277
- raise RuntimeError(
278
- "column layout drifted from demo/lift/projection_operator.column_index "
279
- f"({mine} != {theirs}); every row this driver builds would be scrambled"
280
- )
281
-
282
-
283
- # --- constraint construction -------------------------------------------------
284
-
285
- def build_line_rows(line, *, frames, joint_id):
286
- """Per-frame affine perspective rows for the drawn polyline.
287
-
288
- Returns (rows, values, frame_indices, uv) where `rows` is a list of
289
- (col, coefficient) triples-per-row. See the module docstring for the algebra.
290
- """
291
- cam = line["camera"]
292
- fx = float(cam["fx"])
293
- fy = float(cam["fy"])
294
- cx = float(cam["cx"])
295
- cy = float(cam["cy"])
296
- R = np.asarray(cam["R"], dtype=np.float64)
297
- t = np.asarray(cam["t"], dtype=np.float64).reshape(3)
298
- if R.shape != (3, 3):
299
- raise ValueError(f"camera.R must be 3x3, got {R.shape}")
300
- if fx == 0.0 or fy == 0.0:
301
- raise ValueError("camera.fx and camera.fy must be non-zero")
302
-
303
- start = int(line["frameRange"]["start"])
304
- end = int(line["frameRange"]["end"])
305
- if not (0 <= start <= end < frames):
306
- raise ValueError(
307
- f"frameRange {start}..{end} is outside the source motion's 0..{frames - 1}"
308
- )
309
- span = end - start + 1
310
- uv = resample_polyline(line["points2d"], span)
311
-
312
- R0, R1, R2 = R[0], R[1], R[2]
313
- rows = []
314
- values = []
315
- frame_indices = list(range(start, end + 1))
316
- for k, frame in enumerate(frame_indices):
317
- u = float(uv[k, 0])
318
- v = float(uv[k, 1])
319
- cols = [
320
- column_index(axis, frame, joint_id, num_frames=frames, num_joints=NUM_JOINTS)
321
- for axis in range(3)
322
- ]
323
- # u row
324
- coef_u = fx * R0 - (u - cx) * R2
325
- rows.append(list(zip(cols, coef_u.tolist())))
326
- values.append(-(fx * t[0] - (u - cx) * t[2]))
327
- # v row
328
- coef_v = fy * R1 - (v - cy) * R2
329
- rows.append(list(zip(cols, coef_v.tolist())))
330
- values.append(-(fy * t[1] - (v - cy) * t[2]))
331
- return rows, values, frame_indices, uv
332
-
333
-
334
- def build_pin_rows(line, *, frames, joint_id):
335
- """Hard 3D keyframe rows for `pins3d` — the OTHER shape of a line edit.
336
-
337
- A pin says "this joint is exactly HERE at this generation frame", so it is
338
- three identity rows (x, y, z) with the pinned coordinates as the
339
- measurement. That is the sparsest possible constraint and it is precisely
340
- what the paper measures: ProjFlow's spatial-control table is 1, 2, 5, 49 and
341
- 196 keyframes, with 0.0000 trajectory/location/average error at every
342
- density, and the flow prior — not any interpolation of ours — is what fills
343
- the frames between. So there is deliberately NOTHING between the pins here:
344
- no pseudo-observations, no interpolated guides. The paper's soft
345
- machinery (Sec 4.3.1) exists to rescue the *inpainting* task where hard
346
- observations are sparse across a whole clip; here the preserve rows outside
347
- the range and the seam pin at its edges already bracket the gap, and the
348
- free span between two pins is a couple of seconds at most.
349
-
350
- Same row format as build_line_rows so the two are interchangeable to the
351
- caller: (rows, values, frame_indices). A line's rows are TWO per frame and
352
- depend on the camera; a pin's are THREE and depend on nothing, which is why
353
- a pinned edit needs no lens and survives an orbit.
354
-
355
- Frames arrive already on the generation clock (line-edit-job.mjs scales
356
- them with the same rounding rule it uses for frameRange), so the only
357
- checking here is that they are inside the clip.
358
- """
359
- pins = line["pins3d"]
360
- if not isinstance(pins, list) or not pins:
361
- raise ValueError("pins3d must be a non-empty list of {frame, position}")
362
- rows = []
363
- values = []
364
- frame_indices = []
365
- targets = []
366
- for index, pin in enumerate(pins):
367
- frame = int(pin["frame"])
368
- if not (0 <= frame < frames):
369
- raise ValueError(
370
- f"pins3d[{index}].frame {frame} is outside the source motion's 0..{frames - 1}"
371
- )
372
- position = [float(v) for v in pin["position"]]
373
- if len(position) != 3:
374
- raise ValueError(f"pins3d[{index}].position must be [x, y, z], got {pin['position']}")
375
- for axis in range(3):
376
- col = column_index(axis, frame, joint_id, num_frames=frames, num_joints=NUM_JOINTS)
377
- rows.append([(col, 1.0)])
378
- values.append(position[axis])
379
- frame_indices.append(frame)
380
- targets.append(position)
381
- return rows, values, frame_indices, np.asarray(targets, dtype=np.float64)
382
-
383
-
384
- def build_preserve_rows(source, *, edit_start, edit_end, chain, stride, margin, seam_pin=2, hold_body=False, owned=()):
385
- """Identity rows pinning the SOURCE take outside the edited range.
386
-
387
- SUBSAMPLING POLICY — stated in full because it is the one place this driver
388
- trades exactness for a tractable Cholesky (`A A^T + lambda I` is m x m):
389
-
390
- 1. FRAME STRIDE. Only every `stride`-th frame outside the edit range carries
391
- rows (default 2 => half the rows). Adjacent frames of a real take are
392
- nearly the same pose, so a pinned frame constrains its neighbour almost
393
- as hard as pinning it directly would; the flow prior supplies the rest.
394
- The stride is phase-locked to frame 0 (`range(0, T, stride)`) so two runs
395
- over the same take pin the same frames.
396
-
397
- 2. THE SEAM IS ALWAYS PINNED. The frames immediately either side of the edit
398
- range are added regardless of stride. That is where a pop would show
399
- (gate GP2 measures the blend seams, not the preserved interior), so it is
400
- the one place a missed frame is expensive.
401
-
402
- 3. CHAIN EXCLUSION INSIDE A MARGIN. Within `margin` frames of the edit range
403
- the EDITED LIMB's joints (`chain`) carry no rows, while the rest of the
404
- body still does. The limb has to travel from where the source take left
405
- it to the drawn line; pinning it hard right up to the boundary would ask
406
- for an instantaneous jump and the sampler would split the difference,
407
- missing the line at its start. Outside the margin every joint is pinned.
408
-
409
- 4. SEAM PIN — the edit's OWN first and last `seam_pin` frames carry FULL
410
- rows, chain included, even though they are inside the edited range.
411
-
412
- This is the fix for a seam pop that rules 1-3 cannot reach, and it is
413
- licensed by a property the APP now guarantees rather than by anything
414
- here. src/line-edit.js hands over a curve whose first and last points are
415
- the joint's own projected trajectory, byte-identical: a drag pins them
416
- (PINNED_CURVE_ENDS) and a drawn stroke eases into them (seamEaseWeight is
417
- exactly 0 at both window edges). So at those two frames the drawn line
418
- and the source take ALREADY agree in 2D, and pinning the full 3D pose
419
- there adds no contradiction — it removes the freedom in DEPTH and in the
420
- other 21 joints that was producing the pop.
421
-
422
- Without it those frames are unconstrained except for the line's two
423
- affine rows, and the nearest hard evidence is a whole frame away on the
424
- far side of the boundary: the paper's Plain Masking case (sparse hard
425
- observations, no soft guides), which its own ablation reports as the
426
- worst configuration.
427
-
428
- MEASURED, backfall take, a 102-frame leftFoot reroute (app frames
429
- 20..122, 20-step preview, seed 4242, cold path). Seam figures are the
430
- spliced take's frame-to-frame jump at the boundary as a multiple of the
431
- SOURCE take's own worst frame delta (0.2153 m), so 1.0x means "no more
432
- violent than the take already is":
433
-
434
- seam_pin head tail worst jump in-window edit (max / mean)
435
- 0 2.46x 6.64x 6.64x 2.934 / 1.786 m
436
- 1 0.33x 1.70x 4.51x 2.740 / 1.684 m
437
- 2 0.24x 1.06x 3.96x 2.612 / 1.611 m <- default
438
- 3 0.21x 1.04x 3.00x 2.491 / 1.548 m
439
- 4 0.20x 1.03x 3.03x 2.367 / 1.492 m
440
-
441
- TWO is the default because it is where the seam stops being visible
442
- (1.06x = the take's own worst frame already) while the edit keeps 89% of
443
- its magnitude; past 2 the pin buys almost no seam and starts eating the
444
- edit, since every pinned frame is a frame the drawn line does not own.
445
- Two also matches the app's PINNED_CURVE_ENDS exactly: those are the
446
- frames src/line-edit.js already holds on the source trail in 2D, so this
447
- pins in 3D precisely what the app pinned in 2D. `seam_pin=0` restores the
448
- pre-2026-08-29 behaviour byte for byte.
449
-
450
- Order of the emitted rows is (frame, joint, axis), ascending, so a run is
451
- reproducible and `preservedFrames` in the metadata reads in time order.
452
- """
453
- frames = int(source.shape[0])
454
- lo = max(0, int(edit_start) - int(margin))
455
- hi = min(frames - 1, int(edit_end) + int(margin))
456
-
457
- candidates = set(range(0, frames, max(1, int(stride))))
458
- for seam in (int(edit_start) - 1, int(edit_end) + 1):
459
- if 0 <= seam < frames:
460
- candidates.add(seam)
461
- kept_frames = sorted(f for f in candidates if f < int(edit_start) or f > int(edit_end))
462
-
463
- # 5. HOLD THE BODY (pins only). Inside the range the non-edited joints keep
464
- # their subsampled rows; only the pinned limb's chain is freed.
465
- #
466
- # WHY IT IS ONLY FOR PINS, and why it is not optional for them. A drawn
467
- # line replaces a passage of movement, so the body genuinely has to
468
- # re-time itself around the new path and freeing the range is the point.
469
- # A PIN says the opposite thing — "the take is right, this joint is
470
- # wrong at this instant" — and freeing 1.5 s of whole body to deliver it
471
- # was measured as exactly the wrong trade: on the backfall take a 5 cm
472
- # ask and a 40 cm ask both moved the head ~28 cm and both missed by
473
- # ~25 cm, because the pin was a small perturbation on a window the model
474
- # had otherwise re-imagined (in-window whole-body drift 1.19 m in every
475
- # case, identical to three decimal places).
476
- #
477
- # This is also the paper's own shape for sparse observations. Its
478
- # inpainting formulation surrounds sparse hard keyframes with guidance
479
- # everywhere else rather than leaving the gap free, and its ablation
480
- # reports Plain Masking — hard keyframes alone — as the worst
481
- # configuration. Here the guidance is not interpolated pseudo-
482
- # observations with a fitted variance: it is the SOURCE TAKE, which is
483
- # both hard and correct, so it goes in at Sigma = 0 like every other
484
- # preserve row and needs none of the time-varying machinery.
485
- hold = bool(hold_body)
486
- if hold:
487
- inside = sorted(
488
- f for f in set(range(0, frames, max(1, int(stride))))
489
- if int(edit_start) <= f <= int(edit_end)
490
- )
491
- kept_frames = sorted(set(kept_frames) | set(inside))
492
-
493
- # The seam pin's own frames, taken from INSIDE the range. Capped at half the
494
- # range so a two-frame edit cannot pin itself into a no-op.
495
- pin_span = max(0, int(seam_pin))
496
- span = int(edit_end) - int(edit_start) + 1
497
- pin_span = min(pin_span, span // 2)
498
- seam_frames = []
499
- for offset in range(pin_span):
500
- seam_frames.append(int(edit_start) + offset)
501
- seam_frames.append(int(edit_end) - offset)
502
- seam_frames = sorted(set(f for f in seam_frames if 0 <= f < frames))
503
- kept_frames = sorted(set(kept_frames) | set(seam_frames))
504
-
505
- # Cells the EDIT's own rows already own. A seam-pin frame that happens to
506
- # carry a 3D pin would otherwise get an identity row to the source AND a pin
507
- # row to the target for the same coordinate: two contradictory measurements
508
- # that the ridge quietly averages into a third answer nobody asked for. The
509
- # edit always wins — it is the thing the artist authored.
510
- owned_cells = {(int(f), int(j)) for f, j in owned}
511
-
512
- rows = []
513
- values = []
514
- pinned = [] # (frame, joint) pairs, for the self-check below
515
- for frame in kept_frames:
516
- in_margin = lo <= frame <= hi
517
- for joint in range(NUM_JOINTS):
518
- if (frame, joint) in owned_cells:
519
- continue
520
- # A seam frame is pinned WHOLE — the chain exclusion is about giving
521
- # the limb room to travel, and at the seam it has nowhere to travel
522
- # to: the line already puts it where the source does.
523
- if in_margin and joint in chain and frame not in seam_frames:
524
- continue
525
- for axis in range(3):
526
- col = column_index(axis, frame, joint, num_frames=frames, num_joints=NUM_JOINTS)
527
- rows.append([(col, 1.0)])
528
- values.append(float(source[frame, joint, axis]))
529
- pinned.append((frame, joint))
530
- return rows, values, kept_frames, pinned
531
-
532
-
533
- def densify(rows, values, *, frames):
534
- """Materialise the sparse row list into the dense (m, 3*T*J) operator the
535
- sampler wants. Every row has at most 3 nonzeros, but
536
- `sample_linear_constraints` takes a dense tensor, so this is where the
537
- memory goes: m * 3*T*J * 4 bytes."""
538
- m = len(rows)
539
- n = 3 * int(frames) * NUM_JOINTS
540
- A = np.zeros((m, n), dtype=np.float32)
541
- for index, row in enumerate(rows):
542
- for col, coefficient in row:
543
- A[index, col] = coefficient
544
- return A, np.asarray(values, dtype=np.float32)
545
-
546
-
547
- def project(points_world, cam):
548
- """Perspective-project (N,3) world points with the request's camera, in the
549
- same units the request used. The self-check reprojection number is computed
550
- with THIS, independently of the rows, so a sign error in the row algebra
551
- shows up as a large error rather than cancelling itself out."""
552
- R = np.asarray(cam["R"], dtype=np.float64)
553
- t = np.asarray(cam["t"], dtype=np.float64).reshape(1, 3)
554
- cam_pts = points_world.astype(np.float64) @ R.T + t
555
- z = cam_pts[:, 2]
556
- u = float(cam["fx"]) * cam_pts[:, 0] / z + float(cam["cx"])
557
- v = float(cam["fy"]) * cam_pts[:, 1] / z + float(cam["cy"])
558
- return np.stack([u, v], axis=1)
559
-
560
-
561
- # --- request validation ------------------------------------------------------
562
-
563
- def resolve_joint(line):
564
- track = line.get("track")
565
- if not isinstance(track, str) or not track:
566
- raise ValueError("line.track is required and must be a non-empty string")
567
- if track in UNMAPPABLE_TRACKS:
568
- raise ValueError(f"track {track!r} cannot be line-edited: {UNMAPPABLE_TRACKS[track]}")
569
- if track not in TRACK_TO_JOINT:
570
- known = ", ".join(sorted(TRACK_TO_JOINT))
571
- raise ValueError(f"unknown track {track!r}; known line-edit tracks: {known}")
572
- expected = TRACK_TO_JOINT[track]
573
- declared = line.get("jointId")
574
- if declared is not None and int(declared) != expected:
575
- # See TRACK_TO_JOINT's header: the JS side resolves the joint and this is
576
- # the guard against a wrapper that has drifted from the driver.
577
- raise ValueError(
578
- f"line.jointId {declared} disagrees with track {track!r} -> joint {expected}; "
579
- "the wrapper and the driver disagree about the skeleton"
580
- )
581
- return expected
582
-
583
-
584
- # --- the loaded model, and the one edit both run modes perform ---------------
585
-
586
- def open_repo(repo_path):
587
- """Put the ProjFlow clone on sys.path AND make it the cwd, then return it.
588
-
589
- Both are required and for different reasons: the repo's modules are imported
590
- by package path (`demo.common.*`), and its model loader resolves checkpoint
591
- paths relative to the process's cwd. Called once per process — the serve
592
- loop must not chdir per request, or a relative path in a request would mean
593
- something different depending on what ran before it.
594
- """
595
- repo = Path(repo_path).expanduser().resolve()
596
- if not (repo / "demo").is_dir():
597
- raise SystemExit(f"--repo {repo} does not look like a ProjFlow clone (no demo/)")
598
- sys.path.insert(0, str(repo))
599
- os.chdir(str(repo))
600
- return repo
601
-
602
-
603
- class Engine:
604
- """The loaded checkpoint plus the handful of repo functions an edit needs.
605
-
606
- THE WHOLE POINT of contract C11 is that this object outlives a request. S1
607
- measured the load at 3.9 s against 1.3 s of sampling, so a resident Engine
608
- is the difference between an 8 s round trip and a 2 s one. Nothing about a
609
- request mutates it: the operator, the Cholesky and the sample are all built
610
- per call in `run_line_edit`, and the sampler seeds torch itself, so request
611
- N+1 cannot inherit anything from request N except the weights.
612
-
613
- The repo imports happen HERE rather than at module scope because they are
614
- only importable after `open_repo`, and because `import torch` is most of the
615
- load time — a caller that only wants the argument parser should not pay it.
616
- """
617
-
618
- def __init__(self, repo, *, model_id, model_name, device):
619
- import torch # after open_repo, so the venv's torch is the one that loads
620
- from demo.common.model_loader import load_acmdm_from_config, resolve_device
621
- from demo.common.motion_stats import load_22x3_stats, denormalize_world
622
- from demo.common.normal_equations import precompute_normal_cholesky
623
- from demo.common.sampling import sample_with_linear_constraints
624
- from demo.lift.projection_operator import convert_linear_measurement_to_normalized_space
625
-
626
- started = time.time()
627
- self.torch = torch
628
- self.repo = repo
629
- self.device = resolve_device(str(device))
630
- self.loaded = load_acmdm_from_config(
631
- {"id": model_id, "name": model_name, "dataset": "t2m"},
632
- repo_root=repo,
633
- device=self.device,
634
- )
635
- self.mean_np, self.std_np = load_22x3_stats("t2m", repo)
636
- self.load_seconds = time.time() - started
637
- self.to_normalized = convert_linear_measurement_to_normalized_space
638
- self.precompute_normal_cholesky = precompute_normal_cholesky
639
- self.sample = sample_with_linear_constraints
640
- self.denormalize_world = denormalize_world
641
-
642
-
643
- def run_line_edit(
644
- engine,
645
- source,
646
- line,
647
- *,
648
- steps,
649
- ridge,
650
- preserve_stride,
651
- preserve_margin,
652
- seed,
653
- cfg,
654
- seam_pin=2,
655
- preview=False,
656
- reset_peak=False,
657
- ):
658
- """One line edit against a loaded Engine. Returns (out_world, meta).
659
-
660
- Every number the one-shot mode ever wrote lives in the returned meta, and
661
- the serve mode returns the same dict verbatim, so a warm result and a cold
662
- result are indistinguishable to the wrapper.
663
- """
664
- torch = engine.torch
665
- source = np.asarray(source, dtype=np.float32)
666
- if source.ndim != 3 or source.shape[1] != NUM_JOINTS or source.shape[2] != 3:
667
- raise ValueError(f"source must be (T,22,3); got {source.shape}")
668
- frames = int(source.shape[0])
669
- _assert_column_layout(frames)
670
-
671
- joint_id = resolve_joint(line)
672
- prompt = str(line.get("prompt") or "")
673
-
674
- # --- fuse the operator ---------------------------------------------------
675
- # ONE EDIT IS ONE GESTURE (contract C6 v2): either a drawn line (two affine
676
- # rows per frame, camera-dependent) or 3D pins (three identity rows per pin,
677
- # camera-free). The wrapper validates the exclusivity; here the presence of
678
- # pins3d simply selects the row builder, and everything downstream — the
679
- # preserve rows, the seam pin, the solve, the splice — is identical.
680
- pinned_edit = line.get("pins3d") is not None
681
- if pinned_edit:
682
- line_rows, line_values, line_frames, uv = build_pin_rows(line, frames=frames, joint_id=joint_id)
683
- else:
684
- line_rows, line_values, line_frames, uv = build_line_rows(line, frames=frames, joint_id=joint_id)
685
- chain = limb_chain(joint_id)
686
- keep_rows, keep_values, kept_frames, pinned = build_preserve_rows(
687
- source,
688
- edit_start=int(line["frameRange"]["start"]),
689
- edit_end=int(line["frameRange"]["end"]),
690
- chain=chain,
691
- stride=int(preserve_stride),
692
- margin=int(preserve_margin),
693
- seam_pin=int(seam_pin),
694
- # A pins edit holds the rest of the body through the range; a drawn line
695
- # frees it, because a stroke replaces a passage of movement and a pin
696
- # corrects an instant of one. See build_preserve_rows rule 5.
697
- hold_body=pinned_edit,
698
- owned=[(f, joint_id) for f in line_frames],
699
- )
700
- A_world_np, y_world_np = densify(line_rows + keep_rows, line_values + keep_values, frames=frames)
701
- m = int(A_world_np.shape[0])
702
-
703
- device = engine.device
704
- A_world = torch.from_numpy(A_world_np).to(device=device)
705
- y_world = torch.from_numpy(y_world_np).to(device=device)
706
- # THE normalisation convention, taken from the repo so there is one spelling
707
- # of it: x_world = x_norm*std + mean => A' = A diag(std), y' = y - A mean.
708
- # It is applied to the line rows and the preserve rows TOGETHER, which is the
709
- # whole point of fusing them into one operator.
710
- A_norm, y_norm = engine.to_normalized(
711
- operator_world=A_world,
712
- measurement_world=y_world,
713
- mean=torch.from_numpy(np.asarray(engine.mean_np, dtype=np.float32)).to(device=device),
714
- std=torch.from_numpy(np.asarray(engine.std_np, dtype=np.float32)).to(device=device),
715
- num_frames=frames,
716
- num_joints=NUM_JOINTS,
717
- )
718
-
719
- factor = engine.precompute_normal_cholesky(A_norm, ridge_lambda=float(ridge))
720
-
721
- # An empty prompt with cfg != 1 duplicates the batch to guide toward nothing;
722
- # on a line edit the constraints carry the intent, so guidance is switched off
723
- # rather than paid for. A prompt that IS given keeps the demos' cfg.
724
- cfg_scale = float(cfg) if prompt else 1.0
725
-
726
- if device.type == "cuda":
727
- # Only the resident asks for this: it makes `vramPeakAllocMiB` the peak of
728
- # THIS request (the weights stay resident, so they stay in the baseline)
729
- # instead of a session high-water mark that only ever grows. The one-shot
730
- # process is its own reset and must keep reporting the load peak it always
731
- # has.
732
- if reset_peak:
733
- torch.cuda.reset_peak_memory_stats()
734
- torch.cuda.synchronize()
735
- started = time.time()
736
- result = engine.sample(
737
- engine.loaded.model,
738
- prompt=prompt,
739
- operator=A_norm,
740
- measurement=y_norm.view(1, -1),
741
- normal_cholesky=factor.normal_cholesky.to(device=device),
742
- frames=frames,
743
- num_joints=NUM_JOINTS,
744
- num_samples=1,
745
- seed=int(seed),
746
- steps=int(steps),
747
- cfg=cfg_scale,
748
- device=device,
749
- )
750
- if device.type == "cuda":
751
- torch.cuda.synchronize()
752
- sampling_seconds = time.time() - started
753
-
754
- out_norm = result.samples_norm[0].permute(1, 2, 0).detach().cpu().numpy() # (T,J,3)
755
- out_world = engine.denormalize_world(out_norm, engine.mean_np, engine.std_np).astype(np.float32)
756
-
757
- # --- self-check ----------------------------------------------------------
758
- # Both numbers are computed from the RESULT, not from the sampler's
759
- # internals, and the reprojection uses `project()` rather than the rows, so a
760
- # sign error cannot hide inside its own operator.
761
- if pinned:
762
- pin_f = np.asarray([f for f, _ in pinned])
763
- pin_j = np.asarray([j for _, j in pinned])
764
- preserved_err = np.abs(out_world[pin_f, pin_j] - source[pin_f, pin_j])
765
- preserved_max = float(preserved_err.max())
766
- preserved_mean = float(preserved_err.mean())
767
- else:
768
- preserved_max = preserved_mean = 0.0
769
- if pinned_edit:
770
- # A pin's own self-check is in METRES, not viewport units: it was
771
- # authored in world space and there is no lens to project it through.
772
- # Reported under the same key so the wrapper and the gates read one
773
- # number whichever gesture produced it; `lineErrUnits` says which.
774
- line_err = np.abs(out_world[line_frames, joint_id] - uv)
775
- err_units = "m"
776
- else:
777
- reproj = project(out_world[line_frames, joint_id], line["camera"])
778
- line_err = np.abs(reproj - uv.astype(np.float64))
779
- err_units = "uv"
780
- edit_lo = int(line["frameRange"]["start"])
781
- edit_hi = int(line["frameRange"]["end"])
782
- edited = np.abs(out_world[edit_lo:edit_hi + 1] - source[edit_lo:edit_hi + 1])
783
-
784
- meta = {
785
- "m": m,
786
- "steps": int(steps),
787
- "ridge": float(ridge),
788
- "sampling_seconds": round(sampling_seconds, 4),
789
- "frames": frames,
790
- "fps": 20,
791
- "joints": NUM_JOINTS,
792
- "track": line["track"],
793
- "jointId": joint_id,
794
- "prompt": prompt,
795
- "cfg": cfg_scale,
796
- "seed": int(seed),
797
- "preview": bool(preview),
798
- "gesture": "pins" if pinned_edit else "line",
799
- "pins": [int(f) for f in line_frames] if pinned_edit else None,
800
- "rows": {
801
- "line": len(line_rows),
802
- "preserve": len(keep_rows),
803
- "operatorCols": 3 * frames * NUM_JOINTS,
804
- },
805
- "preserve": {
806
- "stride": int(preserve_stride),
807
- "margin": int(preserve_margin),
808
- "seamPin": int(seam_pin),
809
- "holdBody": bool(pinned_edit),
810
- "frames": [int(f) for f in kept_frames],
811
- "pinnedJointFrames": len(pinned),
812
- "freeChain": sorted(int(j) for j in chain),
813
- },
814
- "editRange": [int(line["frameRange"]["start"]), int(line["frameRange"]["end"])],
815
- "checks": {
816
- # Success criterion (i): pinned source frames come back unchanged.
817
- "preservedMaxAbsDiffM": preserved_max,
818
- "preservedMeanAbsDiffM": preserved_mean,
819
- # Success criterion (ii): the drawn line is followed, in the request's
820
- # own 2D units.
821
- "lineMaxReprojErr": float(line_err.max()),
822
- "lineMeanReprojErr": float(line_err.mean()),
823
- "lineErrUnits": err_units,
824
- # Proof the edit is not a no-op: the range genuinely moved.
825
- "editedMaxAbsDiffM": float(edited.max()),
826
- },
827
- "device": str(device),
828
- "checkpoint": str(engine.loaded.checkpoint_path),
829
- }
830
- if device.type == "cuda":
831
- meta["vramPeakAllocMiB"] = round(torch.cuda.max_memory_allocated() / 2 ** 20, 1)
832
- return out_world, meta
833
-
834
-
835
- # --- the resident service ----------------------------------------------------
836
-
837
- def decode_array(blob):
838
- """A {"shape": [...], "data": "<base64 little-endian float32>"} payload -> ndarray.
839
-
840
- The shape is carried explicitly rather than inferred from the byte count
841
- because a (T,22,3) and a (3,22,T) blob are the same length, and the sampler
842
- would happily edit the transpose.
843
- """
844
- if not isinstance(blob, dict):
845
- raise ValueError("source must be an object with `shape` and base64 `data`")
846
- shape = [int(dim) for dim in blob.get("shape") or []]
847
- if not shape or any(dim < 0 for dim in shape):
848
- raise ValueError(f"source.shape must be positive integers, got {blob.get('shape')!r}")
849
- raw = base64.b64decode(blob.get("data") or "", validate=True)
850
- count = int(np.prod(shape))
851
- if len(raw) != count * 4:
852
- raise ValueError(f"source.data is {len(raw)} bytes; shape {shape} needs {count * 4}")
853
- return np.frombuffer(raw, dtype="<f4").reshape(shape).astype(np.float32)
854
-
855
-
856
- def encode_array(array):
857
- """The inverse. C order and float32 are stated in the payload, not assumed:
858
- the JS side reads the shape back and refuses a result it did not expect."""
859
- contiguous = np.ascontiguousarray(array, dtype="<f4")
860
- return {
861
- "shape": [int(dim) for dim in contiguous.shape],
862
- "dtype": "float32",
863
- "data": base64.b64encode(contiguous.tobytes(order="C")).decode("ascii"),
864
- }
865
-
866
-
867
- def serve(args) -> int:
868
- """Load the model once, then answer one NDJSON request per stdin line.
869
-
870
- The loop is deliberately SEQUENTIAL — one request in flight, no threads. A
871
- single 3070 samples one edit at a time anyway, and the wrapper queues on its
872
- side, so concurrency here would buy nothing and cost the guarantee that a
873
- response line always belongs to the request that preceded it.
874
-
875
- EOF ON STDIN IS THE SHUTDOWN SIGNAL. When the wrapper's ssh child dies (or
876
- is killed on process exit) sshd closes this end and `readline` returns "",
877
- which ends the loop and drops the checkpoint. That is why the resident needs
878
- no supervisor on the box and leaves nothing behind.
879
- """
880
- # Claim the real stdout BEFORE anything can print to it, and point fd 1 at
881
- # stderr so that model loading, tqdm and any native extension write where the
882
- # wrapper treats output as status. See the module docstring.
883
- protocol_fd = os.dup(1)
884
- os.dup2(2, 1)
885
- protocol_out = os.fdopen(protocol_fd, "w", encoding="utf-8", newline="\n")
886
-
887
- def emit(message):
888
- protocol_out.write(json.dumps(message, separators=(",", ":")) + "\n")
889
- protocol_out.flush()
890
-
891
- repo = open_repo(args.repo)
892
- engine = Engine(repo, model_id=args.model_id, model_name=args.model_name, device=args.device)
893
- emit({
894
- "type": "ready",
895
- "protocol": PROTOCOL_VERSION,
896
- "device": str(engine.device),
897
- "loadSeconds": round(engine.load_seconds, 3),
898
- "repo": str(repo),
899
- "pid": os.getpid(),
900
- })
901
- print(f"projflow-serve: ready in {engine.load_seconds:.2f}s on {engine.device}", file=sys.stderr, flush=True)
902
-
903
- served = 0
904
- while True:
905
- text = sys.stdin.readline()
906
- if text == "":
907
- break
908
- text = text.strip()
909
- if not text:
910
- continue
911
- try:
912
- request = json.loads(text)
913
- if not isinstance(request, dict):
914
- raise ValueError("a request must be a JSON object")
915
- except Exception as error:
916
- # No id means no way to pair this with anything. Answering with a null
917
- # id is the honest move: the wrapper reads it as a protocol failure,
918
- # falls back to the cold path and restarts this process.
919
- emit({"id": None, "ok": False, "error": {"type": type(error).__name__, "message": str(error)}})
920
- continue
921
-
922
- request_id = request.get("id")
923
- kind = request.get("type") or "lineEdit"
924
- try:
925
- if kind == "ping":
926
- result = {
927
- "pong": True,
928
- "protocol": PROTOCOL_VERSION,
929
- "device": str(engine.device),
930
- "served": served,
931
- "loadSeconds": round(engine.load_seconds, 3),
932
- }
933
- elif kind == "shutdown":
934
- emit({"id": request_id, "ok": True, "result": {"bye": True}})
935
- break
936
- elif kind == "lineEdit":
937
- source = decode_array(request.get("source"))
938
- line = request.get("line")
939
- if not isinstance(line, dict):
940
- raise ValueError("line must be a C6 line-edit object")
941
- preview = bool(request.get("preview"))
942
- steps = 20 if preview else int(request.get("steps", 100))
943
- out_world, meta = run_line_edit(
944
- engine,
945
- source,
946
- line,
947
- steps=steps,
948
- ridge=float(request.get("ridge", 1e-6)),
949
- preserve_stride=int(request.get("preserveStride", 2)),
950
- preserve_margin=int(request.get("preserveMargin", 20)),
951
- seam_pin=int(request.get("seamPin", 2)),
952
- seed=int(request.get("seed", 0)),
953
- cfg=float(request.get("cfg", 3.0)),
954
- preview=preview,
955
- reset_peak=True,
956
- )
957
- served += 1
958
- result = dict(encode_array(out_world))
959
- result["meta"] = meta
960
- else:
961
- raise ValueError(f"unknown request type {kind!r}")
962
- except Exception as error: # noqa: BLE001 — a bad request must not end the session
963
- # The traceback goes to stderr (where the wrapper forwards it as
964
- # status) and the message goes back on the wire. The loop survives:
965
- # one malformed line edit is not a reason to pay another model load.
966
- traceback.print_exc(file=sys.stderr)
967
- sys.stderr.flush()
968
- emit({"id": request_id, "ok": False, "error": {"type": type(error).__name__, "message": str(error)}})
969
- continue
970
- emit({"id": request_id, "ok": True, "result": result})
971
-
972
- print(f"projflow-serve: stdin closed after {served} edits; exiting", file=sys.stderr, flush=True)
973
- return 0
974
-
975
-
976
- def main() -> int:
977
- ap = argparse.ArgumentParser("ProjFlow line-edit driver")
978
- ap.add_argument("--serve", action="store_true",
979
- help="resident mode: load the model once and answer NDJSON requests on stdin/stdout")
980
- ap.add_argument("--source", help="source motion .npy, (T,22,3) float32 world metres")
981
- ap.add_argument("--line", help="line-edit request .json (contract C6)")
982
- ap.add_argument("--out", help="output .npy; metadata goes to <out stem>.meta.json")
983
- ap.add_argument("--steps", type=int, default=100, help="ODE steps (S1: ~7.3 ms/step at 100 frames)")
984
- ap.add_argument("--ridge", type=float, default=1e-6,
985
- help="ridge_lambda. S1: THIS is the exactness knob, not steps — 1e-3 lands at "
986
- "2.6e-5 while 1e-6 gives 2.4e-7 at half the steps")
987
- ap.add_argument("--preview", action="store_true",
988
- help="20 steps instead of --steps (S1 measured 0.145 s for a 100-frame preview)")
989
- ap.add_argument("--preserve-stride", type=int, default=2, help="pin every Nth frame outside the edit range")
990
- ap.add_argument("--preserve-margin", type=int, default=20,
991
- help="frames either side of the edit range where the edited limb is left free (20 = 1 s)")
992
- ap.add_argument("--seam-pin", type=int, default=2,
993
- help="pin the edit range's own outermost N frames to the source, whole body "
994
- "(0 = off; the app guarantees the curve's ends sit on the source trail)")
995
- ap.add_argument("--seed", type=int, default=0, help="matches demo/run.py's default")
996
- ap.add_argument("--cfg", type=float, default=3.0, help="classifier-free guidance scale")
997
- ap.add_argument("--repo", default=os.environ.get("CCLAY_PROJFLOW_REPO", "/home/yun/projflow-scout/repo"))
998
- ap.add_argument("--model-id", default="ACMDM-Raw-Flow-S-PatchSize22")
999
- ap.add_argument("--model-name", default="ACMDM_Raw_Flow_S_PatchSize22")
1000
- ap.add_argument("--device", default="auto")
1001
- args = ap.parse_args()
1002
-
1003
- if args.serve:
1004
- # The resident takes its whole request on the wire, so the one-shot file
1005
- # flags are meaningless here and passing them is a caller bug worth
1006
- # naming rather than ignoring.
1007
- for flag in ("source", "line", "out"):
1008
- if getattr(args, flag):
1009
- raise SystemExit(f"--serve takes its requests on stdin; --{flag} does not apply")
1010
- return serve(args)
1011
-
1012
- for flag in ("source", "line", "out"):
1013
- if not getattr(args, flag):
1014
- raise SystemExit(f"--{flag} is required (or use --serve)")
1015
-
1016
- # Resolve every caller-supplied path BEFORE the chdir below, or a relative
1017
- # --source silently resolves against the ProjFlow clone instead of the
1018
- # caller's cwd. generate.mjs always passes absolute paths; a human running
1019
- # this by hand should not have to.
1020
- args.source = str(Path(args.source).expanduser().resolve())
1021
- args.line = str(Path(args.line).expanduser().resolve())
1022
- args.out = str(Path(args.out).expanduser().absolute())
1023
-
1024
- repo = open_repo(args.repo)
1025
-
1026
- source = np.load(args.source).astype(np.float32)
1027
- if source.ndim != 3 or source.shape[1] != NUM_JOINTS or source.shape[2] != 3:
1028
- raise SystemExit(f"--source must be (T,22,3); got {source.shape}")
1029
-
1030
- with open(args.line, "r", encoding="utf-8") as handle:
1031
- line = json.load(handle)
1032
-
1033
- steps = 20 if args.preview else int(args.steps)
1034
- engine = Engine(repo, model_id=args.model_id, model_name=args.model_name, device=args.device)
1035
- out_world, meta = run_line_edit(
1036
- engine,
1037
- source,
1038
- line,
1039
- steps=steps,
1040
- ridge=float(args.ridge),
1041
- preserve_stride=int(args.preserve_stride),
1042
- preserve_margin=int(args.preserve_margin),
1043
- seam_pin=int(args.seam_pin),
1044
- seed=int(args.seed),
1045
- cfg=float(args.cfg),
1046
- preview=bool(args.preview),
1047
- )
1048
-
1049
- out_path = Path(args.out).expanduser()
1050
- out_path.parent.mkdir(parents=True, exist_ok=True)
1051
- np.save(str(out_path), out_world)
1052
-
1053
- # `<stem>.meta.json` beside the motion: --out foo.npy => foo.meta.json. The
1054
- # wrapper builds the same name on the JS side, so this spelling is a contract.
1055
- stem = str(out_path)[:-4] if str(out_path).endswith(".npy") else str(out_path)
1056
- meta_path = Path(stem + ".meta.json")
1057
- with open(meta_path, "w", encoding="utf-8") as handle:
1058
- json.dump(meta, handle, indent=2)
1059
-
1060
- # The wrapper greps this line, the same way bridge.mjs greps Kimodo's.
1061
- print(
1062
- f"projflow-line-edit: done - {out_path} "
1063
- f"(m={meta['m']} steps={meta['steps']} sample={meta['sampling_seconds']:.3f}s)"
1064
- )
1065
- return 0
1066
-
1067
-
1068
- if __name__ == "__main__":
1069
- raise SystemExit(main())