cozyclay 2.1.1 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (577) hide show
  1. package/CHANGELOG.md +60 -0
  2. package/README.md +12 -30
  3. package/bin/agent/agent-routes.mjs +40 -113
  4. package/bin/agent/agent-runner.mjs +9 -10
  5. package/bin/agent/cliproxy-provider.mjs +135 -0
  6. package/bin/agent/credential-store.mjs +30 -7
  7. package/bin/agent/pi-tools.mjs +4 -1
  8. package/bin/agent/provider-keys.mjs +32 -0
  9. package/bin/agent/provider-login.mjs +110 -0
  10. package/bin/agent/provider-routes.mjs +99 -0
  11. package/bin/agent/providers.mjs +19 -97
  12. package/bin/agent/script-tool.mjs +17 -4
  13. package/bin/agent/studio-motion-reference.mjs +23 -0
  14. package/bin/agent/studio-prompt.mjs +1 -1
  15. package/bin/agent/studio-rebase-text.mjs +32 -0
  16. package/bin/agent/studio-rebase.mjs +26 -39
  17. package/bin/agent/studio-tools.mjs +9 -28
  18. package/bin/agent/tool-arguments.mjs +49 -0
  19. package/bin/agent/turn-frames.mjs +66 -0
  20. package/bin/codex-auth.mjs +9 -2
  21. package/bin/cozyclay.mjs +10 -116
  22. package/bin/live/cli.mjs +3 -3
  23. package/bin/mcp-runtime.mjs +2 -12
  24. package/bin/package-signature.mjs +0 -4
  25. package/dist/app/index.html +7 -6
  26. package/dist/assets/EXRLoader-CvVIvGVE.js +1 -0
  27. package/dist/assets/analytics-C9sUnndd.js +1 -0
  28. package/dist/assets/app-CCh--1Hc.js +5087 -0
  29. package/dist/assets/app-f-fkV5uP.css +1 -0
  30. package/dist/assets/compile-worker-CkxiD6lH.js +5 -0
  31. package/dist/assets/fflate.module-KbGj5-H6.js +5 -0
  32. package/dist/assets/{first-shot-handoff-msLI71g1.js → first-shot-handoff-CC4xe4HM.js} +1 -1
  33. package/dist/assets/{landing-ChUvB_zL.js → landing-DjHD8u7k.js} +1 -1
  34. package/dist/assets/scene-asset-sync-Hx_FzdOH.js +20 -0
  35. package/dist/assets/scene-asset-sync-_S8NQl5B.css +1 -0
  36. package/dist/assets/workflow-C4yKH1p4.js +179 -0
  37. package/dist/cozyclay-package.json +1 -1
  38. package/dist/index.html +3 -3
  39. package/dist/sw.js +3 -6
  40. package/dist/workflow/index.html +5 -4
  41. package/mcp/LIVE-PROTOCOL.md +7 -47
  42. package/mcp/README.md +16 -26
  43. package/mcp/live-hub.mjs +6 -10
  44. package/mcp/package.json +1 -3
  45. package/mcp/tool-handlers.mjs +4 -168
  46. package/package.json +3 -22
  47. package/src/App.jsx +150 -2624
  48. package/src/agent-progress.js +79 -0
  49. package/src/agent-turn-mark.js +30 -0
  50. package/src/analytics.js +1 -19
  51. package/src/app-context.js +5 -0
  52. package/src/app-stage.jsx +2 -34
  53. package/src/asset-pane.jsx +6 -5
  54. package/src/camera-block.js +21 -0
  55. package/src/camera-direction.js +120 -0
  56. package/src/character-kind.js +3 -9
  57. package/src/command-bus.js +8 -6
  58. package/src/commands/cast.js +5 -34
  59. package/src/commands/elements/character.js +1 -1
  60. package/src/commands/elements/motion.js +1 -6
  61. package/src/commands/elements/object.js +6 -2
  62. package/src/commands/elements/shot.js +4 -1
  63. package/src/commands/elements.js +38 -14
  64. package/src/commands/index.js +3 -1
  65. package/src/commands/kits.js +37 -0
  66. package/src/commands/motion-timing.js +57 -0
  67. package/src/commands/motion.js +9 -152
  68. package/src/commands/motion3-compile.js +65 -0
  69. package/src/commands/motion3.js +207 -0
  70. package/src/commands/objects.js +6 -4
  71. package/src/commands/shot.js +49 -11
  72. package/src/commands/stage.js +14 -1
  73. package/src/commands/view.js +4 -1
  74. package/src/document-mirrors.js +42 -0
  75. package/src/document-store.js +4 -0
  76. package/src/domains/cast.js +19 -194
  77. package/src/domains/motion/motion-range-pin-rules.js +1 -1
  78. package/src/domains/motion/take-source.js +51 -0
  79. package/src/domains/motion.js +60 -1838
  80. package/src/domains/scenes.js +12 -55
  81. package/src/domains/shots.js +2 -2
  82. package/src/domains/stage.js +4 -2
  83. package/src/draft-ghost.js +46 -0
  84. package/src/draft-ghost.jsx +129 -0
  85. package/src/dualview.jsx +4 -3
  86. package/src/effects/EffectsLayer.jsx +112 -0
  87. package/src/effects/effect-meshes.js +232 -0
  88. package/src/effects/effect-record.js +103 -0
  89. package/src/effects/emitter.js +59 -0
  90. package/src/effects/simulate.js +261 -0
  91. package/src/effects/state.js +59 -0
  92. package/src/execution-telemetry.js +2 -3
  93. package/src/first-success-guide.jsx +5 -5
  94. package/src/ghost-set.js +257 -0
  95. package/src/held-object-edit.jsx +113 -0
  96. package/src/held-object-pose.js +175 -0
  97. package/src/hierarchy-model.js +1 -1
  98. package/src/hierarchy-panel.jsx +1 -1
  99. package/src/imported-mesh.jsx +103 -0
  100. package/src/kits/held.js +84 -0
  101. package/src/kits/index.js +70 -0
  102. package/src/kits/smoking.js +57 -0
  103. package/src/light-blockers.js +93 -0
  104. package/src/motion/beat-board-model.js +84 -0
  105. package/src/motion/beat-board.css +124 -0
  106. package/src/motion/beat-board.jsx +59 -0
  107. package/src/motion/center-of-mass.js +66 -0
  108. package/src/motion/fix-collisions.js +850 -0
  109. package/src/motion/physics-review.js +118 -0
  110. package/src/{ardy → motion}/physics-support.js +1 -76
  111. package/src/motion/rig-snapshot.js +22 -0
  112. package/src/motion/take-segments.jsx +140 -0
  113. package/src/{ardy → motion}/timeline-coordinates.js +0 -21
  114. package/src/{ardy → motion}/timeline.css +1 -2
  115. package/src/{ardy → motion}/timeline.jsx +42 -413
  116. package/src/motion/waypoints.js +76 -0
  117. package/src/motion3/beats.js +83 -0
  118. package/src/motion3/body.js +158 -0
  119. package/src/motion3/bones.js +47 -0
  120. package/src/motion3/carried.js +47 -0
  121. package/src/motion3/compile-job.js +75 -0
  122. package/src/motion3/compile-worker.js +10 -0
  123. package/src/motion3/compiler/anchors.js +105 -0
  124. package/src/motion3/compiler/arm-orient.js +128 -0
  125. package/src/motion3/compiler/channels.js +125 -0
  126. package/src/motion3/compiler/compile.js +205 -0
  127. package/src/motion3/compiler/gait-rhythm.js +97 -0
  128. package/src/motion3/compiler/grip.js +171 -0
  129. package/src/motion3/compiler/head-steady.js +48 -0
  130. package/src/motion3/compiler/hold.js +134 -0
  131. package/src/motion3/compiler/idle.js +113 -0
  132. package/src/motion3/compiler/limbs.js +170 -0
  133. package/src/motion3/compiler/math.js +72 -0
  134. package/src/motion3/compiler/pose.js +162 -0
  135. package/src/motion3/compiler/quat.js +49 -0
  136. package/src/motion3/compiler/skeleton.js +74 -0
  137. package/src/motion3/compiler/validate-common.js +32 -0
  138. package/src/motion3/compiler/validate-goals.js +118 -0
  139. package/src/motion3/compiler/validate-objects.js +143 -0
  140. package/src/motion3/compiler/validate-parts.js +116 -0
  141. package/src/motion3/compiler/validate.js +122 -0
  142. package/src/motion3/compiler/vocab.js +73 -0
  143. package/src/motion3/compiler/walk-pelvis.js +275 -0
  144. package/src/motion3/compiler/walk.js +207 -0
  145. package/src/motion3/compiler/wrist-clamp.js +32 -0
  146. package/src/motion3/delta.js +47 -0
  147. package/src/motion3/describe.js +174 -0
  148. package/src/motion3/docs.js +27 -0
  149. package/src/motion3/edit-carry.js +21 -0
  150. package/src/motion3/fix/ease.js +16 -0
  151. package/src/motion3/fix/ground.js +37 -0
  152. package/src/motion3/fix/index.js +52 -0
  153. package/src/motion3/fix/push.js +71 -0
  154. package/src/motion3/fix/reach.js +46 -0
  155. package/src/motion3/fix/retime.js +79 -0
  156. package/src/motion3/fix/self.js +211 -0
  157. package/src/motion3/fix/slide.js +70 -0
  158. package/src/motion3/fix/turn.js +51 -0
  159. package/src/motion3/follow.js +39 -0
  160. package/src/motion3/free-motion.js +173 -0
  161. package/src/motion3/grip-choice.js +130 -0
  162. package/src/motion3/hold-plan.js +157 -0
  163. package/src/motion3/json-patch.js +77 -0
  164. package/src/motion3/key-aim.js +105 -0
  165. package/src/motion3/key-edit.js +82 -0
  166. package/src/motion3/lenient.js +127 -0
  167. package/src/motion3/object-keys.js +236 -0
  168. package/src/motion3/object-pose.js +182 -0
  169. package/src/motion3/object-tracks.js +12 -0
  170. package/src/motion3/pipeline.js +102 -0
  171. package/src/motion3/playback.js +51 -0
  172. package/src/motion3/probes.js +91 -0
  173. package/src/motion3/qa/body-checks.js +34 -0
  174. package/src/motion3/qa/check-names.js +14 -0
  175. package/src/motion3/qa/checks.js +121 -0
  176. package/src/motion3/qa/contact-checks.js +51 -0
  177. package/src/motion3/qa/explain.js +80 -0
  178. package/src/motion3/qa/fmt.js +5 -0
  179. package/src/motion3/qa/foot-checks.js +72 -0
  180. package/src/motion3/qa/geometry.js +77 -0
  181. package/src/motion3/qa/goals.js +191 -0
  182. package/src/motion3/qa/ground.js +60 -0
  183. package/src/motion3/qa/hints-contact.js +179 -0
  184. package/src/motion3/qa/hints-goal.js +160 -0
  185. package/src/motion3/qa/hints-knee.js +24 -0
  186. package/src/motion3/qa/hints-motion.js +209 -0
  187. package/src/motion3/qa/hints-object.js +91 -0
  188. package/src/motion3/qa/index.js +42 -0
  189. package/src/motion3/qa/key-checks.js +82 -0
  190. package/src/motion3/qa/near.js +96 -0
  191. package/src/motion3/qa/object-checks.js +135 -0
  192. package/src/motion3/qa/patch.js +161 -0
  193. package/src/motion3/qa/severity.js +38 -0
  194. package/src/motion3/qa/spec-checks.js +65 -0
  195. package/src/motion3/qa/style.js +218 -0
  196. package/src/motion3/qa/wrist-checks.js +69 -0
  197. package/src/motion3/qa/wrist-obs.js +60 -0
  198. package/src/motion3/refs.js +120 -0
  199. package/src/motion3/repair-tune.js +121 -0
  200. package/src/motion3/repair.js +83 -0
  201. package/src/motion3/report.js +67 -0
  202. package/src/motion3/rigs/index.js +14 -0
  203. package/src/motion3/rigs/x-bot-tpose-body.json +1 -0
  204. package/src/motion3/rigs/x-bot-tpose.json +1 -0
  205. package/src/motion3/rigs/y-bot-tpose-body.json +1 -0
  206. package/src/motion3/rigs/y-bot-tpose.json +1 -0
  207. package/src/motion3/scene.js +60 -0
  208. package/src/motion3/schema/brief.js +89 -0
  209. package/src/motion3/schema/fields.js +44 -0
  210. package/src/motion3/schema/spec-examples.js +46 -0
  211. package/src/motion3/schema/topics.js +111 -0
  212. package/src/motion3/schema/types.js +147 -0
  213. package/src/motion3/search.js +117 -0
  214. package/src/motion3/second-hand.js +112 -0
  215. package/src/motion3/skills/basic.js +82 -0
  216. package/src/motion3/skills/book.js +76 -0
  217. package/src/motion3/skills/car.js +145 -0
  218. package/src/motion3/skills/frames-auto.js +29 -0
  219. package/src/motion3/skills/index.js +83 -0
  220. package/src/motion3/skills/info.js +65 -0
  221. package/src/motion3/skills/smoke-kit.js +109 -0
  222. package/src/motion3/skills/smoke.js +160 -0
  223. package/src/motion3/spec-store.js +59 -0
  224. package/src/motion3/take-edit.js +155 -0
  225. package/src/motion3/take.js +63 -0
  226. package/src/motion3/timeline.js +127 -0
  227. package/src/motion3/views.js +121 -0
  228. package/src/motion3/walk-plan.js +119 -0
  229. package/src/object-catalog.jsx +7 -1
  230. package/src/object-colors.js +98 -0
  231. package/src/object-gizmo-parts.js +115 -0
  232. package/src/object-gizmo.jsx +37 -122
  233. package/src/object-glow.js +139 -0
  234. package/src/object-hidden.js +33 -0
  235. package/src/object-look.js +196 -0
  236. package/src/object-size.js +10 -0
  237. package/src/object-travel.js +71 -6
  238. package/src/panels/CameraPanel.jsx +7 -7
  239. package/src/panels/HdrEnvironmentControls.jsx +59 -0
  240. package/src/panels/LightPanel.jsx +25 -4
  241. package/src/panels/ObjectLookPanel.jsx +88 -0
  242. package/src/panels/PosePanel.jsx +2 -17
  243. package/src/panels/RigControlPanel.jsx +5 -53
  244. package/src/panels/SubjectBox.jsx +4 -1
  245. package/src/panels/details.css +14 -0
  246. package/src/posestudio.jsx +4 -27
  247. package/src/project-browser.jsx +3 -4
  248. package/src/project-resources.js +18 -31
  249. package/src/project.js +13 -104
  250. package/src/props.jsx +25 -280
  251. package/src/rail-dot-drag.js +23 -0
  252. package/src/render-passes.js +10 -0
  253. package/src/resource-status.jsx +1 -0
  254. package/src/result-modal.jsx +4 -19
  255. package/src/room.jsx +20 -12
  256. package/src/scene-assets.js +8 -6
  257. package/src/scene-light-place.js +72 -0
  258. package/src/scene-lights.jsx +148 -0
  259. package/src/scene-objects.js +31 -139
  260. package/src/scenes.js +15 -105
  261. package/src/semantic-edit.js +2 -5
  262. package/src/set-pieces.jsx +183 -0
  263. package/src/settings-menu.jsx +3 -11
  264. package/src/shell/BottomDock.jsx +9 -51
  265. package/src/shell/DetailsSlot.jsx +16 -200
  266. package/src/shell/GhostsMenuGroup.jsx +74 -0
  267. package/src/shell/LibrarySlot.jsx +1 -3
  268. package/src/shell/MenuBar.jsx +3 -3
  269. package/src/shell/PreferencesDialog.jsx +15 -264
  270. package/src/shell/PreferencesSlot.jsx +2 -8
  271. package/src/shell/TopBar.jsx +5 -84
  272. package/src/shell/ViewportToolbar.jsx +10 -70
  273. package/src/shell/agent-glass.css +19 -6
  274. package/src/shell/details-selection.js +42 -0
  275. package/src/shell/ghosts-menu.css +22 -0
  276. package/src/shell/glass-regions.css +29 -3
  277. package/src/shell/preferences.css +6 -1
  278. package/src/shot-authoring.js +4 -7
  279. package/src/soft-shadows.js +37 -0
  280. package/src/stage-attach-frames.js +12 -12
  281. package/src/stage-canvas-rigs.jsx +3 -3
  282. package/src/stage-capture.jsx +42 -8
  283. package/src/stage-characters.jsx +22 -216
  284. package/src/stage-editor-gizmos.jsx +6 -6
  285. package/src/stage-environment.js +93 -0
  286. package/src/stage-hdr-builtins.js +107 -0
  287. package/src/stage-hdr-image.js +116 -0
  288. package/src/stage-hdr-library.js +70 -0
  289. package/src/stage-hdr-map.js +94 -0
  290. package/src/stage-hdr.js +130 -0
  291. package/src/stage-lighting.js +149 -0
  292. package/src/stage-occlusion.js +210 -0
  293. package/src/stage-sky.jsx +159 -0
  294. package/src/stage-startup-defaults.js +5 -9
  295. package/src/studio-actions.js +12 -14
  296. package/src/studio-agent-commands.js +19 -16
  297. package/src/studio-agent-motion.js +9 -9
  298. package/src/studio-agent-protocol.js +41 -71
  299. package/src/studio-app-binding.js +25 -45
  300. package/src/studio-asset-list.js +17 -0
  301. package/src/studio-contact-sheet.js +37 -5
  302. package/src/studio-elements.js +27 -7
  303. package/src/studio-entity-bounds.js +18 -0
  304. package/src/studio-error-codes.js +8 -0
  305. package/src/studio-export-framing.js +152 -0
  306. package/src/studio-lighting-check.js +103 -0
  307. package/src/studio-motion-visual.js +150 -0
  308. package/src/studio-schema-errors.js +80 -0
  309. package/src/studio-support-gap.js +74 -0
  310. package/src/styles.css +1 -19
  311. package/src/timeline-defaults.js +5 -0
  312. package/src/timeline-extent.js +8 -20
  313. package/src/tutorial-walk.js +17 -0
  314. package/src/ui-scrub.js +1 -1
  315. package/src/use-shot-camera-tracks.js +12 -0
  316. package/src/view-ghosts.js +97 -0
  317. package/src/view-ghosts.jsx +49 -0
  318. package/src/workflow/AgentPanel.jsx +47 -166
  319. package/src/workflow/AgentProgressCard.jsx +107 -0
  320. package/src/workflow/AgentReplyText.jsx +37 -0
  321. package/src/workflow/AgentTurnSteps.jsx +84 -0
  322. package/src/workflow/LoginPanel.jsx +140 -0
  323. package/src/workflow/WorkflowBuilder.jsx +7 -23
  324. package/src/workflow/agent-client.js +54 -51
  325. package/src/workflow/agent-panel.css +71 -1
  326. package/src/workflow/agent-progress-card.css +167 -0
  327. package/src/workflow/agent-progress-view.js +47 -0
  328. package/src/workflow/agent-turn-steps.css +113 -0
  329. package/src/workflow/agent-turn.js +191 -0
  330. package/src/workflow/login-client.js +34 -0
  331. package/src/workflow/motion-input.js +18 -31
  332. package/src/workflow/node-schema.js +1 -1
  333. package/src/workflow/reply-markdown.js +69 -0
  334. package/src/workflow/scene-asset-sync.js +8 -8
  335. package/tools/agent-session-stats.mjs +61 -0
  336. package/tools/dev-full.mjs +5 -56
  337. package/tools/eslint-baseline.json +12 -13
  338. package/tools/knip-baseline.json +0 -179
  339. package/tools/loc-baseline.json +31 -81
  340. package/tools/motion3/extract-body.mjs +103 -0
  341. package/tools/motion3/extract-rig.mjs +85 -0
  342. package/tools/process-supervisor.mjs +0 -88
  343. package/tools/run-tests.mjs +8 -124
  344. package/bin/agent/motion-runtime.mjs +0 -76
  345. package/dist/ardy/cskel27-rest.json +0 -557
  346. package/dist/assets/analytics-C9pMpW7s.js +0 -1
  347. package/dist/assets/app-B_2XB9rw.css +0 -1
  348. package/dist/assets/app-DAuvKPP8.js +0 -4920
  349. package/dist/assets/config-Bx7pYsD9.js +0 -1
  350. package/dist/assets/config-DeoFTYzb.css +0 -1
  351. package/dist/assets/demo-CvKuBgy5.js +0 -1
  352. package/dist/assets/scene-asset-sync-DKkpjKBV.js +0 -19
  353. package/dist/assets/scene-asset-sync-DjJ_cJ2R.css +0 -1
  354. package/dist/assets/ticket-BPk2Ni95.js +0 -1
  355. package/dist/assets/workflow-jJRBR94Z.js +0 -179
  356. package/dist/demo/walk-then-stop.npz +0 -0
  357. package/mcp/ardy-prompts.mjs +0 -218
  358. package/src/ardy/auto-fix-panel.css +0 -108
  359. package/src/ardy/auto-physics.js +0 -1194
  360. package/src/ardy/client.js +0 -153
  361. package/src/ardy/convert.js +0 -313
  362. package/src/ardy/cskel27-neutral.js +0 -39
  363. package/src/ardy/cskel27.js +0 -77
  364. package/src/ardy/export.js +0 -165
  365. package/src/ardy/fix-collisions.js +0 -2257
  366. package/src/ardy/foot-lock.js +0 -230
  367. package/src/ardy/motion-calibration.js +0 -118
  368. package/src/ardy/motion-edit.js +0 -213
  369. package/src/ardy/motion-url.js +0 -55
  370. package/src/ardy/mp4-duration.js +0 -102
  371. package/src/ardy/npz.js +0 -595
  372. package/src/ardy/physics-panel.css +0 -21
  373. package/src/ardy/physics-panel.jsx +0 -89
  374. package/src/ardy/physics-refine.js +0 -111
  375. package/src/ardy/physics-review.js +0 -567
  376. package/src/ardy/physics-temporal.js +0 -54
  377. package/src/ardy/platform-fit-panel.css +0 -64
  378. package/src/ardy/platform-fit-panel.jsx +0 -41
  379. package/src/ardy/platform-fit.js +0 -326
  380. package/src/ardy/playback.js +0 -711
  381. package/src/ardy/pose-pin.js +0 -97
  382. package/src/ardy/prompt-clips.js +0 -30
  383. package/src/ardy/retime.js +0 -211
  384. package/src/ardy/root-drop.js +0 -369
  385. package/src/ardy/timeline-resize.js +0 -13
  386. package/src/ardy/to-cskel27.js +0 -225
  387. package/src/ardy/trim.js +0 -33
  388. package/src/ardy/waypoints.js +0 -369
  389. package/src/ardy-bridge-contract.js +0 -281
  390. package/src/domains/motion/motion-video-capture.js +0 -664
  391. package/src/i2v-motion-client.js +0 -77
  392. package/src/i2v-motion-studio.jsx +0 -180
  393. package/src/line-edit.js +0 -1348
  394. package/src/motion/generation.js +0 -105
  395. package/src/motion-readiness-ui.jsx +0 -67
  396. package/src/motion-readiness.js +0 -40
  397. package/src/motion-resources.js +0 -177
  398. package/src/motion-store.js +0 -65
  399. package/src/motion-trail.js +0 -601
  400. package/src/multimodel-ingest.js +0 -394
  401. package/src/panels/PromptBlocksPanel.jsx +0 -444
  402. package/src/panels/VideoCapturePanel.jsx +0 -181
  403. package/src/pose-extract/browser.js +0 -42
  404. package/src/pose-extract/detector.js +0 -100
  405. package/src/pose-extract/fit.js +0 -227
  406. package/src/pose-extract/image-frame.js +0 -51
  407. package/src/pose-extract/index.js +0 -58
  408. package/src/pose-extract/keyframes.js +0 -74
  409. package/src/pose-extract/landmarks.js +0 -162
  410. package/src/pose-extract/mirror.js +0 -153
  411. package/src/pose-extract/take.js +0 -131
  412. package/src/pose-extract/video-frames.js +0 -91
  413. package/src/take-recipe.js +0 -165
  414. package/src/trail-key-conflicts.js +0 -39
  415. package/src/trail-pick.js +0 -59
  416. package/src/use-i2v-capture-modal.js +0 -15
  417. package/tools/ardy/BRIDGE.md +0 -37
  418. package/tools/ardy/GVHMR-NPZ.md +0 -68
  419. package/tools/ardy/README.md +0 -124
  420. package/tools/ardy/__pycache__/cclay_gvhmr_worker.cpython-313.pyc +0 -0
  421. package/tools/ardy/artifacts.mjs +0 -34
  422. package/tools/ardy/bridge.mjs +0 -1732
  423. package/tools/ardy/bvh-cskel27.mjs +0 -1265
  424. package/tools/ardy/cclay_constrained_generate.py +0 -561
  425. package/tools/ardy/cclay_gvhmr_worker.py +0 -264
  426. package/tools/ardy/cclay_motion_edit.py +0 -510
  427. package/tools/ardy/cclay_sequence_generate.py +0 -705
  428. package/tools/ardy/cclay_worker.py +0 -304
  429. package/tools/ardy/dump-npz.py +0 -205
  430. package/tools/ardy/extract-rest.mjs +0 -299
  431. package/tools/ardy/extract.mjs +0 -507
  432. package/tools/ardy/footage.mjs +0 -467
  433. package/tools/ardy/gvhmr-floor.mjs +0 -102
  434. package/tools/ardy/gvhmr-to-cskel27.mjs +0 -21
  435. package/tools/ardy/gvhmr_fastpath.py +0 -168
  436. package/tools/ardy/gvhmr_trajectory.py +0 -240
  437. package/tools/ardy/merge-text-encoder.py +0 -160
  438. package/tools/ardy/mocap-metrics.mjs +0 -200
  439. package/tools/ardy/mocap-quality-gate.mjs +0 -110
  440. package/tools/ardy/motion-stabilize.mjs +0 -348
  441. package/tools/ardy/npz.mjs +0 -414
  442. package/tools/ardy/pose-to-npz.mjs +0 -106
  443. package/tools/ardy/prompt-limits.mjs +0 -5
  444. package/tools/ardy/run-edit-on-box.sh +0 -73
  445. package/tools/ardy/run-local.mjs +0 -233
  446. package/tools/ardy/run-on-box.sh +0 -593
  447. package/tools/ardy/run-sequence-on-box.sh +0 -177
  448. package/tools/ardy/runners/gvhmr-worker.mjs +0 -206
  449. package/tools/ardy/runners/index.mjs +0 -18
  450. package/tools/ardy/runners/local.mjs +0 -455
  451. package/tools/ardy/runners/proc.mjs +0 -125
  452. package/tools/ardy/runners/remote.mjs +0 -252
  453. package/tools/ardy/setup-local.mjs +0 -189
  454. package/tools/ardy/setup-text-encoder-on-box.sh +0 -93
  455. package/tools/ardy/setup-text-encoder.py +0 -306
  456. package/tools/ardy/smpl-cskel27.mjs +0 -178
  457. package/tools/ardy/visual-qa.mjs +0 -196
  458. package/tools/ardy/vq-car.mjs +0 -52
  459. package/tools/bench/EXP3.md +0 -61
  460. package/tools/bench/cclay_bench_extract_incam.py +0 -125
  461. package/tools/bench/cclay_bench_extract_obs.py +0 -204
  462. package/tools/bench/cclay_bench_runner.py +0 -42
  463. package/tools/bench/cube-contact.mjs +0 -162
  464. package/tools/bench/exp3.mjs +0 -152
  465. package/tools/bench/extract-bench-lib.mjs +0 -216
  466. package/tools/bench/extract-bench.mjs +0 -302
  467. package/tools/bench/fal-generate.mjs +0 -88
  468. package/tools/bench/fit/README.md +0 -189
  469. package/tools/bench/fit/camera.mjs +0 -59
  470. package/tools/bench/fit/contact.mjs +0 -419
  471. package/tools/bench/fit/footlock.mjs +0 -234
  472. package/tools/bench/fit/motion.mjs +0 -85
  473. package/tools/bench/fit/pin.mjs +0 -27
  474. package/tools/bench/fit/remote.mjs +0 -105
  475. package/tools/bench/fit-bench.mjs +0 -126
  476. package/tools/bench/fit-sanity.mjs +0 -45
  477. package/tools/bench/metrics.mjs +0 -313
  478. package/tools/bench/obs/depth.mjs +0 -258
  479. package/tools/bench/obs/extrinsics.mjs +0 -175
  480. package/tools/bench/obs/fit_mannequin_betas.py +0 -340
  481. package/tools/bench/obs/ground.mjs +0 -294
  482. package/tools/bench/obs/heading.mjs +0 -151
  483. package/tools/bench/obs/ladder.mjs +0 -555
  484. package/tools/bench/obs/mannequin-betas.json +0 -124
  485. package/tools/bench/obs/remote.mjs +0 -83
  486. package/tools/bench/obs/rest_joints.py +0 -52
  487. package/tools/bench/obs/ybot-targets.mjs +0 -49
  488. package/tools/bench/obs-bench.mjs +0 -520
  489. package/tools/bench/score.mjs +0 -343
  490. package/tools/bench/summarize.mjs +0 -86
  491. package/tools/demo-worker/README.md +0 -137
  492. package/tools/demo-worker/api-client.mjs +0 -233
  493. package/tools/demo-worker/generate.mjs +0 -301
  494. package/tools/demo-worker/index.mjs +0 -466
  495. package/tools/demo-worker/sign-core.mjs +0 -112
  496. package/tools/demo-worker/sign.mjs +0 -161
  497. package/tools/gt-render/browser.mjs +0 -182
  498. package/tools/gt-render/camera-math.mjs +0 -276
  499. package/tools/gt-render/page.mjs +0 -267
  500. package/tools/gt-render/render.mjs +0 -514
  501. package/tools/gt-render/scene-box.mjs +0 -31
  502. package/tools/gt-render/take-transform.mjs +0 -87
  503. package/tools/kimodo/compare-seam.mjs +0 -108
  504. package/tools/kimodo/constraints.mjs +0 -244
  505. package/tools/kimodo/edit.mjs +0 -259
  506. package/tools/kimodo/effector-constraints.mjs +0 -200
  507. package/tools/kimodo/generate.mjs +0 -646
  508. package/tools/kimodo/local-output.mjs +0 -265
  509. package/tools/kimodo/measure-pose.mjs +0 -65
  510. package/tools/kimodo/measure-preserve.mjs +0 -899
  511. package/tools/kimodo/measure-waypoints.mjs +0 -55
  512. package/tools/kimodo/mlx-generate.py +0 -111
  513. package/tools/kimodo/pose-constraints.mjs +0 -248
  514. package/tools/kimodo/preserve-mask.mjs +0 -543
  515. package/tools/kimodo/read-npz.mjs +0 -189
  516. package/tools/kimodo/run-edit-on-box.mjs +0 -155
  517. package/tools/kimodo/run-sequence-on-box.mjs +0 -178
  518. package/tools/kimodo/runner.mjs +0 -284
  519. package/tools/kimodo/seam-batch.mjs +0 -77
  520. package/tools/kimodo/setup-local.mjs +0 -30
  521. package/tools/kimodo/setup-on-box.sh +0 -339
  522. package/tools/kimodo/soma77-to-cskel27.mjs +0 -305
  523. package/tools/morphgs/assets/gen_truth.py +0 -39
  524. package/tools/morphgs/assets/mesh_ori_rig.txt +0 -27
  525. package/tools/morphgs/assets/playback-check.mjs +0 -30
  526. package/tools/morphgs/demo-gate.sh +0 -26
  527. package/tools/morphgs/fbx2morphgs.mjs +0 -68
  528. package/tools/morphgs/morphgs-to-cskel27.mjs +0 -105
  529. package/tools/morphgs/patches/preprocess_src-none-mode.patch +0 -76
  530. package/tools/morphgs/setup-on-cluster.sh +0 -126
  531. package/tools/projflow/driver.py +0 -1069
  532. package/tools/projflow/generate.mjs +0 -805
  533. package/tools/projflow/hml22-to-cskel27.mjs +0 -735
  534. package/tools/projflow/line-edit-job.mjs +0 -547
  535. package/tools/projflow/replay.mjs +0 -452
  536. package/tools/projflow/runner.mjs +0 -269
  537. package/tools/projflow/service.mjs +0 -742
  538. package/tools/track/DESIGN.md +0 -513
  539. package/tools/track/backfill-provenance.mjs +0 -116
  540. package/tools/track/budget.mjs +0 -8
  541. package/tools/track/check-rig.mjs +0 -209
  542. package/tools/track/diagnostics.schema.json +0 -60
  543. package/tools/track/export-rig.mjs +0 -256
  544. package/tools/track/fallback.mjs +0 -7
  545. package/tools/track/fk-parity-fixture.mjs +0 -161
  546. package/tools/track/gate.mjs +0 -349
  547. package/tools/track/masks.mjs +0 -318
  548. package/tools/track/metrics.mjs +0 -201
  549. package/tools/track/publish-obs.mjs +0 -108
  550. package/tools/track/py/check_env.py +0 -61
  551. package/tools/track/py/eval_lr.py +0 -229
  552. package/tools/track/py/lr_viterbi.py +0 -294
  553. package/tools/track/py/masks.py +0 -405
  554. package/tools/track/py/objective.py +0 -382
  555. package/tools/track/py/rig.py +0 -508
  556. package/tools/track/py/scene.py +0 -326
  557. package/tools/track/py/test_joint_indices.py +0 -123
  558. package/tools/track/py/test_lr_viterbi.py +0 -195
  559. package/tools/track/py/test_masks.py +0 -229
  560. package/tools/track/py/test_rig.py +0 -288
  561. package/tools/track/py/test_scene.py +0 -354
  562. package/tools/track/py/test_track.py +0 -328
  563. package/tools/track/py/track.py +0 -411
  564. package/tools/track/remote.mjs +0 -140
  565. package/tools/track/rig-dump.mjs +0 -227
  566. package/tools/track/run-box-tests.mjs +0 -14
  567. package/tools/track/run-box.mjs +0 -168
  568. package/tools/track/setup-box.sh +0 -59
  569. package/tools/track/study-2d.mjs +0 -728
  570. /package/src/{ardy → motion}/collision-blockers.js +0 -0
  571. /package/src/{ardy → motion}/ground.js +0 -0
  572. /package/src/{ardy → motion}/ik-drag.js +0 -0
  573. /package/src/{ardy → motion}/ik-key-json.js +0 -0
  574. /package/src/{ardy → motion}/ik.js +0 -0
  575. /package/src/{ardy → motion}/physics-surface.js +0 -0
  576. /package/src/{ardy → motion}/playback-clock.js +0 -0
  577. /package/src/{ardy → motion}/range-pin.js +0 -0
@@ -1,1069 +0,0 @@
1
- #!/usr/bin/env python3
2
- """driver.py — ProjFlow line-edit driver, shipped to the GPU box per run.
3
-
4
- CozyClay ships THIS file to the box the same way tools/kimodo/generate.mjs
5
- ships its constraint JSONs: nothing is installed on the box, the scout venv is
6
- the only python, and a run leaves a mkdtemp behind that the wrapper deletes.
7
-
8
- WHY A DRIVER AND NOT `demo/run.py`. Contract C7: the two ProjFlow samplers each
9
- carry exactly half of a CozyClay line edit and neither CLI exposes the other
10
- half.
11
-
12
- * `sample_projflow(A=(B,3,T,J) mask, y=values)` preserves source frames
13
- exactly (S1 measured 4.8e-7 m), but its targets are 3D WORLD points — it has
14
- no camera, so a drawn screen-space line cannot be expressed at all.
15
- * `sample_linear_constraints(operator A=(m, 3*T*J), measurement y=(m,))` takes
16
- a GENERAL dense operator, so a camera row is just another row — but
17
- `demo/run.py` only ever builds projection rows plus a frame-0 keyframe, so
18
- "keep the rest of the take" does not exist on that path.
19
-
20
- `sample_linear_constraints` is the general one and the cheaper one (S1: ~2.7x
21
- faster per step than the projflow inpainting sampler), and demo/run.py ALREADY
22
- stacks two kinds of row into one operator (projection rows for the lifted joint
23
- + identity-ish keyframe rows at frame 0). So preserved frames are nothing more
24
- exotic than more rows: one identity row per kept (frame, joint, axis) whose
25
- measurement is the source coordinate. One operator, one pass, both halves.
26
-
27
- WHY THE ROWS ARE EXACTLY LINEAR UNDER A PERSPECTIVE CAMERA (contract C6, as
28
- amended after Phase 0). The paper's demo only accepts an orthographic camera and
29
- the plan originally carried an orthographic approximation of CozyClay's shot
30
- camera as a known risk. It is not needed. With
31
-
32
- X_cam = R @ x + t (R world->camera, t in camera space)
33
- u = fx * X_cam[0]/X_cam[2] + cx
34
- v = fy * X_cam[1]/X_cam[2] + cy
35
-
36
- the depth division is the only nonlinearity, and multiplying it out cancels it:
37
-
38
- (u - cx) * (R2.x + t2) = fx * (R0.x + t0)
39
- => [fx*R0 - (u-cx)*R2] . x = -[fx*t0 - (u-cx)*t2]
40
- and [fy*R1 - (v-cy)*R2] . x = -[fy*t1 - (v-cy)*t2]
41
-
42
- Two rows per constrained frame, each with exactly THREE nonzero entries (the
43
- x/y/z columns of the target joint at that frame). u and v are KNOWN — they are
44
- what the artist drew — so nothing here depends on the unknown depth. This is
45
- exact at every focal length, which is why gate GP1's "<1 px" now applies to the
46
- close-up case too.
47
-
48
- UNITS. `points2d` and the intrinsics must share ONE unit system; the algebra
49
- above is unit-agnostic as long as they agree. CozyClay sends viewport-normalised
50
- 0..1 for both (fx/fy/cx/cy pre-divided by the viewport size), so the reprojection
51
- errors reported in the metadata are in normalised viewport units.
52
-
53
- THE COST OF A ROW, AND WHY PRESERVE ROWS ARE SUBSAMPLED. The caller precomputes
54
- a Cholesky of `A A^T + lambda I`, which is m x m — the ROW count, not the column
55
- count, is what has to stay bounded. Freezing a 196-frame take densely would be
56
- 196 * 22 * 3 = 12936 rows and a 12936^2 factorisation (670 MB fp32) for
57
- information that is 90% redundant: adjacent frames of a real take are nearly the
58
- same pose. The subsampling policy is documented at `build_preserve_rows`.
59
-
60
- OUTPUT. `<out>.npy` (T,22,3) float32 world metres @ 20 fps, HumanML3D 22-joint
61
- order, plus `<out>.meta.json`. POSITIONS ONLY — ACMDM's "Raw" family generates
62
- absolute coordinates and has no rotation channel. Converting to cskel27 (which
63
- must LIFT rotations) is a separate module and deliberately not done here; see
64
- the seam comment in generate.mjs.
65
-
66
- TWO RUN MODES, ONE SAMPLER (contract C11). `--source/--line/--out` is the
67
- ONE-SHOT mode above: load the model, edit once, exit. It is the contract of
68
- record and nothing below changes a number it produces. `--serve` is the
69
- RESIDENT mode: load the model ONCE and then answer requests forever on
70
- stdin/stdout, because the measured cost of an edit is 3.9 s of model load
71
- against 1.3 s of sampling and paying the load per drag is what makes the detail
72
- loop feel like a batch job. Both modes call exactly the same `Engine` and
73
- `run_line_edit`, so a warm edit and a cold edit differ in nothing but when the
74
- weights were read (`sample_with_linear_constraints` calls
75
- `torch.manual_seed(seed)` itself, so a seeded request is bit-identical either
76
- way).
77
-
78
- THE SERVE PROTOCOL. NDJSON, one request object per stdin line, one response
79
- object per stdout line, arrays as base64 little-endian float32 blobs beside an
80
- explicit shape (a 196-frame source is ~52 KB raw / 69 KB encoded — small enough
81
- that a pipe is the whole transport and there is no port, no tunnel, no file):
82
-
83
- ready (unsolicited, once, after the model is loaded):
84
- {"type":"ready","protocol":1,"device":"cuda:0","loadSeconds":3.9}
85
- request:
86
- {"id":"7","type":"lineEdit","line":{...C6...},
87
- "source":{"shape":[T,22,3],"data":"<base64 float32>"},
88
- "steps":100,"ridge":1e-6,"preserveStride":2,"preserveMargin":20,
89
- "seamPin":2,"seed":0,"cfg":3.0,"preview":false}
90
- {"id":"8","type":"ping"} {"id":"9","type":"shutdown"}
91
- response:
92
- {"id":"7","ok":true,"result":{"shape":[T,22,3],"data":"<base64>",
93
- "meta":{...the one-shot meta, verbatim...}}}
94
- {"id":"7","ok":false,"error":{"message":"...","type":"ValueError"}}
95
-
96
- STDOUT IS THE PROTOCOL AND NOTHING ELSE. torch, CLIP and the repo's own loaders
97
- all print to stdout, and one stray progress line would desynchronise the stream
98
- for the rest of the session. So `--serve` DUPLICATES fd 1 to a private handle
99
- and then points fd 1 at fd 2 before importing anything: every `print`, every
100
- tqdm bar, every C-level write from a native extension lands on stderr (where
101
- the wrapper forwards it as status), and the only writer to the real stdout is
102
- `_write_message`. A request whose JSON cannot even be parsed is answered with
103
- `"id": null`, which the wrapper reads as a protocol failure — it must, because
104
- an unparseable line means the two sides no longer agree about framing.
105
- """
106
-
107
- from __future__ import annotations
108
-
109
- import argparse
110
- import base64
111
- import json
112
- import os
113
- import sys
114
- import time
115
- import traceback
116
- from pathlib import Path
117
-
118
- import numpy as np
119
-
120
- # Bumped when a request or response field changes meaning. The wrapper ships
121
- # THIS file to the box per session, so the two sides cannot drift by more than
122
- # one deploy — the number exists so that a stale resident left over from an
123
- # earlier build is refused loudly instead of answering with the old shape.
124
- PROTOCOL_VERSION = 1
125
-
126
-
127
- # --- the hml22 skeleton ------------------------------------------------------
128
- #
129
- # HumanML3D / T2M 22-joint order, from `utils.motion_process.t2m_kinematic_chain`
130
- # and SMPL_JOINT_NAMES[0:22]. S2's mapping table is the citation; this is the
131
- # subset the driver actually needs.
132
- HML22_PARENT = (
133
- -1, # 0 pelvis
134
- 0, # 1 left_hip
135
- 0, # 2 right_hip
136
- 0, # 3 spine1
137
- 1, # 4 left_knee
138
- 2, # 5 right_knee
139
- 3, # 6 spine2
140
- 4, # 7 left_ankle
141
- 5, # 8 right_ankle
142
- 6, # 9 spine3
143
- 7, # 10 left_foot (TOE BASE, not the foot — S2's naming trap)
144
- 8, # 11 right_foot (toe base)
145
- 9, # 12 neck
146
- 9, # 13 left_collar (the CLAVICLE — cskel27 LeftShoulder)
147
- 9, # 14 right_collar
148
- 12, # 15 head
149
- 13, # 16 left_shoulder (the UPPER-ARM ROOT — cskel27 LeftArm)
150
- 14, # 17 right_shoulder
151
- 16, # 18 left_elbow
152
- 17, # 19 right_elbow
153
- 18, # 20 left_wrist
154
- 19, # 21 right_wrist
155
- )
156
-
157
- NUM_JOINTS = 22
158
-
159
- # The joints where independent limbs branch: the pelvis and the spine3 junction
160
- # that carries the neck and both clavicles. An edit frees its limb UP TO its
161
- # anchor and no further, so the body cannot drift while one arm is redrawn.
162
- LIMB_ANCHORS = frozenset({0, 9})
163
-
164
- # CozyClay IK track id -> hml22 joint index.
165
- #
166
- # THE JS SIDE IS THE AUTHORING SOURCE (generate.mjs exports the same table and
167
- # writes the resolved `jointId` into line.json); this copy is a GUARD, re-checked
168
- # below, so a stale wrapper shipping the wrong joint fails loudly on the box
169
- # instead of silently editing the wrong limb.
170
- #
171
- # Derived from S2's mapping table by composing the two hops it documents:
172
- # track -> cskel27 target (the pose studio's own table, ik.js) -> hml22 source.
173
- # Two of those hops look like typos and are not:
174
- # * `leftElbow` is cskel27 LeftForeArm, whose hml22 source is left_elbow (18).
175
- # A mid-joint track therefore constrains its OWN joint, not its chain's
176
- # effector: the artist dragged the elbow handle and drew the ELBOW's path,
177
- # and pinning joint 20 instead would move the wrist along the drawn line and
178
- # leave the elbow wherever the model liked.
179
- # * `spine` is cskel27 Spine1, whose hml22 source is spine2 (6) — cskel27 has
180
- # three spine links between the anchors where hml22 has two.
181
- TRACK_TO_JOINT = {
182
- # IK effectors
183
- "leftHand": 20, # cskel27 LeftHand <- left_wrist
184
- "rightHand": 21, # cskel27 RightHand <- right_wrist
185
- "leftFoot": 7, # cskel27 LeftFoot <- left_ANKLE (10 is the toe base)
186
- "rightFoot": 8, # cskel27 RightFoot <- right_ankle
187
- # mid-chain handles
188
- "leftElbow": 18, # cskel27 LeftForeArm <- left_elbow
189
- "rightElbow": 19,
190
- "leftKnee": 4, # cskel27 LeftLeg <- left_knee
191
- "rightKnee": 5,
192
- # FK handles
193
- "hips": 0, # cskel27 Hips <- pelvis
194
- "spine": 6, # cskel27 Spine1 <- spine2
195
- "neck": 12,
196
- "head": 15,
197
- "leftShoulder": 13, # cskel27 LeftShoulder is the CLAVICLE <- left_collar
198
- "rightShoulder": 14,
199
- }
200
-
201
- # `chest` is cskel27 Spine2, and Spine2 is one of S2's five FILLED joints: hml22
202
- # has no source for it at all (three cskel27 spine links between Hips and Spine3
203
- # against hml22's two). There is no joint to constrain, so a chest line edit is
204
- # refused by name rather than silently retargeted onto a neighbouring spine
205
- # joint, which would put the drawn line on a curve the artist did not draw.
206
- UNMAPPABLE_TRACKS = {
207
- "chest": "cskel27 Spine2 has no hml22 source joint (S2: a FILLED joint)",
208
- }
209
-
210
-
211
- def limb_chain(joint: int) -> frozenset:
212
- """Joints the edit is allowed to move: `joint`, its descendants, and its
213
- ancestors up to (not including) the nearest limb anchor.
214
-
215
- For joint 20 (left wrist) this is {13, 16, 18, 20} — the whole left arm from
216
- the clavicle out. For joint 0 (pelvis) it is the whole body, which is right:
217
- a hips line edit drags the figure.
218
- """
219
- out = {int(joint)}
220
- # descendants
221
- changed = True
222
- while changed:
223
- changed = False
224
- for child, parent in enumerate(HML22_PARENT):
225
- if parent in out and child not in out:
226
- out.add(child)
227
- changed = True
228
- # ancestors, up to the anchor
229
- node = HML22_PARENT[int(joint)]
230
- while node is not None and node >= 0 and node not in LIMB_ANCHORS:
231
- out.add(node)
232
- node = HML22_PARENT[node]
233
- return frozenset(out)
234
-
235
-
236
- def resample_polyline(points, count):
237
- """Resample an (P,2) polyline to `count` points, arc-index-linearly.
238
-
239
- Deliberately the SAME convention as the repo's own
240
- `demo/lift/trajectory_io._resample_xy`: uniform parameter over the point
241
- INDEX (not arc length), linear interpolation. Matching it means a polyline
242
- that the demo would place at frame t lands on frame t here too, so S1's
243
- measured 2.4e-7 reprojection number transfers.
244
- """
245
- pts = np.asarray(points, dtype=np.float32)
246
- if pts.ndim != 2 or pts.shape[1] != 2:
247
- raise ValueError(f"points2d must be (P,2), got {pts.shape}")
248
- if pts.shape[0] < 2:
249
- raise ValueError("points2d needs at least 2 points")
250
- if pts.shape[0] == int(count):
251
- return pts
252
- t_src = np.linspace(0.0, 1.0, num=pts.shape[0], dtype=np.float32)
253
- t_dst = np.linspace(0.0, 1.0, num=int(count), dtype=np.float32)
254
- out = np.empty((int(count), 2), dtype=np.float32)
255
- out[:, 0] = np.interp(t_dst, t_src, pts[:, 0])
256
- out[:, 1] = np.interp(t_dst, t_src, pts[:, 1])
257
- return out.astype(np.float32)
258
-
259
-
260
- def column_index(axis: int, frame: int, joint: int, *, num_frames: int, num_joints: int) -> int:
261
- """Column of x[axis, frame, joint] in the flattened (3, L, J) operand.
262
-
263
- Copied from `demo/lift/projection_operator.column_index` rather than
264
- imported so this file states the layout it depends on; the two are asserted
265
- equal at startup (see `_assert_column_layout`).
266
- """
267
- return (axis * num_frames + frame) * num_joints + joint
268
-
269
-
270
- def _assert_column_layout(num_frames: int) -> None:
271
- from demo.lift.projection_operator import column_index as repo_column_index
272
-
273
- for axis, frame, joint in ((0, 0, 0), (1, 3, 7), (2, num_frames - 1, NUM_JOINTS - 1)):
274
- mine = column_index(axis, frame, joint, num_frames=num_frames, num_joints=NUM_JOINTS)
275
- theirs = repo_column_index(axis, frame, joint, num_frames=num_frames, num_joints=NUM_JOINTS)
276
- if mine != theirs:
277
- raise RuntimeError(
278
- "column layout drifted from demo/lift/projection_operator.column_index "
279
- f"({mine} != {theirs}); every row this driver builds would be scrambled"
280
- )
281
-
282
-
283
- # --- constraint construction -------------------------------------------------
284
-
285
- def build_line_rows(line, *, frames, joint_id):
286
- """Per-frame affine perspective rows for the drawn polyline.
287
-
288
- Returns (rows, values, frame_indices, uv) where `rows` is a list of
289
- (col, coefficient) triples-per-row. See the module docstring for the algebra.
290
- """
291
- cam = line["camera"]
292
- fx = float(cam["fx"])
293
- fy = float(cam["fy"])
294
- cx = float(cam["cx"])
295
- cy = float(cam["cy"])
296
- R = np.asarray(cam["R"], dtype=np.float64)
297
- t = np.asarray(cam["t"], dtype=np.float64).reshape(3)
298
- if R.shape != (3, 3):
299
- raise ValueError(f"camera.R must be 3x3, got {R.shape}")
300
- if fx == 0.0 or fy == 0.0:
301
- raise ValueError("camera.fx and camera.fy must be non-zero")
302
-
303
- start = int(line["frameRange"]["start"])
304
- end = int(line["frameRange"]["end"])
305
- if not (0 <= start <= end < frames):
306
- raise ValueError(
307
- f"frameRange {start}..{end} is outside the source motion's 0..{frames - 1}"
308
- )
309
- span = end - start + 1
310
- uv = resample_polyline(line["points2d"], span)
311
-
312
- R0, R1, R2 = R[0], R[1], R[2]
313
- rows = []
314
- values = []
315
- frame_indices = list(range(start, end + 1))
316
- for k, frame in enumerate(frame_indices):
317
- u = float(uv[k, 0])
318
- v = float(uv[k, 1])
319
- cols = [
320
- column_index(axis, frame, joint_id, num_frames=frames, num_joints=NUM_JOINTS)
321
- for axis in range(3)
322
- ]
323
- # u row
324
- coef_u = fx * R0 - (u - cx) * R2
325
- rows.append(list(zip(cols, coef_u.tolist())))
326
- values.append(-(fx * t[0] - (u - cx) * t[2]))
327
- # v row
328
- coef_v = fy * R1 - (v - cy) * R2
329
- rows.append(list(zip(cols, coef_v.tolist())))
330
- values.append(-(fy * t[1] - (v - cy) * t[2]))
331
- return rows, values, frame_indices, uv
332
-
333
-
334
- def build_pin_rows(line, *, frames, joint_id):
335
- """Hard 3D keyframe rows for `pins3d` — the OTHER shape of a line edit.
336
-
337
- A pin says "this joint is exactly HERE at this generation frame", so it is
338
- three identity rows (x, y, z) with the pinned coordinates as the
339
- measurement. That is the sparsest possible constraint and it is precisely
340
- what the paper measures: ProjFlow's spatial-control table is 1, 2, 5, 49 and
341
- 196 keyframes, with 0.0000 trajectory/location/average error at every
342
- density, and the flow prior — not any interpolation of ours — is what fills
343
- the frames between. So there is deliberately NOTHING between the pins here:
344
- no pseudo-observations, no interpolated guides. The paper's soft
345
- machinery (Sec 4.3.1) exists to rescue the *inpainting* task where hard
346
- observations are sparse across a whole clip; here the preserve rows outside
347
- the range and the seam pin at its edges already bracket the gap, and the
348
- free span between two pins is a couple of seconds at most.
349
-
350
- Same row format as build_line_rows so the two are interchangeable to the
351
- caller: (rows, values, frame_indices). A line's rows are TWO per frame and
352
- depend on the camera; a pin's are THREE and depend on nothing, which is why
353
- a pinned edit needs no lens and survives an orbit.
354
-
355
- Frames arrive already on the generation clock (line-edit-job.mjs scales
356
- them with the same rounding rule it uses for frameRange), so the only
357
- checking here is that they are inside the clip.
358
- """
359
- pins = line["pins3d"]
360
- if not isinstance(pins, list) or not pins:
361
- raise ValueError("pins3d must be a non-empty list of {frame, position}")
362
- rows = []
363
- values = []
364
- frame_indices = []
365
- targets = []
366
- for index, pin in enumerate(pins):
367
- frame = int(pin["frame"])
368
- if not (0 <= frame < frames):
369
- raise ValueError(
370
- f"pins3d[{index}].frame {frame} is outside the source motion's 0..{frames - 1}"
371
- )
372
- position = [float(v) for v in pin["position"]]
373
- if len(position) != 3:
374
- raise ValueError(f"pins3d[{index}].position must be [x, y, z], got {pin['position']}")
375
- for axis in range(3):
376
- col = column_index(axis, frame, joint_id, num_frames=frames, num_joints=NUM_JOINTS)
377
- rows.append([(col, 1.0)])
378
- values.append(position[axis])
379
- frame_indices.append(frame)
380
- targets.append(position)
381
- return rows, values, frame_indices, np.asarray(targets, dtype=np.float64)
382
-
383
-
384
- def build_preserve_rows(source, *, edit_start, edit_end, chain, stride, margin, seam_pin=2, hold_body=False, owned=()):
385
- """Identity rows pinning the SOURCE take outside the edited range.
386
-
387
- SUBSAMPLING POLICY — stated in full because it is the one place this driver
388
- trades exactness for a tractable Cholesky (`A A^T + lambda I` is m x m):
389
-
390
- 1. FRAME STRIDE. Only every `stride`-th frame outside the edit range carries
391
- rows (default 2 => half the rows). Adjacent frames of a real take are
392
- nearly the same pose, so a pinned frame constrains its neighbour almost
393
- as hard as pinning it directly would; the flow prior supplies the rest.
394
- The stride is phase-locked to frame 0 (`range(0, T, stride)`) so two runs
395
- over the same take pin the same frames.
396
-
397
- 2. THE SEAM IS ALWAYS PINNED. The frames immediately either side of the edit
398
- range are added regardless of stride. That is where a pop would show
399
- (gate GP2 measures the blend seams, not the preserved interior), so it is
400
- the one place a missed frame is expensive.
401
-
402
- 3. CHAIN EXCLUSION INSIDE A MARGIN. Within `margin` frames of the edit range
403
- the EDITED LIMB's joints (`chain`) carry no rows, while the rest of the
404
- body still does. The limb has to travel from where the source take left
405
- it to the drawn line; pinning it hard right up to the boundary would ask
406
- for an instantaneous jump and the sampler would split the difference,
407
- missing the line at its start. Outside the margin every joint is pinned.
408
-
409
- 4. SEAM PIN — the edit's OWN first and last `seam_pin` frames carry FULL
410
- rows, chain included, even though they are inside the edited range.
411
-
412
- This is the fix for a seam pop that rules 1-3 cannot reach, and it is
413
- licensed by a property the APP now guarantees rather than by anything
414
- here. src/line-edit.js hands over a curve whose first and last points are
415
- the joint's own projected trajectory, byte-identical: a drag pins them
416
- (PINNED_CURVE_ENDS) and a drawn stroke eases into them (seamEaseWeight is
417
- exactly 0 at both window edges). So at those two frames the drawn line
418
- and the source take ALREADY agree in 2D, and pinning the full 3D pose
419
- there adds no contradiction — it removes the freedom in DEPTH and in the
420
- other 21 joints that was producing the pop.
421
-
422
- Without it those frames are unconstrained except for the line's two
423
- affine rows, and the nearest hard evidence is a whole frame away on the
424
- far side of the boundary: the paper's Plain Masking case (sparse hard
425
- observations, no soft guides), which its own ablation reports as the
426
- worst configuration.
427
-
428
- MEASURED, backfall take, a 102-frame leftFoot reroute (app frames
429
- 20..122, 20-step preview, seed 4242, cold path). Seam figures are the
430
- spliced take's frame-to-frame jump at the boundary as a multiple of the
431
- SOURCE take's own worst frame delta (0.2153 m), so 1.0x means "no more
432
- violent than the take already is":
433
-
434
- seam_pin head tail worst jump in-window edit (max / mean)
435
- 0 2.46x 6.64x 6.64x 2.934 / 1.786 m
436
- 1 0.33x 1.70x 4.51x 2.740 / 1.684 m
437
- 2 0.24x 1.06x 3.96x 2.612 / 1.611 m <- default
438
- 3 0.21x 1.04x 3.00x 2.491 / 1.548 m
439
- 4 0.20x 1.03x 3.03x 2.367 / 1.492 m
440
-
441
- TWO is the default because it is where the seam stops being visible
442
- (1.06x = the take's own worst frame already) while the edit keeps 89% of
443
- its magnitude; past 2 the pin buys almost no seam and starts eating the
444
- edit, since every pinned frame is a frame the drawn line does not own.
445
- Two also matches the app's PINNED_CURVE_ENDS exactly: those are the
446
- frames src/line-edit.js already holds on the source trail in 2D, so this
447
- pins in 3D precisely what the app pinned in 2D. `seam_pin=0` restores the
448
- pre-2026-08-29 behaviour byte for byte.
449
-
450
- Order of the emitted rows is (frame, joint, axis), ascending, so a run is
451
- reproducible and `preservedFrames` in the metadata reads in time order.
452
- """
453
- frames = int(source.shape[0])
454
- lo = max(0, int(edit_start) - int(margin))
455
- hi = min(frames - 1, int(edit_end) + int(margin))
456
-
457
- candidates = set(range(0, frames, max(1, int(stride))))
458
- for seam in (int(edit_start) - 1, int(edit_end) + 1):
459
- if 0 <= seam < frames:
460
- candidates.add(seam)
461
- kept_frames = sorted(f for f in candidates if f < int(edit_start) or f > int(edit_end))
462
-
463
- # 5. HOLD THE BODY (pins only). Inside the range the non-edited joints keep
464
- # their subsampled rows; only the pinned limb's chain is freed.
465
- #
466
- # WHY IT IS ONLY FOR PINS, and why it is not optional for them. A drawn
467
- # line replaces a passage of movement, so the body genuinely has to
468
- # re-time itself around the new path and freeing the range is the point.
469
- # A PIN says the opposite thing — "the take is right, this joint is
470
- # wrong at this instant" — and freeing 1.5 s of whole body to deliver it
471
- # was measured as exactly the wrong trade: on the backfall take a 5 cm
472
- # ask and a 40 cm ask both moved the head ~28 cm and both missed by
473
- # ~25 cm, because the pin was a small perturbation on a window the model
474
- # had otherwise re-imagined (in-window whole-body drift 1.19 m in every
475
- # case, identical to three decimal places).
476
- #
477
- # This is also the paper's own shape for sparse observations. Its
478
- # inpainting formulation surrounds sparse hard keyframes with guidance
479
- # everywhere else rather than leaving the gap free, and its ablation
480
- # reports Plain Masking — hard keyframes alone — as the worst
481
- # configuration. Here the guidance is not interpolated pseudo-
482
- # observations with a fitted variance: it is the SOURCE TAKE, which is
483
- # both hard and correct, so it goes in at Sigma = 0 like every other
484
- # preserve row and needs none of the time-varying machinery.
485
- hold = bool(hold_body)
486
- if hold:
487
- inside = sorted(
488
- f for f in set(range(0, frames, max(1, int(stride))))
489
- if int(edit_start) <= f <= int(edit_end)
490
- )
491
- kept_frames = sorted(set(kept_frames) | set(inside))
492
-
493
- # The seam pin's own frames, taken from INSIDE the range. Capped at half the
494
- # range so a two-frame edit cannot pin itself into a no-op.
495
- pin_span = max(0, int(seam_pin))
496
- span = int(edit_end) - int(edit_start) + 1
497
- pin_span = min(pin_span, span // 2)
498
- seam_frames = []
499
- for offset in range(pin_span):
500
- seam_frames.append(int(edit_start) + offset)
501
- seam_frames.append(int(edit_end) - offset)
502
- seam_frames = sorted(set(f for f in seam_frames if 0 <= f < frames))
503
- kept_frames = sorted(set(kept_frames) | set(seam_frames))
504
-
505
- # Cells the EDIT's own rows already own. A seam-pin frame that happens to
506
- # carry a 3D pin would otherwise get an identity row to the source AND a pin
507
- # row to the target for the same coordinate: two contradictory measurements
508
- # that the ridge quietly averages into a third answer nobody asked for. The
509
- # edit always wins — it is the thing the artist authored.
510
- owned_cells = {(int(f), int(j)) for f, j in owned}
511
-
512
- rows = []
513
- values = []
514
- pinned = [] # (frame, joint) pairs, for the self-check below
515
- for frame in kept_frames:
516
- in_margin = lo <= frame <= hi
517
- for joint in range(NUM_JOINTS):
518
- if (frame, joint) in owned_cells:
519
- continue
520
- # A seam frame is pinned WHOLE — the chain exclusion is about giving
521
- # the limb room to travel, and at the seam it has nowhere to travel
522
- # to: the line already puts it where the source does.
523
- if in_margin and joint in chain and frame not in seam_frames:
524
- continue
525
- for axis in range(3):
526
- col = column_index(axis, frame, joint, num_frames=frames, num_joints=NUM_JOINTS)
527
- rows.append([(col, 1.0)])
528
- values.append(float(source[frame, joint, axis]))
529
- pinned.append((frame, joint))
530
- return rows, values, kept_frames, pinned
531
-
532
-
533
- def densify(rows, values, *, frames):
534
- """Materialise the sparse row list into the dense (m, 3*T*J) operator the
535
- sampler wants. Every row has at most 3 nonzeros, but
536
- `sample_linear_constraints` takes a dense tensor, so this is where the
537
- memory goes: m * 3*T*J * 4 bytes."""
538
- m = len(rows)
539
- n = 3 * int(frames) * NUM_JOINTS
540
- A = np.zeros((m, n), dtype=np.float32)
541
- for index, row in enumerate(rows):
542
- for col, coefficient in row:
543
- A[index, col] = coefficient
544
- return A, np.asarray(values, dtype=np.float32)
545
-
546
-
547
- def project(points_world, cam):
548
- """Perspective-project (N,3) world points with the request's camera, in the
549
- same units the request used. The self-check reprojection number is computed
550
- with THIS, independently of the rows, so a sign error in the row algebra
551
- shows up as a large error rather than cancelling itself out."""
552
- R = np.asarray(cam["R"], dtype=np.float64)
553
- t = np.asarray(cam["t"], dtype=np.float64).reshape(1, 3)
554
- cam_pts = points_world.astype(np.float64) @ R.T + t
555
- z = cam_pts[:, 2]
556
- u = float(cam["fx"]) * cam_pts[:, 0] / z + float(cam["cx"])
557
- v = float(cam["fy"]) * cam_pts[:, 1] / z + float(cam["cy"])
558
- return np.stack([u, v], axis=1)
559
-
560
-
561
- # --- request validation ------------------------------------------------------
562
-
563
- def resolve_joint(line):
564
- track = line.get("track")
565
- if not isinstance(track, str) or not track:
566
- raise ValueError("line.track is required and must be a non-empty string")
567
- if track in UNMAPPABLE_TRACKS:
568
- raise ValueError(f"track {track!r} cannot be line-edited: {UNMAPPABLE_TRACKS[track]}")
569
- if track not in TRACK_TO_JOINT:
570
- known = ", ".join(sorted(TRACK_TO_JOINT))
571
- raise ValueError(f"unknown track {track!r}; known line-edit tracks: {known}")
572
- expected = TRACK_TO_JOINT[track]
573
- declared = line.get("jointId")
574
- if declared is not None and int(declared) != expected:
575
- # See TRACK_TO_JOINT's header: the JS side resolves the joint and this is
576
- # the guard against a wrapper that has drifted from the driver.
577
- raise ValueError(
578
- f"line.jointId {declared} disagrees with track {track!r} -> joint {expected}; "
579
- "the wrapper and the driver disagree about the skeleton"
580
- )
581
- return expected
582
-
583
-
584
- # --- the loaded model, and the one edit both run modes perform ---------------
585
-
586
- def open_repo(repo_path):
587
- """Put the ProjFlow clone on sys.path AND make it the cwd, then return it.
588
-
589
- Both are required and for different reasons: the repo's modules are imported
590
- by package path (`demo.common.*`), and its model loader resolves checkpoint
591
- paths relative to the process's cwd. Called once per process — the serve
592
- loop must not chdir per request, or a relative path in a request would mean
593
- something different depending on what ran before it.
594
- """
595
- repo = Path(repo_path).expanduser().resolve()
596
- if not (repo / "demo").is_dir():
597
- raise SystemExit(f"--repo {repo} does not look like a ProjFlow clone (no demo/)")
598
- sys.path.insert(0, str(repo))
599
- os.chdir(str(repo))
600
- return repo
601
-
602
-
603
- class Engine:
604
- """The loaded checkpoint plus the handful of repo functions an edit needs.
605
-
606
- THE WHOLE POINT of contract C11 is that this object outlives a request. S1
607
- measured the load at 3.9 s against 1.3 s of sampling, so a resident Engine
608
- is the difference between an 8 s round trip and a 2 s one. Nothing about a
609
- request mutates it: the operator, the Cholesky and the sample are all built
610
- per call in `run_line_edit`, and the sampler seeds torch itself, so request
611
- N+1 cannot inherit anything from request N except the weights.
612
-
613
- The repo imports happen HERE rather than at module scope because they are
614
- only importable after `open_repo`, and because `import torch` is most of the
615
- load time — a caller that only wants the argument parser should not pay it.
616
- """
617
-
618
- def __init__(self, repo, *, model_id, model_name, device):
619
- import torch # after open_repo, so the venv's torch is the one that loads
620
- from demo.common.model_loader import load_acmdm_from_config, resolve_device
621
- from demo.common.motion_stats import load_22x3_stats, denormalize_world
622
- from demo.common.normal_equations import precompute_normal_cholesky
623
- from demo.common.sampling import sample_with_linear_constraints
624
- from demo.lift.projection_operator import convert_linear_measurement_to_normalized_space
625
-
626
- started = time.time()
627
- self.torch = torch
628
- self.repo = repo
629
- self.device = resolve_device(str(device))
630
- self.loaded = load_acmdm_from_config(
631
- {"id": model_id, "name": model_name, "dataset": "t2m"},
632
- repo_root=repo,
633
- device=self.device,
634
- )
635
- self.mean_np, self.std_np = load_22x3_stats("t2m", repo)
636
- self.load_seconds = time.time() - started
637
- self.to_normalized = convert_linear_measurement_to_normalized_space
638
- self.precompute_normal_cholesky = precompute_normal_cholesky
639
- self.sample = sample_with_linear_constraints
640
- self.denormalize_world = denormalize_world
641
-
642
-
643
- def run_line_edit(
644
- engine,
645
- source,
646
- line,
647
- *,
648
- steps,
649
- ridge,
650
- preserve_stride,
651
- preserve_margin,
652
- seed,
653
- cfg,
654
- seam_pin=2,
655
- preview=False,
656
- reset_peak=False,
657
- ):
658
- """One line edit against a loaded Engine. Returns (out_world, meta).
659
-
660
- Every number the one-shot mode ever wrote lives in the returned meta, and
661
- the serve mode returns the same dict verbatim, so a warm result and a cold
662
- result are indistinguishable to the wrapper.
663
- """
664
- torch = engine.torch
665
- source = np.asarray(source, dtype=np.float32)
666
- if source.ndim != 3 or source.shape[1] != NUM_JOINTS or source.shape[2] != 3:
667
- raise ValueError(f"source must be (T,22,3); got {source.shape}")
668
- frames = int(source.shape[0])
669
- _assert_column_layout(frames)
670
-
671
- joint_id = resolve_joint(line)
672
- prompt = str(line.get("prompt") or "")
673
-
674
- # --- fuse the operator ---------------------------------------------------
675
- # ONE EDIT IS ONE GESTURE (contract C6 v2): either a drawn line (two affine
676
- # rows per frame, camera-dependent) or 3D pins (three identity rows per pin,
677
- # camera-free). The wrapper validates the exclusivity; here the presence of
678
- # pins3d simply selects the row builder, and everything downstream — the
679
- # preserve rows, the seam pin, the solve, the splice — is identical.
680
- pinned_edit = line.get("pins3d") is not None
681
- if pinned_edit:
682
- line_rows, line_values, line_frames, uv = build_pin_rows(line, frames=frames, joint_id=joint_id)
683
- else:
684
- line_rows, line_values, line_frames, uv = build_line_rows(line, frames=frames, joint_id=joint_id)
685
- chain = limb_chain(joint_id)
686
- keep_rows, keep_values, kept_frames, pinned = build_preserve_rows(
687
- source,
688
- edit_start=int(line["frameRange"]["start"]),
689
- edit_end=int(line["frameRange"]["end"]),
690
- chain=chain,
691
- stride=int(preserve_stride),
692
- margin=int(preserve_margin),
693
- seam_pin=int(seam_pin),
694
- # A pins edit holds the rest of the body through the range; a drawn line
695
- # frees it, because a stroke replaces a passage of movement and a pin
696
- # corrects an instant of one. See build_preserve_rows rule 5.
697
- hold_body=pinned_edit,
698
- owned=[(f, joint_id) for f in line_frames],
699
- )
700
- A_world_np, y_world_np = densify(line_rows + keep_rows, line_values + keep_values, frames=frames)
701
- m = int(A_world_np.shape[0])
702
-
703
- device = engine.device
704
- A_world = torch.from_numpy(A_world_np).to(device=device)
705
- y_world = torch.from_numpy(y_world_np).to(device=device)
706
- # THE normalisation convention, taken from the repo so there is one spelling
707
- # of it: x_world = x_norm*std + mean => A' = A diag(std), y' = y - A mean.
708
- # It is applied to the line rows and the preserve rows TOGETHER, which is the
709
- # whole point of fusing them into one operator.
710
- A_norm, y_norm = engine.to_normalized(
711
- operator_world=A_world,
712
- measurement_world=y_world,
713
- mean=torch.from_numpy(np.asarray(engine.mean_np, dtype=np.float32)).to(device=device),
714
- std=torch.from_numpy(np.asarray(engine.std_np, dtype=np.float32)).to(device=device),
715
- num_frames=frames,
716
- num_joints=NUM_JOINTS,
717
- )
718
-
719
- factor = engine.precompute_normal_cholesky(A_norm, ridge_lambda=float(ridge))
720
-
721
- # An empty prompt with cfg != 1 duplicates the batch to guide toward nothing;
722
- # on a line edit the constraints carry the intent, so guidance is switched off
723
- # rather than paid for. A prompt that IS given keeps the demos' cfg.
724
- cfg_scale = float(cfg) if prompt else 1.0
725
-
726
- if device.type == "cuda":
727
- # Only the resident asks for this: it makes `vramPeakAllocMiB` the peak of
728
- # THIS request (the weights stay resident, so they stay in the baseline)
729
- # instead of a session high-water mark that only ever grows. The one-shot
730
- # process is its own reset and must keep reporting the load peak it always
731
- # has.
732
- if reset_peak:
733
- torch.cuda.reset_peak_memory_stats()
734
- torch.cuda.synchronize()
735
- started = time.time()
736
- result = engine.sample(
737
- engine.loaded.model,
738
- prompt=prompt,
739
- operator=A_norm,
740
- measurement=y_norm.view(1, -1),
741
- normal_cholesky=factor.normal_cholesky.to(device=device),
742
- frames=frames,
743
- num_joints=NUM_JOINTS,
744
- num_samples=1,
745
- seed=int(seed),
746
- steps=int(steps),
747
- cfg=cfg_scale,
748
- device=device,
749
- )
750
- if device.type == "cuda":
751
- torch.cuda.synchronize()
752
- sampling_seconds = time.time() - started
753
-
754
- out_norm = result.samples_norm[0].permute(1, 2, 0).detach().cpu().numpy() # (T,J,3)
755
- out_world = engine.denormalize_world(out_norm, engine.mean_np, engine.std_np).astype(np.float32)
756
-
757
- # --- self-check ----------------------------------------------------------
758
- # Both numbers are computed from the RESULT, not from the sampler's
759
- # internals, and the reprojection uses `project()` rather than the rows, so a
760
- # sign error cannot hide inside its own operator.
761
- if pinned:
762
- pin_f = np.asarray([f for f, _ in pinned])
763
- pin_j = np.asarray([j for _, j in pinned])
764
- preserved_err = np.abs(out_world[pin_f, pin_j] - source[pin_f, pin_j])
765
- preserved_max = float(preserved_err.max())
766
- preserved_mean = float(preserved_err.mean())
767
- else:
768
- preserved_max = preserved_mean = 0.0
769
- if pinned_edit:
770
- # A pin's own self-check is in METRES, not viewport units: it was
771
- # authored in world space and there is no lens to project it through.
772
- # Reported under the same key so the wrapper and the gates read one
773
- # number whichever gesture produced it; `lineErrUnits` says which.
774
- line_err = np.abs(out_world[line_frames, joint_id] - uv)
775
- err_units = "m"
776
- else:
777
- reproj = project(out_world[line_frames, joint_id], line["camera"])
778
- line_err = np.abs(reproj - uv.astype(np.float64))
779
- err_units = "uv"
780
- edit_lo = int(line["frameRange"]["start"])
781
- edit_hi = int(line["frameRange"]["end"])
782
- edited = np.abs(out_world[edit_lo:edit_hi + 1] - source[edit_lo:edit_hi + 1])
783
-
784
- meta = {
785
- "m": m,
786
- "steps": int(steps),
787
- "ridge": float(ridge),
788
- "sampling_seconds": round(sampling_seconds, 4),
789
- "frames": frames,
790
- "fps": 20,
791
- "joints": NUM_JOINTS,
792
- "track": line["track"],
793
- "jointId": joint_id,
794
- "prompt": prompt,
795
- "cfg": cfg_scale,
796
- "seed": int(seed),
797
- "preview": bool(preview),
798
- "gesture": "pins" if pinned_edit else "line",
799
- "pins": [int(f) for f in line_frames] if pinned_edit else None,
800
- "rows": {
801
- "line": len(line_rows),
802
- "preserve": len(keep_rows),
803
- "operatorCols": 3 * frames * NUM_JOINTS,
804
- },
805
- "preserve": {
806
- "stride": int(preserve_stride),
807
- "margin": int(preserve_margin),
808
- "seamPin": int(seam_pin),
809
- "holdBody": bool(pinned_edit),
810
- "frames": [int(f) for f in kept_frames],
811
- "pinnedJointFrames": len(pinned),
812
- "freeChain": sorted(int(j) for j in chain),
813
- },
814
- "editRange": [int(line["frameRange"]["start"]), int(line["frameRange"]["end"])],
815
- "checks": {
816
- # Success criterion (i): pinned source frames come back unchanged.
817
- "preservedMaxAbsDiffM": preserved_max,
818
- "preservedMeanAbsDiffM": preserved_mean,
819
- # Success criterion (ii): the drawn line is followed, in the request's
820
- # own 2D units.
821
- "lineMaxReprojErr": float(line_err.max()),
822
- "lineMeanReprojErr": float(line_err.mean()),
823
- "lineErrUnits": err_units,
824
- # Proof the edit is not a no-op: the range genuinely moved.
825
- "editedMaxAbsDiffM": float(edited.max()),
826
- },
827
- "device": str(device),
828
- "checkpoint": str(engine.loaded.checkpoint_path),
829
- }
830
- if device.type == "cuda":
831
- meta["vramPeakAllocMiB"] = round(torch.cuda.max_memory_allocated() / 2 ** 20, 1)
832
- return out_world, meta
833
-
834
-
835
- # --- the resident service ----------------------------------------------------
836
-
837
- def decode_array(blob):
838
- """A {"shape": [...], "data": "<base64 little-endian float32>"} payload -> ndarray.
839
-
840
- The shape is carried explicitly rather than inferred from the byte count
841
- because a (T,22,3) and a (3,22,T) blob are the same length, and the sampler
842
- would happily edit the transpose.
843
- """
844
- if not isinstance(blob, dict):
845
- raise ValueError("source must be an object with `shape` and base64 `data`")
846
- shape = [int(dim) for dim in blob.get("shape") or []]
847
- if not shape or any(dim < 0 for dim in shape):
848
- raise ValueError(f"source.shape must be positive integers, got {blob.get('shape')!r}")
849
- raw = base64.b64decode(blob.get("data") or "", validate=True)
850
- count = int(np.prod(shape))
851
- if len(raw) != count * 4:
852
- raise ValueError(f"source.data is {len(raw)} bytes; shape {shape} needs {count * 4}")
853
- return np.frombuffer(raw, dtype="<f4").reshape(shape).astype(np.float32)
854
-
855
-
856
- def encode_array(array):
857
- """The inverse. C order and float32 are stated in the payload, not assumed:
858
- the JS side reads the shape back and refuses a result it did not expect."""
859
- contiguous = np.ascontiguousarray(array, dtype="<f4")
860
- return {
861
- "shape": [int(dim) for dim in contiguous.shape],
862
- "dtype": "float32",
863
- "data": base64.b64encode(contiguous.tobytes(order="C")).decode("ascii"),
864
- }
865
-
866
-
867
- def serve(args) -> int:
868
- """Load the model once, then answer one NDJSON request per stdin line.
869
-
870
- The loop is deliberately SEQUENTIAL — one request in flight, no threads. A
871
- single 3070 samples one edit at a time anyway, and the wrapper queues on its
872
- side, so concurrency here would buy nothing and cost the guarantee that a
873
- response line always belongs to the request that preceded it.
874
-
875
- EOF ON STDIN IS THE SHUTDOWN SIGNAL. When the wrapper's ssh child dies (or
876
- is killed on process exit) sshd closes this end and `readline` returns "",
877
- which ends the loop and drops the checkpoint. That is why the resident needs
878
- no supervisor on the box and leaves nothing behind.
879
- """
880
- # Claim the real stdout BEFORE anything can print to it, and point fd 1 at
881
- # stderr so that model loading, tqdm and any native extension write where the
882
- # wrapper treats output as status. See the module docstring.
883
- protocol_fd = os.dup(1)
884
- os.dup2(2, 1)
885
- protocol_out = os.fdopen(protocol_fd, "w", encoding="utf-8", newline="\n")
886
-
887
- def emit(message):
888
- protocol_out.write(json.dumps(message, separators=(",", ":")) + "\n")
889
- protocol_out.flush()
890
-
891
- repo = open_repo(args.repo)
892
- engine = Engine(repo, model_id=args.model_id, model_name=args.model_name, device=args.device)
893
- emit({
894
- "type": "ready",
895
- "protocol": PROTOCOL_VERSION,
896
- "device": str(engine.device),
897
- "loadSeconds": round(engine.load_seconds, 3),
898
- "repo": str(repo),
899
- "pid": os.getpid(),
900
- })
901
- print(f"projflow-serve: ready in {engine.load_seconds:.2f}s on {engine.device}", file=sys.stderr, flush=True)
902
-
903
- served = 0
904
- while True:
905
- text = sys.stdin.readline()
906
- if text == "":
907
- break
908
- text = text.strip()
909
- if not text:
910
- continue
911
- try:
912
- request = json.loads(text)
913
- if not isinstance(request, dict):
914
- raise ValueError("a request must be a JSON object")
915
- except Exception as error:
916
- # No id means no way to pair this with anything. Answering with a null
917
- # id is the honest move: the wrapper reads it as a protocol failure,
918
- # falls back to the cold path and restarts this process.
919
- emit({"id": None, "ok": False, "error": {"type": type(error).__name__, "message": str(error)}})
920
- continue
921
-
922
- request_id = request.get("id")
923
- kind = request.get("type") or "lineEdit"
924
- try:
925
- if kind == "ping":
926
- result = {
927
- "pong": True,
928
- "protocol": PROTOCOL_VERSION,
929
- "device": str(engine.device),
930
- "served": served,
931
- "loadSeconds": round(engine.load_seconds, 3),
932
- }
933
- elif kind == "shutdown":
934
- emit({"id": request_id, "ok": True, "result": {"bye": True}})
935
- break
936
- elif kind == "lineEdit":
937
- source = decode_array(request.get("source"))
938
- line = request.get("line")
939
- if not isinstance(line, dict):
940
- raise ValueError("line must be a C6 line-edit object")
941
- preview = bool(request.get("preview"))
942
- steps = 20 if preview else int(request.get("steps", 100))
943
- out_world, meta = run_line_edit(
944
- engine,
945
- source,
946
- line,
947
- steps=steps,
948
- ridge=float(request.get("ridge", 1e-6)),
949
- preserve_stride=int(request.get("preserveStride", 2)),
950
- preserve_margin=int(request.get("preserveMargin", 20)),
951
- seam_pin=int(request.get("seamPin", 2)),
952
- seed=int(request.get("seed", 0)),
953
- cfg=float(request.get("cfg", 3.0)),
954
- preview=preview,
955
- reset_peak=True,
956
- )
957
- served += 1
958
- result = dict(encode_array(out_world))
959
- result["meta"] = meta
960
- else:
961
- raise ValueError(f"unknown request type {kind!r}")
962
- except Exception as error: # noqa: BLE001 — a bad request must not end the session
963
- # The traceback goes to stderr (where the wrapper forwards it as
964
- # status) and the message goes back on the wire. The loop survives:
965
- # one malformed line edit is not a reason to pay another model load.
966
- traceback.print_exc(file=sys.stderr)
967
- sys.stderr.flush()
968
- emit({"id": request_id, "ok": False, "error": {"type": type(error).__name__, "message": str(error)}})
969
- continue
970
- emit({"id": request_id, "ok": True, "result": result})
971
-
972
- print(f"projflow-serve: stdin closed after {served} edits; exiting", file=sys.stderr, flush=True)
973
- return 0
974
-
975
-
976
- def main() -> int:
977
- ap = argparse.ArgumentParser("ProjFlow line-edit driver")
978
- ap.add_argument("--serve", action="store_true",
979
- help="resident mode: load the model once and answer NDJSON requests on stdin/stdout")
980
- ap.add_argument("--source", help="source motion .npy, (T,22,3) float32 world metres")
981
- ap.add_argument("--line", help="line-edit request .json (contract C6)")
982
- ap.add_argument("--out", help="output .npy; metadata goes to <out stem>.meta.json")
983
- ap.add_argument("--steps", type=int, default=100, help="ODE steps (S1: ~7.3 ms/step at 100 frames)")
984
- ap.add_argument("--ridge", type=float, default=1e-6,
985
- help="ridge_lambda. S1: THIS is the exactness knob, not steps — 1e-3 lands at "
986
- "2.6e-5 while 1e-6 gives 2.4e-7 at half the steps")
987
- ap.add_argument("--preview", action="store_true",
988
- help="20 steps instead of --steps (S1 measured 0.145 s for a 100-frame preview)")
989
- ap.add_argument("--preserve-stride", type=int, default=2, help="pin every Nth frame outside the edit range")
990
- ap.add_argument("--preserve-margin", type=int, default=20,
991
- help="frames either side of the edit range where the edited limb is left free (20 = 1 s)")
992
- ap.add_argument("--seam-pin", type=int, default=2,
993
- help="pin the edit range's own outermost N frames to the source, whole body "
994
- "(0 = off; the app guarantees the curve's ends sit on the source trail)")
995
- ap.add_argument("--seed", type=int, default=0, help="matches demo/run.py's default")
996
- ap.add_argument("--cfg", type=float, default=3.0, help="classifier-free guidance scale")
997
- ap.add_argument("--repo", default=os.environ.get("CCLAY_PROJFLOW_REPO", "/home/yun/projflow-scout/repo"))
998
- ap.add_argument("--model-id", default="ACMDM-Raw-Flow-S-PatchSize22")
999
- ap.add_argument("--model-name", default="ACMDM_Raw_Flow_S_PatchSize22")
1000
- ap.add_argument("--device", default="auto")
1001
- args = ap.parse_args()
1002
-
1003
- if args.serve:
1004
- # The resident takes its whole request on the wire, so the one-shot file
1005
- # flags are meaningless here and passing them is a caller bug worth
1006
- # naming rather than ignoring.
1007
- for flag in ("source", "line", "out"):
1008
- if getattr(args, flag):
1009
- raise SystemExit(f"--serve takes its requests on stdin; --{flag} does not apply")
1010
- return serve(args)
1011
-
1012
- for flag in ("source", "line", "out"):
1013
- if not getattr(args, flag):
1014
- raise SystemExit(f"--{flag} is required (or use --serve)")
1015
-
1016
- # Resolve every caller-supplied path BEFORE the chdir below, or a relative
1017
- # --source silently resolves against the ProjFlow clone instead of the
1018
- # caller's cwd. generate.mjs always passes absolute paths; a human running
1019
- # this by hand should not have to.
1020
- args.source = str(Path(args.source).expanduser().resolve())
1021
- args.line = str(Path(args.line).expanduser().resolve())
1022
- args.out = str(Path(args.out).expanduser().absolute())
1023
-
1024
- repo = open_repo(args.repo)
1025
-
1026
- source = np.load(args.source).astype(np.float32)
1027
- if source.ndim != 3 or source.shape[1] != NUM_JOINTS or source.shape[2] != 3:
1028
- raise SystemExit(f"--source must be (T,22,3); got {source.shape}")
1029
-
1030
- with open(args.line, "r", encoding="utf-8") as handle:
1031
- line = json.load(handle)
1032
-
1033
- steps = 20 if args.preview else int(args.steps)
1034
- engine = Engine(repo, model_id=args.model_id, model_name=args.model_name, device=args.device)
1035
- out_world, meta = run_line_edit(
1036
- engine,
1037
- source,
1038
- line,
1039
- steps=steps,
1040
- ridge=float(args.ridge),
1041
- preserve_stride=int(args.preserve_stride),
1042
- preserve_margin=int(args.preserve_margin),
1043
- seam_pin=int(args.seam_pin),
1044
- seed=int(args.seed),
1045
- cfg=float(args.cfg),
1046
- preview=bool(args.preview),
1047
- )
1048
-
1049
- out_path = Path(args.out).expanduser()
1050
- out_path.parent.mkdir(parents=True, exist_ok=True)
1051
- np.save(str(out_path), out_world)
1052
-
1053
- # `<stem>.meta.json` beside the motion: --out foo.npy => foo.meta.json. The
1054
- # wrapper builds the same name on the JS side, so this spelling is a contract.
1055
- stem = str(out_path)[:-4] if str(out_path).endswith(".npy") else str(out_path)
1056
- meta_path = Path(stem + ".meta.json")
1057
- with open(meta_path, "w", encoding="utf-8") as handle:
1058
- json.dump(meta, handle, indent=2)
1059
-
1060
- # The wrapper greps this line, the same way bridge.mjs greps Kimodo's.
1061
- print(
1062
- f"projflow-line-edit: done - {out_path} "
1063
- f"(m={meta['m']} steps={meta['steps']} sample={meta['sampling_seconds']:.3f}s)"
1064
- )
1065
- return 0
1066
-
1067
-
1068
- if __name__ == "__main__":
1069
- raise SystemExit(main())