cozyclay 2.1.1 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +60 -0
- package/README.md +12 -30
- package/bin/agent/agent-routes.mjs +40 -113
- package/bin/agent/agent-runner.mjs +9 -10
- package/bin/agent/cliproxy-provider.mjs +135 -0
- package/bin/agent/credential-store.mjs +30 -7
- package/bin/agent/pi-tools.mjs +4 -1
- package/bin/agent/provider-keys.mjs +32 -0
- package/bin/agent/provider-login.mjs +110 -0
- package/bin/agent/provider-routes.mjs +99 -0
- package/bin/agent/providers.mjs +19 -97
- package/bin/agent/script-tool.mjs +17 -4
- package/bin/agent/studio-motion-reference.mjs +23 -0
- package/bin/agent/studio-prompt.mjs +1 -1
- package/bin/agent/studio-rebase-text.mjs +32 -0
- package/bin/agent/studio-rebase.mjs +26 -39
- package/bin/agent/studio-tools.mjs +9 -28
- package/bin/agent/tool-arguments.mjs +49 -0
- package/bin/agent/turn-frames.mjs +66 -0
- package/bin/codex-auth.mjs +9 -2
- package/bin/cozyclay.mjs +10 -116
- package/bin/live/cli.mjs +3 -3
- package/bin/mcp-runtime.mjs +2 -12
- package/bin/package-signature.mjs +0 -4
- package/dist/app/index.html +7 -6
- package/dist/assets/EXRLoader-CvVIvGVE.js +1 -0
- package/dist/assets/analytics-C9sUnndd.js +1 -0
- package/dist/assets/app-CCh--1Hc.js +5087 -0
- package/dist/assets/app-f-fkV5uP.css +1 -0
- package/dist/assets/compile-worker-CkxiD6lH.js +5 -0
- package/dist/assets/fflate.module-KbGj5-H6.js +5 -0
- package/dist/assets/{first-shot-handoff-msLI71g1.js → first-shot-handoff-CC4xe4HM.js} +1 -1
- package/dist/assets/{landing-ChUvB_zL.js → landing-DjHD8u7k.js} +1 -1
- package/dist/assets/scene-asset-sync-Hx_FzdOH.js +20 -0
- package/dist/assets/scene-asset-sync-_S8NQl5B.css +1 -0
- package/dist/assets/workflow-C4yKH1p4.js +179 -0
- package/dist/cozyclay-package.json +1 -1
- package/dist/index.html +3 -3
- package/dist/sw.js +3 -6
- package/dist/workflow/index.html +5 -4
- package/mcp/LIVE-PROTOCOL.md +7 -47
- package/mcp/README.md +16 -26
- package/mcp/live-hub.mjs +6 -10
- package/mcp/package.json +1 -3
- package/mcp/tool-handlers.mjs +4 -168
- package/package.json +3 -22
- package/src/App.jsx +150 -2624
- package/src/agent-progress.js +79 -0
- package/src/agent-turn-mark.js +30 -0
- package/src/analytics.js +1 -19
- package/src/app-context.js +5 -0
- package/src/app-stage.jsx +2 -34
- package/src/asset-pane.jsx +6 -5
- package/src/camera-block.js +21 -0
- package/src/camera-direction.js +120 -0
- package/src/character-kind.js +3 -9
- package/src/command-bus.js +8 -6
- package/src/commands/cast.js +5 -34
- package/src/commands/elements/character.js +1 -1
- package/src/commands/elements/motion.js +1 -6
- package/src/commands/elements/object.js +6 -2
- package/src/commands/elements/shot.js +4 -1
- package/src/commands/elements.js +38 -14
- package/src/commands/index.js +3 -1
- package/src/commands/kits.js +37 -0
- package/src/commands/motion-timing.js +57 -0
- package/src/commands/motion.js +9 -152
- package/src/commands/motion3-compile.js +65 -0
- package/src/commands/motion3.js +207 -0
- package/src/commands/objects.js +6 -4
- package/src/commands/shot.js +49 -11
- package/src/commands/stage.js +14 -1
- package/src/commands/view.js +4 -1
- package/src/document-mirrors.js +42 -0
- package/src/document-store.js +4 -0
- package/src/domains/cast.js +19 -194
- package/src/domains/motion/motion-range-pin-rules.js +1 -1
- package/src/domains/motion/take-source.js +51 -0
- package/src/domains/motion.js +60 -1838
- package/src/domains/scenes.js +12 -55
- package/src/domains/shots.js +2 -2
- package/src/domains/stage.js +4 -2
- package/src/draft-ghost.js +46 -0
- package/src/draft-ghost.jsx +129 -0
- package/src/dualview.jsx +4 -3
- package/src/effects/EffectsLayer.jsx +112 -0
- package/src/effects/effect-meshes.js +232 -0
- package/src/effects/effect-record.js +103 -0
- package/src/effects/emitter.js +59 -0
- package/src/effects/simulate.js +261 -0
- package/src/effects/state.js +59 -0
- package/src/execution-telemetry.js +2 -3
- package/src/first-success-guide.jsx +5 -5
- package/src/ghost-set.js +257 -0
- package/src/held-object-edit.jsx +113 -0
- package/src/held-object-pose.js +175 -0
- package/src/hierarchy-model.js +1 -1
- package/src/hierarchy-panel.jsx +1 -1
- package/src/imported-mesh.jsx +103 -0
- package/src/kits/held.js +84 -0
- package/src/kits/index.js +70 -0
- package/src/kits/smoking.js +57 -0
- package/src/light-blockers.js +93 -0
- package/src/motion/beat-board-model.js +84 -0
- package/src/motion/beat-board.css +124 -0
- package/src/motion/beat-board.jsx +59 -0
- package/src/motion/center-of-mass.js +66 -0
- package/src/motion/fix-collisions.js +850 -0
- package/src/motion/physics-review.js +118 -0
- package/src/{ardy → motion}/physics-support.js +1 -76
- package/src/motion/rig-snapshot.js +22 -0
- package/src/motion/take-segments.jsx +140 -0
- package/src/{ardy → motion}/timeline-coordinates.js +0 -21
- package/src/{ardy → motion}/timeline.css +1 -2
- package/src/{ardy → motion}/timeline.jsx +42 -413
- package/src/motion/waypoints.js +76 -0
- package/src/motion3/beats.js +83 -0
- package/src/motion3/body.js +158 -0
- package/src/motion3/bones.js +47 -0
- package/src/motion3/carried.js +47 -0
- package/src/motion3/compile-job.js +75 -0
- package/src/motion3/compile-worker.js +10 -0
- package/src/motion3/compiler/anchors.js +105 -0
- package/src/motion3/compiler/arm-orient.js +128 -0
- package/src/motion3/compiler/channels.js +125 -0
- package/src/motion3/compiler/compile.js +205 -0
- package/src/motion3/compiler/gait-rhythm.js +97 -0
- package/src/motion3/compiler/grip.js +171 -0
- package/src/motion3/compiler/head-steady.js +48 -0
- package/src/motion3/compiler/hold.js +134 -0
- package/src/motion3/compiler/idle.js +113 -0
- package/src/motion3/compiler/limbs.js +170 -0
- package/src/motion3/compiler/math.js +72 -0
- package/src/motion3/compiler/pose.js +162 -0
- package/src/motion3/compiler/quat.js +49 -0
- package/src/motion3/compiler/skeleton.js +74 -0
- package/src/motion3/compiler/validate-common.js +32 -0
- package/src/motion3/compiler/validate-goals.js +118 -0
- package/src/motion3/compiler/validate-objects.js +143 -0
- package/src/motion3/compiler/validate-parts.js +116 -0
- package/src/motion3/compiler/validate.js +122 -0
- package/src/motion3/compiler/vocab.js +73 -0
- package/src/motion3/compiler/walk-pelvis.js +275 -0
- package/src/motion3/compiler/walk.js +207 -0
- package/src/motion3/compiler/wrist-clamp.js +32 -0
- package/src/motion3/delta.js +47 -0
- package/src/motion3/describe.js +174 -0
- package/src/motion3/docs.js +27 -0
- package/src/motion3/edit-carry.js +21 -0
- package/src/motion3/fix/ease.js +16 -0
- package/src/motion3/fix/ground.js +37 -0
- package/src/motion3/fix/index.js +52 -0
- package/src/motion3/fix/push.js +71 -0
- package/src/motion3/fix/reach.js +46 -0
- package/src/motion3/fix/retime.js +79 -0
- package/src/motion3/fix/self.js +211 -0
- package/src/motion3/fix/slide.js +70 -0
- package/src/motion3/fix/turn.js +51 -0
- package/src/motion3/follow.js +39 -0
- package/src/motion3/free-motion.js +173 -0
- package/src/motion3/grip-choice.js +130 -0
- package/src/motion3/hold-plan.js +157 -0
- package/src/motion3/json-patch.js +77 -0
- package/src/motion3/key-aim.js +105 -0
- package/src/motion3/key-edit.js +82 -0
- package/src/motion3/lenient.js +127 -0
- package/src/motion3/object-keys.js +236 -0
- package/src/motion3/object-pose.js +182 -0
- package/src/motion3/object-tracks.js +12 -0
- package/src/motion3/pipeline.js +102 -0
- package/src/motion3/playback.js +51 -0
- package/src/motion3/probes.js +91 -0
- package/src/motion3/qa/body-checks.js +34 -0
- package/src/motion3/qa/check-names.js +14 -0
- package/src/motion3/qa/checks.js +121 -0
- package/src/motion3/qa/contact-checks.js +51 -0
- package/src/motion3/qa/explain.js +80 -0
- package/src/motion3/qa/fmt.js +5 -0
- package/src/motion3/qa/foot-checks.js +72 -0
- package/src/motion3/qa/geometry.js +77 -0
- package/src/motion3/qa/goals.js +191 -0
- package/src/motion3/qa/ground.js +60 -0
- package/src/motion3/qa/hints-contact.js +179 -0
- package/src/motion3/qa/hints-goal.js +160 -0
- package/src/motion3/qa/hints-knee.js +24 -0
- package/src/motion3/qa/hints-motion.js +209 -0
- package/src/motion3/qa/hints-object.js +91 -0
- package/src/motion3/qa/index.js +42 -0
- package/src/motion3/qa/key-checks.js +82 -0
- package/src/motion3/qa/near.js +96 -0
- package/src/motion3/qa/object-checks.js +135 -0
- package/src/motion3/qa/patch.js +161 -0
- package/src/motion3/qa/severity.js +38 -0
- package/src/motion3/qa/spec-checks.js +65 -0
- package/src/motion3/qa/style.js +218 -0
- package/src/motion3/qa/wrist-checks.js +69 -0
- package/src/motion3/qa/wrist-obs.js +60 -0
- package/src/motion3/refs.js +120 -0
- package/src/motion3/repair-tune.js +121 -0
- package/src/motion3/repair.js +83 -0
- package/src/motion3/report.js +67 -0
- package/src/motion3/rigs/index.js +14 -0
- package/src/motion3/rigs/x-bot-tpose-body.json +1 -0
- package/src/motion3/rigs/x-bot-tpose.json +1 -0
- package/src/motion3/rigs/y-bot-tpose-body.json +1 -0
- package/src/motion3/rigs/y-bot-tpose.json +1 -0
- package/src/motion3/scene.js +60 -0
- package/src/motion3/schema/brief.js +89 -0
- package/src/motion3/schema/fields.js +44 -0
- package/src/motion3/schema/spec-examples.js +46 -0
- package/src/motion3/schema/topics.js +111 -0
- package/src/motion3/schema/types.js +147 -0
- package/src/motion3/search.js +117 -0
- package/src/motion3/second-hand.js +112 -0
- package/src/motion3/skills/basic.js +82 -0
- package/src/motion3/skills/book.js +76 -0
- package/src/motion3/skills/car.js +145 -0
- package/src/motion3/skills/frames-auto.js +29 -0
- package/src/motion3/skills/index.js +83 -0
- package/src/motion3/skills/info.js +65 -0
- package/src/motion3/skills/smoke-kit.js +109 -0
- package/src/motion3/skills/smoke.js +160 -0
- package/src/motion3/spec-store.js +59 -0
- package/src/motion3/take-edit.js +155 -0
- package/src/motion3/take.js +63 -0
- package/src/motion3/timeline.js +127 -0
- package/src/motion3/views.js +121 -0
- package/src/motion3/walk-plan.js +119 -0
- package/src/object-catalog.jsx +7 -1
- package/src/object-colors.js +98 -0
- package/src/object-gizmo-parts.js +115 -0
- package/src/object-gizmo.jsx +37 -122
- package/src/object-glow.js +139 -0
- package/src/object-hidden.js +33 -0
- package/src/object-look.js +196 -0
- package/src/object-size.js +10 -0
- package/src/object-travel.js +71 -6
- package/src/panels/CameraPanel.jsx +7 -7
- package/src/panels/HdrEnvironmentControls.jsx +59 -0
- package/src/panels/LightPanel.jsx +25 -4
- package/src/panels/ObjectLookPanel.jsx +88 -0
- package/src/panels/PosePanel.jsx +2 -17
- package/src/panels/RigControlPanel.jsx +5 -53
- package/src/panels/SubjectBox.jsx +4 -1
- package/src/panels/details.css +14 -0
- package/src/posestudio.jsx +4 -27
- package/src/project-browser.jsx +3 -4
- package/src/project-resources.js +18 -31
- package/src/project.js +13 -104
- package/src/props.jsx +25 -280
- package/src/rail-dot-drag.js +23 -0
- package/src/render-passes.js +10 -0
- package/src/resource-status.jsx +1 -0
- package/src/result-modal.jsx +4 -19
- package/src/room.jsx +20 -12
- package/src/scene-assets.js +8 -6
- package/src/scene-light-place.js +72 -0
- package/src/scene-lights.jsx +148 -0
- package/src/scene-objects.js +31 -139
- package/src/scenes.js +15 -105
- package/src/semantic-edit.js +2 -5
- package/src/set-pieces.jsx +183 -0
- package/src/settings-menu.jsx +3 -11
- package/src/shell/BottomDock.jsx +9 -51
- package/src/shell/DetailsSlot.jsx +16 -200
- package/src/shell/GhostsMenuGroup.jsx +74 -0
- package/src/shell/LibrarySlot.jsx +1 -3
- package/src/shell/MenuBar.jsx +3 -3
- package/src/shell/PreferencesDialog.jsx +15 -264
- package/src/shell/PreferencesSlot.jsx +2 -8
- package/src/shell/TopBar.jsx +5 -84
- package/src/shell/ViewportToolbar.jsx +10 -70
- package/src/shell/agent-glass.css +19 -6
- package/src/shell/details-selection.js +42 -0
- package/src/shell/ghosts-menu.css +22 -0
- package/src/shell/glass-regions.css +29 -3
- package/src/shell/preferences.css +6 -1
- package/src/shot-authoring.js +4 -7
- package/src/soft-shadows.js +37 -0
- package/src/stage-attach-frames.js +12 -12
- package/src/stage-canvas-rigs.jsx +3 -3
- package/src/stage-capture.jsx +42 -8
- package/src/stage-characters.jsx +22 -216
- package/src/stage-editor-gizmos.jsx +6 -6
- package/src/stage-environment.js +93 -0
- package/src/stage-hdr-builtins.js +107 -0
- package/src/stage-hdr-image.js +116 -0
- package/src/stage-hdr-library.js +70 -0
- package/src/stage-hdr-map.js +94 -0
- package/src/stage-hdr.js +130 -0
- package/src/stage-lighting.js +149 -0
- package/src/stage-occlusion.js +210 -0
- package/src/stage-sky.jsx +159 -0
- package/src/stage-startup-defaults.js +5 -9
- package/src/studio-actions.js +12 -14
- package/src/studio-agent-commands.js +19 -16
- package/src/studio-agent-motion.js +9 -9
- package/src/studio-agent-protocol.js +41 -71
- package/src/studio-app-binding.js +25 -45
- package/src/studio-asset-list.js +17 -0
- package/src/studio-contact-sheet.js +37 -5
- package/src/studio-elements.js +27 -7
- package/src/studio-entity-bounds.js +18 -0
- package/src/studio-error-codes.js +8 -0
- package/src/studio-export-framing.js +152 -0
- package/src/studio-lighting-check.js +103 -0
- package/src/studio-motion-visual.js +150 -0
- package/src/studio-schema-errors.js +80 -0
- package/src/studio-support-gap.js +74 -0
- package/src/styles.css +1 -19
- package/src/timeline-defaults.js +5 -0
- package/src/timeline-extent.js +8 -20
- package/src/tutorial-walk.js +17 -0
- package/src/ui-scrub.js +1 -1
- package/src/use-shot-camera-tracks.js +12 -0
- package/src/view-ghosts.js +97 -0
- package/src/view-ghosts.jsx +49 -0
- package/src/workflow/AgentPanel.jsx +47 -166
- package/src/workflow/AgentProgressCard.jsx +107 -0
- package/src/workflow/AgentReplyText.jsx +37 -0
- package/src/workflow/AgentTurnSteps.jsx +84 -0
- package/src/workflow/LoginPanel.jsx +140 -0
- package/src/workflow/WorkflowBuilder.jsx +7 -23
- package/src/workflow/agent-client.js +54 -51
- package/src/workflow/agent-panel.css +71 -1
- package/src/workflow/agent-progress-card.css +167 -0
- package/src/workflow/agent-progress-view.js +47 -0
- package/src/workflow/agent-turn-steps.css +113 -0
- package/src/workflow/agent-turn.js +191 -0
- package/src/workflow/login-client.js +34 -0
- package/src/workflow/motion-input.js +18 -31
- package/src/workflow/node-schema.js +1 -1
- package/src/workflow/reply-markdown.js +69 -0
- package/src/workflow/scene-asset-sync.js +8 -8
- package/tools/agent-session-stats.mjs +61 -0
- package/tools/dev-full.mjs +5 -56
- package/tools/eslint-baseline.json +12 -13
- package/tools/knip-baseline.json +0 -179
- package/tools/loc-baseline.json +31 -81
- package/tools/motion3/extract-body.mjs +103 -0
- package/tools/motion3/extract-rig.mjs +85 -0
- package/tools/process-supervisor.mjs +0 -88
- package/tools/run-tests.mjs +8 -124
- package/bin/agent/motion-runtime.mjs +0 -76
- package/dist/ardy/cskel27-rest.json +0 -557
- package/dist/assets/analytics-C9pMpW7s.js +0 -1
- package/dist/assets/app-B_2XB9rw.css +0 -1
- package/dist/assets/app-DAuvKPP8.js +0 -4920
- package/dist/assets/config-Bx7pYsD9.js +0 -1
- package/dist/assets/config-DeoFTYzb.css +0 -1
- package/dist/assets/demo-CvKuBgy5.js +0 -1
- package/dist/assets/scene-asset-sync-DKkpjKBV.js +0 -19
- package/dist/assets/scene-asset-sync-DjJ_cJ2R.css +0 -1
- package/dist/assets/ticket-BPk2Ni95.js +0 -1
- package/dist/assets/workflow-jJRBR94Z.js +0 -179
- package/dist/demo/walk-then-stop.npz +0 -0
- package/mcp/ardy-prompts.mjs +0 -218
- package/src/ardy/auto-fix-panel.css +0 -108
- package/src/ardy/auto-physics.js +0 -1194
- package/src/ardy/client.js +0 -153
- package/src/ardy/convert.js +0 -313
- package/src/ardy/cskel27-neutral.js +0 -39
- package/src/ardy/cskel27.js +0 -77
- package/src/ardy/export.js +0 -165
- package/src/ardy/fix-collisions.js +0 -2257
- package/src/ardy/foot-lock.js +0 -230
- package/src/ardy/motion-calibration.js +0 -118
- package/src/ardy/motion-edit.js +0 -213
- package/src/ardy/motion-url.js +0 -55
- package/src/ardy/mp4-duration.js +0 -102
- package/src/ardy/npz.js +0 -595
- package/src/ardy/physics-panel.css +0 -21
- package/src/ardy/physics-panel.jsx +0 -89
- package/src/ardy/physics-refine.js +0 -111
- package/src/ardy/physics-review.js +0 -567
- package/src/ardy/physics-temporal.js +0 -54
- package/src/ardy/platform-fit-panel.css +0 -64
- package/src/ardy/platform-fit-panel.jsx +0 -41
- package/src/ardy/platform-fit.js +0 -326
- package/src/ardy/playback.js +0 -711
- package/src/ardy/pose-pin.js +0 -97
- package/src/ardy/prompt-clips.js +0 -30
- package/src/ardy/retime.js +0 -211
- package/src/ardy/root-drop.js +0 -369
- package/src/ardy/timeline-resize.js +0 -13
- package/src/ardy/to-cskel27.js +0 -225
- package/src/ardy/trim.js +0 -33
- package/src/ardy/waypoints.js +0 -369
- package/src/ardy-bridge-contract.js +0 -281
- package/src/domains/motion/motion-video-capture.js +0 -664
- package/src/i2v-motion-client.js +0 -77
- package/src/i2v-motion-studio.jsx +0 -180
- package/src/line-edit.js +0 -1348
- package/src/motion/generation.js +0 -105
- package/src/motion-readiness-ui.jsx +0 -67
- package/src/motion-readiness.js +0 -40
- package/src/motion-resources.js +0 -177
- package/src/motion-store.js +0 -65
- package/src/motion-trail.js +0 -601
- package/src/multimodel-ingest.js +0 -394
- package/src/panels/PromptBlocksPanel.jsx +0 -444
- package/src/panels/VideoCapturePanel.jsx +0 -181
- package/src/pose-extract/browser.js +0 -42
- package/src/pose-extract/detector.js +0 -100
- package/src/pose-extract/fit.js +0 -227
- package/src/pose-extract/image-frame.js +0 -51
- package/src/pose-extract/index.js +0 -58
- package/src/pose-extract/keyframes.js +0 -74
- package/src/pose-extract/landmarks.js +0 -162
- package/src/pose-extract/mirror.js +0 -153
- package/src/pose-extract/take.js +0 -131
- package/src/pose-extract/video-frames.js +0 -91
- package/src/take-recipe.js +0 -165
- package/src/trail-key-conflicts.js +0 -39
- package/src/trail-pick.js +0 -59
- package/src/use-i2v-capture-modal.js +0 -15
- package/tools/ardy/BRIDGE.md +0 -37
- package/tools/ardy/GVHMR-NPZ.md +0 -68
- package/tools/ardy/README.md +0 -124
- package/tools/ardy/__pycache__/cclay_gvhmr_worker.cpython-313.pyc +0 -0
- package/tools/ardy/artifacts.mjs +0 -34
- package/tools/ardy/bridge.mjs +0 -1732
- package/tools/ardy/bvh-cskel27.mjs +0 -1265
- package/tools/ardy/cclay_constrained_generate.py +0 -561
- package/tools/ardy/cclay_gvhmr_worker.py +0 -264
- package/tools/ardy/cclay_motion_edit.py +0 -510
- package/tools/ardy/cclay_sequence_generate.py +0 -705
- package/tools/ardy/cclay_worker.py +0 -304
- package/tools/ardy/dump-npz.py +0 -205
- package/tools/ardy/extract-rest.mjs +0 -299
- package/tools/ardy/extract.mjs +0 -507
- package/tools/ardy/footage.mjs +0 -467
- package/tools/ardy/gvhmr-floor.mjs +0 -102
- package/tools/ardy/gvhmr-to-cskel27.mjs +0 -21
- package/tools/ardy/gvhmr_fastpath.py +0 -168
- package/tools/ardy/gvhmr_trajectory.py +0 -240
- package/tools/ardy/merge-text-encoder.py +0 -160
- package/tools/ardy/mocap-metrics.mjs +0 -200
- package/tools/ardy/mocap-quality-gate.mjs +0 -110
- package/tools/ardy/motion-stabilize.mjs +0 -348
- package/tools/ardy/npz.mjs +0 -414
- package/tools/ardy/pose-to-npz.mjs +0 -106
- package/tools/ardy/prompt-limits.mjs +0 -5
- package/tools/ardy/run-edit-on-box.sh +0 -73
- package/tools/ardy/run-local.mjs +0 -233
- package/tools/ardy/run-on-box.sh +0 -593
- package/tools/ardy/run-sequence-on-box.sh +0 -177
- package/tools/ardy/runners/gvhmr-worker.mjs +0 -206
- package/tools/ardy/runners/index.mjs +0 -18
- package/tools/ardy/runners/local.mjs +0 -455
- package/tools/ardy/runners/proc.mjs +0 -125
- package/tools/ardy/runners/remote.mjs +0 -252
- package/tools/ardy/setup-local.mjs +0 -189
- package/tools/ardy/setup-text-encoder-on-box.sh +0 -93
- package/tools/ardy/setup-text-encoder.py +0 -306
- package/tools/ardy/smpl-cskel27.mjs +0 -178
- package/tools/ardy/visual-qa.mjs +0 -196
- package/tools/ardy/vq-car.mjs +0 -52
- package/tools/bench/EXP3.md +0 -61
- package/tools/bench/cclay_bench_extract_incam.py +0 -125
- package/tools/bench/cclay_bench_extract_obs.py +0 -204
- package/tools/bench/cclay_bench_runner.py +0 -42
- package/tools/bench/cube-contact.mjs +0 -162
- package/tools/bench/exp3.mjs +0 -152
- package/tools/bench/extract-bench-lib.mjs +0 -216
- package/tools/bench/extract-bench.mjs +0 -302
- package/tools/bench/fal-generate.mjs +0 -88
- package/tools/bench/fit/README.md +0 -189
- package/tools/bench/fit/camera.mjs +0 -59
- package/tools/bench/fit/contact.mjs +0 -419
- package/tools/bench/fit/footlock.mjs +0 -234
- package/tools/bench/fit/motion.mjs +0 -85
- package/tools/bench/fit/pin.mjs +0 -27
- package/tools/bench/fit/remote.mjs +0 -105
- package/tools/bench/fit-bench.mjs +0 -126
- package/tools/bench/fit-sanity.mjs +0 -45
- package/tools/bench/metrics.mjs +0 -313
- package/tools/bench/obs/depth.mjs +0 -258
- package/tools/bench/obs/extrinsics.mjs +0 -175
- package/tools/bench/obs/fit_mannequin_betas.py +0 -340
- package/tools/bench/obs/ground.mjs +0 -294
- package/tools/bench/obs/heading.mjs +0 -151
- package/tools/bench/obs/ladder.mjs +0 -555
- package/tools/bench/obs/mannequin-betas.json +0 -124
- package/tools/bench/obs/remote.mjs +0 -83
- package/tools/bench/obs/rest_joints.py +0 -52
- package/tools/bench/obs/ybot-targets.mjs +0 -49
- package/tools/bench/obs-bench.mjs +0 -520
- package/tools/bench/score.mjs +0 -343
- package/tools/bench/summarize.mjs +0 -86
- package/tools/demo-worker/README.md +0 -137
- package/tools/demo-worker/api-client.mjs +0 -233
- package/tools/demo-worker/generate.mjs +0 -301
- package/tools/demo-worker/index.mjs +0 -466
- package/tools/demo-worker/sign-core.mjs +0 -112
- package/tools/demo-worker/sign.mjs +0 -161
- package/tools/gt-render/browser.mjs +0 -182
- package/tools/gt-render/camera-math.mjs +0 -276
- package/tools/gt-render/page.mjs +0 -267
- package/tools/gt-render/render.mjs +0 -514
- package/tools/gt-render/scene-box.mjs +0 -31
- package/tools/gt-render/take-transform.mjs +0 -87
- package/tools/kimodo/compare-seam.mjs +0 -108
- package/tools/kimodo/constraints.mjs +0 -244
- package/tools/kimodo/edit.mjs +0 -259
- package/tools/kimodo/effector-constraints.mjs +0 -200
- package/tools/kimodo/generate.mjs +0 -646
- package/tools/kimodo/local-output.mjs +0 -265
- package/tools/kimodo/measure-pose.mjs +0 -65
- package/tools/kimodo/measure-preserve.mjs +0 -899
- package/tools/kimodo/measure-waypoints.mjs +0 -55
- package/tools/kimodo/mlx-generate.py +0 -111
- package/tools/kimodo/pose-constraints.mjs +0 -248
- package/tools/kimodo/preserve-mask.mjs +0 -543
- package/tools/kimodo/read-npz.mjs +0 -189
- package/tools/kimodo/run-edit-on-box.mjs +0 -155
- package/tools/kimodo/run-sequence-on-box.mjs +0 -178
- package/tools/kimodo/runner.mjs +0 -284
- package/tools/kimodo/seam-batch.mjs +0 -77
- package/tools/kimodo/setup-local.mjs +0 -30
- package/tools/kimodo/setup-on-box.sh +0 -339
- package/tools/kimodo/soma77-to-cskel27.mjs +0 -305
- package/tools/morphgs/assets/gen_truth.py +0 -39
- package/tools/morphgs/assets/mesh_ori_rig.txt +0 -27
- package/tools/morphgs/assets/playback-check.mjs +0 -30
- package/tools/morphgs/demo-gate.sh +0 -26
- package/tools/morphgs/fbx2morphgs.mjs +0 -68
- package/tools/morphgs/morphgs-to-cskel27.mjs +0 -105
- package/tools/morphgs/patches/preprocess_src-none-mode.patch +0 -76
- package/tools/morphgs/setup-on-cluster.sh +0 -126
- package/tools/projflow/driver.py +0 -1069
- package/tools/projflow/generate.mjs +0 -805
- package/tools/projflow/hml22-to-cskel27.mjs +0 -735
- package/tools/projflow/line-edit-job.mjs +0 -547
- package/tools/projflow/replay.mjs +0 -452
- package/tools/projflow/runner.mjs +0 -269
- package/tools/projflow/service.mjs +0 -742
- package/tools/track/DESIGN.md +0 -513
- package/tools/track/backfill-provenance.mjs +0 -116
- package/tools/track/budget.mjs +0 -8
- package/tools/track/check-rig.mjs +0 -209
- package/tools/track/diagnostics.schema.json +0 -60
- package/tools/track/export-rig.mjs +0 -256
- package/tools/track/fallback.mjs +0 -7
- package/tools/track/fk-parity-fixture.mjs +0 -161
- package/tools/track/gate.mjs +0 -349
- package/tools/track/masks.mjs +0 -318
- package/tools/track/metrics.mjs +0 -201
- package/tools/track/publish-obs.mjs +0 -108
- package/tools/track/py/check_env.py +0 -61
- package/tools/track/py/eval_lr.py +0 -229
- package/tools/track/py/lr_viterbi.py +0 -294
- package/tools/track/py/masks.py +0 -405
- package/tools/track/py/objective.py +0 -382
- package/tools/track/py/rig.py +0 -508
- package/tools/track/py/scene.py +0 -326
- package/tools/track/py/test_joint_indices.py +0 -123
- package/tools/track/py/test_lr_viterbi.py +0 -195
- package/tools/track/py/test_masks.py +0 -229
- package/tools/track/py/test_rig.py +0 -288
- package/tools/track/py/test_scene.py +0 -354
- package/tools/track/py/test_track.py +0 -328
- package/tools/track/py/track.py +0 -411
- package/tools/track/remote.mjs +0 -140
- package/tools/track/rig-dump.mjs +0 -227
- package/tools/track/run-box-tests.mjs +0 -14
- package/tools/track/run-box.mjs +0 -168
- package/tools/track/setup-box.sh +0 -59
- package/tools/track/study-2d.mjs +0 -728
- /package/src/{ardy → motion}/collision-blockers.js +0 -0
- /package/src/{ardy → motion}/ground.js +0 -0
- /package/src/{ardy → motion}/ik-drag.js +0 -0
- /package/src/{ardy → motion}/ik-key-json.js +0 -0
- /package/src/{ardy → motion}/ik.js +0 -0
- /package/src/{ardy → motion}/physics-surface.js +0 -0
- /package/src/{ardy → motion}/playback-clock.js +0 -0
- /package/src/{ardy → motion}/range-pin.js +0 -0
package/tools/track/py/rig.py
DELETED
|
@@ -1,508 +0,0 @@
|
|
|
1
|
-
"""Differentiable Studio rig for the known-character tracker (plan mocap-rearch todo 6).
|
|
2
|
-
|
|
3
|
-
Two deliberately separate layers:
|
|
4
|
-
|
|
5
|
-
1. `cskel27_fk(state, bone_scale, rig)` -> posedJoints. cskel27 forward kinematics
|
|
6
|
-
over the canonical neutral skeleton grown by `bone_scale` - exactly the
|
|
7
|
-
quantity a motion NPZ stores as `posed_joints` (tools/ardy/smpl-cskel27.mjs,
|
|
8
|
-
tools/bench/fit/motion.mjs regenerateJoints) and the scorers read. Keypoint and
|
|
9
|
-
contact terms use these joints.
|
|
10
|
-
2. `studio_skin(globals, posed_joints, rig, ...)` -> bone world transforms ->
|
|
11
|
-
vertices. A torch port of Studio playback's positional skinning
|
|
12
|
-
(src/ardy/playback.js applyMotionFrame, prepOf, scaledOffsets, boneStretch)
|
|
13
|
-
followed by linear-blend skinning of the exported rig mesh. Silhouette and box
|
|
14
|
-
terms use these vertices.
|
|
15
|
-
|
|
16
|
-
State convention (NPZ `local_rot_mats` convention): per frame
|
|
17
|
-
transl (T, 3) root/pelvis world position = posed_joints[:, 0]
|
|
18
|
-
root6d (T, 6) a_t, the root rotation = local_rot_mats[:, 0]
|
|
19
|
-
local6d (T, 26, 6) u_t,j for cskel27 joints 1..26 (local to the parent)
|
|
20
|
-
Zero pose = identity locals = the neutral T-pose, so 6D vectors are the rotations
|
|
21
|
-
themselves (no rest delta). `rot6d_to_matrix` follows the ultrabrain design
|
|
22
|
-
section 1: the 6D vector is the first two COLUMNS of the matrix, Gram-Schmidt.
|
|
23
|
-
|
|
24
|
-
Rig input: the npz written by tools/track/export-rig.mjs (layout and units in
|
|
25
|
-
tools/track/rig-dump.mjs). `load_rig` validates it (`RigError.code`:
|
|
26
|
-
`bad-rig`, `bad-skin-weights`) and decimates the merged mesh to <= 8,000 faces,
|
|
27
|
-
carrying skin weights from the nearest original vertex (renormalised).
|
|
28
|
-
|
|
29
|
-
World frame: Studio world, metres, +Y up. `studio_skin` returns the rig as Studio
|
|
30
|
-
places it under a Character group at the origin with no clip-to-scene yaw:
|
|
31
|
-
rig-space bone = s * (posed - anchor_xz) + R_global @ offset
|
|
32
|
-
with s = prep scale (rig leg height / canonical leg height, rig units) and the
|
|
33
|
-
0.01 rig root scale taking rig units to metres. Pass `anchor_xz=None` for the
|
|
34
|
-
tracker (no anchoring: vertices live in the posedJoints frame, scaled by the
|
|
35
|
-
rig's leg ratio about the XZ origin exactly as Studio scales them about the
|
|
36
|
-
anchor), or the anchor frame's root XZ to reproduce Studio playback bit-for-bit.
|
|
37
|
-
Nothing here optimises bone lengths, shape or camera.
|
|
38
|
-
"""
|
|
39
|
-
from __future__ import annotations
|
|
40
|
-
|
|
41
|
-
import json
|
|
42
|
-
from dataclasses import dataclass
|
|
43
|
-
from pathlib import Path
|
|
44
|
-
|
|
45
|
-
import numpy as np
|
|
46
|
-
import torch
|
|
47
|
-
|
|
48
|
-
J = 27
|
|
49
|
-
INFLUENCES = 4
|
|
50
|
-
WEIGHT_TOLERANCE = 1e-5 # same as tools/track/rig-dump.mjs
|
|
51
|
-
MAX_FACES = 8000
|
|
52
|
-
EPS = 1e-8
|
|
53
|
-
|
|
54
|
-
REQUIRED_MEMBERS = (
|
|
55
|
-
"meta_json", "rig_root_scale", "vertices", "faces", "skin_index", "skin_weight",
|
|
56
|
-
"bone_parent", "bone_rest_local", "bone_bind_inverse",
|
|
57
|
-
"prep_bone", "prep_scale", "prep_offsets", "prep_bind_pos", "prep_bind_quat",
|
|
58
|
-
"prep_bind_scale", "prep_bind_local_pos", "prep_parent_bind_world",
|
|
59
|
-
"prep_chain_parent", "prep_chain_rel", "prep_canonical_bone_length",
|
|
60
|
-
"prep_rig_bone_length", "prep_stretched_leaves", "prep_stretched_leaf_local_pos",
|
|
61
|
-
"prep_stretched_leaf_local_quat", "hierarchy_preserved", "girdle_joints",
|
|
62
|
-
"ardy_neutral_min_y", "cskel27_parents", "cskel27_neutral",
|
|
63
|
-
)
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
class RigError(ValueError):
|
|
67
|
-
"""Malformed rig input. `code` is the machine-readable reason."""
|
|
68
|
-
|
|
69
|
-
def __init__(self, code: str, detail: str):
|
|
70
|
-
super().__init__(f"{code}: {detail}")
|
|
71
|
-
self.code = code
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
# --- rotations -----------------------------------------------------------------
|
|
75
|
-
|
|
76
|
-
def rot6d_to_matrix(a: torch.Tensor) -> torch.Tensor:
|
|
77
|
-
"""(..., 6) -> (..., 3, 3). a = [a1, a2]; columns b1, b2, b3 (Gram-Schmidt)."""
|
|
78
|
-
# Clamp (not add) the epsilon: columns stay exactly unit length unless degenerate.
|
|
79
|
-
a1, a2 = a[..., 0:3], a[..., 3:6]
|
|
80
|
-
b1 = a1 / a1.norm(dim=-1, keepdim=True).clamp_min(EPS)
|
|
81
|
-
a2 = a2 - b1 * (b1 * a2).sum(dim=-1, keepdim=True)
|
|
82
|
-
b2 = a2 / a2.norm(dim=-1, keepdim=True).clamp_min(EPS)
|
|
83
|
-
b3 = torch.cross(b1, b2, dim=-1)
|
|
84
|
-
return torch.stack((b1, b2, b3), dim=-1)
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
def matrix_to_rot6d(R: torch.Tensor) -> torch.Tensor:
|
|
88
|
-
"""(..., 3, 3) -> (..., 6): the first two columns."""
|
|
89
|
-
return torch.cat((R[..., :, 0], R[..., :, 1]), dim=-1)
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
def quat_to_matrix(q: torch.Tensor) -> torch.Tensor:
|
|
93
|
-
"""(..., 4) three.js [x, y, z, w] (normalised here) -> (..., 3, 3)."""
|
|
94
|
-
q = q / q.norm(dim=-1, keepdim=True)
|
|
95
|
-
x, y, z, w = q.unbind(-1)
|
|
96
|
-
return torch.stack((
|
|
97
|
-
1 - 2 * (y * y + z * z), 2 * (x * y - w * z), 2 * (x * z + w * y),
|
|
98
|
-
2 * (x * y + w * z), 1 - 2 * (x * x + z * z), 2 * (y * z - w * x),
|
|
99
|
-
2 * (x * z - w * y), 2 * (y * z + w * x), 1 - 2 * (x * x + y * y),
|
|
100
|
-
), dim=-1).reshape(q.shape[:-1] + (3, 3))
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
def _affine(R: torch.Tensor, t: torch.Tensor) -> torch.Tensor:
|
|
104
|
-
"""(..., 3, 3), (..., 3) -> (..., 4, 4)."""
|
|
105
|
-
top = torch.cat((R, t.unsqueeze(-1)), dim=-1)
|
|
106
|
-
bottom = torch.zeros(top.shape[:-2] + (1, 4), dtype=top.dtype, device=top.device)
|
|
107
|
-
bottom[..., 0, 3] = 1
|
|
108
|
-
return torch.cat((top, bottom), dim=-2)
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
def _apply(M: torch.Tensor, p: torch.Tensor) -> torch.Tensor:
|
|
112
|
-
"""Affine (..., 4, 4) applied to points (..., 3)."""
|
|
113
|
-
return (M[..., :3, :3] @ p.unsqueeze(-1)).squeeze(-1) + M[..., :3, 3]
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
# --- rig -----------------------------------------------------------------------
|
|
117
|
-
|
|
118
|
-
@dataclass
|
|
119
|
-
class Rig:
|
|
120
|
-
"""Exported Studio rig as torch tensors (float, on `device`)."""
|
|
121
|
-
|
|
122
|
-
meta: dict
|
|
123
|
-
parents: list # cskel27 parent per joint, -1 for the root
|
|
124
|
-
neutral: torch.Tensor # (27, 3) canonical neutral, hips at the origin
|
|
125
|
-
min_y: float # ARDY_NEUTRAL_MIN_Y
|
|
126
|
-
root_scale: float # rig units -> metres (0.01)
|
|
127
|
-
bone_names: list
|
|
128
|
-
bone_parent: list # per exported bone, -1 for bones under the (static) rig root
|
|
129
|
-
bone_rest_local: torch.Tensor # (B, 4, 4) rig units
|
|
130
|
-
bone_rest_scale: torch.Tensor # (B, 3)
|
|
131
|
-
bind_inverse: torch.Tensor # (B, 4, 4) metres
|
|
132
|
-
# prepOf
|
|
133
|
-
prep_bone: list # cskel27 -> bone index or -1
|
|
134
|
-
prep_scale: float
|
|
135
|
-
prep_offsets: torch.Tensor # (27, 3)
|
|
136
|
-
prep_bind_pos: torch.Tensor # (27, 3)
|
|
137
|
-
prep_bind_rot: torch.Tensor # (27, 3, 3)
|
|
138
|
-
prep_bind_scale: torch.Tensor # (27, 3)
|
|
139
|
-
prep_bind_local_pos: torch.Tensor # (27, 3)
|
|
140
|
-
prep_parent_bind_world: torch.Tensor # (27, 4, 4)
|
|
141
|
-
prep_chain_parent: list
|
|
142
|
-
prep_chain_rel: torch.Tensor # (27, 4, 4)
|
|
143
|
-
canonical_bone_length: np.ndarray # (27,)
|
|
144
|
-
rig_bone_length: np.ndarray # (27,)
|
|
145
|
-
leaves: list # [(bone, joint, local_pos (3,), local_rot (3, 3))]
|
|
146
|
-
hierarchy_preserved: np.ndarray # (27,) bool
|
|
147
|
-
girdle: np.ndarray # (27,) bool
|
|
148
|
-
# full-resolution merged mesh (rest pose, metres)
|
|
149
|
-
vertices: torch.Tensor # (V, 3)
|
|
150
|
-
skin_index: torch.Tensor # (V, 4) long
|
|
151
|
-
skin_weight: torch.Tensor # (V, 4)
|
|
152
|
-
# decimated mesh used by the tracker's silhouette/box terms
|
|
153
|
-
dec_vertices: torch.Tensor # (Vd, 3)
|
|
154
|
-
dec_faces: torch.Tensor # (Fd, 3) long
|
|
155
|
-
dec_skin_index: torch.Tensor # (Vd, 4) long
|
|
156
|
-
dec_skin_weight: torch.Tensor # (Vd, 4)
|
|
157
|
-
dec_weights: torch.Tensor # (Vd, B) dense blend weights
|
|
158
|
-
dec_source: np.ndarray # (Vd,) nearest original vertex
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
def _require(z, name, shape=None, dtype_kind=None):
|
|
162
|
-
if name not in z:
|
|
163
|
-
raise RigError("bad-rig", f"missing member {name}")
|
|
164
|
-
value = np.asarray(z[name])
|
|
165
|
-
if shape is not None and (value.ndim != len(shape) or any(s is not None and s != d for s, d in zip(shape, value.shape))):
|
|
166
|
-
raise RigError("bad-rig", f"{name} has shape {value.shape}, expected {shape}")
|
|
167
|
-
if dtype_kind == "f" and not np.all(np.isfinite(value)):
|
|
168
|
-
raise RigError("bad-rig", f"{name} has non-finite values")
|
|
169
|
-
return value
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
def validate_skin_weights(skin_weight: np.ndarray, tol: float = WEIGHT_TOLERANCE) -> None:
|
|
173
|
-
"""Raise `bad-skin-weights` unless every row is non-negative and sums to 1."""
|
|
174
|
-
w = np.asarray(skin_weight, dtype=np.float64)
|
|
175
|
-
if not np.all(np.isfinite(w)):
|
|
176
|
-
raise RigError("bad-skin-weights", "non-finite skin weight")
|
|
177
|
-
if (w < 0).any():
|
|
178
|
-
v = int(np.argwhere(w < 0)[0, 0])
|
|
179
|
-
raise RigError("bad-skin-weights", f"vertex {v} has a negative weight")
|
|
180
|
-
err = np.abs(w.sum(axis=1) - 1)
|
|
181
|
-
if (err > tol).any():
|
|
182
|
-
v = int(err.argmax())
|
|
183
|
-
raise RigError("bad-skin-weights", f"vertex {v} weights sum to {w[v].sum():.7f}")
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
def decimate(vertices: np.ndarray, faces: np.ndarray, skin_index: np.ndarray, skin_weight: np.ndarray,
|
|
187
|
-
max_faces: int = MAX_FACES):
|
|
188
|
-
"""Weld, simplify to <= max_faces, transfer skin weights from the nearest original vertex.
|
|
189
|
-
|
|
190
|
-
Returns (vertices (Vd,3), faces (Fd,3), skin_index (Vd,4), skin_weight (Vd,4), source (Vd,)).
|
|
191
|
-
The export is unindexed per triangle, so identical positions are welded first
|
|
192
|
-
(fast_simplification needs shared vertices to collapse edges).
|
|
193
|
-
"""
|
|
194
|
-
import fast_simplification
|
|
195
|
-
from scipy.spatial import cKDTree
|
|
196
|
-
|
|
197
|
-
welded, first, inverse = np.unique(vertices, axis=0, return_index=True, return_inverse=True)
|
|
198
|
-
inverse = inverse.reshape(-1)
|
|
199
|
-
wf = inverse[faces]
|
|
200
|
-
keep = (wf[:, 0] != wf[:, 1]) & (wf[:, 1] != wf[:, 2]) & (wf[:, 0] != wf[:, 2])
|
|
201
|
-
wf = wf[keep]
|
|
202
|
-
reduction = 1.0 - max_faces / len(wf) if len(wf) > max_faces else 0.0
|
|
203
|
-
for _ in range(20):
|
|
204
|
-
if reduction <= 0:
|
|
205
|
-
points, out_faces = welded, wf
|
|
206
|
-
else:
|
|
207
|
-
points, out_faces = fast_simplification.simplify(
|
|
208
|
-
welded.astype(np.float32), wf.astype(np.int32), target_reduction=min(reduction, 0.999))
|
|
209
|
-
if len(out_faces) <= max_faces:
|
|
210
|
-
break
|
|
211
|
-
reduction += (1 - reduction) * 0.05
|
|
212
|
-
else:
|
|
213
|
-
raise RigError("bad-rig", f"could not decimate below {max_faces} faces")
|
|
214
|
-
points = np.asarray(points, dtype=np.float32)
|
|
215
|
-
out_faces = np.asarray(out_faces, dtype=np.int64)
|
|
216
|
-
used = np.unique(out_faces) # drop vertices no face references
|
|
217
|
-
remap = np.full(len(points), -1, dtype=np.int64)
|
|
218
|
-
remap[used] = np.arange(len(used))
|
|
219
|
-
points, out_faces = points[used], remap[out_faces]
|
|
220
|
-
_, nearest = cKDTree(vertices).query(points, k=1)
|
|
221
|
-
source = np.asarray(nearest, dtype=np.int64)
|
|
222
|
-
w = skin_weight[source].astype(np.float64)
|
|
223
|
-
w = w / w.sum(axis=1, keepdims=True)
|
|
224
|
-
return points, out_faces, skin_index[source].astype(np.int64), w.astype(np.float32), source
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
def load_rig(path, device="cpu", dtype=torch.float32, max_faces: int = MAX_FACES) -> Rig:
|
|
228
|
-
path = Path(path)
|
|
229
|
-
if not path.is_file():
|
|
230
|
-
raise RigError("bad-rig", f"rig npz not found: {path}")
|
|
231
|
-
with np.load(path) as z:
|
|
232
|
-
missing = [name for name in REQUIRED_MEMBERS if name not in z]
|
|
233
|
-
if missing:
|
|
234
|
-
raise RigError("bad-rig", f"missing members: {', '.join(missing)}")
|
|
235
|
-
meta = json.loads(bytes(z["meta_json"].astype(np.uint8)))
|
|
236
|
-
vertices = _require(z, "vertices", (None, 3), "f").astype(np.float32)
|
|
237
|
-
V = len(vertices)
|
|
238
|
-
faces = _require(z, "faces", (None, 3)).astype(np.int64)
|
|
239
|
-
skin_index = _require(z, "skin_index", (V, INFLUENCES)).astype(np.int64)
|
|
240
|
-
skin_weight = _require(z, "skin_weight", (V, INFLUENCES), "f").astype(np.float32)
|
|
241
|
-
bone_parent = _require(z, "bone_parent", (None,)).astype(np.int64)
|
|
242
|
-
B = len(bone_parent)
|
|
243
|
-
arrays = {name: _require(z, name, shape, "f") for name, shape in (
|
|
244
|
-
("bone_rest_local", (B, 4, 4)), ("bone_bind_inverse", (B, 4, 4)),
|
|
245
|
-
("prep_offsets", (J, 3)), ("prep_bind_pos", (J, 3)), ("prep_bind_quat", (J, 4)),
|
|
246
|
-
("prep_bind_scale", (J, 3)), ("prep_bind_local_pos", (J, 3)),
|
|
247
|
-
("prep_parent_bind_world", (J, 4, 4)), ("prep_chain_rel", (J, 4, 4)),
|
|
248
|
-
("prep_canonical_bone_length", (J,)), ("prep_rig_bone_length", (J,)),
|
|
249
|
-
("cskel27_neutral", (J, 3)), ("rig_root_scale", (1,)), ("prep_scale", (1,)),
|
|
250
|
-
("ardy_neutral_min_y", (1,)),
|
|
251
|
-
)}
|
|
252
|
-
prep_bone = _require(z, "prep_bone", (J,)).astype(np.int64)
|
|
253
|
-
chain_parent = _require(z, "prep_chain_parent", (J,)).astype(np.int64)
|
|
254
|
-
parents = _require(z, "cskel27_parents", (J,)).astype(np.int64)
|
|
255
|
-
leaves_idx = _require(z, "prep_stretched_leaves", (None, 2)).astype(np.int64)
|
|
256
|
-
leaf_pos = _require(z, "prep_stretched_leaf_local_pos", (len(leaves_idx), 3), "f")
|
|
257
|
-
leaf_quat = _require(z, "prep_stretched_leaf_local_quat", (len(leaves_idx), 4), "f")
|
|
258
|
-
preserved = _require(z, "hierarchy_preserved", (J,)).astype(bool)
|
|
259
|
-
girdle = _require(z, "girdle_joints", (J,)).astype(bool)
|
|
260
|
-
|
|
261
|
-
if len(meta.get("boneNames", [])) != B:
|
|
262
|
-
raise RigError("bad-rig", f"meta boneNames has {len(meta.get('boneNames', []))} names for {B} bones")
|
|
263
|
-
if faces.size and (faces.min() < 0 or faces.max() >= V):
|
|
264
|
-
raise RigError("bad-rig", "face index out of range")
|
|
265
|
-
if skin_index.min() < 0 or skin_index.max() >= B:
|
|
266
|
-
raise RigError("bad-rig", "skin index out of range")
|
|
267
|
-
for i, p in enumerate(bone_parent):
|
|
268
|
-
if not -1 <= p < i:
|
|
269
|
-
raise RigError("bad-rig", f"bone {i} parent {p} does not precede it")
|
|
270
|
-
for j, p in enumerate(parents):
|
|
271
|
-
if not (p == -1 if j == 0 else 0 <= p < j):
|
|
272
|
-
raise RigError("bad-rig", f"cskel27 parent {p} of joint {j} is not topological")
|
|
273
|
-
for j in range(J):
|
|
274
|
-
if not -1 <= prep_bone[j] < B:
|
|
275
|
-
raise RigError("bad-rig", f"prep_bone[{j}] out of range")
|
|
276
|
-
cp = chain_parent[j]
|
|
277
|
-
if cp >= 0 and (cp >= j or prep_bone[cp] < 0 or prep_bone[j] < 0):
|
|
278
|
-
raise RigError("bad-rig", f"prep_chain_parent[{j}]={cp} must be an earlier mapped joint")
|
|
279
|
-
validate_skin_weights(skin_weight)
|
|
280
|
-
|
|
281
|
-
dv, df, dsi, dsw, source = decimate(vertices, faces, skin_index, skin_weight, max_faces)
|
|
282
|
-
dense = np.zeros((len(dv), B), dtype=np.float32)
|
|
283
|
-
np.add.at(dense, (np.repeat(np.arange(len(dv)), INFLUENCES), dsi.reshape(-1)), dsw.reshape(-1))
|
|
284
|
-
|
|
285
|
-
t = lambda a: torch.as_tensor(np.asarray(a, dtype=np.float32), device=device).to(dtype) # noqa: E731
|
|
286
|
-
rest_local = arrays["bone_rest_local"]
|
|
287
|
-
return Rig(
|
|
288
|
-
meta=meta,
|
|
289
|
-
parents=[int(p) for p in parents],
|
|
290
|
-
neutral=t(arrays["cskel27_neutral"]),
|
|
291
|
-
min_y=float(arrays["ardy_neutral_min_y"][0]),
|
|
292
|
-
root_scale=float(arrays["rig_root_scale"][0]),
|
|
293
|
-
bone_names=list(meta["boneNames"]),
|
|
294
|
-
bone_parent=[int(p) for p in bone_parent],
|
|
295
|
-
bone_rest_local=t(rest_local),
|
|
296
|
-
bone_rest_scale=t(np.linalg.norm(rest_local[:, :3, :3], axis=1)),
|
|
297
|
-
bind_inverse=t(arrays["bone_bind_inverse"]),
|
|
298
|
-
prep_bone=[int(b) for b in prep_bone],
|
|
299
|
-
prep_scale=float(arrays["prep_scale"][0]),
|
|
300
|
-
prep_offsets=t(arrays["prep_offsets"]),
|
|
301
|
-
prep_bind_pos=t(arrays["prep_bind_pos"]),
|
|
302
|
-
prep_bind_rot=quat_to_matrix(t(arrays["prep_bind_quat"])),
|
|
303
|
-
prep_bind_scale=t(arrays["prep_bind_scale"]),
|
|
304
|
-
prep_bind_local_pos=t(arrays["prep_bind_local_pos"]),
|
|
305
|
-
prep_parent_bind_world=t(arrays["prep_parent_bind_world"]),
|
|
306
|
-
prep_chain_parent=[int(c) for c in chain_parent],
|
|
307
|
-
prep_chain_rel=t(arrays["prep_chain_rel"]),
|
|
308
|
-
canonical_bone_length=np.asarray(arrays["prep_canonical_bone_length"], dtype=np.float64),
|
|
309
|
-
rig_bone_length=np.asarray(arrays["prep_rig_bone_length"], dtype=np.float64),
|
|
310
|
-
leaves=[(int(b), int(j), t(p), quat_to_matrix(t(q))) for (b, j), p, q in zip(leaves_idx, leaf_pos, leaf_quat)],
|
|
311
|
-
hierarchy_preserved=preserved,
|
|
312
|
-
girdle=girdle,
|
|
313
|
-
vertices=t(vertices),
|
|
314
|
-
skin_index=torch.as_tensor(skin_index, device=device),
|
|
315
|
-
skin_weight=t(skin_weight),
|
|
316
|
-
dec_vertices=t(dv),
|
|
317
|
-
dec_faces=torch.as_tensor(df, device=device),
|
|
318
|
-
dec_skin_index=torch.as_tensor(dsi, device=device),
|
|
319
|
-
dec_skin_weight=t(dsw),
|
|
320
|
-
dec_weights=t(dense),
|
|
321
|
-
dec_source=source,
|
|
322
|
-
)
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
# --- layer 1: cskel27 FK ---------------------------------------------------------
|
|
326
|
-
|
|
327
|
-
@dataclass
|
|
328
|
-
class State:
|
|
329
|
-
"""Optimisable per-frame pose (see module docstring for the convention)."""
|
|
330
|
-
|
|
331
|
-
transl: torch.Tensor # (T, 3)
|
|
332
|
-
root6d: torch.Tensor # (T, 6)
|
|
333
|
-
local6d: torch.Tensor # (T, 26, 6), joints 1..26
|
|
334
|
-
|
|
335
|
-
@staticmethod
|
|
336
|
-
def from_motion(local_rot_mats, root_positions, device="cpu", dtype=torch.float32) -> "State":
|
|
337
|
-
R = torch.as_tensor(np.asarray(local_rot_mats), device=device).to(dtype)
|
|
338
|
-
x = torch.as_tensor(np.asarray(root_positions), device=device).to(dtype)
|
|
339
|
-
if R.ndim != 4 or R.shape[1:] != (J, 3, 3) or x.shape != (R.shape[0], 3):
|
|
340
|
-
raise RigError("bad-motion", f"expected local_rot_mats (T,27,3,3) and root (T,3), got {tuple(R.shape)} {tuple(x.shape)}")
|
|
341
|
-
return State(x.clone(), matrix_to_rot6d(R[:, 0]), matrix_to_rot6d(R[:, 1:]))
|
|
342
|
-
|
|
343
|
-
@staticmethod
|
|
344
|
-
def identity(frames: int, root=(0.0, 0.0, 0.0), device="cpu", dtype=torch.float32) -> "State":
|
|
345
|
-
eye6 = torch.tensor([1.0, 0, 0, 0, 1, 0], device=device, dtype=dtype)
|
|
346
|
-
return State(torch.tensor(root, device=device, dtype=dtype).expand(frames, 3).clone(),
|
|
347
|
-
eye6.expand(frames, 6).clone(), eye6.expand(frames, J - 1, 6).clone())
|
|
348
|
-
|
|
349
|
-
def local_rot_mats(self) -> torch.Tensor:
|
|
350
|
-
"""(T, 27, 3, 3) in the NPZ local_rot_mats convention."""
|
|
351
|
-
return torch.cat((rot6d_to_matrix(self.root6d).unsqueeze(1), rot6d_to_matrix(self.local6d)), dim=1)
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
def global_rotations(local: torch.Tensor, parents) -> torch.Tensor:
|
|
355
|
-
"""(T, 27, 3, 3) local -> global rotations down the cskel27 tree."""
|
|
356
|
-
out = [None] * J
|
|
357
|
-
for j in range(J):
|
|
358
|
-
p = parents[j]
|
|
359
|
-
out[j] = local[:, j] if p < 0 else out[p] @ local[:, j]
|
|
360
|
-
return torch.stack(out, dim=1)
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
def cskel27_offsets(rig: Rig, bone_scale=None) -> torch.Tensor:
|
|
364
|
-
"""(27, 3) bone offsets in the parent frame: boneScale[j] * (neutral[j] - neutral[parent])."""
|
|
365
|
-
n = rig.neutral
|
|
366
|
-
parents = torch.tensor([max(p, 0) for p in rig.parents], device=n.device)
|
|
367
|
-
offsets = n - n[parents] # the root row is n[0] - n[0] = 0
|
|
368
|
-
if bone_scale is not None:
|
|
369
|
-
offsets = offsets * _bone_scale(bone_scale, n).unsqueeze(-1)
|
|
370
|
-
return offsets
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
def _bone_scale(bone_scale, like: torch.Tensor) -> torch.Tensor:
|
|
374
|
-
bs = torch.as_tensor(np.asarray(bone_scale) if not torch.is_tensor(bone_scale) else bone_scale, device=like.device).to(like.dtype)
|
|
375
|
-
if bs.shape != (J,) or not torch.isfinite(bs).all() or (bs <= 0).any():
|
|
376
|
-
raise RigError("bad-bone-scale", "bone_scale must be 27 positive finite factors")
|
|
377
|
-
return bs
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
def cskel27_fk(state: State, bone_scale, rig: Rig):
|
|
381
|
-
"""Layer 1. Returns (posed_joints (T, 27, 3), globals (T, 27, 3, 3)).
|
|
382
|
-
|
|
383
|
-
posed_joints is the motion NPZ `posed_joints`: FK over the neutral skeleton
|
|
384
|
-
grown by `bone_scale` (None == all ones) with the root at `state.transl`.
|
|
385
|
-
"""
|
|
386
|
-
globals_ = global_rotations(state.local_rot_mats(), rig.parents)
|
|
387
|
-
offsets = cskel27_offsets(rig, bone_scale)
|
|
388
|
-
pos = [None] * J
|
|
389
|
-
for j in range(J):
|
|
390
|
-
p = rig.parents[j]
|
|
391
|
-
pos[j] = state.transl if p < 0 else pos[p] + (globals_[:, p] @ offsets[j].unsqueeze(-1)).squeeze(-1)
|
|
392
|
-
return torch.stack(pos, dim=1), globals_
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
# --- layer 2: Studio positional skinning + LBS ------------------------------------
|
|
396
|
-
|
|
397
|
-
def _studio_offsets(rig: Rig, bone_scale) -> torch.Tensor:
|
|
398
|
-
"""playback.js prep.offsets, or scaledOffsets(prep, boneScale) for a take with boneScale."""
|
|
399
|
-
if bone_scale is None:
|
|
400
|
-
return rig.prep_offsets
|
|
401
|
-
grown = cskel27_offsets(rig, bone_scale)
|
|
402
|
-
g = [None] * J
|
|
403
|
-
for j in range(J):
|
|
404
|
-
p = rig.parents[j]
|
|
405
|
-
g[j] = rig.neutral[j] if p < 0 else g[p] + grown[j]
|
|
406
|
-
g = torch.stack(g)
|
|
407
|
-
shift = torch.tensor([0.0, rig.min_y, 0.0], dtype=g.dtype, device=g.device)
|
|
408
|
-
return rig.prep_bind_pos - rig.prep_scale * (g - shift)
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
def _stretch(rig: Rig, bone_scale, j: int):
|
|
412
|
-
"""playback.js boneStretch: 1 without boneScale; girdle joints take the raw factor."""
|
|
413
|
-
if bone_scale is None:
|
|
414
|
-
return 1.0
|
|
415
|
-
if rig.girdle[j]:
|
|
416
|
-
return bone_scale[j]
|
|
417
|
-
canonical, rig_bind = rig.canonical_bone_length[j], rig.rig_bone_length[j]
|
|
418
|
-
if not (canonical > 1e-6) or not (rig_bind > 1e-6):
|
|
419
|
-
return bone_scale[j]
|
|
420
|
-
return bone_scale[j] * float(canonical / rig_bind)
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
def studio_bone_worlds(globals_: torch.Tensor, posed_joints: torch.Tensor, rig: Rig,
|
|
424
|
-
bone_scale=None, anchor_xz=None) -> torch.Tensor:
|
|
425
|
-
"""Every exported bone's world matrix (T, B, 4, 4) in metres under Studio playback.
|
|
426
|
-
|
|
427
|
-
globals_: (T, 27, 3, 3) cskel27 global rotations; posed_joints: (T, 27, 3).
|
|
428
|
-
bone_scale: None reproduces a take without bone_scale (no arm stretch); a
|
|
429
|
-
27-vector reproduces a mocap take (grown offsets, stretched arm chain/hands).
|
|
430
|
-
anchor_xz: None (no anchoring) or (2,)/(T, 2) root XZ subtracted like the
|
|
431
|
-
anchor frame's root in applyMotionFrame.
|
|
432
|
-
"""
|
|
433
|
-
T = globals_.shape[0]
|
|
434
|
-
dtype, device = globals_.dtype, globals_.device
|
|
435
|
-
bs = None if bone_scale is None else _bone_scale(bone_scale, globals_)
|
|
436
|
-
offsets = _studio_offsets(rig, bs)
|
|
437
|
-
s = rig.prep_scale
|
|
438
|
-
posed = posed_joints
|
|
439
|
-
if anchor_xz is not None:
|
|
440
|
-
a = torch.as_tensor(anchor_xz, dtype=dtype, device=device).reshape(-1, 2)
|
|
441
|
-
zero = torch.zeros_like(a[:, :1])
|
|
442
|
-
posed = posed - torch.cat((a[:, :1], zero, a[:, 1:]), dim=-1).unsqueeze(1)
|
|
443
|
-
|
|
444
|
-
desired = [None] * J
|
|
445
|
-
assumed_parent = {}
|
|
446
|
-
for j in range(J):
|
|
447
|
-
b = rig.prep_bone[j]
|
|
448
|
-
if b < 0:
|
|
449
|
-
continue
|
|
450
|
-
R = globals_[:, j]
|
|
451
|
-
pos = s * posed[:, j] + (R @ offsets[j].unsqueeze(-1)).squeeze(-1)
|
|
452
|
-
cp = rig.prep_chain_parent[j]
|
|
453
|
-
parent = desired[cp] @ rig.prep_chain_rel[j] if cp >= 0 else rig.prep_parent_bind_world[j].expand(T, 4, 4)
|
|
454
|
-
if rig.hierarchy_preserved[j]:
|
|
455
|
-
pos = _apply(parent, (rig.prep_bind_local_pos[j] * _stretch(rig, bs, j)).expand(T, 3))
|
|
456
|
-
rot = R @ rig.prep_bind_rot[j] * rig.prep_bind_scale[j] # compose(pos, quat, scale)
|
|
457
|
-
desired[j] = _affine(rot, pos)
|
|
458
|
-
assumed_parent[b] = (parent, j)
|
|
459
|
-
|
|
460
|
-
leaves = {b: (joint, p, Rq) for b, joint, p, Rq in rig.leaves}
|
|
461
|
-
worlds = []
|
|
462
|
-
for b, parent_index in enumerate(rig.bone_parent):
|
|
463
|
-
if b in assumed_parent:
|
|
464
|
-
parent, j = assumed_parent[b]
|
|
465
|
-
local = torch.linalg.inv(parent) @ desired[j]
|
|
466
|
-
elif b in leaves:
|
|
467
|
-
joint, p, Rq = leaves[b]
|
|
468
|
-
local = _affine(Rq * rig.bone_rest_scale[b], p * _stretch(rig, bs, joint)).expand(T, 4, 4)
|
|
469
|
-
else:
|
|
470
|
-
local = rig.bone_rest_local[b].expand(T, 4, 4)
|
|
471
|
-
worlds.append(local if parent_index < 0 else worlds[parent_index] @ local)
|
|
472
|
-
W = torch.stack(worlds, dim=1)
|
|
473
|
-
root = torch.diag(torch.tensor([rig.root_scale] * 3 + [1.0], dtype=dtype, device=device))
|
|
474
|
-
return root @ W
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
def lbs(bone_worlds: torch.Tensor, rig: Rig, rest: torch.Tensor, weights: torch.Tensor) -> torch.Tensor:
|
|
478
|
-
"""Linear-blend skinning: v = sum_b w_vb * (W_b @ bindInverse_b) @ v_rest.
|
|
479
|
-
|
|
480
|
-
bone_worlds (T, B, 4, 4) metres; rest (V, 3); weights dense (V, B). -> (T, V, 3).
|
|
481
|
-
"""
|
|
482
|
-
skin = bone_worlds @ rig.bind_inverse # (T, B, 4, 4)
|
|
483
|
-
blended = torch.einsum("vb,tbij->tvij", weights, skin[:, :, :3, :]) # (T, V, 3, 4)
|
|
484
|
-
return (blended[..., :3] @ rest.unsqueeze(-1)).squeeze(-1) + blended[..., 3]
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
def dense_weights(rig: Rig, indices=None) -> tuple:
|
|
488
|
-
"""(rest (V, 3), dense weights (V, B)) of full-resolution vertices `indices` (all when None)."""
|
|
489
|
-
idx = slice(None) if indices is None else torch.as_tensor(indices, device=rig.vertices.device, dtype=torch.long)
|
|
490
|
-
si, sw = rig.skin_index[idx], rig.skin_weight[idx]
|
|
491
|
-
dense = torch.zeros(si.shape[0], len(rig.bone_names), dtype=sw.dtype, device=sw.device)
|
|
492
|
-
dense.scatter_add_(1, si, sw)
|
|
493
|
-
return rig.vertices[idx], dense
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
def studio_skin(globals_: torch.Tensor, posed_joints: torch.Tensor, rig: Rig, bone_scale=None,
|
|
497
|
-
anchor_xz=None, vertex_indices=None):
|
|
498
|
-
"""Layer 2. Returns (bone_worlds (T, B, 4, 4), vertices (T, V, 3)), metres.
|
|
499
|
-
|
|
500
|
-
Vertices are the decimated tracker mesh by default, or the full-resolution
|
|
501
|
-
export vertices `vertex_indices` (parity checks against Studio).
|
|
502
|
-
"""
|
|
503
|
-
W = studio_bone_worlds(globals_, posed_joints, rig, bone_scale, anchor_xz)
|
|
504
|
-
if vertex_indices is None:
|
|
505
|
-
rest, weights = rig.dec_vertices, rig.dec_weights
|
|
506
|
-
else:
|
|
507
|
-
rest, weights = dense_weights(rig, vertex_indices)
|
|
508
|
-
return W, lbs(W, rig, rest, weights)
|