cozyclay 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +85 -0
- package/README.md +31 -0
- package/THIRD_PARTY_NOTICES.md +37 -1
- package/bin/cozyclay.mjs +51 -2
- package/dist/ai-camera-control/index.html +406 -0
- package/dist/app/index.html +5 -5
- package/dist/assets/app-B3U5aut1.js +4811 -0
- package/dist/assets/app-BrRF0wso.css +1 -0
- package/dist/assets/vision_bundle-jFkh-fIS.js +41 -0
- package/dist/fonts/InstrumentSerif-OFL.txt +93 -0
- package/dist/fonts/Inter-OFL.txt +92 -0
- package/dist/fonts/README.md +15 -0
- package/dist/index.html +60 -16
- package/dist/sitemap.xml +7 -1
- package/mcp/LIVE-PROTOCOL.md +63 -0
- package/mcp/README.md +142 -0
- package/mcp/ardy-prompts.mjs +170 -0
- package/mcp/live-hub.mjs +105 -0
- package/mcp/package.json +24 -0
- package/mcp/server.mjs +1394 -0
- package/package.json +122 -90
- package/src/App.jsx +2957 -514
- package/src/ardy/cskel27.js +7 -2
- package/src/ardy/ik.js +25 -15
- package/src/ardy/npz.js +64 -3
- package/src/ardy/playback.js +31 -1
- package/src/ardy/prompt-clips.js +7 -2
- package/src/ardy/retime.js +211 -0
- package/src/ardy/timeline-coordinates.js +13 -0
- package/src/ardy/timeline.jsx +124 -5
- package/src/ardy/to-cskel27.js +34 -12
- package/src/ardy/trim.js +33 -0
- package/src/asset-pane.jsx +36 -0
- package/src/dualview.jsx +14 -8
- package/src/hierarchy-model.js +95 -13
- package/src/hierarchy-panel.jsx +139 -6
- package/src/live-control.js +122 -0
- package/src/matte-editor.js +543 -0
- package/src/matte.js +503 -0
- package/src/multimodel-ingest.js +344 -0
- package/src/object-gizmo.jsx +43 -15
- package/src/planview.jsx +43 -32
- package/src/pose-extract/detector.js +75 -0
- package/src/pose-extract/index.js +3 -0
- package/src/pose-extract/take.js +87 -0
- package/src/pose-extract/video-frames.js +91 -0
- package/src/pose-thumbs.js +152 -0
- package/src/posestudio.jsx +361 -11
- package/src/project-browser.jsx +135 -0
- package/src/project.js +289 -0
- package/src/props.jsx +69 -3
- package/src/room.jsx +14 -35
- package/src/scene-asset-cache.js +125 -0
- package/src/scene-assets.js +288 -0
- package/src/scene-objects.js +245 -7
- package/src/scenes.js +207 -26
- package/src/shot-authoring.js +55 -13
- package/src/styles.css +1296 -129
- package/tools/ardy/BRIDGE.md +3 -2
- package/tools/ardy/README.md +9 -5
- package/tools/ardy/bridge.mjs +57 -1
- package/tools/ardy/bvh-cskel27.mjs +1209 -0
- package/tools/ardy/cclay_constrained_generate.py +123 -11
- package/tools/ardy/cclay_sequence_generate.py +49 -0
- package/tools/ardy/extract.mjs +367 -0
- package/tools/ardy/footage.mjs +462 -0
- package/tools/ardy/npz.mjs +74 -9
- package/tools/ardy/run-on-box.sh +25 -0
- package/tools/ardy/run-sequence-on-box.sh +15 -0
- package/tools/ardy/runners/remote.mjs +8 -2
- package/dist/assets/app-Cgpk2hwX.js +0 -4803
- package/dist/assets/app-DgZvaAE1.css +0 -1
package/src/ardy/cskel27.js
CHANGED
|
@@ -62,7 +62,12 @@ export const COZYCLAY_BONES = [
|
|
|
62
62
|
"LeftFoot", "RightFoot",
|
|
63
63
|
];
|
|
64
64
|
|
|
65
|
-
/** CozyClay bone name -> cskel27 joint index.
|
|
65
|
+
/** CozyClay/Mixamo bone name -> playback-equivalent cskel27 joint index.
|
|
66
|
+
* Core has one more torso segment, so its first animated spine joint is
|
|
67
|
+
* Spine1 and the Mixamo chest lands on Spine2. */
|
|
66
68
|
export const COZYCLAY_TO_CSKEL27 = Object.fromEntries(
|
|
67
|
-
COZYCLAY_BONES.map((name) => [
|
|
69
|
+
COZYCLAY_BONES.map((name) => [
|
|
70
|
+
name,
|
|
71
|
+
JOINT_INDEX[name === "Spine" ? "Spine1" : name === "Spine1" ? "Spine2" : name],
|
|
72
|
+
])
|
|
68
73
|
);
|
package/src/ardy/ik.js
CHANGED
|
@@ -20,20 +20,20 @@ import { normalizeBoneName } from "../poses.js";
|
|
|
20
20
|
/** The four IK chain handles, mapped to the timeline lanes of the same
|
|
21
21
|
* names. */
|
|
22
22
|
export const IK_TRACKS = [
|
|
23
|
-
{ id: "leftHand", label: "Left Hand", kind: "arm", side: "Left" },
|
|
24
|
-
{ id: "rightHand", label: "Right Hand", kind: "arm", side: "Right" },
|
|
25
|
-
{ id: "leftFoot", label: "Left Foot", kind: "leg", side: "Left" },
|
|
26
|
-
{ id: "rightFoot", label: "Right Foot", kind: "leg", side: "Right" },
|
|
23
|
+
{ id: "leftHand", label: "Left Hand", kind: "arm", side: "Left", visibilityDepth: 0.14 },
|
|
24
|
+
{ id: "rightHand", label: "Right Hand", kind: "arm", side: "Right", visibilityDepth: 0.14 },
|
|
25
|
+
{ id: "leftFoot", label: "Left Foot", kind: "leg", side: "Left", visibilityDepth: 0.14 },
|
|
26
|
+
{ id: "rightFoot", label: "Right Foot", kind: "leg", side: "Right", visibilityDepth: 0.14 },
|
|
27
27
|
];
|
|
28
28
|
|
|
29
29
|
/** Mid-joint position handles: dragging repositions the elbow/knee with
|
|
30
30
|
* BOTH ends pinned (shoulder+wrist / hip+ankle) — the classic mid-chain
|
|
31
31
|
* handle. `chain` links to the IK chain whose bones it edits. */
|
|
32
32
|
export const MID_TRACKS = [
|
|
33
|
-
{ id: "leftElbow", label: "Left Elbow", chain: "leftHand" },
|
|
34
|
-
{ id: "rightElbow", label: "Right Elbow", chain: "rightHand" },
|
|
35
|
-
{ id: "leftKnee", label: "Left Knee", chain: "leftFoot" },
|
|
36
|
-
{ id: "rightKnee", label: "Right Knee", chain: "rightFoot" },
|
|
33
|
+
{ id: "leftElbow", label: "Left Elbow", chain: "leftHand", visibilityDepth: 0.12 },
|
|
34
|
+
{ id: "rightElbow", label: "Right Elbow", chain: "rightHand", visibilityDepth: 0.12 },
|
|
35
|
+
{ id: "leftKnee", label: "Left Knee", chain: "leftFoot", visibilityDepth: 0.12 },
|
|
36
|
+
{ id: "rightKnee", label: "Right Knee", chain: "rightFoot", visibilityDepth: 0.12 },
|
|
37
37
|
];
|
|
38
38
|
|
|
39
39
|
/** FK swing handles: dragging swings the part toward the pointer (rotation
|
|
@@ -42,15 +42,25 @@ export const MID_TRACKS = [
|
|
|
42
42
|
* follow the FK PoseHandles coding: torso yellow, head purple, arms orange,
|
|
43
43
|
* legs blue. */
|
|
44
44
|
export const FK_TRACKS = [
|
|
45
|
-
{ id: "hips", label: "Hips", bone: "mixamorigHips", child: "mixamorigSpine", color: "#ffd23d" },
|
|
46
|
-
{ id: "spine", label: "Spine", bone: "mixamorigSpine", child: "mixamorigSpine1", color: "#ffd23d" },
|
|
47
|
-
{ id: "chest", label: "Chest", bone: "mixamorigSpine1", child: "mixamorigSpine2", color: "#ffd23d" },
|
|
48
|
-
{ id: "neck", label: "Neck", bone: "mixamorigNeck", child: "mixamorigHead", color: "#b98cff" },
|
|
49
|
-
{ id: "head", label: "Head", bone: "mixamorigHead", child: null, color: "#b98cff" },
|
|
50
|
-
{ id: "leftShoulder", label: "Left Shoulder", bone: "mixamorigLeftShoulder", child: "mixamorigLeftArm", color: "#ff8a3d" },
|
|
51
|
-
{ id: "rightShoulder", label: "Right Shoulder", bone: "mixamorigRightShoulder", child: "mixamorigRightArm", color: "#ff8a3d" },
|
|
45
|
+
{ id: "hips", label: "Hips", bone: "mixamorigHips", child: "mixamorigSpine", color: "#ffd23d", group: "torso", visibilityDepth: 0.34 },
|
|
46
|
+
{ id: "spine", label: "Spine", bone: "mixamorigSpine", child: "mixamorigSpine1", color: "#ffd23d", group: "torso", visibilityDepth: 0.32 },
|
|
47
|
+
{ id: "chest", label: "Chest", bone: "mixamorigSpine1", child: "mixamorigSpine2", color: "#ffd23d", group: "torso", visibilityDepth: 0.3 },
|
|
48
|
+
{ id: "neck", label: "Neck", bone: "mixamorigNeck", child: "mixamorigHead", color: "#b98cff", group: "head", visibilityDepth: 0.2 },
|
|
49
|
+
{ id: "head", label: "Head", bone: "mixamorigHead", child: null, color: "#b98cff", group: "head", visibilityDepth: 0.24 },
|
|
50
|
+
{ id: "leftShoulder", label: "Left Shoulder", bone: "mixamorigLeftShoulder", child: "mixamorigLeftArm", color: "#ff8a3d", group: "shoulder", visibilityDepth: 0.2 },
|
|
51
|
+
{ id: "rightShoulder", label: "Right Shoulder", bone: "mixamorigRightShoulder", child: "mixamorigRightArm", color: "#ff8a3d", group: "shoulder", visibilityDepth: 0.2 },
|
|
52
52
|
];
|
|
53
53
|
|
|
54
|
+
/** Whether a control centre is close enough to the first visible surface.
|
|
55
|
+
* Centreline controls (torso/head) intentionally have larger allowances than
|
|
56
|
+
* side-specific limbs, so front-facing body controls remain available while
|
|
57
|
+
* the far shoulder/arm/leg is rejected. */
|
|
58
|
+
export function ikControlIsExposed(targetDistance, blockerDistance, visibilityDepth) {
|
|
59
|
+
if (!Number.isFinite(targetDistance) || targetDistance <= 0) return false;
|
|
60
|
+
if (!Number.isFinite(blockerDistance)) return true;
|
|
61
|
+
return targetDistance - blockerDistance <= visibilityDepth;
|
|
62
|
+
}
|
|
63
|
+
|
|
54
64
|
/** Bone chains per handle, root → effector. Mixamo spelling; the matcher
|
|
55
65
|
* accepts the `mixamorig:` prefix and prefix-less rigs. Shoulder stays out of
|
|
56
66
|
* the arm chain — clavicle rotation swings the whole shoulder mass and reads
|
package/src/ardy/npz.js
CHANGED
|
@@ -32,6 +32,12 @@ const FPS_MAX = 240; // matches CozyClay motion_retarget.FPS_BOUNDS
|
|
|
32
32
|
// Same tolerance motion_retarget uses for squared row/column norms, pairwise
|
|
33
33
|
// dots, and determinant; float32 serialization noise sits far below 1e-3.
|
|
34
34
|
const ROTATION_MATRIX_TOLERANCE = 1e-3;
|
|
35
|
+
// Bounds on the stature a take may ask the character to take on. The estimate
|
|
36
|
+
// is a ratio of leg lengths measured off a video, so a bad detection must
|
|
37
|
+
// never be able to produce a giant or a gnome; the clamp lives next to the
|
|
38
|
+
// decode because that is where the number enters the app.
|
|
39
|
+
export const CHARACTER_SCALE_MIN = 0.6;
|
|
40
|
+
export const CHARACTER_SCALE_MAX = 1.5;
|
|
35
41
|
|
|
36
42
|
const ZIP_EOCD_SIG = 0x06054b50;
|
|
37
43
|
const ZIP_CENTRAL_SIG = 0x02014b50;
|
|
@@ -345,6 +351,25 @@ function scalarIntOf(parsed, label) {
|
|
|
345
351
|
return value;
|
|
346
352
|
}
|
|
347
353
|
|
|
354
|
+
/** scalar float value (person_scale); returns Number. An integer member is
|
|
355
|
+
* accepted too, so a writer that stored a whole-number scale still reads. */
|
|
356
|
+
function scalarFloatOf(parsed, label) {
|
|
357
|
+
if (parsed.kind === "i" || parsed.kind === "u") return scalarIntOf(parsed, label);
|
|
358
|
+
if (parsed.kind !== "f") {
|
|
359
|
+
throw new NpzError(`${label} must be a float scalar, got descr kind '${parsed.kind}'`);
|
|
360
|
+
}
|
|
361
|
+
if (parsed.shape.length > 1 || (parsed.shape.length === 1 && parsed.shape[0] !== 1)) {
|
|
362
|
+
throw new NpzError(`${label} must be a scalar (shape () or (1,)), got shape (${parsed.shape.join(", ")})`);
|
|
363
|
+
}
|
|
364
|
+
const view = new DataView(parsed.data.buffer, parsed.data.byteOffset, parsed.data.byteLength);
|
|
365
|
+
let value;
|
|
366
|
+
if (parsed.itemsize === 4) value = view.getFloat32(0, parsed.littleEndian);
|
|
367
|
+
else if (parsed.itemsize === 8) value = view.getFloat64(0, parsed.littleEndian);
|
|
368
|
+
else throw new NpzError(`${label} has unsupported float itemsize ${parsed.itemsize}`);
|
|
369
|
+
if (!Number.isFinite(value)) throw new NpzError(`${label} is not a finite number`);
|
|
370
|
+
return value;
|
|
371
|
+
}
|
|
372
|
+
|
|
348
373
|
/** Validate one 3x3 matrix: finite, orthonormal rows/columns, det +1. */
|
|
349
374
|
function checkRotationMatrix(rows, frame, joint) {
|
|
350
375
|
const columns = [
|
|
@@ -383,12 +408,19 @@ function checkRotationMatrix(rows, frame, joint) {
|
|
|
383
408
|
/**
|
|
384
409
|
* Decode an ARDY motion npz from raw bytes.
|
|
385
410
|
*
|
|
386
|
-
* Returns { frames, fps, rotMats, rootPos, posedJoints } with `rotMats` a
|
|
411
|
+
* Returns { frames, fps, personScale, rotMats, rootPos, posedJoints } with `rotMats` a
|
|
387
412
|
* Float32Array of frames*27*9 row-major 3x3 matrices (cskel27 joint order),
|
|
388
413
|
* `rootPos` a Float32Array of frames*3 (x, y, z) ARDY-world root positions,
|
|
389
414
|
* and `posedJoints` a Float32Array of frames*27*3 ARDY-world joint
|
|
390
415
|
* positions (the positional-skinning reference — playback.js drives bone
|
|
391
416
|
* positions straight from it). Throws NpzError on anything malformed.
|
|
417
|
+
*
|
|
418
|
+
* `personScale` is the filmed performer's stature relative to the canonical
|
|
419
|
+
* body, carried by the optional `person_scale` member. The extraction divided
|
|
420
|
+
* the root translation by it, so applying the frames without applying the
|
|
421
|
+
* scale exaggerates every stride — the two must move together. An archive
|
|
422
|
+
* without the member (an ARDY-generated take, or one written before the
|
|
423
|
+
* member existed) reports 1: canonical body, unscaled travel.
|
|
392
424
|
*/
|
|
393
425
|
export async function decodeMotionNpz(bytes) {
|
|
394
426
|
if (bytes.byteLength > MAX_ARCHIVE_BYTES) {
|
|
@@ -401,9 +433,12 @@ export async function decodeMotionNpz(bytes) {
|
|
|
401
433
|
const names = ["local_rot_mats.npy", "root_positions.npy", "fps.npy", "posed_joints.npy"];
|
|
402
434
|
const missing = names.filter((n) => !entries.has(n));
|
|
403
435
|
if (missing.length) throw new NpzError(`motion npz is missing members: ${missing.join(", ")}`);
|
|
436
|
+
// person_scale is optional: only a filmed take has a performer to measure,
|
|
437
|
+
// and archives predating the member are still valid motions.
|
|
438
|
+
const read = [...names, ...(entries.has("person_scale.npy") ? ["person_scale.npy"] : [])];
|
|
404
439
|
|
|
405
440
|
const members = {};
|
|
406
|
-
for (const name of
|
|
441
|
+
for (const name of read) {
|
|
407
442
|
const entry = entries.get(name);
|
|
408
443
|
if (entry.uncompSize > MAX_MEMBER_BYTES) {
|
|
409
444
|
throw new NpzError(`motion npz member ${name} decompresses to ${entry.uncompSize} bytes, over the ${MAX_MEMBER_BYTES} byte cap`);
|
|
@@ -449,6 +484,13 @@ export async function decodeMotionNpz(bytes) {
|
|
|
449
484
|
if (fps < FPS_MIN || fps > FPS_MAX) {
|
|
450
485
|
throw new NpzError(`motion fps ${fps} is outside ${FPS_MIN}..${FPS_MAX}`);
|
|
451
486
|
}
|
|
487
|
+
let personScale = 1;
|
|
488
|
+
if (members["person_scale.npy"]) {
|
|
489
|
+
personScale = scalarFloatOf(parseNpyMember(members["person_scale.npy"], "person_scale"), "person_scale");
|
|
490
|
+
// A stored zero or negative would mirror the trajectory or collapse the
|
|
491
|
+
// body; that is corrupt data, not a scale to clamp into range.
|
|
492
|
+
if (!(personScale > 0)) throw new NpzError(`motion person_scale ${personScale} must be positive`);
|
|
493
|
+
}
|
|
452
494
|
|
|
453
495
|
const rotMats = float32Of(rotParsed, "local_rot_mats");
|
|
454
496
|
const rootPos = float32Of(posParsed, "root_positions");
|
|
@@ -482,7 +524,26 @@ export async function decodeMotionNpz(bytes) {
|
|
|
482
524
|
if (!Number.isFinite(posedJoints[i])) throw new NpzError(`motion npz posed_joints contains a non-finite value at index ${i}`);
|
|
483
525
|
}
|
|
484
526
|
|
|
485
|
-
return { frames, fps, rotMats, rootPos, posedJoints };
|
|
527
|
+
return { frames, fps, personScale, rotMats, rootPos, posedJoints };
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
/**
|
|
531
|
+
* The character scale a decoded take implies, clamped to the sane stature
|
|
532
|
+
* band. THE INVARIANT: a take's root travel was authored against this scale,
|
|
533
|
+
* so every path that applies a motion must apply this too — leaving them apart
|
|
534
|
+
* is what makes a filmed stride overshoot and the feet slide.
|
|
535
|
+
*
|
|
536
|
+
* `fallback` is for a take whose npz predates the member but whose scale
|
|
537
|
+
* arrived some other way (the extraction response). No scale anywhere means
|
|
538
|
+
* canonical: 1.
|
|
539
|
+
*/
|
|
540
|
+
export function characterScaleFor(motion, fallback = 1) {
|
|
541
|
+
const raw = Number.isFinite(motion?.personScale) && motion.personScale > 0
|
|
542
|
+
? motion.personScale
|
|
543
|
+
: Number.isFinite(fallback) && fallback > 0
|
|
544
|
+
? fallback
|
|
545
|
+
: 1;
|
|
546
|
+
return Math.max(CHARACTER_SCALE_MIN, Math.min(CHARACTER_SCALE_MAX, raw));
|
|
486
547
|
}
|
|
487
548
|
|
|
488
549
|
/**
|
package/src/ardy/playback.js
CHANGED
|
@@ -35,6 +35,8 @@
|
|
|
35
35
|
* HandEnd leaves drive no bone — their motion folds into the mapped joints'
|
|
36
36
|
* global transforms, and HeadTop_End/Toe_End/finger bones simply follow
|
|
37
37
|
* their parents). Bones the map does not touch keep their bind transforms.
|
|
38
|
+
* Shoulder and arm chains keep their Mixamo bind translations and consume
|
|
39
|
+
* ARDY rotations only, avoiding clavicle shear between incongruent rigs.
|
|
38
40
|
*/
|
|
39
41
|
|
|
40
42
|
import * as THREE from "three";
|
|
@@ -76,6 +78,18 @@ const SKINNING_MAP = CSKEL27_JOINTS.map((name) => {
|
|
|
76
78
|
}
|
|
77
79
|
});
|
|
78
80
|
|
|
81
|
+
// CoreSkeleton27 and the Mixamo bots are not congruent around the clavicles:
|
|
82
|
+
// their neutral shoulder directions differ by about 30 degrees. Positional
|
|
83
|
+
// skinning therefore moves the arm root away from the direction encoded by
|
|
84
|
+
// the parent bone's rotation and visibly shears the shoulder. Keep the
|
|
85
|
+
// Mixamo arm chains' authored local translations and retarget rotations only;
|
|
86
|
+
// the rest of the body still uses ARDY positional skinning for root motion
|
|
87
|
+
// and foot placement.
|
|
88
|
+
const HIERARCHY_PRESERVED_JOINTS = new Set([
|
|
89
|
+
"RightShoulder", "RightArm", "RightForeArm",
|
|
90
|
+
"LeftShoulder", "LeftArm", "LeftForeArm",
|
|
91
|
+
]);
|
|
92
|
+
|
|
79
93
|
const ARDY_NEUTRAL_MIN_Y = -0.9544128; // toe depth under the hips-origin neutral pose
|
|
80
94
|
|
|
81
95
|
/* No finger forcing: ARDY's cskel27 stops at the wrist, and the fingers
|
|
@@ -160,6 +174,7 @@ function prepOf(rig) {
|
|
|
160
174
|
const bindPos = new Array(CSKEL27_JOINTS.length).fill(null);
|
|
161
175
|
const bindQuat = new Array(CSKEL27_JOINTS.length).fill(null);
|
|
162
176
|
const bindScale = new Array(CSKEL27_JOINTS.length).fill(null);
|
|
177
|
+
const bindLocalPos = new Array(CSKEL27_JOINTS.length).fill(null);
|
|
163
178
|
const parentBindWorld = new Array(CSKEL27_JOINTS.length).fill(null);
|
|
164
179
|
for (let j = 0; j < CSKEL27_JOINTS.length; j += 1) {
|
|
165
180
|
const mixamoName = SKINNING_MAP[j];
|
|
@@ -171,6 +186,7 @@ function prepOf(rig) {
|
|
|
171
186
|
bindPos[j] = new THREE.Vector3().setFromMatrixPosition(world);
|
|
172
187
|
bindQuat[j] = new THREE.Quaternion().setFromRotationMatrix(world);
|
|
173
188
|
bindScale[j] = bone.scale.clone();
|
|
189
|
+
bindLocalPos[j] = bone.position.clone();
|
|
174
190
|
parentBindWorld[j] = worldByNode.get(bone.parent) ?? new THREE.Matrix4();
|
|
175
191
|
}
|
|
176
192
|
|
|
@@ -228,7 +244,17 @@ function prepOf(rig) {
|
|
|
228
244
|
}
|
|
229
245
|
}
|
|
230
246
|
|
|
231
|
-
prep = {
|
|
247
|
+
prep = {
|
|
248
|
+
bones,
|
|
249
|
+
bindQuat,
|
|
250
|
+
bindScale,
|
|
251
|
+
bindLocalPos,
|
|
252
|
+
parentBindWorld,
|
|
253
|
+
chainParent,
|
|
254
|
+
chainRel,
|
|
255
|
+
scale,
|
|
256
|
+
offsets,
|
|
257
|
+
};
|
|
232
258
|
rigPreps.set(rig, prep);
|
|
233
259
|
return prep;
|
|
234
260
|
}
|
|
@@ -373,6 +399,10 @@ export function applyMotionFrame(rig, motion, frame) {
|
|
|
373
399
|
mParentInv.copy(prep.parentBindWorld[j]);
|
|
374
400
|
}
|
|
375
401
|
|
|
402
|
+
if (HIERARCHY_PRESERVED_JOINTS.has(CSKEL27_JOINTS[j])) {
|
|
403
|
+
vWorld.copy(prep.bindLocalPos[j]).applyMatrix4(mParentInv);
|
|
404
|
+
}
|
|
405
|
+
|
|
376
406
|
mWorld.compose(vWorld, qWorld, prep.bindScale[j]);
|
|
377
407
|
desiredWorld[j] = mWorld.clone();
|
|
378
408
|
mLocal.copy(mParentInv).invert().multiply(mWorld);
|
package/src/ardy/prompt-clips.js
CHANGED
|
@@ -1,6 +1,11 @@
|
|
|
1
|
-
/** Move one prompt clip on the fixed
|
|
1
|
+
/** Move one prompt clip on the fixed prompt-block grid without changing its
|
|
2
2
|
* duration. Overlaps are rejected because segment generation needs one prompt
|
|
3
|
-
* per frame range; gaps remain valid and inherit the main prompt.
|
|
3
|
+
* per frame range; gaps remain valid and inherit the main prompt.
|
|
4
|
+
*
|
|
5
|
+
* Clock-agnostic: `blockFrames` is the caller's grid, and App.jsx always
|
|
6
|
+
* passes ARDY_PROMPT_HORIZON_FRAMES (2 s on the 24 fps timeline clock = 48).
|
|
7
|
+
* The 40 default is only the bare-call fallback and means nothing on the
|
|
8
|
+
* wire — nothing here ever reaches the bridge unconverted. */
|
|
4
9
|
export function movePromptClipFrames(clips, id, rawStartFrame, blockFrames = 40) {
|
|
5
10
|
const target = clips.find((clip) => clip.id === id);
|
|
6
11
|
if (!target) return clips;
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
// retime.js — duration-preserving resample of a decoded motion between frame
|
|
2
|
+
// clocks. ARDY Core generates on its trained 20 fps clock, filmed takes arrive
|
|
3
|
+
// at 30 or 60; the app timeline runs the production 24 fps one. A 20 fps take
|
|
4
|
+
// counted against a 24 fps reference reads 1.2× fast, so inbound takes are
|
|
5
|
+
// resampled here: rotations slerp between neighbouring source frames, the root
|
|
6
|
+
// lerps, frame 0 stays identical, and the clip's length in seconds does not
|
|
7
|
+
// move — only how densely it is sampled.
|
|
8
|
+
//
|
|
9
|
+
// World joint positions are NOT interpolated. A slerped rotation walks the arc
|
|
10
|
+
// between two frames while a lerped position cuts the chord, so the two used to
|
|
11
|
+
// disagree at every blended frame — and since most 24 fps frames land between
|
|
12
|
+
// source frames, that disagreement was phase-locked to the resample (a 6 Hz
|
|
13
|
+
// pulse from 30 fps, alternating frames from 60). Playback drives bone.position
|
|
14
|
+
// straight from posedJoints, so on screen the shin visibly shortened by up to
|
|
15
|
+
// 2.6 cm and sprang back: the trembling feet. Positions are regenerated by
|
|
16
|
+
// forward kinematics from the retimed rotations instead, which makes constant
|
|
17
|
+
// bone length a property of the output rather than a hope.
|
|
18
|
+
|
|
19
|
+
import { CSKEL27_JOINTS } from "./cskel27.js";
|
|
20
|
+
import { deriveBoneOffsets, forwardKinematics, matToQuat, quatToMat } from "./convert.js";
|
|
21
|
+
import { canonicalCskel27Reference } from "./to-cskel27.js";
|
|
22
|
+
|
|
23
|
+
/** w-first quaternion slerp with hemisphere correction; falls back to a
|
|
24
|
+
* normalised lerp when the arc is too small for a stable sine. Shared with
|
|
25
|
+
* the bridge's BVH conversion (torso stabilization, root-spin repair). */
|
|
26
|
+
export function slerpQuat(a, b, t) {
|
|
27
|
+
let dot = a[0] * b[0] + a[1] * b[1] + a[2] * b[2] + a[3] * b[3];
|
|
28
|
+
const sign = dot < 0 ? -1 : 1;
|
|
29
|
+
dot *= sign;
|
|
30
|
+
let wa;
|
|
31
|
+
let wb;
|
|
32
|
+
if (dot > 0.9995) {
|
|
33
|
+
wa = 1 - t;
|
|
34
|
+
wb = t;
|
|
35
|
+
} else {
|
|
36
|
+
const theta = Math.acos(Math.min(1, dot));
|
|
37
|
+
const sinTheta = Math.sin(theta);
|
|
38
|
+
wa = Math.sin((1 - t) * theta) / sinTheta;
|
|
39
|
+
wb = Math.sin(t * theta) / sinTheta;
|
|
40
|
+
}
|
|
41
|
+
const out = [
|
|
42
|
+
a[0] * wa + b[0] * sign * wb,
|
|
43
|
+
a[1] * wa + b[1] * sign * wb,
|
|
44
|
+
a[2] * wa + b[2] * sign * wb,
|
|
45
|
+
a[3] * wa + b[3] * sign * wb,
|
|
46
|
+
];
|
|
47
|
+
const norm = Math.hypot(...out);
|
|
48
|
+
return out.map((component) => component / norm);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function readMat(flat, offset) {
|
|
52
|
+
return [
|
|
53
|
+
[flat[offset], flat[offset + 1], flat[offset + 2]],
|
|
54
|
+
[flat[offset + 3], flat[offset + 4], flat[offset + 5]],
|
|
55
|
+
[flat[offset + 6], flat[offset + 7], flat[offset + 8]],
|
|
56
|
+
];
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function writeMat(flat, offset, m) {
|
|
60
|
+
flat[offset] = m[0][0]; flat[offset + 1] = m[0][1]; flat[offset + 2] = m[0][2];
|
|
61
|
+
flat[offset + 3] = m[1][0]; flat[offset + 4] = m[1][1]; flat[offset + 5] = m[1][2];
|
|
62
|
+
flat[offset + 6] = m[2][0]; flat[offset + 7] = m[2][1]; flat[offset + 8] = m[2][2];
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const JOINTS = CSKEL27_JOINTS.length;
|
|
66
|
+
|
|
67
|
+
// Bone lengths are a property of the skeleton, not of the frame, so the offsets
|
|
68
|
+
// are derived once for the module rather than per take or per frame. Same
|
|
69
|
+
// pairing the BVH bridge uses (tools/ardy/bvh-cskel27.mjs): the canonical
|
|
70
|
+
// cskel27 reference, not a clip's frame zero, so a take whose first frame is
|
|
71
|
+
// noisy cannot smear its noise across the whole clip's proportions.
|
|
72
|
+
const BONE_OFFSETS = (() => {
|
|
73
|
+
const skeleton = canonicalCskel27Reference();
|
|
74
|
+
return deriveBoneOffsets(skeleton.posed_joints, skeleton.local_rot_mats);
|
|
75
|
+
})();
|
|
76
|
+
|
|
77
|
+
// Above this (in npz metres) a take's own posedJoints are not the forward
|
|
78
|
+
// kinematics of its own rotations, so regenerating them would move the take
|
|
79
|
+
// rather than repair it. Measured across every take in tools/ardy/out — filmed
|
|
80
|
+
// extractions and ARDY-generated clips alike — the residual is at most 1e-6 m,
|
|
81
|
+
// pure float32 serialization noise, so 1e-4 m sits 100x above the honest floor
|
|
82
|
+
// and far below any disagreement worth calling one.
|
|
83
|
+
const FK_CONSISTENCY_TOLERANCE = 1e-4;
|
|
84
|
+
|
|
85
|
+
// The 27 local rotations of one frame, as the nested matrices convert.js takes.
|
|
86
|
+
function readLocals(flat, frame) {
|
|
87
|
+
const locals = new Array(JOINTS);
|
|
88
|
+
for (let j = 0; j < JOINTS; j += 1) locals[j] = readMat(flat, (frame * JOINTS + j) * 9);
|
|
89
|
+
return locals;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Does this take's posedJoints already agree with FK of its own rotations?
|
|
94
|
+
* Sampled rather than exhaustive: an inconsistent take is inconsistent by
|
|
95
|
+
* construction (wrong proportions, hand-authored positions, a legacy writer),
|
|
96
|
+
* not on one unlucky frame, and the check must not cost a second FK sweep.
|
|
97
|
+
*/
|
|
98
|
+
function posedJointsFollowRotations(motion) {
|
|
99
|
+
const step = Math.max(1, Math.floor(motion.frames / 16));
|
|
100
|
+
for (let f = 0; f < motion.frames; f += step) {
|
|
101
|
+
const positions = forwardKinematics(
|
|
102
|
+
readLocals(motion.rotMats, f),
|
|
103
|
+
BONE_OFFSETS,
|
|
104
|
+
[motion.rootPos[f * 3], motion.rootPos[f * 3 + 1], motion.rootPos[f * 3 + 2]]
|
|
105
|
+
);
|
|
106
|
+
for (let j = 0; j < JOINTS; j += 1) {
|
|
107
|
+
const p = (f * JOINTS + j) * 3;
|
|
108
|
+
const dx = positions[j][0] - motion.posedJoints[p];
|
|
109
|
+
const dy = positions[j][1] - motion.posedJoints[p + 1];
|
|
110
|
+
const dz = positions[j][2] - motion.posedJoints[p + 2];
|
|
111
|
+
if (Math.hypot(dx, dy, dz) > FK_CONSISTENCY_TOLERANCE) return false;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
return true;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Resample `{ frames, fps, rotMats, rootPos, posedJoints, ... }` onto
|
|
119
|
+
* `targetFps`. Every other field rides through untouched (`personScale`
|
|
120
|
+
* included — changing the sampling clock does not change how big the filmed
|
|
121
|
+
* performer was), except a present `anchorFrame`, which is rescaled onto the
|
|
122
|
+
* new clock. Same-rate input is
|
|
123
|
+
* returned as-is — retiming is never a copy for its own sake.
|
|
124
|
+
*
|
|
125
|
+
* `posedJoints` comes out of forward kinematics on the retimed rotations, so
|
|
126
|
+
* every bone keeps its length exactly. The one exception is a take whose own
|
|
127
|
+
* posedJoints do not already follow its own rotations: there the positions are
|
|
128
|
+
* the authority and FK would move the take rather than repair it, so those
|
|
129
|
+
* lerp between source frames as before. Nothing in tools/ardy/out is such a
|
|
130
|
+
* take; the branch exists so a legacy or hand-built one is not rewritten.
|
|
131
|
+
*/
|
|
132
|
+
export function retimeMotion(motion, targetFps) {
|
|
133
|
+
if (!motion || !Number.isFinite(motion.fps) || motion.fps <= 0) {
|
|
134
|
+
throw new Error("retimeMotion: motion.fps must be a positive finite number");
|
|
135
|
+
}
|
|
136
|
+
if (!Number.isFinite(targetFps) || targetFps <= 0) {
|
|
137
|
+
throw new Error("retimeMotion: targetFps must be a positive finite number");
|
|
138
|
+
}
|
|
139
|
+
if (motion.fps === targetFps) return motion;
|
|
140
|
+
const srcFrames = motion.frames;
|
|
141
|
+
if (!Number.isInteger(srcFrames) || srcFrames < 1) {
|
|
142
|
+
throw new Error("retimeMotion: motion.frames must be a positive integer");
|
|
143
|
+
}
|
|
144
|
+
const ratio = targetFps / motion.fps;
|
|
145
|
+
const frames = Math.max(1, Math.round(srcFrames * ratio));
|
|
146
|
+
const rotMats = new Float32Array(frames * JOINTS * 9);
|
|
147
|
+
const rootPos = new Float32Array(frames * 3);
|
|
148
|
+
const posedJoints = new Float32Array(frames * JOINTS * 3);
|
|
149
|
+
const regenerate = posedJointsFollowRotations(motion);
|
|
150
|
+
|
|
151
|
+
for (let f = 0; f < frames; f += 1) {
|
|
152
|
+
const s = f / ratio;
|
|
153
|
+
const i0 = Math.min(Math.floor(s), srcFrames - 1);
|
|
154
|
+
const i1 = Math.min(i0 + 1, srcFrames - 1);
|
|
155
|
+
const t = Math.min(Math.max(s - i0, 0), 1);
|
|
156
|
+
const exact = t === 0 || i0 === i1;
|
|
157
|
+
|
|
158
|
+
for (let j = 0; j < JOINTS; j += 1) {
|
|
159
|
+
const src0 = (i0 * JOINTS + j) * 9;
|
|
160
|
+
const dst = (f * JOINTS + j) * 9;
|
|
161
|
+
if (exact) {
|
|
162
|
+
for (let k = 0; k < 9; k += 1) rotMats[dst + k] = motion.rotMats[src0 + k];
|
|
163
|
+
} else {
|
|
164
|
+
const src1 = (i1 * JOINTS + j) * 9;
|
|
165
|
+
const q = slerpQuat(
|
|
166
|
+
matToQuat(readMat(motion.rotMats, src0)),
|
|
167
|
+
matToQuat(readMat(motion.rotMats, src1)),
|
|
168
|
+
t
|
|
169
|
+
);
|
|
170
|
+
writeMat(rotMats, dst, quatToMat(q));
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
for (let axis = 0; axis < 3; axis += 1) {
|
|
174
|
+
rootPos[f * 3 + axis] =
|
|
175
|
+
motion.rootPos[i0 * 3 + axis] * (1 - t) + motion.rootPos[i1 * 3 + axis] * t;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
if (regenerate) {
|
|
179
|
+
// The root is the only position left in the interpolation; every joint
|
|
180
|
+
// below it is placed rigidly off the retimed rotations, which is what
|
|
181
|
+
// keeps bone lengths out of the blend entirely.
|
|
182
|
+
const positions = forwardKinematics(readLocals(rotMats, f), BONE_OFFSETS, [
|
|
183
|
+
rootPos[f * 3],
|
|
184
|
+
rootPos[f * 3 + 1],
|
|
185
|
+
rootPos[f * 3 + 2],
|
|
186
|
+
]);
|
|
187
|
+
for (let j = 0; j < JOINTS; j += 1) {
|
|
188
|
+
const pd = (f * JOINTS + j) * 3;
|
|
189
|
+
posedJoints[pd] = positions[j][0];
|
|
190
|
+
posedJoints[pd + 1] = positions[j][1];
|
|
191
|
+
posedJoints[pd + 2] = positions[j][2];
|
|
192
|
+
}
|
|
193
|
+
} else {
|
|
194
|
+
for (let j = 0; j < JOINTS; j += 1) {
|
|
195
|
+
const p0 = (i0 * JOINTS + j) * 3;
|
|
196
|
+
const p1 = (i1 * JOINTS + j) * 3;
|
|
197
|
+
const pd = (f * JOINTS + j) * 3;
|
|
198
|
+
for (let axis = 0; axis < 3; axis += 1) {
|
|
199
|
+
posedJoints[pd + axis] =
|
|
200
|
+
motion.posedJoints[p0 + axis] * (1 - t) + motion.posedJoints[p1 + axis] * t;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
const retimed = { ...motion, frames, fps: targetFps, rotMats, rootPos, posedJoints };
|
|
207
|
+
if (Number.isFinite(motion.anchorFrame)) {
|
|
208
|
+
retimed.anchorFrame = Math.min(frames - 1, Math.round(motion.anchorFrame * ratio));
|
|
209
|
+
}
|
|
210
|
+
return retimed;
|
|
211
|
+
}
|
|
@@ -16,6 +16,19 @@ export function promptMoveStartFrame(startFrame, startClientX, clientX, laneWidt
|
|
|
16
16
|
return startFrame + (clientX - startClientX) * framesPerPixel;
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
+
/** Half a second at 24 fps: a shorter take is a pose, not a motion. */
|
|
20
|
+
export const TRIM_MIN_FRAMES = 12;
|
|
21
|
+
|
|
22
|
+
/** Clamp one trim-handle drag of the loaded take into a legal in/out range.
|
|
23
|
+
* `preview` is the range the gesture currently shows, `frame` the frame under
|
|
24
|
+
* the pointer and `max` the take's last displayed frame. The opposite edge
|
|
25
|
+
* never moves, and the remaining take never drops below `minFrames`. */
|
|
26
|
+
export function motionTrimRange(edge, frame, preview, max, minFrames = TRIM_MIN_FRAMES) {
|
|
27
|
+
return edge === "start"
|
|
28
|
+
? { ...preview, start: Math.max(0, Math.min(frame, preview.end - minFrames)) }
|
|
29
|
+
: { ...preview, end: Math.min(max, Math.max(frame, preview.start + minFrames)) };
|
|
30
|
+
}
|
|
31
|
+
|
|
19
32
|
/** Optional Shot blocks use their own explicit inclusive range. Gaps remain
|
|
20
33
|
* visible free-camera time instead of being inferred from the next card. */
|
|
21
34
|
export function shotBlockGeometry(shots, index, frameCount, displayFrameCount = frameCount) {
|