@melite/formats 0.0.0-stage → 0.1.4-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +557 -0
- package/lib/b64.d.ts +2 -0
- package/lib/b64.js +7 -0
- package/lib/bodyPresets.d.ts +40 -0
- package/lib/bodyPresets.js +77 -0
- package/lib/capabilityCards.d.ts +54 -0
- package/lib/capabilityCards.js +127 -0
- package/lib/captionSplitter.d.ts +49 -0
- package/lib/captionSplitter.js +172 -0
- package/lib/cardMutations.d.ts +52 -0
- package/lib/cardMutations.js +67 -0
- package/lib/downloads.d.ts +18 -0
- package/lib/downloads.js +64 -0
- package/lib/extractVideoFrame.d.ts +23 -0
- package/lib/extractVideoFrame.js +114 -0
- package/lib/glbLoad.d.ts +54 -0
- package/lib/glbLoad.js +96 -0
- package/lib/glbSkin.d.ts +14 -0
- package/lib/glbSkin.js +124 -0
- package/lib/handPose.d.ts +46 -0
- package/lib/handPose.js +122 -0
- package/lib/index.d.ts +26 -0
- package/lib/index.js +26 -0
- package/lib/keyframeExtract.d.ts +46 -0
- package/lib/keyframeExtract.js +89 -0
- package/lib/maskPrep.d.ts +88 -0
- package/lib/maskPrep.js +386 -0
- package/lib/motionConstraints.d.ts +237 -0
- package/lib/motionConstraints.js +369 -0
- package/lib/motionEnvelope.d.ts +56 -0
- package/lib/motionEnvelope.js +59 -0
- package/lib/motionNpz.d.ts +120 -0
- package/lib/motionNpz.js +497 -0
- package/lib/npz.d.ts +24 -0
- package/lib/npz.js +158 -0
- package/lib/openPoseCanvas.d.ts +17 -0
- package/lib/openPoseCanvas.js +90 -0
- package/lib/pngMeta.d.ts +42 -0
- package/lib/pngMeta.js +212 -0
- package/lib/pngMetaInfo.d.ts +14 -0
- package/lib/pngMetaInfo.js +127 -0
- package/lib/poseCodec.d.ts +20 -0
- package/lib/poseCodec.js +61 -0
- package/lib/poseTransforms.d.ts +73 -0
- package/lib/poseTransforms.js +237 -0
- package/lib/poserSkin.d.ts +75 -0
- package/lib/poserSkin.js +126 -0
- package/lib/prompt-tags.json +132 -0
- package/lib/promptTags.d.ts +42 -0
- package/lib/promptTags.js +116 -0
- package/lib/psdExport.d.ts +28 -0
- package/lib/psdExport.js +272 -0
- package/lib/readFile.d.ts +9 -0
- package/lib/readFile.js +16 -0
- package/lib/renderSnapshot.d.ts +34 -0
- package/lib/renderSnapshot.js +55 -0
- package/lib/somaSkin.d.ts +82 -0
- package/lib/somaSkin.js +267 -0
- package/lib/textToPose.d.ts +388 -0
- package/lib/textToPose.js +604 -0
- package/lib/uploadToEstate.d.ts +25 -0
- package/lib/uploadToEstate.js +97 -0
- package/package.json +148 -4
- package/README.md +0 -3
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Minimal PNG tEXt chunk injector / reader.
|
|
3
|
+
*
|
|
4
|
+
* PNG format: 8-byte signature, then a sequence of chunks.
|
|
5
|
+
* Each chunk: [4-byte length][4-byte type][N-byte data][4-byte CRC].
|
|
6
|
+
* tEXt chunk data: <keyword>\0<text> (both Latin-1, no compression).
|
|
7
|
+
*
|
|
8
|
+
* We use the key "tech_noir" to store generation params as JSON.
|
|
9
|
+
*/
|
|
10
|
+
const PNG_SIG = new Uint8Array([137, 80, 78, 71, 13, 10, 26, 10]);
|
|
11
|
+
const KEY = "tech_noir";
|
|
12
|
+
// ── CRC-32 table ────────────────────────────────────────────────────────────
|
|
13
|
+
const CRC_TABLE = (() => {
|
|
14
|
+
const t = new Uint32Array(256);
|
|
15
|
+
for (let i = 0; i < 256; i++) {
|
|
16
|
+
let c = i;
|
|
17
|
+
for (let k = 0; k < 8; k++)
|
|
18
|
+
c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
|
|
19
|
+
t[i] = c;
|
|
20
|
+
}
|
|
21
|
+
return t;
|
|
22
|
+
})();
|
|
23
|
+
function crc32(buf) {
|
|
24
|
+
let c = 0xffffffff;
|
|
25
|
+
for (let i = 0; i < buf.length; i++)
|
|
26
|
+
c = CRC_TABLE[(c ^ buf[i]) & 0xff] ^ (c >>> 8);
|
|
27
|
+
return (c ^ 0xffffffff) >>> 0;
|
|
28
|
+
}
|
|
29
|
+
function u32be(n) {
|
|
30
|
+
return new Uint8Array([(n >>> 24) & 0xff, (n >>> 16) & 0xff, (n >>> 8) & 0xff, n & 0xff]);
|
|
31
|
+
}
|
|
32
|
+
function readU32BE(buf, offset) {
|
|
33
|
+
return ((buf[offset] << 24) | (buf[offset + 1] << 16) | (buf[offset + 2] << 8) | buf[offset + 3]) >>> 0;
|
|
34
|
+
}
|
|
35
|
+
function makeChunk(type, data) {
|
|
36
|
+
const typeBytes = new TextEncoder().encode(type);
|
|
37
|
+
const crcInput = new Uint8Array(4 + data.length);
|
|
38
|
+
crcInput.set(typeBytes, 0);
|
|
39
|
+
crcInput.set(data, 4);
|
|
40
|
+
const crc = crc32(crcInput);
|
|
41
|
+
const chunk = new Uint8Array(4 + 4 + data.length + 4);
|
|
42
|
+
chunk.set(u32be(data.length), 0);
|
|
43
|
+
chunk.set(typeBytes, 4);
|
|
44
|
+
chunk.set(data, 8);
|
|
45
|
+
chunk.set(u32be(crc), 8 + data.length);
|
|
46
|
+
return chunk;
|
|
47
|
+
}
|
|
48
|
+
/** Inject generation params into a PNG as a tEXt chunk (inserted after IHDR). */
|
|
49
|
+
export function injectPngMeta(b64, params) {
|
|
50
|
+
try {
|
|
51
|
+
const bin = atob(b64);
|
|
52
|
+
const buf = new Uint8Array(bin.length);
|
|
53
|
+
for (let i = 0; i < bin.length; i++)
|
|
54
|
+
buf[i] = bin.charCodeAt(i);
|
|
55
|
+
// Verify PNG signature
|
|
56
|
+
for (let i = 0; i < 8; i++)
|
|
57
|
+
if (buf[i] !== PNG_SIG[i])
|
|
58
|
+
return b64;
|
|
59
|
+
// Find end of IHDR chunk (sig=8, length=4, type=4, data=13, crc=4 → 33 bytes)
|
|
60
|
+
const ihdrEnd = 8 + 4 + 4 + 13 + 4; // = 33
|
|
61
|
+
// Build tEXt chunk: "tech_noir\0<json>"
|
|
62
|
+
const enc = new TextEncoder();
|
|
63
|
+
const keyBytes = enc.encode(KEY);
|
|
64
|
+
const valBytes = enc.encode(JSON.stringify(params));
|
|
65
|
+
const chunkData = new Uint8Array(keyBytes.length + 1 + valBytes.length);
|
|
66
|
+
chunkData.set(keyBytes, 0);
|
|
67
|
+
chunkData[keyBytes.length] = 0;
|
|
68
|
+
chunkData.set(valBytes, keyBytes.length + 1);
|
|
69
|
+
const textChunk = makeChunk("tEXt", chunkData);
|
|
70
|
+
// Splice in after IHDR
|
|
71
|
+
const out = new Uint8Array(buf.length + textChunk.length);
|
|
72
|
+
out.set(buf.slice(0, ihdrEnd), 0);
|
|
73
|
+
out.set(textChunk, ihdrEnd);
|
|
74
|
+
out.set(buf.slice(ihdrEnd), ihdrEnd + textChunk.length);
|
|
75
|
+
// Re-encode to base64
|
|
76
|
+
let result = "";
|
|
77
|
+
for (let i = 0; i < out.length; i++)
|
|
78
|
+
result += String.fromCharCode(out[i]);
|
|
79
|
+
return btoa(result);
|
|
80
|
+
}
|
|
81
|
+
catch (e) {
|
|
82
|
+
// NO SILENT SWALLOW — chunk injection failed (corrupt PNG, bad base64,
|
|
83
|
+
// OOM on huge buffers). Log so we know metadata is being dropped, then
|
|
84
|
+
// return the original unmodified so the image itself still renders.
|
|
85
|
+
console.error('[png-meta] injectPngMeta failed, returning original:', e);
|
|
86
|
+
return b64;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
/** Read generation params from a PNG's tEXt chunk, or null if not present. */
|
|
90
|
+
export function readPngMeta(b64OrDataUrl) {
|
|
91
|
+
try {
|
|
92
|
+
const b64 = b64OrDataUrl.startsWith("data:") ? b64OrDataUrl.split(",")[1] : b64OrDataUrl;
|
|
93
|
+
const bin = atob(b64);
|
|
94
|
+
const buf = new Uint8Array(bin.length);
|
|
95
|
+
for (let i = 0; i < bin.length; i++)
|
|
96
|
+
buf[i] = bin.charCodeAt(i);
|
|
97
|
+
// Walk chunks from offset 8
|
|
98
|
+
let pos = 8;
|
|
99
|
+
while (pos + 12 <= buf.length) {
|
|
100
|
+
const length = readU32BE(buf, pos);
|
|
101
|
+
const type = String.fromCharCode(buf[pos + 4], buf[pos + 5], buf[pos + 6], buf[pos + 7]);
|
|
102
|
+
if (type === "IEND")
|
|
103
|
+
break;
|
|
104
|
+
if (type === "tEXt" && length > KEY.length) {
|
|
105
|
+
const data = buf.slice(pos + 8, pos + 8 + length);
|
|
106
|
+
const nullIdx = data.indexOf(0);
|
|
107
|
+
if (nullIdx !== -1) {
|
|
108
|
+
const key = new TextDecoder().decode(data.slice(0, nullIdx));
|
|
109
|
+
if (key === KEY) {
|
|
110
|
+
const val = new TextDecoder().decode(data.slice(nullIdx + 1));
|
|
111
|
+
return JSON.parse(val);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
pos += 4 + 4 + length + 4;
|
|
116
|
+
}
|
|
117
|
+
return null;
|
|
118
|
+
}
|
|
119
|
+
catch (e) {
|
|
120
|
+
// NO SILENT SWALLOW — parse failure means either the input wasn't a
|
|
121
|
+
// valid PNG/data-URL, or the tEXt payload was malformed. Log so we know
|
|
122
|
+
// metadata is being dropped, then return null ("no params found").
|
|
123
|
+
console.error('[png-meta] readPngMeta failed:', e);
|
|
124
|
+
return null;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
//# sourceMappingURL=pngMetaInfo.js.map
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/** poseCodec — the PURE codec half of the old lib/text-to-pose:
|
|
2
|
+
* byte/b64 codecs + the SOMA-77 bone topology. Fetch-free by law —
|
|
3
|
+
* the fetch-based off-main-thread decode stays app-side (lib/text-to-pose.ts). */
|
|
4
|
+
/** Canonical SOMA-77 mesh vertex count.
|
|
5
|
+
*
|
|
6
|
+
* The LBS skinning weights are keyed to this exact vertex topology, so every
|
|
7
|
+
* shaped mesh — anny, mhr, soma, custom GLB — must have exactly this many
|
|
8
|
+
* vertices. The backend echoes the actual count via PoseMotion.vertex_count;
|
|
9
|
+
* this constant is the fallback for old responses that don't include it and
|
|
10
|
+
* the source of truth for UI prompts (CustomMeshLoader, etc.).
|
|
11
|
+
*/
|
|
12
|
+
export declare const SOMA_VERTEX_COUNT = 18056;
|
|
13
|
+
export declare const SOMA77_BONE_CONNECTIONS: [number, number][];
|
|
14
|
+
/** Encode a Uint8Array to a base64 string.
|
|
15
|
+
*
|
|
16
|
+
* Processes the input in 32KB chunks because String.fromCharCode.apply has
|
|
17
|
+
* a ~65K argument limit on some engines (the historical V8 stack-arg limit).
|
|
18
|
+
* Uses Array.from() rather than the brittle `as unknown as number[]` cast. */
|
|
19
|
+
export declare function encodeBufferToB64(bytes: Uint8Array): string;
|
|
20
|
+
//# sourceMappingURL=poseCodec.d.ts.map
|
package/lib/poseCodec.js
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/** poseCodec — the PURE codec half of the old lib/text-to-pose:
|
|
2
|
+
* byte/b64 codecs + the SOMA-77 bone topology. Fetch-free by law —
|
|
3
|
+
* the fetch-based off-main-thread decode stays app-side (lib/text-to-pose.ts). */
|
|
4
|
+
/** Canonical SOMA-77 mesh vertex count.
|
|
5
|
+
*
|
|
6
|
+
* The LBS skinning weights are keyed to this exact vertex topology, so every
|
|
7
|
+
* shaped mesh — anny, mhr, soma, custom GLB — must have exactly this many
|
|
8
|
+
* vertices. The backend echoes the actual count via PoseMotion.vertex_count;
|
|
9
|
+
* this constant is the fallback for old responses that don't include it and
|
|
10
|
+
* the source of truth for UI prompts (CustomMeshLoader, etc.).
|
|
11
|
+
*/
|
|
12
|
+
export const SOMA_VERTEX_COUNT = 18056;
|
|
13
|
+
// Generated from SOMASkeleton77.bone_order_names_with_parents
|
|
14
|
+
// (vendor/kimodo/kimodo/skeleton/definitions.py). 77 joints → 76 bones.
|
|
15
|
+
// Joint order: Hips(0) Spine1(1) Spine2(2) Chest(3) Neck1(4) Neck2(5) Head(6)
|
|
16
|
+
// HeadEnd(7) Jaw(8) LeftEye(9) RightEye(10) LeftShoulder(11) LeftArm(12)
|
|
17
|
+
// LeftForeArm(13) LeftHand(14) [LeftHand fingers 15..38]
|
|
18
|
+
// RightShoulder(39) RightArm(40) RightForeArm(41) RightHand(42)
|
|
19
|
+
// [RightHand fingers 43..66] LeftLeg(67) LeftShin(68) LeftFoot(69)
|
|
20
|
+
// LeftToeBase(70) LeftToeEnd(71) RightLeg(72) RightShin(73) RightFoot(74)
|
|
21
|
+
// RightToeBase(75) RightToeEnd(76).
|
|
22
|
+
export const SOMA77_BONE_CONNECTIONS = [
|
|
23
|
+
// Spine + head
|
|
24
|
+
[0, 1], [1, 2], [2, 3], [3, 4], [4, 5], [5, 6],
|
|
25
|
+
[6, 7], [6, 8], [6, 9], [6, 10],
|
|
26
|
+
// Left shoulder + arm + thumb chain
|
|
27
|
+
[3, 11], [11, 12], [12, 13], [13, 14],
|
|
28
|
+
[14, 15], [15, 16], [16, 17], [17, 18],
|
|
29
|
+
// Left index / middle / ring / pinky
|
|
30
|
+
[14, 19], [19, 20], [20, 21], [21, 22], [22, 23],
|
|
31
|
+
[14, 24], [24, 25], [25, 26], [26, 27], [27, 28],
|
|
32
|
+
[14, 29], [29, 30], [30, 31], [31, 32], [32, 33],
|
|
33
|
+
[14, 34], [34, 35], [35, 36], [36, 37], [37, 38],
|
|
34
|
+
// Right shoulder + arm + thumb chain
|
|
35
|
+
[3, 39], [39, 40], [40, 41], [41, 42],
|
|
36
|
+
[42, 43], [43, 44], [44, 45], [45, 46],
|
|
37
|
+
// Right index / middle / ring / pinky
|
|
38
|
+
[42, 47], [47, 48], [48, 49], [49, 50], [50, 51],
|
|
39
|
+
[42, 52], [52, 53], [53, 54], [54, 55], [55, 56],
|
|
40
|
+
[42, 57], [57, 58], [58, 59], [59, 60], [60, 61],
|
|
41
|
+
[42, 62], [62, 63], [63, 64], [64, 65], [65, 66],
|
|
42
|
+
// Left leg + foot + toes
|
|
43
|
+
[0, 67], [67, 68], [68, 69], [69, 70], [70, 71],
|
|
44
|
+
// Right leg + foot + toes
|
|
45
|
+
[0, 72], [72, 73], [73, 74], [74, 75], [75, 76],
|
|
46
|
+
];
|
|
47
|
+
// ── Base64 encode (32KB chunks — engine stack-arg limits) ────────────────
|
|
48
|
+
/** Encode a Uint8Array to a base64 string.
|
|
49
|
+
*
|
|
50
|
+
* Processes the input in 32KB chunks because String.fromCharCode.apply has
|
|
51
|
+
* a ~65K argument limit on some engines (the historical V8 stack-arg limit).
|
|
52
|
+
* Uses Array.from() rather than the brittle `as unknown as number[]` cast. */
|
|
53
|
+
export function encodeBufferToB64(bytes) {
|
|
54
|
+
let bin = "";
|
|
55
|
+
const CHUNK = 0x8000;
|
|
56
|
+
for (let i = 0; i < bytes.length; i += CHUNK) {
|
|
57
|
+
bin += String.fromCharCode.apply(null, Array.from(bytes.subarray(i, Math.min(i + CHUNK, bytes.length))));
|
|
58
|
+
}
|
|
59
|
+
return btoa(bin);
|
|
60
|
+
}
|
|
61
|
+
//# sourceMappingURL=poseCodec.js.map
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
export declare const KP: {
|
|
2
|
+
readonly NOSE: 0;
|
|
3
|
+
readonly NECK: 1;
|
|
4
|
+
readonly R_SHOULDER: 2;
|
|
5
|
+
readonly R_ELBOW: 3;
|
|
6
|
+
readonly R_WRIST: 4;
|
|
7
|
+
readonly L_SHOULDER: 5;
|
|
8
|
+
readonly L_ELBOW: 6;
|
|
9
|
+
readonly L_WRIST: 7;
|
|
10
|
+
readonly R_HIP: 8;
|
|
11
|
+
readonly R_KNEE: 9;
|
|
12
|
+
readonly R_ANKLE: 10;
|
|
13
|
+
readonly L_HIP: 11;
|
|
14
|
+
readonly L_KNEE: 12;
|
|
15
|
+
readonly L_ANKLE: 13;
|
|
16
|
+
readonly R_EYE: 14;
|
|
17
|
+
readonly L_EYE: 15;
|
|
18
|
+
readonly R_EAR: 16;
|
|
19
|
+
readonly L_EAR: 17;
|
|
20
|
+
};
|
|
21
|
+
export declare const NUM_KEYPOINTS = 18;
|
|
22
|
+
export declare const FLAT_LENGTH: number;
|
|
23
|
+
export type Keypoints = number[];
|
|
24
|
+
export interface PosePreset {
|
|
25
|
+
id: string;
|
|
26
|
+
name: string;
|
|
27
|
+
category: string;
|
|
28
|
+
description: string;
|
|
29
|
+
tags: string[];
|
|
30
|
+
render_url: string;
|
|
31
|
+
keypoints: Keypoints;
|
|
32
|
+
}
|
|
33
|
+
export interface PoseAdjustments {
|
|
34
|
+
leftArmRaise: number;
|
|
35
|
+
rightArmRaise: number;
|
|
36
|
+
leftElbowBend: number;
|
|
37
|
+
rightElbowBend: number;
|
|
38
|
+
weightShift: number;
|
|
39
|
+
crouch: number;
|
|
40
|
+
headTilt: number;
|
|
41
|
+
spineLean: number;
|
|
42
|
+
}
|
|
43
|
+
export declare const DEFAULT_ADJUSTMENTS: PoseAdjustments;
|
|
44
|
+
export declare function applyAdjustments(base: Keypoints, adj: PoseAdjustments): Keypoints;
|
|
45
|
+
export declare function fetchPosePresets(): Promise<PosePreset[]>;
|
|
46
|
+
/**
|
|
47
|
+
* Render keypoints to a PNG skeleton image via the backend.
|
|
48
|
+
* Returns a Blob (image/png) suitable for object URLs.
|
|
49
|
+
*/
|
|
50
|
+
export declare function renderCustomPose(keypoints: Keypoints, opts?: {
|
|
51
|
+
width?: number;
|
|
52
|
+
height?: number;
|
|
53
|
+
}): Promise<Blob>;
|
|
54
|
+
/**
|
|
55
|
+
* Convert a Blob to a base64 data URL — for saving to the asset store
|
|
56
|
+
* (which expects data URLs, not object URLs, for persistence).
|
|
57
|
+
* Delegates to the shared readAsDataURL (@melite/ui SlotSquare — the
|
|
58
|
+
* ONE read path every slot-shaped attach uses).
|
|
59
|
+
*/
|
|
60
|
+
export declare function blobToDataUrl(blob: Blob): Promise<string>;
|
|
61
|
+
/**
|
|
62
|
+
* Convert 2D flat COCO-18 keypoints to approximate 3D positions.
|
|
63
|
+
*
|
|
64
|
+
* The 2D keypoints are flattened [x, y, confidence, ...] in image-pixel space
|
|
65
|
+
* (typically 1024×1024). This function maps them into the SOMA coordinate
|
|
66
|
+
* frame: +X = subject's left, +Y = up, +Z = forward.
|
|
67
|
+
*
|
|
68
|
+
* Z-values are estimated from joint category (shoulders forward, hips back).
|
|
69
|
+
* The result is suitable for the 3D PoseViewer — not physically accurate but
|
|
70
|
+
* visually coherent for preset poses.
|
|
71
|
+
*/
|
|
72
|
+
export declare function flatTo3D(flat: number[]): [number, number, number][];
|
|
73
|
+
//# sourceMappingURL=poseTransforms.d.ts.map
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pose Studio — keypoint transform utilities.
|
|
3
|
+
*
|
|
4
|
+
* COCO-18 keypoints: 18 joints × (x, y, confidence) = 54 numbers in a flat
|
|
5
|
+
* array. This is the same format DWPose outputs (see media/workflows/utils/dwpose.py)
|
|
6
|
+
* and the same format the /poser/* backend endpoints accept.
|
|
7
|
+
*
|
|
8
|
+
* The Pose Studio UI lets the user pick a base preset, then tweak it with
|
|
9
|
+
* semantic sliders (arm raise, elbow bend, head tilt, etc.). Each slider
|
|
10
|
+
* maps to a transform on the base keypoint array. The result is POSTed to
|
|
11
|
+
* /poser/render, which returns a PNG skeleton image.
|
|
12
|
+
*
|
|
13
|
+
* Convention: subject's RIGHT side = image LEFT (so right-side keypoints
|
|
14
|
+
* have smaller X values than left-side ones).
|
|
15
|
+
*/
|
|
16
|
+
import { apiFetch } from "@melite/catalog";
|
|
17
|
+
import { readAsDataURL } from "./readFile.js";
|
|
18
|
+
// ── COCO-18 keypoint indices ─────────────────────────────────────────────
|
|
19
|
+
export const KP = {
|
|
20
|
+
NOSE: 0, NECK: 1,
|
|
21
|
+
R_SHOULDER: 2, R_ELBOW: 3, R_WRIST: 4,
|
|
22
|
+
L_SHOULDER: 5, L_ELBOW: 6, L_WRIST: 7,
|
|
23
|
+
R_HIP: 8, R_KNEE: 9, R_ANKLE: 10,
|
|
24
|
+
L_HIP: 11, L_KNEE: 12, L_ANKLE: 13,
|
|
25
|
+
R_EYE: 14, L_EYE: 15, R_EAR: 16, L_EAR: 17,
|
|
26
|
+
};
|
|
27
|
+
export const NUM_KEYPOINTS = 18;
|
|
28
|
+
export const FLAT_LENGTH = NUM_KEYPOINTS * 3; // 54
|
|
29
|
+
export const DEFAULT_ADJUSTMENTS = {
|
|
30
|
+
leftArmRaise: 0,
|
|
31
|
+
rightArmRaise: 0,
|
|
32
|
+
leftElbowBend: 0,
|
|
33
|
+
rightElbowBend: 0,
|
|
34
|
+
weightShift: 0,
|
|
35
|
+
crouch: 0,
|
|
36
|
+
headTilt: 0,
|
|
37
|
+
spineLean: 0,
|
|
38
|
+
};
|
|
39
|
+
// ── Math helpers ─────────────────────────────────────────────────────────
|
|
40
|
+
function lerp(a, b, t) {
|
|
41
|
+
return a + (b - a) * t;
|
|
42
|
+
}
|
|
43
|
+
function getX(kp, idx) {
|
|
44
|
+
return kp[idx * 3];
|
|
45
|
+
}
|
|
46
|
+
function getY(kp, idx) {
|
|
47
|
+
return kp[idx * 3 + 1];
|
|
48
|
+
}
|
|
49
|
+
function setXY(kp, idx, x, y) {
|
|
50
|
+
kp[idx * 3] = x;
|
|
51
|
+
kp[idx * 3 + 1] = y;
|
|
52
|
+
}
|
|
53
|
+
function translate(kp, idx, dx, dy) {
|
|
54
|
+
kp[idx * 3] += dx;
|
|
55
|
+
kp[idx * 3 + 1] += dy;
|
|
56
|
+
}
|
|
57
|
+
// ── Transform flow ───────────────────────────────────────────────────
|
|
58
|
+
// Each transform mutates the (already-cloned) keypoint array.
|
|
59
|
+
// Order matters: crouch and weight shift move everything; arm/elbow transforms
|
|
60
|
+
// then operate on the post-shifted shoulder positions; head tilt and spine
|
|
61
|
+
// lean are last because they're independent of limb positions.
|
|
62
|
+
export function applyAdjustments(base, adj) {
|
|
63
|
+
const kp = [...base]; // shallow clone — numbers are immutable so this is enough
|
|
64
|
+
// 1. Crouch — compress Y of all keypoints around the hip pivot.
|
|
65
|
+
if (adj.crouch !== 0) {
|
|
66
|
+
const hipY = getY(kp, KP.R_HIP); // pivot at hip height
|
|
67
|
+
const scale = 1 - adj.crouch * 0.30; // up to 30% compression
|
|
68
|
+
for (let i = 0; i < NUM_KEYPOINTS; i++) {
|
|
69
|
+
const y = getY(kp, i);
|
|
70
|
+
setXY(kp, i, getX(kp, i), hipY + (y - hipY) * scale);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
// 2. Weight shift — translate all keypoints horizontally.
|
|
74
|
+
if (adj.weightShift !== 0) {
|
|
75
|
+
const dx = adj.weightShift * 80;
|
|
76
|
+
for (let i = 0; i < NUM_KEYPOINTS; i++) {
|
|
77
|
+
translate(kp, i, dx, 0);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
// 3. Spine lean — translate head/neck/shoulders forward (down in image space)
|
|
81
|
+
// or backward (up in image space). Arms follow shoulders partially.
|
|
82
|
+
if (adj.spineLean !== 0) {
|
|
83
|
+
const dy = adj.spineLean * 60;
|
|
84
|
+
const upperBody = [
|
|
85
|
+
KP.NOSE, KP.NECK, KP.R_SHOULDER, KP.L_SHOULDER,
|
|
86
|
+
KP.R_EYE, KP.L_EYE, KP.R_EAR, KP.L_EAR,
|
|
87
|
+
];
|
|
88
|
+
for (const idx of upperBody)
|
|
89
|
+
translate(kp, idx, 0, dy);
|
|
90
|
+
// Elbows and wrists trail the shoulder motion (less, since arms have inertia)
|
|
91
|
+
translate(kp, KP.R_ELBOW, 0, dy * 0.5);
|
|
92
|
+
translate(kp, KP.L_ELBOW, 0, dy * 0.5);
|
|
93
|
+
translate(kp, KP.R_WRIST, 0, dy * 0.3);
|
|
94
|
+
translate(kp, KP.L_WRIST, 0, dy * 0.3);
|
|
95
|
+
}
|
|
96
|
+
// 4. Head tilt — rotate the head cluster (nose, eyes, ears) around the neck.
|
|
97
|
+
if (adj.headTilt !== 0) {
|
|
98
|
+
const nx = getX(kp, KP.NECK);
|
|
99
|
+
const ny = getY(kp, KP.NECK);
|
|
100
|
+
const angle = adj.headTilt * 0.35; // ~20° max
|
|
101
|
+
const cos = Math.cos(angle);
|
|
102
|
+
const sin = Math.sin(angle);
|
|
103
|
+
for (const idx of [KP.NOSE, KP.R_EYE, KP.L_EYE, KP.R_EAR, KP.L_EAR]) {
|
|
104
|
+
const dx = getX(kp, idx) - nx;
|
|
105
|
+
const dy = getY(kp, idx) - ny;
|
|
106
|
+
setXY(kp, idx, nx + dx * cos - dy * sin, ny + dx * sin + dy * cos);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
// 5. Arm raises — pull elbow + wrist toward a "raised above head" target.
|
|
110
|
+
// Slider > 0 raises; slider < 0 is a no-op (can't go lower than "at side").
|
|
111
|
+
raiseArm(kp, KP.L_SHOULDER, KP.L_ELBOW, KP.L_WRIST, Math.max(0, adj.leftArmRaise));
|
|
112
|
+
raiseArm(kp, KP.R_SHOULDER, KP.R_ELBOW, KP.R_WRIST, Math.max(0, adj.rightArmRaise));
|
|
113
|
+
// 6. Elbow bends — draw wrist toward the midline (neck X).
|
|
114
|
+
bendElbow(kp, KP.L_WRIST, KP.NECK, adj.leftElbowBend);
|
|
115
|
+
bendElbow(kp, KP.R_WRIST, KP.NECK, adj.rightElbowBend);
|
|
116
|
+
return kp;
|
|
117
|
+
}
|
|
118
|
+
function raiseArm(kp, shoulderIdx, elbowIdx, wristIdx, amount) {
|
|
119
|
+
if (amount === 0)
|
|
120
|
+
return;
|
|
121
|
+
const shx = getX(kp, shoulderIdx);
|
|
122
|
+
// Target wrist position: directly above shoulder, near top of frame.
|
|
123
|
+
const targetWristX = shx;
|
|
124
|
+
const targetWristY = 80;
|
|
125
|
+
// Target elbow position: halfway between current and target wrist.
|
|
126
|
+
const targetElbowX = lerp(getX(kp, elbowIdx), targetWristX, 0.5);
|
|
127
|
+
const targetElbowY = lerp(getY(kp, elbowIdx), targetWristY, 0.5);
|
|
128
|
+
// Lerp wrist fully, elbow partially (it trails).
|
|
129
|
+
setXY(kp, wristIdx, lerp(getX(kp, wristIdx), targetWristX, amount), lerp(getY(kp, wristIdx), targetWristY, amount));
|
|
130
|
+
setXY(kp, elbowIdx, lerp(getX(kp, elbowIdx), targetElbowX, amount * 0.7), lerp(getY(kp, elbowIdx), targetElbowY, amount * 0.7));
|
|
131
|
+
}
|
|
132
|
+
function bendElbow(kp, wristIdx, neckIdx, amount) {
|
|
133
|
+
if (amount === 0)
|
|
134
|
+
return;
|
|
135
|
+
const targetX = lerp(getX(kp, wristIdx), getX(kp, neckIdx), 0.65);
|
|
136
|
+
const targetY = lerp(getY(kp, wristIdx), getY(kp, neckIdx), 0.30);
|
|
137
|
+
setXY(kp, wristIdx, lerp(getX(kp, wristIdx), targetX, amount), lerp(getY(kp, wristIdx), targetY, amount));
|
|
138
|
+
}
|
|
139
|
+
// ── Preset fetch / render helpers ────────────────────────────────────────
|
|
140
|
+
export async function fetchPosePresets() {
|
|
141
|
+
const res = await apiFetch("/v1/poser/presets");
|
|
142
|
+
if (!res.ok)
|
|
143
|
+
throw new Error(`Failed to fetch pose presets: ${res.status}`);
|
|
144
|
+
return res.json();
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Render keypoints to a PNG skeleton image via the backend.
|
|
148
|
+
* Returns a Blob (image/png) suitable for object URLs.
|
|
149
|
+
*/
|
|
150
|
+
export async function renderCustomPose(keypoints, opts = {}) {
|
|
151
|
+
const res = await apiFetch("/v1/poser/render", {
|
|
152
|
+
method: "POST",
|
|
153
|
+
headers: { "Content-Type": "application/json" },
|
|
154
|
+
body: JSON.stringify({
|
|
155
|
+
keypoints,
|
|
156
|
+
width: opts.width ?? 1024,
|
|
157
|
+
height: opts.height ?? 1024,
|
|
158
|
+
line_width: 5,
|
|
159
|
+
point_radius: 6,
|
|
160
|
+
}),
|
|
161
|
+
});
|
|
162
|
+
if (!res.ok) {
|
|
163
|
+
const txt = await res.text();
|
|
164
|
+
throw new Error(`Pose render failed: ${res.status} ${txt}`);
|
|
165
|
+
}
|
|
166
|
+
return res.blob();
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* Convert a Blob to a base64 data URL — for saving to the asset store
|
|
170
|
+
* (which expects data URLs, not object URLs, for persistence).
|
|
171
|
+
* Delegates to the shared readAsDataURL (@melite/ui SlotSquare — the
|
|
172
|
+
* ONE read path every slot-shaped attach uses).
|
|
173
|
+
*/
|
|
174
|
+
export function blobToDataUrl(blob) {
|
|
175
|
+
return readAsDataURL(blob);
|
|
176
|
+
}
|
|
177
|
+
// ── 2D → approximate 3D projection ──────────────────────────────────────────
|
|
178
|
+
/** SOMA coordinate space constants. */
|
|
179
|
+
const SOMA_SCALE = 0.6; // maps 1024px width to ~0.6 SOMA units (hip-to-hip)
|
|
180
|
+
const SOMA_CENTER_X = 0.5; // image center as fraction of width
|
|
181
|
+
const SOMA_CENTER_Y = 0.55; // approximate hip center as fraction of height
|
|
182
|
+
/** Default z-depths per COCO-18 joint (in SOMA units). Negative = toward viewer. */
|
|
183
|
+
const DEFAULT_Z = {
|
|
184
|
+
[KP.NOSE]: 0.10,
|
|
185
|
+
[KP.NECK]: 0.08,
|
|
186
|
+
[KP.R_SHOULDER]: 0.05,
|
|
187
|
+
[KP.R_ELBOW]: 0.02,
|
|
188
|
+
[KP.R_WRIST]: 0.00,
|
|
189
|
+
[KP.L_SHOULDER]: 0.05,
|
|
190
|
+
[KP.L_ELBOW]: 0.02,
|
|
191
|
+
[KP.L_WRIST]: 0.00,
|
|
192
|
+
[KP.R_HIP]: -0.05,
|
|
193
|
+
[KP.R_KNEE]: -0.02,
|
|
194
|
+
[KP.R_ANKLE]: 0.00,
|
|
195
|
+
[KP.L_HIP]: -0.05,
|
|
196
|
+
[KP.L_KNEE]: -0.02,
|
|
197
|
+
[KP.L_ANKLE]: 0.00,
|
|
198
|
+
[KP.R_EYE]: 0.15,
|
|
199
|
+
[KP.L_EYE]: 0.15,
|
|
200
|
+
[KP.R_EAR]: 0.12,
|
|
201
|
+
[KP.L_EAR]: 0.12,
|
|
202
|
+
};
|
|
203
|
+
/**
|
|
204
|
+
* Convert 2D flat COCO-18 keypoints to approximate 3D positions.
|
|
205
|
+
*
|
|
206
|
+
* The 2D keypoints are flattened [x, y, confidence, ...] in image-pixel space
|
|
207
|
+
* (typically 1024×1024). This function maps them into the SOMA coordinate
|
|
208
|
+
* frame: +X = subject's left, +Y = up, +Z = forward.
|
|
209
|
+
*
|
|
210
|
+
* Z-values are estimated from joint category (shoulders forward, hips back).
|
|
211
|
+
* The result is suitable for the 3D PoseViewer — not physically accurate but
|
|
212
|
+
* visually coherent for preset poses.
|
|
213
|
+
*/
|
|
214
|
+
export function flatTo3D(flat) {
|
|
215
|
+
const result = [];
|
|
216
|
+
for (let i = 0; i < NUM_KEYPOINTS; i++) {
|
|
217
|
+
const x = flat[i * 3] ?? 0;
|
|
218
|
+
const y = flat[i * 3 + 1] ?? 0;
|
|
219
|
+
const c = flat[i * 3 + 2] ?? 0;
|
|
220
|
+
if (c < 0.1) {
|
|
221
|
+
// Low confidence — place joint at a neutral position
|
|
222
|
+
result.push([0, 0, 0]);
|
|
223
|
+
continue;
|
|
224
|
+
}
|
|
225
|
+
// Normalize from image coords to SOMA coords
|
|
226
|
+
const sx = (x / 1024 - SOMA_CENTER_X) * SOMA_SCALE * 2;
|
|
227
|
+
// Y is inverted in image coords (0 = top)
|
|
228
|
+
const sy = (SOMA_CENTER_Y - y / 1024) * SOMA_SCALE * 2;
|
|
229
|
+
// Every COCO-18 joint 0..17 is tabled in DEFAULT_Z above (the loop
|
|
230
|
+
// bound is NUM_KEYPOINTS = 18), so the lookup is always present —
|
|
231
|
+
// no fallback to cover. Keep the table and loop in lockstep.
|
|
232
|
+
const sz = DEFAULT_Z[i];
|
|
233
|
+
result.push([sx, sy, sz]);
|
|
234
|
+
}
|
|
235
|
+
return result;
|
|
236
|
+
}
|
|
237
|
+
//# sourceMappingURL=poseTransforms.js.map
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
export interface SkinMeshData {
|
|
2
|
+
vertices: number[][];
|
|
3
|
+
faces: number[][];
|
|
4
|
+
}
|
|
5
|
+
export interface IdentityGroup {
|
|
6
|
+
label: string;
|
|
7
|
+
start: number;
|
|
8
|
+
count: number;
|
|
9
|
+
}
|
|
10
|
+
export interface LocalChangeGroup {
|
|
11
|
+
label: string;
|
|
12
|
+
indices: number[];
|
|
13
|
+
category: "body" | "face";
|
|
14
|
+
}
|
|
15
|
+
export interface LocalChangeSpace {
|
|
16
|
+
dims: number;
|
|
17
|
+
labels: string[];
|
|
18
|
+
defaults: number[];
|
|
19
|
+
range: [number, number];
|
|
20
|
+
groups: LocalChangeGroup[];
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* A community MakeHuman .target blendshape exposed as a single slider.
|
|
24
|
+
*
|
|
25
|
+
* Each entry corresponds to a .target file in media/motion's
|
|
26
|
+
* references/makehuman_targets/ directory. The `id` is the slider key sent
|
|
27
|
+
* back to POST /poser/skin/mesh as `custom_targets[id]`. The `name` and
|
|
28
|
+
* `description` power the slider row UI.
|
|
29
|
+
*/
|
|
30
|
+
export interface CustomTargetItem {
|
|
31
|
+
id: string;
|
|
32
|
+
name: string;
|
|
33
|
+
description: string;
|
|
34
|
+
glyph: string;
|
|
35
|
+
/** Absolute path on the server (debug only - not used by the UI). */
|
|
36
|
+
file?: string;
|
|
37
|
+
}
|
|
38
|
+
export interface CustomTargetSpace {
|
|
39
|
+
/** Ordered list - one slider row per entry. */
|
|
40
|
+
items: CustomTargetItem[];
|
|
41
|
+
defaults: number[];
|
|
42
|
+
range: [number, number];
|
|
43
|
+
}
|
|
44
|
+
export interface IdentitySpace {
|
|
45
|
+
identity_model: string;
|
|
46
|
+
dims: number;
|
|
47
|
+
labels: string[];
|
|
48
|
+
defaults: number[];
|
|
49
|
+
range: [number, number];
|
|
50
|
+
groups: IdentityGroup[] | null;
|
|
51
|
+
local_changes: LocalChangeSpace | null;
|
|
52
|
+
/**
|
|
53
|
+
* Community .target blendshapes (Anny only). null for MHR/SOMA - the UI
|
|
54
|
+
* hides the Stylization tab in that case.
|
|
55
|
+
*/
|
|
56
|
+
custom_targets: CustomTargetSpace | null;
|
|
57
|
+
}
|
|
58
|
+
export type PreviewPose = "tpose" | "arms_down";
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
export interface BindMeshQuery {
|
|
64
|
+
identityModel?: string;
|
|
65
|
+
identityCoeffs?: number[] | null;
|
|
66
|
+
localChanges?: Record<string, number> | null;
|
|
67
|
+
previewPose?: PreviewPose | null;
|
|
68
|
+
customTargets?: Record<string, number> | null;
|
|
69
|
+
}
|
|
70
|
+
export declare function fetchBindMesh(query?: BindMeshQuery): Promise<SkinMeshData>;
|
|
71
|
+
export declare function fetchIdentitySpace(model: string): Promise<IdentitySpace>;
|
|
72
|
+
/** SkinMeshData → flat Float32Array positions (V*3) in METERS with the
|
|
73
|
+
* floor lift applied (the body-studio preview geometry path). */
|
|
74
|
+
export declare function buildFlatPositions(meshData: SkinMeshData): Float32Array;
|
|
75
|
+
//# sourceMappingURL=poserSkin.d.ts.map
|