@genex-ai/cli-demo 0.36.0 → 0.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -5
- package/dist/index.js +382 -28
- package/package.json +1 -1
- package/templates/controllers/NOTICE.md +8 -5
- package/templates/controllers/assets/animation-library.glb +0 -0
- package/templates/controllers/assets/anims-manifest.json +1306 -0
- package/templates/controllers/character/animation-packs.ts +70 -0
- package/templates/controllers/character/character-animations.ts +51 -8
- package/templates/controllers/character/character-controller.ts +153 -5
- package/templates/controllers/character/keyboard-input.ts +12 -1
- package/templates/controllers/character/presets.ts +19 -1
- package/templates/controllers/character/vrm/foot-ik.ts +71 -15
- package/templates/controllers/character/vrm/vrm-retarget.ts +72 -12
- package/templates/skills/genex-ai-image/SKILL.md +181 -0
- package/templates/skills/genex-ai-video/SKILL.md +148 -0
- package/templates/skills/genex-getting-started/SKILL.md +4 -1
- package/templates/skills/genex-threejs-character-controller/SKILL.md +55 -22
- package/templates/skills/genex-threejs-character-controller/references/animations.md +91 -46
- package/templates/skills/genex-threejs-character-controller/references/tuning-and-presets.md +3 -0
- package/templates/skills/genex-threejs-multiplayer/references/host-physics.md +7 -6
- package/templates/skills/genex-threejs-skill-router/SKILL.md +9 -8
- package/templates/skills/genex-threejs-vehicle-controllers/references/enter-exit.md +3 -2
- package/templates/controllers/assets/character.glb +0 -0
|
@@ -68,6 +68,16 @@ export interface FootIKOptions {
|
|
|
68
68
|
* so airborne legs keep their jump pose). Default: always active.
|
|
69
69
|
*/
|
|
70
70
|
isActive?: () => boolean;
|
|
71
|
+
/**
|
|
72
|
+
* Gate for the terrain-relative reach (downward offsets + pelvis drop, and
|
|
73
|
+
* full plant-up). Return false while a full-body one-shot is playing
|
|
74
|
+
* (`() => !anims.oneShotActive`): those clips are choreography — a foot
|
|
75
|
+
* swinging over a lower step must not drag the pelvis down into a staircase,
|
|
76
|
+
* and a raised stance foot must not be yanked a full step up. While gated,
|
|
77
|
+
* feet are lifted only as much as needed to keep the sole out of the contact
|
|
78
|
+
* under them (anti dig-in). Default: always allowed.
|
|
79
|
+
*/
|
|
80
|
+
allowReachDown?: () => boolean;
|
|
71
81
|
}
|
|
72
82
|
|
|
73
83
|
// three-vrm's own node type — same THREE.Object3D at runtime, but using the
|
|
@@ -96,6 +106,12 @@ const NORMAL_DAMPING = 12; // ground-normal smoothing
|
|
|
96
106
|
const PLANTED_LIFT_MIN = 0.04; // below this lift the foot is fully planted (align at full weight)
|
|
97
107
|
const PLANTED_LIFT_MAX = 0.16; // above this lift the foot is fully lifted (no align)
|
|
98
108
|
const MAX_FOOT_TILT = 0.6; // clamp foot-to-slope tilt (rad)
|
|
109
|
+
const PENETRATION_SLACK = 0.02; // ankle may dip this far below rest before the no-sink lift kicks in
|
|
110
|
+
// Max anti-dig-in lift while reach-down is gated (one-shots). Retarget dips are
|
|
111
|
+
// a few cm; a hard cap keeps a stance foot carried INTO a stair riser by the
|
|
112
|
+
// choreography from being lifted a whole step ("knee to the chest") — beyond
|
|
113
|
+
// the cap it stays buried, which the step itself mostly occludes.
|
|
114
|
+
const GATED_LIFT_CAP = 0.12;
|
|
99
115
|
const MIN_BONE_LENGTH = 1e-4;
|
|
100
116
|
const IK_EPSILON = 1e-4;
|
|
101
117
|
|
|
@@ -136,6 +152,7 @@ export class FootIK {
|
|
|
136
152
|
#pelvisDrop: boolean;
|
|
137
153
|
#alignFeet: boolean;
|
|
138
154
|
#isActive: (() => boolean) | undefined;
|
|
155
|
+
#allowReachDown: (() => boolean) | undefined;
|
|
139
156
|
#enabled = true;
|
|
140
157
|
#weight = 0;
|
|
141
158
|
#restFootHeight = 0;
|
|
@@ -148,6 +165,7 @@ export class FootIK {
|
|
|
148
165
|
this.#pelvisDrop = options.pelvisDrop ?? true;
|
|
149
166
|
this.#alignFeet = options.alignFeet ?? true;
|
|
150
167
|
this.#isActive = options.isActive;
|
|
168
|
+
this.#allowReachDown = options.allowReachDown;
|
|
151
169
|
this.#modelRoot = vrm.scene;
|
|
152
170
|
|
|
153
171
|
const h = vrm.humanoid;
|
|
@@ -157,7 +175,14 @@ export class FootIK {
|
|
|
157
175
|
const lower = h.getNormalizedBoneNode(l);
|
|
158
176
|
const foot = h.getNormalizedBoneNode(f);
|
|
159
177
|
return upper && lower && foot
|
|
160
|
-
? {
|
|
178
|
+
? {
|
|
179
|
+
upper,
|
|
180
|
+
lower,
|
|
181
|
+
foot,
|
|
182
|
+
offset: 0,
|
|
183
|
+
normal: new THREE.Vector3(0, 1, 0),
|
|
184
|
+
animFootPos: new THREE.Vector3(),
|
|
185
|
+
}
|
|
161
186
|
: null;
|
|
162
187
|
};
|
|
163
188
|
const left = mk(VRMHumanBoneName.LeftUpperLeg, VRMHumanBoneName.LeftLowerLeg, VRMHumanBoneName.LeftFoot);
|
|
@@ -192,14 +217,27 @@ export class FootIK {
|
|
|
192
217
|
|
|
193
218
|
const k = 1 - Math.exp(-this.#smoothing * dt);
|
|
194
219
|
const kNormal = 1 - Math.exp(-NORMAL_DAMPING * dt);
|
|
220
|
+
const reachDown = this.#allowReachDown?.() ?? true;
|
|
195
221
|
|
|
196
222
|
// 1. Sample the ground under each ANIMATED foot; smooth offset + normal.
|
|
197
|
-
//
|
|
198
|
-
//
|
|
199
|
-
//
|
|
200
|
-
//
|
|
201
|
-
//
|
|
202
|
-
//
|
|
223
|
+
// Two lift signals, merged:
|
|
224
|
+
// - TERRAIN-relative plant: the ground height under the foot vs the body
|
|
225
|
+
// root (the VRM origin = its floor/sole level), NOT vs the animated
|
|
226
|
+
// foot. Stride-phase independent — a lifted swing foot keeps its
|
|
227
|
+
// animation (no dragging/sinking while running) while a planted foot
|
|
228
|
+
// still lands on its step, and it self-corrects any residual
|
|
229
|
+
// capsule-float gap (feet reach true ground).
|
|
230
|
+
// - needLift: raise the ankle ONLY as much as needed so the sole clears
|
|
231
|
+
// the contact under it — the anti-dig-in floor for poses that dip the
|
|
232
|
+
// animated feet below the clip's own ground (weapon recoil etc. on
|
|
233
|
+
// differently-proportioned avatars).
|
|
234
|
+
// With reach-down gated off (full-body one-shots), ONLY needLift
|
|
235
|
+
// applies, capped at GATED_LIFT_CAP: choreography feet are left alone
|
|
236
|
+
// unless they'd clip into a step, and a foot the stance carries INTO a
|
|
237
|
+
// riser gets at most a small hop, not a knee-to-the-chest fold. The
|
|
238
|
+
// lift-only offsets also mean min(0,offset)=0 keeps the pelvis drop
|
|
239
|
+
// off. Smoothing eases every transition (one-shot start/end, a ray
|
|
240
|
+
// crossing a step edge) without pops.
|
|
203
241
|
for (const leg of this.#legs) {
|
|
204
242
|
leg.foot.getWorldPosition(leg.animFootPos);
|
|
205
243
|
const sample = active ? this.#query(leg.animFootPos) : null;
|
|
@@ -207,14 +245,30 @@ export class FootIK {
|
|
|
207
245
|
_normalTarget.copy(
|
|
208
246
|
sample !== null && typeof sample !== "number" && sample.normal ? sample.normal : UP,
|
|
209
247
|
);
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
248
|
+
let desired = 0;
|
|
249
|
+
if (groundY !== null) {
|
|
250
|
+
// Anti-dig-in signal: how much lift (if any) the ankle needs so the
|
|
251
|
+
// sole clears the contact under it. NEGATIVE when the foot is safely
|
|
252
|
+
// above the contact — and it must stay negative: flooring it at zero
|
|
253
|
+
// here would win every Math.max below and silently kill the downward
|
|
254
|
+
// reach (and with it the pelvis drop) for the whole planter.
|
|
255
|
+
const needLift =
|
|
256
|
+
groundY +
|
|
257
|
+
this.#soleClearance +
|
|
258
|
+
this.#restFootHeight -
|
|
259
|
+
PENETRATION_SLACK -
|
|
260
|
+
leg.animFootPos.y;
|
|
261
|
+
desired = reachDown
|
|
262
|
+
? Math.max(
|
|
263
|
+
THREE.MathUtils.clamp(
|
|
264
|
+
groundY + this.#soleClearance - _rootPos.y,
|
|
265
|
+
-this.#maxOffset,
|
|
266
|
+
this.#maxOffset,
|
|
267
|
+
),
|
|
268
|
+
Math.min(needLift, this.#maxOffset),
|
|
269
|
+
)
|
|
270
|
+
: THREE.MathUtils.clamp(needLift, 0, GATED_LIFT_CAP);
|
|
271
|
+
}
|
|
218
272
|
leg.offset += (desired - leg.offset) * k;
|
|
219
273
|
leg.normal.lerp(_normalTarget, kNormal).normalize();
|
|
220
274
|
}
|
|
@@ -229,6 +283,8 @@ export class FootIK {
|
|
|
229
283
|
}
|
|
230
284
|
|
|
231
285
|
// 3. Per-leg two-bone IK to the grounded target, then flatten planted feet.
|
|
286
|
+
// (Anti-dig-in is already folded into the smoothed offset — needLift in
|
|
287
|
+
// step 1 — so the target needs no extra instant clamp here.)
|
|
232
288
|
for (const leg of this.#legs) {
|
|
233
289
|
_target.copy(leg.animFootPos);
|
|
234
290
|
_target.y += leg.offset * this.#weight;
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
// SPDX-License-Identifier: MIT
|
|
2
|
-
// Retarget Quaternius Universal Animation Library clips
|
|
3
|
-
//
|
|
2
|
+
// Retarget Quaternius Universal Animation Library clips onto a three-vrm
|
|
3
|
+
// normalized humanoid rig (Genex AG-747/AG-775). Two source rigs are supported
|
|
4
|
+
// and auto-detected: the free UAL's Blender Rigify `DEF-*` skeleton and the UAL
|
|
5
|
+
// Pro's UE-mannequin-style skeleton (`pelvis`/`spine_01`/…) that the bundled
|
|
6
|
+
// core library and every CDN animation pack use. Adapted from the
|
|
4
7
|
// official three-vrm Mixamo retarget recipe (@pixiv/three-vrm examples, MIT):
|
|
5
8
|
// rewrite each bone track into the VRM's normalized-bone local space using the
|
|
6
9
|
// SOURCE rig's rest-pose world rotations, and scale the hips translation by the
|
|
@@ -56,26 +59,81 @@ const DEF_TO_VRM: Record<string, VrmBone> = {
|
|
|
56
59
|
"DEF-toe.R": VRMHumanBoneName.RightToes,
|
|
57
60
|
};
|
|
58
61
|
|
|
62
|
+
// Quaternius UAL Pro (UE-mannequin-style skeleton) -> VRM humanoid bone. Bone
|
|
63
|
+
// names verified against the UAL1 master GLB ("Head" really is capitalized).
|
|
64
|
+
// Fingers are omitted, same policy as the DEF map.
|
|
65
|
+
const UE_TO_VRM: Record<string, VrmBone> = {
|
|
66
|
+
pelvis: VRMHumanBoneName.Hips,
|
|
67
|
+
spine_01: VRMHumanBoneName.Spine,
|
|
68
|
+
spine_02: VRMHumanBoneName.Chest,
|
|
69
|
+
spine_03: VRMHumanBoneName.UpperChest,
|
|
70
|
+
neck_01: VRMHumanBoneName.Neck,
|
|
71
|
+
Head: VRMHumanBoneName.Head,
|
|
72
|
+
clavicle_l: VRMHumanBoneName.LeftShoulder,
|
|
73
|
+
upperarm_l: VRMHumanBoneName.LeftUpperArm,
|
|
74
|
+
lowerarm_l: VRMHumanBoneName.LeftLowerArm,
|
|
75
|
+
hand_l: VRMHumanBoneName.LeftHand,
|
|
76
|
+
clavicle_r: VRMHumanBoneName.RightShoulder,
|
|
77
|
+
upperarm_r: VRMHumanBoneName.RightUpperArm,
|
|
78
|
+
lowerarm_r: VRMHumanBoneName.RightLowerArm,
|
|
79
|
+
hand_r: VRMHumanBoneName.RightHand,
|
|
80
|
+
thigh_l: VRMHumanBoneName.LeftUpperLeg,
|
|
81
|
+
calf_l: VRMHumanBoneName.LeftLowerLeg,
|
|
82
|
+
foot_l: VRMHumanBoneName.LeftFoot,
|
|
83
|
+
ball_l: VRMHumanBoneName.LeftToes,
|
|
84
|
+
thigh_r: VRMHumanBoneName.RightUpperLeg,
|
|
85
|
+
calf_r: VRMHumanBoneName.RightLowerLeg,
|
|
86
|
+
foot_r: VRMHumanBoneName.RightFoot,
|
|
87
|
+
ball_r: VRMHumanBoneName.RightToes,
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Pick the source-rig bone map by looking for each rig's hips bone in the
|
|
92
|
+
* scene: `DEF-hips` -> Rigify (free UAL), `pelvis` -> UE-style (UAL Pro).
|
|
93
|
+
* Returns null for unrecognized rigs.
|
|
94
|
+
*/
|
|
95
|
+
export function detectBoneMap(sourceRoot: THREE.Object3D): Record<string, VrmBone> | null {
|
|
96
|
+
let map: Record<string, VrmBone> | null = null;
|
|
97
|
+
sourceRoot.traverse((object) => {
|
|
98
|
+
if (map !== null) return;
|
|
99
|
+
if (object.name === "DEF-hips") map = DEF_TO_VRM;
|
|
100
|
+
else if (object.name === "pelvis") map = UE_TO_VRM;
|
|
101
|
+
});
|
|
102
|
+
return map;
|
|
103
|
+
}
|
|
104
|
+
|
|
59
105
|
/**
|
|
60
106
|
* Retarget UAL clips onto `vrm`.
|
|
61
107
|
* @param vrm the loaded VRM (already through {@link loadVrm}).
|
|
62
|
-
* @param animationRoot the animation
|
|
63
|
-
*
|
|
64
|
-
* @param clips that GLB's animations
|
|
108
|
+
* @param animationRoot the animation GLB's scene — its bones in rest pose
|
|
109
|
+
* supply the source frame the tracks are relative to.
|
|
110
|
+
* @param clips that GLB's animations.
|
|
111
|
+
* @param boneMap source-bone-name -> VRM humanoid bone. Defaults to
|
|
112
|
+
* {@link detectBoneMap} on `animationRoot` (Rigify or
|
|
113
|
+
* UE-style rigs bind automatically).
|
|
65
114
|
* @returns new clips whose tracks target the VRM's normalized humanoid bones.
|
|
66
115
|
*/
|
|
67
116
|
export function retargetClips(
|
|
68
117
|
vrm: VRM,
|
|
69
118
|
animationRoot: THREE.Object3D,
|
|
70
119
|
clips: THREE.AnimationClip[],
|
|
120
|
+
boneMap?: Record<string, VrmBone>,
|
|
71
121
|
): THREE.AnimationClip[] {
|
|
122
|
+
const sourceToVrm = boneMap ?? detectBoneMap(animationRoot);
|
|
123
|
+
if (sourceToVrm === null) {
|
|
124
|
+
console.warn(
|
|
125
|
+
"[vrm-retarget] unrecognized animation rig (no DEF-hips or pelvis bone found) — returning no clips; pass an explicit boneMap to retargetClips.",
|
|
126
|
+
);
|
|
127
|
+
return [];
|
|
128
|
+
}
|
|
129
|
+
|
|
72
130
|
animationRoot.updateWorldMatrix(true, true);
|
|
73
131
|
vrm.scene.updateWorldMatrix(true, true);
|
|
74
132
|
|
|
75
133
|
// three's GLTFLoader SANITIZES node names in animation track targets
|
|
76
134
|
// (PropertyBinding strips `[].:/ ` and turns spaces into `_`), so a Rigify bone
|
|
77
135
|
// "DEF-upper_arm.L" shows up in tracks as "DEF-upper_armL". Map those sanitized
|
|
78
|
-
// names back to the real bones, whose actual names carry the dots
|
|
136
|
+
// names back to the real bones, whose actual names carry the dots the map
|
|
79
137
|
// keys on. (Mixamo names are dotless, so the upstream recipe never needed this.)
|
|
80
138
|
const sanitize = (name: string): string => name.replace(/\s/g, "_").replace(/[[\]./:]/g, "");
|
|
81
139
|
const sourceByTrackName = new Map<string, THREE.Object3D>();
|
|
@@ -83,9 +141,9 @@ export function retargetClips(
|
|
|
83
141
|
if (o.name) sourceByTrackName.set(sanitize(o.name), o);
|
|
84
142
|
});
|
|
85
143
|
// glTF load sanitizes bone names too, so `source.name` is already dot-stripped —
|
|
86
|
-
// key the VRM-bone lookup by the sanitized
|
|
87
|
-
const
|
|
88
|
-
for (const [
|
|
144
|
+
// key the VRM-bone lookup by the sanitized source name, matching the track's nodeName.
|
|
145
|
+
const sanitizedSourceToVrm: Record<string, VrmBone> = {};
|
|
146
|
+
for (const [src, bone] of Object.entries(sourceToVrm)) sanitizedSourceToVrm[sanitize(src)] = bone;
|
|
89
147
|
|
|
90
148
|
// Reusable bind-pose quaternions. Both rigs are at rest here (nothing has
|
|
91
149
|
// animated them yet), so getWorldQuaternion reads the bind pose.
|
|
@@ -99,8 +157,10 @@ export function retargetClips(
|
|
|
99
157
|
// hips' vertical bob/crouch on the target hips as a rest-relative DELTA, so
|
|
100
158
|
// the physics controller still owns the whole-body base translation while the
|
|
101
159
|
// pose keeps the pelvis (and therefore the feet) at the right height.
|
|
102
|
-
const
|
|
103
|
-
|
|
160
|
+
const hipsSourceName = Object.entries(sanitizedSourceToVrm).find(
|
|
161
|
+
([, bone]) => bone === VRMHumanBoneName.Hips,
|
|
162
|
+
)?.[0];
|
|
163
|
+
const srcHips = (hipsSourceName ? sourceByTrackName.get(hipsSourceName) : undefined) ?? null;
|
|
104
164
|
const tgtHips = vrm.humanoid.getNormalizedBoneNode(VRMHumanBoneName.Hips);
|
|
105
165
|
const hipsSrcParentWorld = new THREE.Quaternion();
|
|
106
166
|
const hipsTgtParentWorldInv = new THREE.Quaternion();
|
|
@@ -128,7 +188,7 @@ export function retargetClips(
|
|
|
128
188
|
const nodeName = track.name.slice(0, lastDot);
|
|
129
189
|
const prop = track.name.slice(lastDot + 1);
|
|
130
190
|
const source = sourceByTrackName.get(nodeName);
|
|
131
|
-
const vrmBone =
|
|
191
|
+
const vrmBone = sanitizedSourceToVrm[nodeName];
|
|
132
192
|
if (!source || !vrmBone) continue;
|
|
133
193
|
|
|
134
194
|
const target = vrm.humanoid.getNormalizedBoneNode(vrmBone);
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: genex-ai-image
|
|
3
|
+
description: Generate a real image (PNG/JPEG) from a text prompt with `npx genex image`, then load it into Three.js on any mesh, plane, or sprite. Use for posters, paintings, billboards, signs, logos, sprites, card/item art, loading screens, textures for in-game screens, and decals/stickers ("wanted poster", "arcade cabinet marquee", "neon graffiti tag") rather than a procedural/shader look. Pass `--transparent` for anything with an alpha channel.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Genex AI · Image
|
|
7
|
+
|
|
8
|
+
Turn a prompt into a real raster image and put it anywhere in the game — a poster,
|
|
9
|
+
a painting, a billboard, a sign, a logo, a sprite, card/item art, a loading screen,
|
|
10
|
+
art on an in-game screen, or a decal/sticker.
|
|
11
|
+
|
|
12
|
+
## When to use this vs. procedural materials
|
|
13
|
+
|
|
14
|
+
- **Use `npx genex image`** for a specific, recognizable picture you can describe —
|
|
15
|
+
"vintage travel poster of Mars", "guild crest with crossed swords", "arcade
|
|
16
|
+
marquee art". You get a real image.
|
|
17
|
+
- **Use `$genex-threejs-procedural-materials`** for stylized/abstract or fully
|
|
18
|
+
parametric surfaces authored in shaders. Use `$genex-ai-texture` for a *tiling*
|
|
19
|
+
PBR surface (floors, ground, walls) — this skill is for a single flat picture.
|
|
20
|
+
|
|
21
|
+
## Run
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
npx genex image "<prompt>"
|
|
25
|
+
npx genex image "neon graffiti tag, spray-paint style" --transparent # PNG with alpha (decals/stickers/logos)
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Blocks until ready, then prints its public URL:
|
|
29
|
+
|
|
30
|
+
```
|
|
31
|
+
https://assets.genex.technology/generations/<id>/image-main
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
The image lives in Genex storage (R2) and loads straight from that URL — you don't
|
|
35
|
+
download it and nothing is committed to your repo. The URL is permanent (local dev,
|
|
36
|
+
published game, and remixes alike).
|
|
37
|
+
|
|
38
|
+
## Load it into the scene
|
|
39
|
+
|
|
40
|
+
Load the image and apply it to any mesh, plane, or sprite:
|
|
41
|
+
|
|
42
|
+
```ts
|
|
43
|
+
import * as THREE from "three";
|
|
44
|
+
|
|
45
|
+
// the URL `npx genex image` printed (R2 sends CORS headers, so cross-origin works):
|
|
46
|
+
const IMAGE_URL = "https://assets.genex.technology/generations/<id>/image-main";
|
|
47
|
+
const map = await new THREE.TextureLoader().loadAsync(IMAGE_URL);
|
|
48
|
+
map.colorSpace = THREE.SRGBColorSpace; // pictures are sRGB — without this they look washed/dark
|
|
49
|
+
map.anisotropy = renderer.capabilities.getMaxAnisotropy(); // stays sharp at grazing angles
|
|
50
|
+
|
|
51
|
+
// a poster/painting/sign on a wall — a flat plane:
|
|
52
|
+
const poster = new THREE.Mesh(
|
|
53
|
+
new THREE.PlaneGeometry(2, 3), // match the image aspect (w:h)
|
|
54
|
+
new THREE.MeshStandardMaterial({ map, roughness: 0.9, metalness: 0 }),
|
|
55
|
+
);
|
|
56
|
+
scene.add(poster);
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
For a `--transparent` PNG, set `transparent: true` on the material so the alpha shows.
|
|
60
|
+
For a screen-space sprite (HUD art, an icon) use `new THREE.Sprite(new THREE.SpriteMaterial({ map }))`.
|
|
61
|
+
For art that must glow (a lit sign, a screen) use `MeshBasicMaterial` (unlit) so scene lighting doesn't darken it.
|
|
62
|
+
|
|
63
|
+
## Decals & stickers
|
|
64
|
+
|
|
65
|
+
**Anything applied *on top* of a surface — a decal, sticker, spray tag, logo, or
|
|
66
|
+
bullet hole — needs an alpha channel, so always generate it with `--transparent`.**
|
|
67
|
+
Without alpha you get an opaque rectangle instead of a shaped mark.
|
|
68
|
+
|
|
69
|
+
Project it onto the target mesh with `DecalGeometry` (the canonical spray-paint look —
|
|
70
|
+
clips to the surface and wraps around corners):
|
|
71
|
+
|
|
72
|
+
```ts
|
|
73
|
+
import * as THREE from "three";
|
|
74
|
+
import { DecalGeometry } from "three/addons/geometries/DecalGeometry.js";
|
|
75
|
+
|
|
76
|
+
// ONE shared texture + material for all sprays (generated with --transparent):
|
|
77
|
+
const map = await new THREE.TextureLoader().loadAsync(IMAGE_URL);
|
|
78
|
+
map.colorSpace = THREE.SRGBColorSpace;
|
|
79
|
+
map.anisotropy = renderer.capabilities.getMaxAnisotropy();
|
|
80
|
+
const sprayMat = new THREE.MeshStandardMaterial({
|
|
81
|
+
map, transparent: true, depthTest: true, depthWrite: false,
|
|
82
|
+
polygonOffset: true, polygonOffsetFactor: -4, // pulls the decal forward — kills z-fighting
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
const raycaster = new THREE.Raycaster();
|
|
86
|
+
const helper = new THREE.Object3D(); // orientation scratch — never added to the scene
|
|
87
|
+
const decals: THREE.Mesh[] = [];
|
|
88
|
+
const MAX_DECALS = 20;
|
|
89
|
+
|
|
90
|
+
function spray(): void {
|
|
91
|
+
raycaster.setFromCamera(new THREE.Vector2(0, 0), camera); // screen centre
|
|
92
|
+
const hit = raycaster.intersectObjects(sprayables, false)[0]; // sprayables = your wall meshes
|
|
93
|
+
if (!hit || !hit.face) return;
|
|
94
|
+
const n = hit.face.normal.clone() // face.normal is LOCAL space —
|
|
95
|
+
.applyNormalMatrix(new THREE.Matrix3().getNormalMatrix(hit.object.matrixWorld)); // must transform it
|
|
96
|
+
helper.position.copy(hit.point);
|
|
97
|
+
helper.lookAt(hit.point.clone().add(n));
|
|
98
|
+
const S = 1; // metres; for non-square art size.y = S * imgH/imgW
|
|
99
|
+
const geom = new DecalGeometry(hit.object as THREE.Mesh, hit.point,
|
|
100
|
+
helper.rotation.clone(), new THREE.Vector3(S, S, S * 0.5)); // size.z = wrap depth
|
|
101
|
+
const decal = new THREE.Mesh(geom, sprayMat);
|
|
102
|
+
decal.renderOrder = 100 + decals.length; // newer sprays draw on top
|
|
103
|
+
(hit.object as THREE.Mesh).attach(decal); // verts are world-space; attach keeps them correct
|
|
104
|
+
decals.push(decal);
|
|
105
|
+
if (decals.length > MAX_DECALS) { // FIFO cap — dispose the oldest
|
|
106
|
+
const old = decals.shift()!;
|
|
107
|
+
old.removeFromParent();
|
|
108
|
+
old.geometry.dispose(); // geometry is per-spray — MUST dispose; the shared material/map are never disposed here
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
window.addEventListener("keydown", (e) => {
|
|
113
|
+
if (e.code === "KeyT" && !e.repeat) spray();
|
|
114
|
+
});
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
Gotchas (all bite in practice):
|
|
118
|
+
|
|
119
|
+
- **`hit.face.normal` is LOCAL space** — apply the normal-matrix from `matrixWorld`.
|
|
120
|
+
Skipping it "works by accident" only on an unrotated, unscaled wall; any rotated
|
|
121
|
+
wall gets the decal facing the wrong way.
|
|
122
|
+
- **`depthWrite: false` means overlap order is draw order** — increment `renderOrder`
|
|
123
|
+
per spray, or stacked decals flicker and sort wrongly.
|
|
124
|
+
- **One decal clips against ONE mesh** — a spray straddling two wall meshes gets cut
|
|
125
|
+
at the boundary. Pass the single `hit.object`, or make walls separate meshes.
|
|
126
|
+
- **`size.z` too large on a thin wall wraps the decal onto the back face** (visible
|
|
127
|
+
from behind). Keep `size.z` below the wall thickness.
|
|
128
|
+
- **`DecalGeometry` is static-mesh only** — no `SkinnedMesh`/morph targets. For a
|
|
129
|
+
moving or skinned target, use a small `PlaneGeometry` quad offset along the normal
|
|
130
|
+
(`hit.point + n * 0.01`, `quad.lookAt(hit.point.clone().add(n))`) instead.
|
|
131
|
+
|
|
132
|
+
## Multiplayer
|
|
133
|
+
|
|
134
|
+
The asset URL is public, permanent, and CORS-open, so it is safe to broadcast the
|
|
135
|
+
string to every player. A placed-at-runtime mark (a decal included) is static shared
|
|
136
|
+
world state — put it on `room.shared`, **not** `objects` (no movement to smooth) and
|
|
137
|
+
**not** `send` (late joiners would see a bare wall; `shared` keys replay on connect).
|
|
138
|
+
|
|
139
|
+
Use a **fixed ring of slots** and overwrite the oldest — **never a fresh key per
|
|
140
|
+
spray**. The relay caps game-writable `shared` keys at **256 per room, keys are
|
|
141
|
+
permanent and undeletable, and new keys past the cap are silently dropped forever**,
|
|
142
|
+
so a new key per spray eventually breaks the game:
|
|
143
|
+
|
|
144
|
+
```ts
|
|
145
|
+
let next = 0;
|
|
146
|
+
const RING = 32; // decal:0 .. decal:31
|
|
147
|
+
function placeShared(url: string, hit: { point: THREE.Vector3; normal: THREE.Vector3 }) {
|
|
148
|
+
room.shared.set(`decal:${next % RING}`, { url, p: hit.point.toArray(), n: hit.normal.toArray() });
|
|
149
|
+
next++;
|
|
150
|
+
}
|
|
151
|
+
room.on("shared", (key, value) => { // every player (incl. late joiners) rebuilds the decal
|
|
152
|
+
if (key.startsWith("decal:") && value) spawnDecalFromShared(value);
|
|
153
|
+
});
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
See `$genex-threejs-multiplayer` for the `shared` channel rules and the room API.
|
|
157
|
+
|
|
158
|
+
## Publish checklist
|
|
159
|
+
|
|
160
|
+
- Load it from the **URL** the command printed — absolute and permanent, so it resolves
|
|
161
|
+
the same in local dev, the published game, and remixes. Nothing to commit.
|
|
162
|
+
- Don't copy the image into `public/assets/` — generated assets live in R2, not the repo.
|
|
163
|
+
|
|
164
|
+
## Options
|
|
165
|
+
|
|
166
|
+
- `--transparent` — PNG with an alpha channel (mandatory for decals/stickers/logos —
|
|
167
|
+
anything laid on top of a surface).
|
|
168
|
+
- `--aspect <ratio>` — image shape (e.g. `square`, `16:9`, `9:16`); default is square.
|
|
169
|
+
- `--no-wait` — enqueue and return immediately (the file won't be downloaded;
|
|
170
|
+
re-run without `--no-wait` to fetch it).
|
|
171
|
+
- `--api-url <url>` — override the API base (local dev).
|
|
172
|
+
|
|
173
|
+
## Troubleshooting
|
|
174
|
+
|
|
175
|
+
- **"Not authorized"** — run `npx @genex-ai/cli-demo@latest init` first (it writes your `GENEX_TOKEN`).
|
|
176
|
+
- **"Prompt rejected"** — the provider's content-safety filter blocked the prompt.
|
|
177
|
+
This is non-retryable; retrying the same wording fails again. Rewrite the prompt.
|
|
178
|
+
- **Decal is an opaque rectangle** — the image has no alpha. Regenerate with
|
|
179
|
+
`--transparent`.
|
|
180
|
+
- **Colors look washed/dark** — ensure `map.colorSpace = THREE.SRGBColorSpace`.
|
|
181
|
+
- **Decal blurs at oblique angles** — set `map.anisotropy = renderer.capabilities.getMaxAnisotropy()`.
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: genex-ai-video
|
|
3
|
+
description: Generate a real video clip (H.264 mp4) from a text prompt with `npx genex video`, then play it in Three.js on any surface via VideoTexture. Use for in-game TVs/screens/monitors, animated billboards, cutscene clips, ambient backdrops, portals, and video decals ("news broadcast on a TV", "swirling portal", "animated arcade attract screen") rather than a static image or shader effect. Pass `--loop` for a seamless loop.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Genex AI · Video
|
|
7
|
+
|
|
8
|
+
Turn a prompt into a real mp4 clip and play it anywhere in the game — an in-game
|
|
9
|
+
TV/screen/monitor, an animated billboard, a cutscene, an ambient backdrop, a portal,
|
|
10
|
+
or a video decal.
|
|
11
|
+
|
|
12
|
+
## When to use this vs. a static image or shader
|
|
13
|
+
|
|
14
|
+
- **Use `npx genex video`** for moving footage you can describe — "static-y CRT
|
|
15
|
+
news broadcast", "swirling neon portal", "rain running down glass". You get a real
|
|
16
|
+
mp4.
|
|
17
|
+
- **Use `$genex-ai-image`** for a single still picture, or
|
|
18
|
+
`$genex-threejs-procedural-vfx` for parametric real-time effects (particles,
|
|
19
|
+
trails, shockwaves) authored in code.
|
|
20
|
+
|
|
21
|
+
## Run
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
npx genex video "<prompt>"
|
|
25
|
+
npx genex video "swirling neon plasma, seamless loop" --loop # seamless loop for screens/backdrops
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Video takes a **minute or two** end to end (queue + generation). If you're building
|
|
29
|
+
other things meanwhile, add `--no-wait` to enqueue and come back for it. Blocks until
|
|
30
|
+
ready, then prints its public URL:
|
|
31
|
+
|
|
32
|
+
```
|
|
33
|
+
https://assets.genex.technology/generations/<id>/video-mp4
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
The clip lives in Genex storage (R2) and loads straight from that URL — you don't
|
|
37
|
+
download it and nothing is committed to your repo. The URL is permanent (local dev,
|
|
38
|
+
published game, and remixes alike).
|
|
39
|
+
|
|
40
|
+
> **Cost & length:** the default is a **5-second, 720p** clip, and that IS the right
|
|
41
|
+
> default for a looping screen or backdrop — only pass `--duration` when the content
|
|
42
|
+
> genuinely needs to be longer (a cutscene). Longer, higher-res clips cost more and
|
|
43
|
+
> take longer. mp4 has **no alpha channel**, so a video is always a full rectangle
|
|
44
|
+
> (there are no transparent video decals).
|
|
45
|
+
|
|
46
|
+
## Play it in Three.js
|
|
47
|
+
|
|
48
|
+
Video needs an `HTMLVideoElement`, and browsers block autoplay — the first
|
|
49
|
+
`video.play()` must run inside a user gesture (any click or keypress). Wrap it in a
|
|
50
|
+
`VideoTexture` and put it on any surface:
|
|
51
|
+
|
|
52
|
+
```ts
|
|
53
|
+
import * as THREE from "three";
|
|
54
|
+
|
|
55
|
+
// the URL `npx genex video` printed (R2 sends CORS headers, so cross-origin works):
|
|
56
|
+
const VIDEO_URL = "https://assets.genex.technology/generations/<id>/video-mp4";
|
|
57
|
+
const video = document.createElement("video");
|
|
58
|
+
video.src = VIDEO_URL;
|
|
59
|
+
video.crossOrigin = "anonymous"; // REQUIRED to upload cross-origin video to WebGL (else a tainted-source error)
|
|
60
|
+
video.muted = true; // muted is what lets it play under autoplay policy
|
|
61
|
+
video.loop = true;
|
|
62
|
+
video.playsInline = true; // iOS: no fullscreen takeover
|
|
63
|
+
|
|
64
|
+
const texture = new THREE.VideoTexture(video); // auto-updates every frame — no per-frame code
|
|
65
|
+
texture.colorSpace = THREE.SRGBColorSpace;
|
|
66
|
+
|
|
67
|
+
// an in-game TV / animated billboard — MeshBasicMaterial so it glows regardless of scene lighting:
|
|
68
|
+
const screen = new THREE.Mesh(
|
|
69
|
+
new THREE.PlaneGeometry(3.2, 1.8), // 16:9
|
|
70
|
+
new THREE.MeshBasicMaterial({ map: texture }),
|
|
71
|
+
);
|
|
72
|
+
scene.add(screen);
|
|
73
|
+
|
|
74
|
+
// start playback on a user gesture (never swallow the rejection):
|
|
75
|
+
window.addEventListener("keydown", () => {
|
|
76
|
+
video.play().catch((e) => console.warn("video play", e));
|
|
77
|
+
}, { once: true });
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
One `VideoTexture` can feed **many** surfaces — a bank of monitors, N sprays. Cost is
|
|
81
|
+
per distinct `<video>` element (media decode + one GPU upload per frame), **not** per
|
|
82
|
+
surface, so sharing one element for every screen is nearly free (all play in sync).
|
|
83
|
+
Keep clips **≤720p**. Pause the element (`video.pause()`) when no video surface is
|
|
84
|
+
visible.
|
|
85
|
+
|
|
86
|
+
## Video decals
|
|
87
|
+
|
|
88
|
+
A video decal is the same `DecalGeometry` projection as an image decal — read the
|
|
89
|
+
**Decals & stickers** section of `$genex-ai-image` for the full recipe and gotchas —
|
|
90
|
+
with one change: swap the `TextureLoader` map for the `VideoTexture` above, and call
|
|
91
|
+
`video.play().catch(...)` inside the spray handler (the keypress is the user gesture).
|
|
92
|
+
All decals can share the one `VideoTexture`. Because mp4 has no alpha, a video decal
|
|
93
|
+
is a full rectangle (fine for a screen-shaped mark; use `$genex-ai-image --transparent`
|
|
94
|
+
for a shaped sticker). **Layering gotcha:** an opaque video decal draws in the opaque
|
|
95
|
+
pass — *before* any `transparent: true` image decal — so image decals always land on
|
|
96
|
+
top regardless of spray order or `renderOrder`. If you mix both and need strict
|
|
97
|
+
newest-on-top, set `transparent: true` on the video decal material too.
|
|
98
|
+
|
|
99
|
+
## Multiplayer
|
|
100
|
+
|
|
101
|
+
The asset URL is public, permanent, and CORS-open, so it is safe to broadcast the
|
|
102
|
+
string to every player. A placed-at-runtime surface (a video decal included) is static
|
|
103
|
+
shared world state — put it on `room.shared`, **not** `objects` (no movement to smooth)
|
|
104
|
+
and **not** `send` (late joiners would see a bare wall; `shared` keys replay on connect).
|
|
105
|
+
|
|
106
|
+
Use a **fixed ring of slots** and overwrite the oldest — **never a fresh key per
|
|
107
|
+
placement**. The relay caps game-writable `shared` keys at **256 per room, keys are
|
|
108
|
+
permanent and undeletable, and new keys past the cap are silently dropped forever**:
|
|
109
|
+
|
|
110
|
+
```ts
|
|
111
|
+
let next = 0;
|
|
112
|
+
const RING = 32; // decal:0 .. decal:31
|
|
113
|
+
function placeShared(url: string, hit: { point: THREE.Vector3; normal: THREE.Vector3 }) {
|
|
114
|
+
room.shared.set(`decal:${next % RING}`, { url, p: hit.point.toArray(), n: hit.normal.toArray() });
|
|
115
|
+
next++;
|
|
116
|
+
}
|
|
117
|
+
room.on("shared", (key, value) => { // every player (incl. late joiners) rebuilds it
|
|
118
|
+
if (key.startsWith("decal:") && value) spawnVideoDecalFromShared(value);
|
|
119
|
+
});
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
See `$genex-threejs-multiplayer` for the `shared` channel rules and the room API.
|
|
123
|
+
|
|
124
|
+
## Publish checklist
|
|
125
|
+
|
|
126
|
+
- Load it from the **URL** the command printed — absolute and permanent, so it resolves
|
|
127
|
+
the same in local dev, the published game, and remixes. Nothing to commit.
|
|
128
|
+
- Don't copy the mp4 into `public/assets/` — generated assets live in R2, not the repo.
|
|
129
|
+
|
|
130
|
+
## Options
|
|
131
|
+
|
|
132
|
+
- `--loop` — a seamless loop (for screens, ambient backdrops, video decals).
|
|
133
|
+
- `--duration <sec>` — clip length 1–15; default 5. Only raise it when the content
|
|
134
|
+
genuinely needs more — longer clips cost more and take longer.
|
|
135
|
+
- `--no-wait` — enqueue and return immediately (the file won't be downloaded;
|
|
136
|
+
re-run without `--no-wait` to fetch it). Handy for video since it takes a minute or two.
|
|
137
|
+
- `--api-url <url>` — override the API base (local dev).
|
|
138
|
+
|
|
139
|
+
## Troubleshooting
|
|
140
|
+
|
|
141
|
+
- **"Not authorized"** — run `npx @genex-ai/cli-demo@latest init` first (it writes your `GENEX_TOKEN`).
|
|
142
|
+
- **"Prompt rejected"** — the provider's content-safety filter blocked the prompt.
|
|
143
|
+
This is non-retryable; retrying the same wording fails again. Rewrite the prompt.
|
|
144
|
+
- **Nothing plays / black surface** — the first `video.play()` must run inside a user
|
|
145
|
+
gesture (click/keydown); confirm it's called and its promise rejection is logged.
|
|
146
|
+
- **Tainted-source / security error** — set `video.crossOrigin = "anonymous"` before
|
|
147
|
+
`video.src`.
|
|
148
|
+
- **Colors look washed/dark** — ensure `texture.colorSpace = THREE.SRGBColorSpace`.
|
|
@@ -37,13 +37,16 @@ npx genex model "weathered wooden barrel" # a 3D mesh (GLB)
|
|
|
37
37
|
npx genex skybox "golden hour over mountains" # a 360° sky + lighting
|
|
38
38
|
npx genex sfx "punchy laser zap" --duration 2 # a sound effect (mp3)
|
|
39
39
|
npx genex texture "mossy cobblestone" --terrain # a tiling surface texture
|
|
40
|
+
npx genex image "vintage travel poster" # a picture (poster/sign/sprite/decal)
|
|
41
|
+
npx genex video "swirling neon portal" --loop # a video clip (screen/backdrop)
|
|
40
42
|
```
|
|
41
43
|
|
|
42
44
|
(Run them inside your project — the `@genex-ai/cli-demo` dev dependency makes
|
|
43
45
|
`npx genex` resolve to the right CLI.)
|
|
44
46
|
|
|
45
47
|
Each has a focused skill with the exact loader code — `$genex-ai-model`,
|
|
46
|
-
`$genex-ai-skybox`, `$genex-ai-sfx`, `$genex-ai-texture
|
|
48
|
+
`$genex-ai-skybox`, `$genex-ai-sfx`, `$genex-ai-texture`, `$genex-ai-image`,
|
|
49
|
+
`$genex-ai-video`.
|
|
47
50
|
|
|
48
51
|
## Identity & saves (every game)
|
|
49
52
|
|