@dcl-regenesislabs/opendcl 0.2.1-26165320302.commit-e6effe4 → 0.2.1-26238928766.commit-28648d7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/context/sdk7-cheat-sheet.md +4 -0
- package/dist/index.js +0 -12
- package/dist/index.js.map +1 -1
- package/extensions/dcl-init.ts +58 -6
- package/package.json +3 -3
- package/prompts/system.md +71 -41
- package/skills/add-3d-models/SKILL.md +120 -70
- package/skills/add-interactivity/SKILL.md +74 -2
- package/skills/advanced-input/SKILL.md +34 -1
- package/skills/advanced-rendering/SKILL.md +82 -9
- package/skills/animations-tweens/SKILL.md +203 -98
- package/skills/audio-analysis/SKILL.md +164 -0
- package/skills/audio-video/SKILL.md +184 -83
- package/skills/build-ui/SKILL.md +25 -2
- package/skills/camera-control/SKILL.md +78 -7
- package/skills/create-scene/SKILL.md +56 -13
- package/skills/deploy-scene/SKILL.md +12 -0
- package/skills/deploy-worlds/SKILL.md +35 -0
- package/skills/editor-gizmo/.gitignore +11 -0
- package/skills/editor-gizmo/SKILL.md +222 -0
- package/skills/editor-gizmo/src/__editor/camera.ts +277 -0
- package/skills/editor-gizmo/src/__editor/discovery.ts +186 -0
- package/skills/editor-gizmo/src/__editor/drag.ts +265 -0
- package/skills/editor-gizmo/src/__editor/gizmo.ts +496 -0
- package/skills/editor-gizmo/src/__editor/history.ts +72 -0
- package/skills/editor-gizmo/src/__editor/index.ts +137 -0
- package/skills/editor-gizmo/src/__editor/input.ts +55 -0
- package/skills/editor-gizmo/src/__editor/math-utils.ts +114 -0
- package/skills/editor-gizmo/src/__editor/persistence.ts +113 -0
- package/skills/editor-gizmo/src/__editor/selection.ts +157 -0
- package/skills/editor-gizmo/src/__editor/state.ts +117 -0
- package/skills/editor-gizmo/src/__editor/ui.tsx +697 -0
- package/skills/game-design/SKILL.md +1 -2
- package/skills/lighting-environment/SKILL.md +103 -56
- package/skills/multiplayer-sync/SKILL.md +31 -2
- package/skills/nft-blockchain/SKILL.md +45 -40
- package/skills/npcs/SKILL.md +180 -0
- package/skills/optimize-scene/SKILL.md +7 -2
- package/skills/particle-system/SKILL.md +222 -0
- package/skills/player-avatar/SKILL.md +133 -7
- package/skills/player-physics/SKILL.md +93 -0
- package/skills/scene-runtime/SKILL.md +9 -5
- package/skills/visual-feedback/SKILL.md +1 -0
- package/extensions/dcl-setup-ollama.ts +0 -312
|
@@ -5,6 +5,10 @@ description: Animate objects in Decentraland scenes. Play GLTF model animations
|
|
|
5
5
|
|
|
6
6
|
# Animations and Tweens in Decentraland
|
|
7
7
|
|
|
8
|
+
## Authoring split
|
|
9
|
+
|
|
10
|
+
`Animator`, `Tween`, and `TweenSequence` are all supported in `main-entities.ts` — declare the entity, its visual components, and its initial animation state in the literal. Switch clips or replace tweens at runtime in `src/index.ts` via `getMutable`.
|
|
11
|
+
|
|
8
12
|
## When to Use Which Animation Approach
|
|
9
13
|
|
|
10
14
|
| Need | Approach | When |
|
|
@@ -21,142 +25,214 @@ description: Animate objects in Decentraland scenes. Play GLTF model animations
|
|
|
21
25
|
|
|
22
26
|
## GLTF Animations (Animator)
|
|
23
27
|
|
|
24
|
-
|
|
28
|
+
Declare the character and its animation states in `main-entities.ts`. The `states` array is part of the `Animator` shape and is JSON-compatible.
|
|
25
29
|
|
|
26
30
|
```typescript
|
|
27
|
-
|
|
28
|
-
import {
|
|
29
|
-
|
|
30
|
-
const
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
Animator
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
}
|
|
31
|
+
// main-entities.ts
|
|
32
|
+
import type { Scene } from '@dcl/sdk/scene-types'
|
|
33
|
+
|
|
34
|
+
export const scene = {
|
|
35
|
+
character: {
|
|
36
|
+
components: {
|
|
37
|
+
Transform: { position: { x: 8, y: 0, z: 8 } },
|
|
38
|
+
GltfContainer: { src: 'models/character.glb' },
|
|
39
|
+
Animator: {
|
|
40
|
+
states: [
|
|
41
|
+
{ clip: 'idle', playing: true, loop: true, speed: 1 },
|
|
42
|
+
{ clip: 'walk', playing: false, loop: true, speed: 1 },
|
|
43
|
+
{ clip: 'attack', playing: false, loop: false, speed: 1.5 }
|
|
44
|
+
]
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
} satisfies Scene
|
|
49
|
+
```
|
|
42
50
|
|
|
43
|
-
|
|
44
|
-
Animator.playSingleAnimation(character, 'walk')
|
|
51
|
+
Trigger and switch animations at runtime in `src/index.ts`:
|
|
45
52
|
|
|
46
|
-
|
|
47
|
-
Animator
|
|
53
|
+
```typescript
|
|
54
|
+
import { engine, Animator } from '@dcl/sdk/ecs'
|
|
55
|
+
|
|
56
|
+
export function main() {
|
|
57
|
+
const character = engine.getEntityOrNullByName('character')
|
|
58
|
+
if (!character) return
|
|
59
|
+
|
|
60
|
+
// Play a specific animation
|
|
61
|
+
Animator.playSingleAnimation(character, 'walk')
|
|
62
|
+
|
|
63
|
+
// Stop all animations
|
|
64
|
+
// Animator.stopAllAnimations(character)
|
|
65
|
+
}
|
|
48
66
|
```
|
|
49
67
|
|
|
50
68
|
### Switching Animations
|
|
51
69
|
```typescript
|
|
70
|
+
import { Entity, Animator } from '@dcl/sdk/ecs'
|
|
71
|
+
|
|
52
72
|
function playAnimation(entity: Entity, clipName: string) {
|
|
53
73
|
const animator = Animator.getMutable(entity)
|
|
54
|
-
|
|
55
|
-
for (const state of animator.states) {
|
|
56
|
-
state.playing = false
|
|
57
|
-
}
|
|
58
|
-
// Play the desired one
|
|
74
|
+
for (const state of animator.states) state.playing = false
|
|
59
75
|
const state = animator.states.find(s => s.clip === clipName)
|
|
60
|
-
if (state)
|
|
61
|
-
state.playing = true
|
|
62
|
-
}
|
|
76
|
+
if (state) state.playing = true
|
|
63
77
|
}
|
|
64
78
|
```
|
|
65
79
|
|
|
66
|
-
## Tweens
|
|
80
|
+
## Tweens
|
|
67
81
|
|
|
68
|
-
|
|
82
|
+
`Tween` and `TweenSequence` are JSON-compatible. The `mode` is a `$case`-tagged union; positions/quaternions are plain object literals; easing is a numeric enum.
|
|
69
83
|
|
|
70
84
|
### Move
|
|
71
85
|
```typescript
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
}
|
|
86
|
+
// main-entities.ts
|
|
87
|
+
sliding_box: {
|
|
88
|
+
components: {
|
|
89
|
+
Transform: { position: { x: 2, y: 1, z: 8 } },
|
|
90
|
+
MeshRenderer: { mesh: { $case: 'box', box: { uvs: [] } } },
|
|
91
|
+
Tween: {
|
|
92
|
+
duration: 2000, // milliseconds
|
|
93
|
+
easingFunction: 6, // EasingFunction.EF_EASESINE
|
|
94
|
+
mode: {
|
|
95
|
+
$case: 'move',
|
|
96
|
+
move: { start: { x: 2, y: 1, z: 8 }, end: { x: 14, y: 1, z: 8 } }
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
86
101
|
```
|
|
87
102
|
|
|
103
|
+
Common easing values: `0 = EF_LINEAR`, `1 = EF_EASEINQUAD`, `2 = EF_EASEOUTQUAD`, `3 = EF_EASEQUAD`, `6 = EF_EASESINE`. To replace or stop a tween at runtime, use `Tween.createOrReplace` / `Tween.deleteFrom` on the named entity in `src/index.ts`.
|
|
104
|
+
|
|
88
105
|
### Rotate
|
|
106
|
+
|
|
89
107
|
```typescript
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
108
|
+
// main-entities.ts — quaternions are { x, y, z, w } literals.
|
|
109
|
+
// 360° around Y end-state is { x: 0, y: 1, z: 0, w: 0 }
|
|
110
|
+
spinning_obj: {
|
|
111
|
+
components: {
|
|
112
|
+
Transform: { position: { x: 8, y: 1, z: 8 } },
|
|
113
|
+
MeshRenderer: { mesh: { $case: 'box', box: { uvs: [] } } },
|
|
114
|
+
Tween: {
|
|
115
|
+
duration: 3000,
|
|
116
|
+
easingFunction: 0, // EF_LINEAR
|
|
117
|
+
mode: {
|
|
118
|
+
$case: 'rotate',
|
|
119
|
+
rotate: {
|
|
120
|
+
start: { x: 0, y: 0, z: 0, w: 1 },
|
|
121
|
+
end: { x: 0, y: 1, z: 0, w: 0 }
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
For exact-degree rotations without hand-computing quaternions, leave the tween out of `main-entities.ts` and create it at runtime in `src/index.ts` where `Quaternion.fromEulerDegrees()` is available:
|
|
130
|
+
|
|
131
|
+
```typescript
|
|
132
|
+
// src/index.ts — runtime tween authoring (helpers allowed)
|
|
133
|
+
import { engine, Tween, EasingFunction } from '@dcl/sdk/ecs'
|
|
134
|
+
import { Quaternion } from '@dcl/sdk/math'
|
|
135
|
+
|
|
136
|
+
export function main() {
|
|
137
|
+
const box = engine.getEntityOrNullByName('spinning_obj')
|
|
138
|
+
if (!box) return
|
|
139
|
+
Tween.createOrReplace(box, {
|
|
140
|
+
mode: Tween.Mode.Rotate({
|
|
141
|
+
start: Quaternion.fromEulerDegrees(0, 0, 0),
|
|
142
|
+
end: Quaternion.fromEulerDegrees(0, 360, 0)
|
|
143
|
+
}),
|
|
144
|
+
duration: 3000,
|
|
145
|
+
easingFunction: EasingFunction.EF_LINEAR
|
|
146
|
+
})
|
|
147
|
+
}
|
|
98
148
|
```
|
|
99
149
|
|
|
100
150
|
### Scale
|
|
151
|
+
|
|
101
152
|
```typescript
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
153
|
+
// main-entities.ts
|
|
154
|
+
grow_box: {
|
|
155
|
+
components: {
|
|
156
|
+
Transform: { position: { x: 8, y: 1, z: 8 } },
|
|
157
|
+
MeshRenderer: { mesh: { $case: 'box', box: { uvs: [] } } },
|
|
158
|
+
Tween: {
|
|
159
|
+
duration: 1000,
|
|
160
|
+
easingFunction: 14, // EF_EASEOUTBOUNCE
|
|
161
|
+
mode: {
|
|
162
|
+
$case: 'scale',
|
|
163
|
+
scale: { start: { x: 1, y: 1, z: 1 }, end: { x: 2, y: 2, z: 2 } }
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
}
|
|
110
168
|
```
|
|
111
169
|
|
|
112
170
|
## Tween Sequences (Chained Animations)
|
|
113
171
|
|
|
114
|
-
Chain multiple tweens to play one after another
|
|
172
|
+
Chain multiple tweens to play one after another. `TweenSequence` is supported in `main-entities.ts`:
|
|
115
173
|
|
|
116
174
|
```typescript
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
end: Vector3.create(14, 1, 8)
|
|
124
|
-
}),
|
|
125
|
-
duration: 2000,
|
|
126
|
-
easingFunction: EasingFunction.EF_EASESINE
|
|
127
|
-
})
|
|
128
|
-
|
|
129
|
-
// Chain sequence
|
|
130
|
-
TweenSequence.create(box, {
|
|
131
|
-
sequence: [
|
|
132
|
-
// Second: move back
|
|
133
|
-
{
|
|
134
|
-
mode: Tween.Mode.Move({
|
|
135
|
-
start: Vector3.create(14, 1, 8),
|
|
136
|
-
end: Vector3.create(2, 1, 8)
|
|
137
|
-
}),
|
|
175
|
+
// main-entities.ts
|
|
176
|
+
patrol_box: {
|
|
177
|
+
components: {
|
|
178
|
+
Transform: { position: { x: 2, y: 1, z: 8 } },
|
|
179
|
+
MeshRenderer: { mesh: { $case: 'box', box: { uvs: [] } } },
|
|
180
|
+
Tween: {
|
|
138
181
|
duration: 2000,
|
|
139
|
-
easingFunction:
|
|
182
|
+
easingFunction: 6, // EF_EASESINE
|
|
183
|
+
mode: { $case: 'move', move: { start: { x: 2, y: 1, z: 8 }, end: { x: 14, y: 1, z: 8 } } }
|
|
184
|
+
},
|
|
185
|
+
TweenSequence: {
|
|
186
|
+
loop: 0, // TweenLoop.TL_RESTART (1 = TL_YOYO)
|
|
187
|
+
sequence: [
|
|
188
|
+
{
|
|
189
|
+
duration: 2000,
|
|
190
|
+
easingFunction: 6,
|
|
191
|
+
mode: { $case: 'move', move: { start: { x: 14, y: 1, z: 8 }, end: { x: 2, y: 1, z: 8 } } }
|
|
192
|
+
}
|
|
193
|
+
]
|
|
140
194
|
}
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
})
|
|
195
|
+
}
|
|
196
|
+
}
|
|
144
197
|
```
|
|
145
198
|
|
|
146
199
|
## Easing Functions
|
|
147
200
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
201
|
+
Inside `main-entities.ts` use the integer values (left). In `src/index.ts` you can reference `EasingFunction.<NAME>` directly.
|
|
202
|
+
|
|
203
|
+
| value | enum | shape |
|
|
204
|
+
|---|---|---|
|
|
205
|
+
| 0 | EF_LINEAR | Constant speed |
|
|
206
|
+
| 1 | EF_EASEINQUAD | Quadratic in |
|
|
207
|
+
| 2 | EF_EASEOUTQUAD | Quadratic out |
|
|
208
|
+
| 3 | EF_EASEQUAD | Quadratic in-out |
|
|
209
|
+
| 4 | EF_EASEINSINE | Sinusoidal in |
|
|
210
|
+
| 5 | EF_EASEOUTSINE | Sinusoidal out |
|
|
211
|
+
| 6 | EF_EASESINE | Sinusoidal in-out (smooth) |
|
|
212
|
+
| 7 | EF_EASEINEXPO | Exponential in |
|
|
213
|
+
| 8 | EF_EASEOUTEXPO | Exponential out |
|
|
214
|
+
| 9 | EF_EASEEXPO | Exponential in-out |
|
|
215
|
+
| 10 | EF_EASEINELASTIC | Elastic in |
|
|
216
|
+
| 11 | EF_EASEOUTELASTIC | Elastic out |
|
|
217
|
+
| 12 | EF_EASEELASTIC | Elastic in-out |
|
|
218
|
+
| 13 | EF_EASEINBOUNCE | Bounce in |
|
|
219
|
+
| 14 | EF_EASEOUTBOUNCE | Bounce out |
|
|
220
|
+
| 15 | EF_EASEBOUNCE | Bounce in-out |
|
|
221
|
+
| 16 | EF_EASEINCUBIC | Cubic in |
|
|
222
|
+
| 17 | EF_EASEOUTCUBIC | Cubic out |
|
|
223
|
+
| 18 | EF_EASECUBIC | Cubic in-out |
|
|
224
|
+
| 19 | EF_EASEINQUART | Quartic in |
|
|
225
|
+
| 20 | EF_EASEOUTQUART | Quartic out |
|
|
226
|
+
| 21 | EF_EASEQUART | Quartic in-out |
|
|
227
|
+
| 22 | EF_EASEINQUINT | Quintic in |
|
|
228
|
+
| 23 | EF_EASEOUTQUINT | Quintic out |
|
|
229
|
+
| 24 | EF_EASEQUINT | Quintic in-out |
|
|
230
|
+
| 25 | EF_EASEINCIRC | Circular in |
|
|
231
|
+
| 26 | EF_EASEOUTCIRC | Circular out |
|
|
232
|
+
| 27 | EF_EASECIRC | Circular in-out |
|
|
233
|
+
| 28 | EF_EASEINBACK | Overshoot in |
|
|
234
|
+
| 29 | EF_EASEOUTBACK | Overshoot out |
|
|
235
|
+
| 30 | EF_EASEBACK | Overshoot in-out |
|
|
160
236
|
|
|
161
237
|
## Custom Animation Systems
|
|
162
238
|
|
|
@@ -205,8 +281,23 @@ Tween.setScale(entity,
|
|
|
205
281
|
Vector3.One(), Vector3.create(2, 2, 2),
|
|
206
282
|
1000, EasingFunction.EF_LINEAR
|
|
207
283
|
)
|
|
284
|
+
|
|
285
|
+
// Combined Move + Rotate + Scale in one Tween
|
|
286
|
+
Tween.setMoveRotateScale(entity, {
|
|
287
|
+
positionStart: Vector3.create(0, 0, 0), positionEnd: Vector3.create(0, 5, 0),
|
|
288
|
+
rotationStart: Quaternion.fromEulerDegrees(0, 0, 0), rotationEnd: Quaternion.fromEulerDegrees(0, 360, 0),
|
|
289
|
+
scaleStart: Vector3.One(), scaleEnd: Vector3.create(0.5, 0.5, 0.5)
|
|
290
|
+
}, 2000, EasingFunction.EF_EASEINOUTCUBIC)
|
|
291
|
+
|
|
292
|
+
// Continuous spin — applies a per-frame Quaternion delta. No end state.
|
|
293
|
+
Tween.setRotateContinuous(entity, Quaternion.fromEulerDegrees(0, -1, 0), 700)
|
|
294
|
+
|
|
295
|
+
// Continuous move — applies a per-frame Vector3 delta scaled by speed.
|
|
296
|
+
Tween.setMoveContinuous(entity, Vector3.create(0, 0, 1), 0.5)
|
|
208
297
|
```
|
|
209
298
|
|
|
299
|
+
`*Continuous` variants don't end — they apply a constant delta until the Tween is removed (`Tween.deleteFrom(entity)`).
|
|
300
|
+
|
|
210
301
|
### Yoyo Loop Mode
|
|
211
302
|
|
|
212
303
|
`TL_YOYO` reverses the tween at each end instead of restarting:
|
|
@@ -247,6 +338,20 @@ anim.states[0].weight = 0.5 // blend walk at 50%
|
|
|
247
338
|
anim.states[1].weight = 0.5 // blend idle at 50%
|
|
248
339
|
```
|
|
249
340
|
|
|
341
|
+
### Morph Targets (Blend Shapes)
|
|
342
|
+
|
|
343
|
+
If a GLTF model exports morph targets (e.g., facial expressions, shape blends), drive them via the `weights` array on each `Animator.state`. Indices match the order morph targets were exported. Values range `0`–`1`.
|
|
344
|
+
|
|
345
|
+
```typescript
|
|
346
|
+
const anim = Animator.getMutable(entity)
|
|
347
|
+
const state = anim.states.find(s => s.clip === 'Idle')
|
|
348
|
+
if (state) {
|
|
349
|
+
state.weights = state.weights ?? []
|
|
350
|
+
state.weights[0] = 0.8 // morph target 0 at 80%
|
|
351
|
+
state.weights[1] = 0.3 // morph target 1 at 30%
|
|
352
|
+
}
|
|
353
|
+
```
|
|
354
|
+
|
|
250
355
|
## Troubleshooting
|
|
251
356
|
|
|
252
357
|
| Problem | Cause | Solution |
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: audio-analysis
|
|
3
|
+
description: Read real-time amplitude and 8-band frequency data from any AudioSource, AudioStream, or VideoPlayer entity in a Decentraland SDK7 scene with the AudioAnalysis component. Renderer fills the component each frame; scenes copy values into a plain JS view via readIntoView/tryReadIntoView and drive entity scale, color, lights, materials, particles, or UI from amplitude (overall loudness) and bands[0..7] (low→high frequency bins). Use when the user asks for music visualizers, beat reactivity, audio-reactive scenes, equalizers, dancing lights, scaling cubes that pulse to music, audio-driven materials, or anything that should react to sound. Do NOT use to play sound (see audio-video) or to detect player-emitted audio (this reads only entity-attached AudioSource/AudioStream/VideoPlayer audio).
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# AudioAnalysis
|
|
7
|
+
|
|
8
|
+
Real-time audio signal analysis attached to any entity that already has an `AudioSource`, `AudioStream`, or `VideoPlayer`. The renderer analyzes the audio frame buffer and writes results back into the component each tick. Scenes read those results to drive visualizers, beat-reactive geometry, audio-driven lights, etc.
|
|
9
|
+
|
|
10
|
+
## Authoring split
|
|
11
|
+
|
|
12
|
+
The **audio-emitting entity** (a speaker / radio / video screen) is static and belongs in `main-entities.ts` with its `AudioSource` / `AudioStream` / `VideoPlayer` component. `AudioAnalysis` itself is **not** in the supported declarative list — attach it at runtime in `src/index.ts` via `getEntityOrNullByName`. Reactive entities (pulsing cubes, EQ bars) are likewise added in code since their visuals are driven dynamically.
|
|
13
|
+
|
|
14
|
+
```typescript
|
|
15
|
+
// main-entities.ts — the audio source
|
|
16
|
+
dj_speaker: {
|
|
17
|
+
components: {
|
|
18
|
+
Transform: { position: { x: 8, y: 1, z: 8 } },
|
|
19
|
+
AudioSource: { audioClipUrl: 'sounds/track.mp3', playing: true, loop: true, volume: 0.8 }
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
```typescript
|
|
25
|
+
// src/index.ts — attach analysis + reactive systems
|
|
26
|
+
import { engine, AudioAnalysis, PBAudioAnalysisMode, Transform } from '@dcl/sdk/ecs'
|
|
27
|
+
import { Vector3 } from '@dcl/sdk/math'
|
|
28
|
+
|
|
29
|
+
export function main() {
|
|
30
|
+
const speaker = engine.getEntityOrNullByName('dj_speaker')
|
|
31
|
+
if (!speaker) return
|
|
32
|
+
AudioAnalysis.createAudioAnalysis(speaker, PBAudioAnalysisMode.MODE_LOGARITHMIC)
|
|
33
|
+
// ... reactive systems below
|
|
34
|
+
}
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
## RULE: Requires an audio-emitting component on the same entity
|
|
38
|
+
|
|
39
|
+
`AudioAnalysis` does nothing on its own. The entity MUST also have one of: `AudioSource`, `AudioStream`, or `VideoPlayer`. The renderer taps that component's audio frame buffer to compute amplitude/bands. An entity with only `AudioAnalysis` produces no data.
|
|
40
|
+
|
|
41
|
+
## RULE: Audio must be playing for non-zero output
|
|
42
|
+
|
|
43
|
+
Values are derived from live audio frames. If the source is paused, muted, or not yet loaded, `amplitude` and all `bands[]` stay at `0`. There is no "ready" event — start your reactive systems unconditionally, they will simply animate toward `0` while silent.
|
|
44
|
+
|
|
45
|
+
## RULE: Only the Unity explorer implements this
|
|
46
|
+
|
|
47
|
+
Bevy and the mobile Godot explorer ignore the component (no analysis written). Treat `AudioAnalysis` as a Unity-explorer-only enhancement and design fallbacks (e.g. a base scale that doesn't depend on `amplitude`) so the scene still looks reasonable elsewhere.
|
|
48
|
+
|
|
49
|
+
## RULE: Read via `readIntoView` into a pre-allocated view
|
|
50
|
+
|
|
51
|
+
`readIntoView` / `tryReadIntoView` write into a caller-owned `AudioAnalysisView = { amplitude: number, bands: number[] }`. Allocate the view once at scene init and reuse it every frame — do not `new` it inside the system. The `bands` array MUST be pre-sized to 8.
|
|
52
|
+
|
|
53
|
+
## RULE: Use `createAudioAnalysis`, not `create`
|
|
54
|
+
|
|
55
|
+
Use the helper `AudioAnalysis.createAudioAnalysis(entity, mode?, amplitudeGain?, bandsGain?)`. It pre-fills all required protobuf fields (8 bands + amplitude + mode) with safe defaults. Calling the raw `AudioAnalysis.create(entity, {...})` requires you to provide every band/amplitude field manually. Use `createOrReplaceAudioAnalysis` to overwrite an existing one without throwing.
|
|
56
|
+
|
|
57
|
+
## Import
|
|
58
|
+
|
|
59
|
+
```typescript
|
|
60
|
+
import {
|
|
61
|
+
AudioAnalysis,
|
|
62
|
+
AudioAnalysisView,
|
|
63
|
+
PBAudioAnalysisMode,
|
|
64
|
+
AudioSource, // or AudioStream / VideoPlayer
|
|
65
|
+
engine,
|
|
66
|
+
Transform
|
|
67
|
+
} from '@dcl/sdk/ecs'
|
|
68
|
+
import { Vector3 } from '@dcl/sdk/math'
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
`AudioAnalysisView` is a TypeScript type alias (not a component), exported from `@dcl/sdk/ecs`.
|
|
72
|
+
|
|
73
|
+
## Component fields
|
|
74
|
+
|
|
75
|
+
Output (filled by the renderer; read in your systems):
|
|
76
|
+
|
|
77
|
+
| Field | Type | Range | Notes |
|
|
78
|
+
|---|---|---|---|
|
|
79
|
+
| `amplitude` | `number` | `0..~1` (mode-dep.) | Aggregate signal strength of the current audio frame. |
|
|
80
|
+
| `band0` | `number` | `0..~1` (mode-dep.) | Lowest frequency bin (sub-bass). |
|
|
81
|
+
| `band1..6` | `number` | `0..~1` (mode-dep.) | Increasing frequency bins, log-spaced under MODE_LOGARITHMIC. |
|
|
82
|
+
| `band7` | `number` | `0..~1` (mode-dep.) | Highest frequency bin (treble/air). |
|
|
83
|
+
|
|
84
|
+
Inputs (configure once at create time):
|
|
85
|
+
|
|
86
|
+
| Field | Type | Default | Notes |
|
|
87
|
+
|---|---|---|---|
|
|
88
|
+
| `mode` | `PBAudioAnalysisMode` | `MODE_LOGARITHMIC` | `MODE_RAW = 0` (raw FFT magnitudes) / `MODE_LOGARITHMIC = 1` (perceptual log mapping, recommended). |
|
|
89
|
+
| `amplitudeGain` | `number?` | `5.0` | Multiplier applied to `amplitude`. Only used in MODE_LOGARITHMIC. |
|
|
90
|
+
| `bandsGain` | `number?` | `0.05` | Multiplier applied to all 8 bands. Only used in MODE_LOGARITHMIC. |
|
|
91
|
+
|
|
92
|
+
> Values are unbounded floats — gains can push them above `1`. Clamp or scale in your system if the visual you drive needs `0..1`. For typical music at default gains, expect peaks roughly in `0..1` with normal content sitting `0..0.5`.
|
|
93
|
+
|
|
94
|
+
## AudioAnalysisView
|
|
95
|
+
|
|
96
|
+
```typescript
|
|
97
|
+
type AudioAnalysisView = {
|
|
98
|
+
amplitude: number
|
|
99
|
+
bands: number[] // length 8 — bands[0] = lowest freq, bands[7] = highest
|
|
100
|
+
}
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
## Reading the data
|
|
104
|
+
|
|
105
|
+
```typescript
|
|
106
|
+
const view: AudioAnalysisView = { amplitude: 0, bands: new Array<number>(8) }
|
|
107
|
+
|
|
108
|
+
engine.addSystem(() => {
|
|
109
|
+
AudioAnalysis.readIntoView(audioEntity, view)
|
|
110
|
+
// Or, defensive variant — returns false if the component is missing:
|
|
111
|
+
// if (!AudioAnalysis.tryReadIntoView(audioEntity, view)) return
|
|
112
|
+
|
|
113
|
+
// view.amplitude and view.bands[0..7] are now populated
|
|
114
|
+
})
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
## Common patterns
|
|
118
|
+
|
|
119
|
+
```typescript
|
|
120
|
+
// 1. Pulse an entity's scale to overall amplitude
|
|
121
|
+
const view: AudioAnalysisView = { amplitude: 0, bands: new Array<number>(8) }
|
|
122
|
+
engine.addSystem(() => {
|
|
123
|
+
AudioAnalysis.readIntoView(audioEntity, view)
|
|
124
|
+
const t = Transform.getMutable(pulseEntity)
|
|
125
|
+
const s = 1 + view.amplitude * 10
|
|
126
|
+
t.scale = Vector3.create(s, s, s)
|
|
127
|
+
})
|
|
128
|
+
|
|
129
|
+
// 2. 8-bar equalizer (one entity per band, scale Y by bands[i])
|
|
130
|
+
for (const [entity, _] of engine.getEntitiesWith(VisualBar, Transform)) {
|
|
131
|
+
const i = VisualBar.get(entity).index // 0..7
|
|
132
|
+
Transform.getMutable(entity).scale = Vector3.create(1, view.bands[i] * BAR_HEIGHT, 1)
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// 3. Bass-only kick
|
|
136
|
+
const bass = view.bands[0] + view.bands[1]
|
|
137
|
+
if (bass > 0.7) { /* trigger flash */ }
|
|
138
|
+
|
|
139
|
+
// 4. Custom gains (less sensitive amplitude, punchier bands)
|
|
140
|
+
AudioAnalysis.createAudioAnalysis(
|
|
141
|
+
audioEntity,
|
|
142
|
+
PBAudioAnalysisMode.MODE_LOGARITHMIC,
|
|
143
|
+
2.0,
|
|
144
|
+
0.1
|
|
145
|
+
)
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
## Mode selection
|
|
149
|
+
|
|
150
|
+
- `MODE_LOGARITHMIC` (default) — bands are log-spaced and gain-scaled to roughly fit `0..1` for typical music. Use for visualizers, scaling, color reactivity. `amplitudeGain` / `bandsGain` apply.
|
|
151
|
+
- `MODE_RAW` — raw FFT-derived magnitudes, linearly spaced. Lower bands dominate visually because most musical energy is there. Gains are ignored. Use only if you intend to do your own normalization.
|
|
152
|
+
|
|
153
|
+
## Gotchas
|
|
154
|
+
|
|
155
|
+
- **Output values can exceed `1.0`** with high gains or loud sources. Clamp downstream if you feed UI bars or alpha channels expecting `0..1`.
|
|
156
|
+
- **Throttled updates.** Renderer runs analysis under a frame-time budget — values may skip frames under load. Drive smooth animations with `dt` interpolation.
|
|
157
|
+
- **Multi-source scenes.** Each audio-emitting entity needs its own `AudioAnalysis`. No global mix-down.
|
|
158
|
+
- **Works on `VideoPlayer` audio too.** The same audio frame buffer interface backs `AudioStream` and `VideoPlayer`, so you can react to a video's soundtrack.
|
|
159
|
+
- **No `pitch` interaction.** `AudioSource.pitch` changes playback speed; analysis runs on the actual played frames, so a higher pitch shifts perceived band energy upward.
|
|
160
|
+
- **Don't call `readIntoView` before `createAudioAnalysis`.** Reading without the component throws. Use `tryReadIntoView` if the component may not be attached yet.
|
|
161
|
+
|
|
162
|
+
## Permissions
|
|
163
|
+
|
|
164
|
+
No special scene permission beyond what `AudioSource` / `AudioStream` / `VideoPlayer` already requires. Streamed audio still needs `ALLOW_MEDIA_HOSTNAMES` in `scene.json` for its hostname (see `audio-video` skill).
|