@viji-dev/core 0.3.39 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,534 +4,542 @@
4
4
  */
5
5
 
6
6
  /**
7
- * Configuration for slider parameters
7
+ * Configuration object passed to `viji.slider()`. Defines the slider's bounds, increment, label, and UI grouping/visibility.
8
8
  * @typedef {Object} SliderConfig
9
- * @property {number} [min] - min property
10
- * @property {number} [max] - max property
11
- * @property {number} [step] - step property
12
- * @property {string} label - label property
13
- * @property {string} [description] - description property
14
- * @property {string} [group] - group property
15
- * @property {ParameterCategory} [category] - category property
9
+ * @property {number} [min] - Inclusive minimum value of the slider. Default: `0`.
10
+ * @property {number} [max] - Inclusive maximum value of the slider. Default: `100`.
11
+ * @property {number} [step] - Increment between values. Default: `1`.
12
+ * @property {string} label - Display name shown next to the control in the parameter UI. Required.
13
+ * @property {string} [description] - Optional tooltip / help text shown next to the control.
14
+ * @property {string} [group] - Group name for organizing related parameters under a shared heading. Default: `'general'`.
15
+ * @property {ParameterCategory} [category] - Visibility category — the control hides when its capability is unavailable. Default: `'general'`.
16
16
  */
17
17
 
18
18
  /**
19
- * Configuration for color parameters
19
+ * Configuration object passed to `viji.color()`. Defines the color picker's label and UI grouping/visibility.
20
20
  * @typedef {Object} ColorConfig
21
- * @property {string} label - label property
22
- * @property {string} [description] - description property
23
- * @property {string} [group] - group property
24
- * @property {ParameterCategory} [category] - category property
21
+ * @property {string} label - Display name shown next to the color swatch in the parameter UI. Required.
22
+ * @property {string} [description] - Optional tooltip / help text shown next to the control.
23
+ * @property {string} [group] - Group name for organizing related parameters under a shared heading. Default: `'general'`.
24
+ * @property {ParameterCategory} [category] - Visibility category — the control hides when its capability is unavailable. Default: `'general'`.
25
25
  */
26
26
 
27
27
  /**
28
- * Configuration for toggle parameters
28
+ * Configuration object passed to `viji.toggle()`. Defines the on/off switch's label and UI grouping/visibility.
29
29
  * @typedef {Object} ToggleConfig
30
- * @property {string} label - label property
31
- * @property {string} [description] - description property
32
- * @property {string} [group] - group property
33
- * @property {ParameterCategory} [category] - category property
30
+ * @property {string} label - Display name shown next to the switch in the parameter UI. Required.
31
+ * @property {string} [description] - Optional tooltip / help text shown next to the control.
32
+ * @property {string} [group] - Group name for organizing related parameters under a shared heading. Default: `'general'`.
33
+ * @property {ParameterCategory} [category] - Visibility category — the control hides when its capability is unavailable. Default: `'general'`.
34
34
  */
35
35
 
36
36
  /**
37
- * Configuration for select parameters
37
+ * Configuration object passed to `viji.select()`. The host renders the choices as a dropdown or segmented control.
38
38
  * @typedef {Object} SelectConfig
39
- * @property {string[] | number[]} options - options property
40
- * @property {string} label - label property
41
- * @property {string} [description] - description property
42
- * @property {string} [group] - group property
43
- * @property {ParameterCategory} [category] - category property
39
+ * @property {string[] | number[]} options - Available choices. The element type (`string` or `number`) becomes the parameter's value type. Required.
40
+ * @property {string} label - Display name shown next to the dropdown in the parameter UI. Required.
41
+ * @property {string} [description] - Optional tooltip / help text shown next to the control.
42
+ * @property {string} [group] - Group name for organizing related parameters under a shared heading. Default: `'general'`.
43
+ * @property {ParameterCategory} [category] - Visibility category — the control hides when its capability is unavailable. Default: `'general'`.
44
44
  */
45
45
 
46
46
  /**
47
- * Configuration for text parameters
47
+ * Configuration object passed to `viji.text()`. The host renders a single-line text input field.
48
48
  * @typedef {Object} TextConfig
49
- * @property {string} label - label property
50
- * @property {string} [description] - description property
51
- * @property {string} [group] - group property
52
- * @property {ParameterCategory} [category] - category property
53
- * @property {number} [maxLength] - maxLength property
49
+ * @property {string} label - Display name shown next to the text field in the parameter UI. Required.
50
+ * @property {string} [description] - Optional tooltip / help text shown next to the control.
51
+ * @property {string} [group] - Group name for organizing related parameters under a shared heading. Default: `'general'`.
52
+ * @property {ParameterCategory} [category] - Visibility category — the control hides when its capability is unavailable. Default: `'general'`.
53
+ * @property {number} [maxLength] - Maximum character count enforced by the host UI. Default: `1000`.
54
54
  */
55
55
 
56
56
  /**
57
- * Configuration for number parameters
57
+ * Configuration object passed to `viji.number()`. Defines the bounds, step, label, and UI grouping/visibility for a precise numeric input.
58
58
  * @typedef {Object} NumberConfig
59
- * @property {number} [min] - min property
60
- * @property {number} [max] - max property
61
- * @property {number} [step] - step property
62
- * @property {string} label - label property
63
- * @property {string} [description] - description property
64
- * @property {string} [group] - group property
65
- * @property {ParameterCategory} [category] - category property
59
+ * @property {number} [min] - Inclusive minimum value. Default: `0`.
60
+ * @property {number} [max] - Inclusive maximum value. Default: `100`.
61
+ * @property {number} [step] - Increment between values. Default: `1`.
62
+ * @property {string} label - Display name shown next to the input in the parameter UI. Required.
63
+ * @property {string} [description] - Optional tooltip / help text shown next to the control.
64
+ * @property {string} [group] - Group name for organizing related parameters under a shared heading. Default: `'general'`.
65
+ * @property {ParameterCategory} [category] - Visibility category — the control hides when its capability is unavailable. Default: `'general'`.
66
66
  */
67
67
 
68
68
  /**
69
- * Configuration for image parameters
69
+ * Configuration object passed to `viji.image()`. The host renders an upload field; the user-supplied image becomes available as an `ImageBitmap` (Native) or a P5 image wrapper (P5 renderer).
70
70
  * @typedef {Object} ImageConfig
71
- * @property {string} label - label property
72
- * @property {string} [description] - description property
73
- * @property {string} [group] - group property
74
- * @property {ParameterCategory} [category] - category property
71
+ * @property {string} label - Display name shown next to the upload field in the parameter UI. Required.
72
+ * @property {string} [description] - Optional tooltip / help text shown next to the control.
73
+ * @property {string} [group] - Group name for organizing related parameters under a shared heading. Default: `'general'`.
74
+ * @property {ParameterCategory} [category] - Visibility category — the control hides when its capability is unavailable. Default: `'general'`.
75
75
  */
76
76
 
77
77
  /**
78
- * Configuration for button parameters
78
+ * Configuration object passed to `viji.button()`. The host renders a clickable momentary button — the resulting `value` is `true` for one frame after press, then auto-resets to `false`.
79
79
  * @typedef {Object} ButtonConfig
80
- * @property {string} label - label property
81
- * @property {string} [description] - description property
82
- * @property {string} [group] - group property
83
- * @property {ParameterCategory} [category] - category property
80
+ * @property {string} label - Display name shown on the button in the parameter UI. Required.
81
+ * @property {string} [description] - Optional tooltip / help text shown next to the control.
82
+ * @property {string} [group] - Group name for organizing related parameters under a shared heading. Default: `'general'`.
83
+ * @property {ParameterCategory} [category] - Visibility category — the control hides when its capability is unavailable. Default: `'general'`.
84
84
  */
85
85
 
86
86
  /**
87
- * Configuration for coordinate parameters
87
+ * Configuration object passed to `viji.coordinate()`. The host renders a 2D draggable pad; both `x` and `y` are clamped to `-1..1`.
88
88
  * @typedef {Object} CoordinateConfig
89
- * @property {number} [step] - step property
90
- * @property {string} label - label property
91
- * @property {string} [description] - description property
92
- * @property {string} [group] - group property
93
- * @property {ParameterCategory} [category] - category property
89
+ * @property {number} [step] - Snap increment for both `x` and `y` components. Default: `0.01`.
90
+ * @property {string} label - Display name shown next to the pad in the parameter UI. Required.
91
+ * @property {string} [description] - Optional tooltip / help text shown next to the control.
92
+ * @property {string} [group] - Group name for organizing related parameters under a shared heading. Default: `'general'`.
93
+ * @property {ParameterCategory} [category] - Visibility category — the control hides when its capability is unavailable. Default: `'general'`.
94
94
  */
95
95
 
96
96
  /**
97
- * Parameter object for slider parameters
97
+ * Reactive object returned by `viji.slider()`. `value` updates in real-time as the user drags the control; the remaining fields mirror the configuration.
98
98
  * @typedef {Object} SliderParameter
99
- * @property {number} value - value property
100
- * @property {number} min - min property
101
- * @property {number} max - max property
102
- * @property {number} step - step property
103
- * @property {string} label - label property
104
- * @property {string} [description] - description property
105
- * @property {string} group - group property
106
- * @property {ParameterCategory} category - category property
99
+ * @property {number} value - Current slider value in the configured `min..max` range. Updates in real-time.
100
+ * @property {number} min - Inclusive minimum value of the slider.
101
+ * @property {number} max - Inclusive maximum value of the slider.
102
+ * @property {number} step - Increment between values.
103
+ * @property {string} label - Display name shown next to the control in the parameter UI.
104
+ * @property {string} [description] - Tooltip / help text for the control (empty string when not configured).
105
+ * @property {string} group - Group name under which the control appears in the UI.
106
+ * @property {ParameterCategory} category - Visibility category — controls when the parameter is shown.
107
107
  */
108
108
 
109
109
  /**
110
- * Parameter object for color parameters
110
+ * Reactive object returned by `viji.color()`. `value` is always a canonical lowercase 6-digit hex string; `rgb` and `hsb` are derived accessors that are recomputed (and frozen) every time the user changes the color.
111
111
  * @typedef {Object} ColorParameter
112
- * @property {string} value - value property
113
- * @property {string} label - label property
114
- * @property {string} [description] - description property
115
- * @property {string} group - group property
116
- * @property {ParameterCategory} category - category property
117
- */
118
-
119
- /**
120
- * Parameter object for toggle parameters
112
+ * @property {string} value - Canonical hex color string `#rrggbb` (lowercase). Updates in real-time.
113
+ * @property {Object} rgb - RGB components as integers in 0..255 (frozen, recomputed when value changes).
114
+ * @property {number} rgb.r - Red component, integer in 0..255.
115
+ * @property {number} rgb.g - Green component, integer in 0..255.
116
+ * @property {number} rgb.b - Blue component, integer in 0..255.
117
+ * @property {Object} hsb - HSB components: `h` in 0..360, `s` and `b` in 0..100 (frozen, recomputed when value changes).
118
+ * @property {number} hsb.h - Hue in 0..360.
119
+ * @property {number} hsb.s - Saturation in 0..100.
120
+ * @property {number} hsb.b - Brightness in 0..100.
121
+ * @property {string} label - Display name shown next to the swatch in the parameter UI.
122
+ * @property {string} [description] - Tooltip / help text for the control (empty string when not configured).
123
+ * @property {string} group - Group name under which the control appears in the UI.
124
+ * @property {ParameterCategory} category - Visibility category — controls when the parameter is shown.
125
+ */
126
+
127
+ /**
128
+ * Reactive object returned by `viji.toggle()`. `value` flips between `true` and `false` as the user interacts with the switch.
121
129
  * @typedef {Object} ToggleParameter
122
- * @property {boolean} value - value property
123
- * @property {string} label - label property
124
- * @property {string} [description] - description property
125
- * @property {string} group - group property
126
- * @property {ParameterCategory} category - category property
130
+ * @property {boolean} value - Current state of the switch. Updates in real-time when the user toggles it.
131
+ * @property {string} label - Display name shown next to the switch in the parameter UI.
132
+ * @property {string} [description] - Tooltip / help text for the control (empty string when not configured).
133
+ * @property {string} group - Group name under which the control appears in the UI.
134
+ * @property {ParameterCategory} category - Visibility category — controls when the parameter is shown.
127
135
  */
128
136
 
129
137
  /**
130
- * Parameter object for select parameters
138
+ * Reactive object returned by `viji.select()`. `value` matches the element type of the configured `options` array (`string` or `number`).
131
139
  * @typedef {Object} SelectParameter
132
- * @property {string | number} value - value property
133
- * @property {string[] | number[]} options - options property
134
- * @property {string} label - label property
135
- * @property {string} [description] - description property
136
- * @property {string} group - group property
137
- * @property {ParameterCategory} category - category property
140
+ * @property {string | number} value - Currently selected option. Updates in real-time when the user changes the selection.
141
+ * @property {string[] | number[]} options - The full list of choices originally configured.
142
+ * @property {string} label - Display name shown next to the dropdown in the parameter UI.
143
+ * @property {string} [description] - Tooltip / help text for the control (empty string when not configured).
144
+ * @property {string} group - Group name under which the control appears in the UI.
145
+ * @property {ParameterCategory} category - Visibility category — controls when the parameter is shown.
138
146
  */
139
147
 
140
148
  /**
141
- * Parameter object for text parameters
149
+ * Reactive object returned by `viji.text()`. `value` updates as the user types and is enforced to be no longer than `maxLength` by the host UI.
142
150
  * @typedef {Object} TextParameter
143
- * @property {string} value - value property
144
- * @property {number} [maxLength] - maxLength property
145
- * @property {string} label - label property
146
- * @property {string} [description] - description property
147
- * @property {string} group - group property
148
- * @property {ParameterCategory} category - category property
151
+ * @property {string} value - Current text content. Updates in real-time as the user types.
152
+ * @property {number} [maxLength] - Maximum character count enforced by the host UI.
153
+ * @property {string} label - Display name shown next to the input in the parameter UI.
154
+ * @property {string} [description] - Tooltip / help text for the control (empty string when not configured).
155
+ * @property {string} group - Group name under which the control appears in the UI.
156
+ * @property {ParameterCategory} category - Visibility category — controls when the parameter is shown.
149
157
  */
150
158
 
151
159
  /**
152
- * Parameter object for number parameters
160
+ * Reactive object returned by `viji.number()`. Like `SliderParameter` but the host UI renders a precise numeric input rather than a draggable slider.
153
161
  * @typedef {Object} NumberParameter
154
- * @property {number} value - value property
155
- * @property {number} min - min property
156
- * @property {number} max - max property
157
- * @property {number} step - step property
158
- * @property {string} label - label property
159
- * @property {string} [description] - description property
160
- * @property {string} group - group property
161
- * @property {ParameterCategory} category - category property
162
+ * @property {number} value - Current numeric value in the configured `min..max` range. Updates in real-time.
163
+ * @property {number} min - Inclusive minimum value.
164
+ * @property {number} max - Inclusive maximum value.
165
+ * @property {number} step - Increment between values.
166
+ * @property {string} label - Display name shown next to the input in the parameter UI.
167
+ * @property {string} [description] - Tooltip / help text for the control (empty string when not configured).
168
+ * @property {string} group - Group name under which the control appears in the UI.
169
+ * @property {ParameterCategory} category - Visibility category — controls when the parameter is shown.
162
170
  */
163
171
 
164
172
  /**
165
- * Parameter object for image parameters
173
+ * Reactive object returned by `viji.image()`. `value` is `null` until the user uploads an image; in P5 scenes a `.p5` accessor is also added at runtime so the image works directly with `image()`, `texture()`, etc.
166
174
  * @typedef {Object} ImageParameter
167
- * @property {ImageBitmap |null} value - value property
168
- * @property {string} label - P5-compatible image wrapper. Only available in the P5 renderer — added at runtime by the P5 adapter.
169
- * @property {string} [description] - description property
170
- * @property {string} group - group property
171
- * @property {ParameterCategory} category - category property
175
+ * @property {ImageBitmap |null} value - Uploaded image as an `ImageBitmap`, or `null` until the user provides one.
176
+ * @property {string} label - P5-compatible image wrapper. Only available in the P5 renderer — added at runtime by the P5 adapter. Display name shown next to the upload field in the parameter UI.
177
+ * @property {string} [description] - Tooltip / help text for the control (empty string when not configured).
178
+ * @property {string} group - Group name under which the control appears in the UI.
179
+ * @property {ParameterCategory} category - Visibility category — controls when the parameter is shown.
172
180
  */
173
181
 
174
182
  /**
175
- * Parameter object for button parameters
183
+ * Reactive object returned by `viji.button()`. `value` is a momentary flag — `true` for exactly one frame after the user clicks, then auto-resets to `false`. Use it to trigger discrete events from `render()`.
176
184
  * @typedef {Object} ButtonParameter
177
- * @property {boolean} value - value property
178
- * @property {string} label - label property
179
- * @property {string} [description] - description property
180
- * @property {string} group - group property
181
- * @property {ParameterCategory} category - category property
185
+ * @property {boolean} value - `true` for the single frame after the user clicks the button, `false` otherwise.
186
+ * @property {string} label - Display name shown on the button in the parameter UI.
187
+ * @property {string} [description] - Tooltip / help text for the control (empty string when not configured).
188
+ * @property {string} group - Group name under which the control appears in the UI.
189
+ * @property {ParameterCategory} category - Visibility category — controls when the parameter is shown.
182
190
  */
183
191
 
184
192
  /**
185
- * Parameter object for coordinate parameters
193
+ * Reactive object returned by `viji.coordinate()`. `value.x` and `value.y` are always in `-1..1`; the host UI renders a draggable 2D pad.
186
194
  * @typedef {Object} CoordinateParameter
187
- * @property {CoordinateValue} value - value property
188
- * @property {number} step - step property
189
- * @property {string} label - label property
190
- * @property {string} [description] - description property
191
- * @property {string} group - group property
192
- * @property {ParameterCategory} category - category property
195
+ * @property {CoordinateValue} value - Current `{ x, y }` position. Both components are in `-1..1`. Updates in real-time.
196
+ * @property {number} step - Snap increment for both `x` and `y` components.
197
+ * @property {string} label - Display name shown next to the pad in the parameter UI.
198
+ * @property {string} [description] - Tooltip / help text for the control (empty string when not configured).
199
+ * @property {string} group - Group name under which the control appears in the UI.
200
+ * @property {ParameterCategory} category - Visibility category — controls when the parameter is shown.
193
201
  */
194
202
 
195
203
  /**
196
- * Audio analysis API - provides real-time audio data and frequency analysis
204
+ * Real-time audio analysis for the main audio stream (`viji.audio`). Provides volume, frequency bands, beat detection (with triggers, events, BPM), spectral features, and access to raw FFT / waveform data. All numeric outputs are zeroed when `isConnected` is `false`.
197
205
  * @typedef {Object} AudioAPI
198
- * @property {boolean} isConnected - isConnected property
199
- * @property {Object} volume - volume property
200
- * @property {number} volume.current - current property
201
- * @property {number} volume.peak - peak property
202
- * @property {number} volume.smoothed - smoothed property
203
- * @property {Object} bands - bands property
204
- * @property {number} bands.low - low property
205
- * @property {number} bands.lowMid - lowMid property
206
- * @property {number} bands.mid - mid property
207
- * @property {number} bands.highMid - highMid property
208
- * @property {number} bands.high - high property
209
- * @property {number} bands.lowSmoothed - lowSmoothed property
210
- * @property {number} bands.lowMidSmoothed - lowMidSmoothed property
211
- * @property {number} bands.midSmoothed - midSmoothed property
212
- * @property {number} bands.highMidSmoothed - highMidSmoothed property
213
- * @property {number} bands.highSmoothed - highSmoothed property
214
- * @property {Object} beat - beat property
215
- * @property {number} beat.kick - kick property
216
- * @property {number} beat.snare - snare property
217
- * @property {number} beat.hat - hat property
218
- * @property {number} beat.any - any property
219
- * @property {number} beat.kickSmoothed - kickSmoothed property
220
- * @property {number} beat.snareSmoothed - snareSmoothed property
221
- * @property {number} beat.hatSmoothed - hatSmoothed property
222
- * @property {number} beat.anySmoothed - anySmoothed property
223
- * @property {number} beat.bpm - bpm property
224
- * @property {number} beat.confidence - confidence property
225
- * @property {boolean} beat.isLocked - isLocked property
226
- * @property {Object} spectral - spectral property
227
- * @property {number} spectral.brightness - brightness property
228
- * @property {number} spectral.flatness - flatness property
229
- * @property {() => Uint8Array} getFrequencyData - getFrequencyData property
230
- * @property {() => Float32Array} getWaveform - getWaveform property
231
- */
232
-
233
- /**
234
- * Additional audio stream API - provides per-stream audio analysis
206
+ * @property {boolean} isConnected - `true` when an audio source is connected; otherwise all numeric fields read as `0` and helper methods return empty data.
207
+ * @property {Object} volume - Overall loudness across the stream.
208
+ * @property {number} volume.current - Instantaneous RMS volume in 0..1.
209
+ * @property {number} volume.peak - Instantaneous peak amplitude in 0..1.
210
+ * @property {number} volume.smoothed - Volume with a 200ms decay envelope; rises fast, falls smoothly. Use for flicker-free animations.
211
+ * @property {Object} bands - Energy in five fixed frequency bands, plus a smoothed companion (~150ms decay) for each. All in 0..1.
212
+ * @property {number} bands.low - Instant low-band energy (20–120 Hz, bass/kick range). 0..1.
213
+ * @property {number} bands.lowMid - Instant low-mid energy (120–400 Hz). 0..1.
214
+ * @property {number} bands.mid - Instant mid energy (400–1600 Hz, vocals/instruments). 0..1.
215
+ * @property {number} bands.highMid - Instant high-mid energy (1600–6000 Hz, cymbals/hi-hats). 0..1.
216
+ * @property {number} bands.high - Instant high-band energy (6000–16000 Hz, air/brilliance). 0..1.
217
+ * @property {number} bands.lowSmoothed - Smoothed `low` (~150ms decay). 0..1.
218
+ * @property {number} bands.lowMidSmoothed - Smoothed `lowMid` (~150ms decay). 0..1.
219
+ * @property {number} bands.midSmoothed - Smoothed `mid` (~150ms decay). 0..1.
220
+ * @property {number} bands.highMidSmoothed - Smoothed `highMid` (~150ms decay). 0..1.
221
+ * @property {number} bands.highSmoothed - Smoothed `high` (~150ms decay). 0..1.
222
+ * @property {Object} beat - Beat detection: fast/smoothed energy curves, single-frame boolean triggers, raw event list, and BPM tracking.
223
+ * @property {number} beat.kick - Kick energy curve in 0..1 with a 300ms decay. Peaks on each detected kick.
224
+ * @property {number} beat.snare - Snare energy curve in 0..1 with a 300ms decay.
225
+ * @property {number} beat.hat - Hi-hat energy curve in 0..1 with a 300ms decay.
226
+ * @property {number} beat.any - Energy curve for any beat type in 0..1 with a 300ms decay.
227
+ * @property {number} beat.kickSmoothed - Smoothed `kick` curve in 0..1 with a 500ms decay. Use for ambient pulses.
228
+ * @property {number} beat.snareSmoothed - Smoothed `snare` curve in 0..1 with a 500ms decay.
229
+ * @property {number} beat.hatSmoothed - Smoothed `hat` curve in 0..1 with a 500ms decay.
230
+ * @property {number} beat.anySmoothed - Smoothed `any` curve in 0..1 with a 500ms decay.
231
+ * @property {number} beat.bpm - Currently detected tempo in beats per minute. Defaults to `120` when no audio.
232
+ * @property {number} beat.confidence - Confidence of the BPM tracker in 0..1.
233
+ * @property {boolean} beat.isLocked - `true` when the BPM tracker has a stable lock on the current tempo.
234
+ * @property {Object} spectral - High-level spectral features derived from the FFT.
235
+ * @property {number} spectral.brightness - Normalized spectral centroid in 0..1 — higher values mean brighter, more treble-heavy sound.
236
+ * @property {number} spectral.flatness - Normalized spectral flatness in 0..1 — higher values mean noisier (white-noise-like) sound; lower values mean tonal.
237
+ * @property {() => Uint8Array} getFrequencyData - Returns the raw FFT magnitude spectrum as a `Uint8Array` (1024 bins, each 0..255). Bin `i` covers frequency `i × (sampleRate / fftSize)`. The returned array is a snapshot from the latest analysis tick — repeated calls in the same frame return the same data. @example const fft = viji.audio.getFrequencyData(); for (let i = 0; i < fft.length; i++) drawBar(i, fft[i] / 255);
238
+ * @property {() => Float32Array} getWaveform - Returns the raw time-domain waveform as a `Float32Array` (2048 PCM samples in -1..1). The returned array is a snapshot — repeated calls in the same frame return the same data. @example const wave = viji.audio.getWaveform(); for (let i = 0; i < wave.length; i++) drawSample(i, wave[i]);
239
+ */
240
+
241
+ /**
242
+ * Lightweight audio analysis for additional or device audio streams. A strict subset of `AudioAPI` — exposes volume, frequency bands, spectral features, and raw FFT/waveform access, but **no beat detection** (no `beat`, no BPM, no triggers, no events).
235
243
  * @typedef {Object} AudioStreamAPI
236
- * @property {boolean} isConnected - isConnected property
237
- * @property {Object} volume - volume property
238
- * @property {number} volume.current - current property
239
- * @property {number} volume.peak - peak property
240
- * @property {number} volume.smoothed - smoothed property
241
- * @property {Object} bands - bands property
242
- * @property {number} bands.low - low property
243
- * @property {number} bands.lowMid - lowMid property
244
- * @property {number} bands.mid - mid property
245
- * @property {number} bands.highMid - highMid property
246
- * @property {number} bands.high - high property
247
- * @property {number} bands.lowSmoothed - lowSmoothed property
248
- * @property {number} bands.lowMidSmoothed - lowMidSmoothed property
249
- * @property {number} bands.midSmoothed - midSmoothed property
250
- * @property {number} bands.highMidSmoothed - highMidSmoothed property
251
- * @property {number} bands.highSmoothed - highSmoothed property
252
- * @property {Object} spectral - spectral property
253
- * @property {number} spectral.brightness - brightness property
254
- * @property {number} spectral.flatness - flatness property
255
- * @property {() => Uint8Array} getFrequencyData - getFrequencyData property
256
- * @property {() => Float32Array} getWaveform - getWaveform property
257
- */
258
-
259
- /**
260
- * Video frame API - provides access to video stream and computer vision data
244
+ * @property {boolean} isConnected - `true` when this audio stream is connected; otherwise all numeric fields read as `0`.
245
+ * @property {Object} volume - Overall loudness across the stream.
246
+ * @property {number} volume.current - Instantaneous RMS volume in 0..1.
247
+ * @property {number} volume.peak - Instantaneous peak amplitude in 0..1.
248
+ * @property {number} volume.smoothed - Volume with a 200ms decay envelope.
249
+ * @property {Object} bands - Energy in five fixed frequency bands, plus a smoothed companion (~150ms decay) for each. All in 0..1.
250
+ * @property {number} bands.low - Instant low-band energy (20–120 Hz). 0..1.
251
+ * @property {number} bands.lowMid - Instant low-mid energy (120–400 Hz). 0..1.
252
+ * @property {number} bands.mid - Instant mid energy (400–1600 Hz). 0..1.
253
+ * @property {number} bands.highMid - Instant high-mid energy (1600–6000 Hz). 0..1.
254
+ * @property {number} bands.high - Instant high-band energy (6000–16000 Hz). 0..1.
255
+ * @property {number} bands.lowSmoothed - Smoothed `low` (~150ms decay). 0..1.
256
+ * @property {number} bands.lowMidSmoothed - Smoothed `lowMid` (~150ms decay). 0..1.
257
+ * @property {number} bands.midSmoothed - Smoothed `mid` (~150ms decay). 0..1.
258
+ * @property {number} bands.highMidSmoothed - Smoothed `highMid` (~150ms decay). 0..1.
259
+ * @property {number} bands.highSmoothed - Smoothed `high` (~150ms decay). 0..1.
260
+ * @property {Object} spectral - High-level spectral features derived from the FFT.
261
+ * @property {number} spectral.brightness - Normalized spectral centroid in 0..1.
262
+ * @property {number} spectral.flatness - Normalized spectral flatness in 0..1.
263
+ * @property {() => Uint8Array} getFrequencyData - Returns the raw FFT magnitude spectrum for this stream as a `Uint8Array` (1024 bins, each 0..255). Snapshot semantics — see `AudioAPI.getFrequencyData`.
264
+ * @property {() => Float32Array} getWaveform - Returns the raw time-domain waveform for this stream as a `Float32Array` (2048 samples, each -1..1). Snapshot semantics — see `AudioAPI.getWaveform`.
265
+ */
266
+
267
+ /**
268
+ * Real-time video API: drawable frame, dimensions, and computer-vision results (faces, hands, pose, segmentation). Activate individual CV features through `cv.*`. When `isConnected` is `false`, `currentFrame` is `null`, all dimensions are `0`, and CV result fields hold their empty defaults.
261
269
  * @typedef {Object} VideoAPI
262
- * @property {boolean} isConnected - isConnected property
263
- * @property {OffscreenCanvas | ImageBitmap |null} currentFrame - currentFrame property
264
- * @property {number} frameWidth - frameWidth property
265
- * @property {number} frameHeight - frameHeight property
266
- * @property {number} frameRate - frameRate property
267
- * @property {() => ImageData |null} getFrameData - getFrameData property
268
- * @property {FaceData[]} faces - faces property
269
- * @property {HandData[]} hands - hands property
270
- * @property {PoseData |null} pose - pose property
271
- * @property {SegmentationData |null} segmentation - segmentation property
272
- * @property {Object} cv - cv property
270
+ * @property {boolean} isConnected - `true` when a video stream is connected. When `false`, all other fields hold their default empty values.
271
+ * @property {OffscreenCanvas | ImageBitmap |null} currentFrame - Latest video frame, drawable directly with `ctx.drawImage()`. `null` when disconnected.
272
+ * @property {number} frameWidth - Source video frame width in pixels. `0` when disconnected.
273
+ * @property {number} frameHeight - Source video frame height in pixels. `0` when disconnected.
274
+ * @property {number} frameRate - Source video frame rate in Hz (frames per second). `0` when disconnected.
275
+ * @property {() => ImageData |null} getFrameData - Returns the latest frame as raw RGBA `ImageData` for per-pixel CPU analysis, or `null` when disconnected. Slow compared to drawing `currentFrame` directly — use only when you need to read individual pixel values.
276
+ * @property {FaceData[]} faces - Detected faces (empty array when face detection is off or no faces are visible).
277
+ * @property {HandData[]} hands - Detected hands (up to 2; empty array when hand tracking is off or no hands are visible).
278
+ * @property {PoseData |null} pose - Detected body pose, or `null` when pose detection is off or no pose is visible.
279
+ * @property {SegmentationData |null} segmentation - Body segmentation mask, or `null` when segmentation is off.
280
+ * @property {Object} cv - Computer-vision feature control. Each `enable*` method toggles a specific CV pipeline; results land in the corresponding `faces` / `hands` / `pose` / `segmentation` field on the next available frame. Multiple features can be active simultaneously, but each adds CPU/GPU and WebGL-context cost.
273
281
  */
274
282
 
275
283
  /**
276
- * Mouse interaction API
284
+ * Mouse input state for the current frame: position, individual button states, per-frame movement, scroll wheel, and frame-based press / release events. Coordinates are in canvas-space pixels with `(0, 0)` at the top-left corner. For interactions that should also work on touch devices, prefer `viji.pointer`.
277
285
  * @typedef {Object} MouseAPI
278
- * @property {number} x - x property
279
- * @property {number} y - y property
280
- * @property {boolean} isInCanvas - isInCanvas property
281
- * @property {boolean} isPressed - isPressed property
282
- * @property {boolean} leftButton - leftButton property
283
- * @property {boolean} rightButton - rightButton property
284
- * @property {boolean} middleButton - middleButton property
285
- * @property {number} deltaX - deltaX property
286
- * @property {number} deltaY - deltaY property
287
- * @property {number} wheelDelta - wheelDelta property
288
- * @property {number} wheelX - wheelX property
289
- * @property {number} wheelY - wheelY property
290
- * @property {boolean} wasPressed - wasPressed property
291
- * @property {boolean} wasReleased - wasReleased property
292
- * @property {boolean} wasMoved - wasMoved property
293
- */
294
-
295
- /**
296
- * Keyboard interaction API
286
+ * @property {number} x - Canvas-space X position in pixels.
287
+ * @property {number} y - Canvas-space Y position in pixels.
288
+ * @property {boolean} isInCanvas - `true` when the cursor is currently over the canvas.
289
+ * @property {boolean} isPressed - `true` if any mouse button is currently held.
290
+ * @property {boolean} leftButton - `true` while the left mouse button is held.
291
+ * @property {boolean} rightButton - `true` while the right mouse button is held. The browser context menu is suppressed on the canvas.
292
+ * @property {boolean} middleButton - `true` while the middle (wheel) mouse button is held.
293
+ * @property {number} deltaX - Horizontal movement this frame in pixels. Resets to `0` each frame.
294
+ * @property {number} deltaY - Vertical movement this frame in pixels. Resets to `0` each frame.
295
+ * @property {number} wheelDelta - Vertical scroll accumulated this frame (alias of `wheelY`). Resets to `0` each frame.
296
+ * @property {number} wheelX - Horizontal scroll accumulated this frame (e.g., trackpad gestures, tilt-wheel mice). Resets to `0` each frame.
297
+ * @property {number} wheelY - Vertical scroll accumulated this frame. Resets to `0` each frame.
298
+ * @property {boolean} wasPressed - `true` for exactly one frame when any button is first pressed, then auto-resets.
299
+ * @property {boolean} wasReleased - `true` for exactly one frame when any button is released, then auto-resets.
300
+ * @property {boolean} wasMoved - `true` if the mouse moved this frame (any non-zero `deltaX` / `deltaY`). Resets each frame.
301
+ */
302
+
303
+ /**
304
+ * Keyboard input state for the current frame: per-key press / hold / release queries (case-insensitive), the live set of held keys, modifier flags, and a Shadertoy-compatible texture buffer for shader-side keyboard reads. Key names follow the browser's `event.key` standard (`'a'`, `'arrowup'`, `' '` for Space, `'enter'`, `'escape'`, etc.).
297
305
  * @typedef {Object} KeyboardAPI
298
- * @property {(key: string) => boolean} isPressed - isPressed property
299
- * @property {(key: string) => boolean} wasPressed - wasPressed property
300
- * @property {(key: string) => boolean} wasReleased - wasReleased property
301
- * @property {Set<string>} activeKeys - activeKeys property
302
- * @property {Set<string>} pressedThisFrame - pressedThisFrame property
303
- * @property {Set<string>} releasedThisFrame - releasedThisFrame property
304
- * @property {string} lastKeyPressed - lastKeyPressed property
305
- * @property {string} lastKeyReleased - lastKeyReleased property
306
- * @property {boolean} shift - shift property
307
- * @property {boolean} ctrl - ctrl property
308
- * @property {boolean} alt - alt property
309
- * @property {boolean} meta - meta property
310
- * @property {Uint8Array} textureData - textureData property
311
- */
312
-
313
- /**
314
- * Touch interaction API
306
+ * @property {(key: string) => boolean} isPressed - Returns `true` while the named key is held. Case-insensitive.
307
+ * @property {(key: string) => boolean} wasPressed - Returns `true` for exactly one frame on the rising edge of the named key (no key-repeat). Case-insensitive.
308
+ * @property {(key: string) => boolean} wasReleased - Returns `true` for exactly one frame on the falling edge of the named key. Case-insensitive.
309
+ * @property {Set<string>} activeKeys - Set of all currently held keys, lowercased. Persists across frames.
310
+ * @property {Set<string>} pressedThisFrame - Set of keys whose key-down fired this frame, lowercased. Cleared each frame.
311
+ * @property {Set<string>} releasedThisFrame - Set of keys whose key-up fired this frame, lowercased. Cleared each frame.
312
+ * @property {string} lastKeyPressed - Most recently pressed key in its original case (e.g., `'A'` when Shift is held).
313
+ * @property {string} lastKeyReleased - Most recently released key in its original case.
314
+ * @property {boolean} shift - `true` while the Shift key is held.
315
+ * @property {boolean} ctrl - `true` while the Ctrl key is held.
316
+ * @property {boolean} alt - `true` while the Alt (or Option) key is held.
317
+ * @property {boolean} meta - `true` while the Meta (Windows / Cmd) key is held.
318
+ * @property {Uint8Array} textureData - Shadertoy-compatible 256×3 keyboard texture (256 keycodes × 3 rows). Row 0: held state (1 / 0). Row 1: pressed-this-frame (1 / 0). Row 2: per-key toggle. Bound to a uniform sampler in shader scenes; rarely needed in JS.
319
+ */
320
+
321
+ /**
322
+ * Multi-touch input state. Lists all active touches, plus per-frame `started` / `moved` / `ended` arrays for frame-based gesture handling. For single-point interactions that should also work with a mouse, prefer `viji.pointer` — it switches automatically between mouse and primary touch.
315
323
  * @typedef {Object} TouchAPI
316
- * @property {TouchPoint[]} points - points property
317
- * @property {number} count - count property
318
- * @property {TouchPoint[]} started - started property
319
- * @property {TouchPoint[]} moved - moved property
320
- * @property {TouchPoint[]} ended - ended property
321
- * @property {TouchPoint |null} primary - primary property
324
+ * @property {TouchPoint[]} points - All currently active touch points. Order is stable but not ordered by id.
325
+ * @property {number} count - Number of currently active touches (equivalent to `points.length`).
326
+ * @property {TouchPoint[]} started - Touches that started this frame (also visible in `points`). Cleared each frame.
327
+ * @property {TouchPoint[]} moved - Touches that moved this frame. Cleared each frame.
328
+ * @property {TouchPoint[]} ended - Touches that ended this frame. Each entry has `isEnding === true`. Cleared each frame.
329
+ * @property {TouchPoint |null} primary - Convenience shortcut to `points[0]`, or `null` when no touches are active.
322
330
  */
323
331
 
324
332
  /**
325
- * Main Viji Artist API - provides access to canvas, timing, audio, video, and interactions
333
+ * The Viji API surface available to artist code as the global `viji` object. Groups canvas access, timing, connected media APIs, user interaction, device sensors, parameter helpers, and the `useContext()` selector.
326
334
  * @typedef {Object} VijiAPI
327
- * @property {OffscreenCanvas} canvas - canvas property
328
- * @property {OffscreenCanvasRenderingContext2D} [ctx] - ctx property
329
- * @property {WebGLRenderingContext | WebGL2RenderingContext} [gl] - gl property
330
- * @property {number} width - width property
331
- * @property {number} height - height property
332
- * @property {number} time - time property
333
- * @property {number} deltaTime - deltaTime property
334
- * @property {number} frameCount - frameCount property
335
- * @property {number} fps - fps property
336
- * @property {AudioAPI} audio - audio property
337
- * @property {VideoAPI} video - video property
338
- * @property {VideoAPI[]} videoStreams - videoStreams property
339
- * @property {AudioStreamAPI[]} audioStreams - audioStreams property
340
- * @property {MouseAPI} mouse - mouse property
341
- * @property {KeyboardAPI} keyboard - keyboard property
342
- * @property {TouchAPI} touches - touches property
343
- * @property {PointerAPI} pointer - pointer property
344
- * @property {DeviceSensorState} device - device property
345
- * @property {DeviceState[]} devices - devices property
346
- * @property {(defaultValue: number, config: SliderConfig) => SliderParameter} slider - slider property
347
- * @property {(defaultValue: string, config: ColorConfig) => ColorParameter} color - color property
348
- * @property {(defaultValue: boolean, config: ToggleConfig) => ToggleParameter} toggle - toggle property
349
- * @property {(defaultValue: string | number, config: SelectConfig) => SelectParameter} select - select property
350
- * @property {(defaultValue: string, config: TextConfig) => TextParameter} text - text property
351
- * @property {(defaultValue: number, config: NumberConfig) => NumberParameter} number - number property
352
- * @property {(defaultValue: null, config: ImageConfig) => ImageParameter} image - image property
353
- * @property {(config: ButtonConfig) => ButtonParameter} button - button property
354
- * @property {(defaultValue: CoordinateValue, config: CoordinateConfig) => CoordinateParameter} coordinate - coordinate property
355
- * @property {(type: '2d') => OffscreenCanvasRenderingContext2D} useContext - useContext property
356
- * @property {(type: 'webgl') => WebGLRenderingContext} useContext - useContext property
357
- * @property {(type: 'webgl2') => WebGL2RenderingContext} useContext - useContext property
358
- */
359
-
360
- /**
361
- * FaceData interface
335
+ * @property {OffscreenCanvas} canvas - The `OffscreenCanvas` driving the scene. The host owns its lifecycle and dimensions; prefer `useContext()` over touching this directly.
336
+ * @property {OffscreenCanvasRenderingContext2D} [ctx] - 2D rendering context. `undefined` until `viji.useContext('2d')` is called; afterwards equals the cached context.
337
+ * @property {WebGLRenderingContext | WebGL2RenderingContext} [gl] - WebGL rendering context. `undefined` until `viji.useContext('webgl')` or `'webgl2'` is called; afterwards equals the cached context.
338
+ * @property {number} width - Current canvas width in pixels. Updates automatically when the host resizes the canvas — read every frame.
339
+ * @property {number} height - Current canvas height in pixels. Updates automatically when the host resizes the canvas — read every frame.
340
+ * @property {number} time - Seconds elapsed since the scene started (monotonically increasing float). Use for oscillations and absolute-time effects.
341
+ * @property {number} deltaTime - Seconds since the previous frame. Use for accumulation: movement, physics, fades — anything that should progress per second regardless of FPS.
342
+ * @property {number} frameCount - Integer frame counter starting at `0` and incrementing by `1` each frame.
343
+ * @property {number} fps - Target frames-per-second for the host's current frame-rate mode (`60` for `'full'`, `30` for `'half'`).
344
+ * @property {AudioAPI} audio - Real-time analysis of the main audio stream: volume, frequency bands, beat detection, spectral features. Fields are zeroed when no audio source is connected.
345
+ * @property {VideoAPI} video - Main video stream API: pixel access, frame metadata, and computer-vision results (face, hands, pose, segmentation).
346
+ * @property {VideoAPI[]} videoStreams - Additional video streams provided by the host. These do not run CV processing — use them for raw pixel access only.
347
+ * @property {AudioStreamAPI[]} audioStreams - Additional audio streams (lightweight analysis: volume, bands, spectral). No beat detection — use the main `audio` for that.
348
+ * @property {MouseAPI} mouse - Mouse position, buttons, wheel, and per-frame movement state. Coordinates are in canvas pixels.
349
+ * @property {KeyboardAPI} keyboard - Keyboard state: per-frame press/release queries, currently held keys, and modifier flags.
350
+ * @property {TouchAPI} touches - Multi-touch state: count, primary touch shortcut, and the per-frame `started` / `moved` / `ended` arrays.
351
+ * @property {PointerAPI} pointer - Unified pointer (mouse + primary touch) — convenient single-point input that works on both desktop and mobile.
352
+ * @property {DeviceSensorState} device - Internal device sensors (motion + orientation) reported by the device running the scene.
353
+ * @property {DeviceState[]} devices - External connected devices reporting sensor / camera / audio. Each entry has its own `id`, `name`, and per-device API surface.
354
+ * @property {(defaultValue: number, config: SliderConfig) => SliderParameter} slider - Declares a numeric slider parameter. The host UI renders a draggable slider. Must be called at the top level of the scene — never inside `render()`. @example const radius = viji.slider(50, { min: 10, max: 200, step: 1, label: 'Radius' }); // ... function render(viji) { drawCircle(radius.value); }
355
+ * @property {(defaultValue: ColorInput, config: ColorConfig) => ColorParameter} color - Declares a color parameter. Accepts hex strings, CSS color functions, or RGB / HSB objects; the value is always normalized to a canonical lowercase hex string and exposes derived `.rgb` / `.hsb` accessors. Must be called at the top level of the scene — never inside `render()`. @example const tint = viji.color('#ff6600', { label: 'Tint' }); // tint.value -> '#ff6600', tint.rgb -> { r: 255, g: 102, b: 0 }, tint.hsb -> { h: 24, s: 100, b: 100 }
356
+ * @property {(defaultValue: boolean, config: ToggleConfig) => ToggleParameter} toggle - Declares a boolean toggle parameter. The host UI renders an on/off switch. Must be called at the top level of the scene — never inside `render()`. @example const showTrail = viji.toggle(true, { label: 'Show Trail' });
357
+ * @property {(defaultValue: string | number, config: SelectConfig) => SelectParameter} select - Declares a dropdown selection parameter. The element type of `options` (`string` or `number`) determines the parameter's value type. Must be called at the top level of the scene — never inside `render()`. @example const shape = viji.select('circle', { options: ['circle', 'square', 'triangle'], label: 'Shape' });
358
+ * @property {(defaultValue: string, config: TextConfig) => TextParameter} text - Declares a text input parameter. The host UI renders a single-line text field. Must be called at the top level of the scene — never inside `render()`. @example const caption = viji.text('Hello', { label: 'Caption', maxLength: 64 });
359
+ * @property {(defaultValue: number, config: NumberConfig) => NumberParameter} number - Declares a precise numeric input parameter. Like `slider()` but the host UI shows a number field instead of a draggable handle. Must be called at the top level of the scene — never inside `render()`. @example const count = viji.number(12, { min: 1, max: 64, step: 1, label: 'Count' });
360
+ * @property {(defaultValue: null, config: ImageConfig) => ImageParameter} image - Declares an image upload parameter. The user-supplied image becomes available as `value` (an `ImageBitmap`); in P5 scenes a `.p5` accessor is also added at runtime. Must be called at the top level of the scene — never inside `render()`. @example const tex = viji.image(null, { label: 'Texture' }); if (tex.value) ctx.drawImage(tex.value, 0, 0, viji.width, viji.height);
361
+ * @property {(config: ButtonConfig) => ButtonParameter} button - Declares a momentary button parameter. `value` is `true` for one frame after the user clicks, then auto-resets — perfect for triggering discrete events. Must be called at the top level of the scene — never inside `render()`. @example const reset = viji.button({ label: 'Reset' }); function render(viji) { if (reset.value) state = initialState; }
362
+ * @property {(defaultValue: CoordinateValue, config: CoordinateConfig) => CoordinateParameter} coordinate - Declares a 2D coordinate parameter. Both `value.x` and `value.y` are in `-1..1`. Must be called at the top level of the scene — never inside `render()`. @example const origin = viji.coordinate({ x: 0, y: 0 }, { label: 'Origin' }); const cx = viji.width / 2 + origin.value.x * viji.width * 0.4;
363
+ * @property {(type: '2d') => OffscreenCanvasRenderingContext2D} useContext - Selects the 2D canvas rendering context. Returns the same instance on subsequent calls and also stores it on `viji.ctx`. A canvas only supports one context type — if a different context type was already requested, the call returns `null`. Choose ONE type and use it for the entire scene. @example const ctx = viji.useContext('2d'); ctx.fillRect(0, 0, viji.width, viji.height);
364
+ * @property {(type: 'webgl') => WebGLRenderingContext} useContext - Selects a WebGL 1 rendering context. Returns the same instance on subsequent calls and also stores it on `viji.gl`. A canvas only supports one context type — if a different context type was already requested, the call returns `null`. Choose ONE type and use it for the entire scene.
365
+ * @property {(type: 'webgl2') => WebGL2RenderingContext} useContext - Selects a WebGL 2 rendering context. Returns the same instance on subsequent calls and also stores it on `viji.gl`. A canvas only supports one context type — if a different context type was already requested, the call returns `null`. Choose ONE type and use it for the entire scene.
366
+ */
367
+
368
+ /**
369
+ * One detected face produced by the video CV pipeline. Always carries `id`, `bounds`, `center`, and `confidence`; the rest is populated only when the matching CV feature is enabled (face mesh for `landmarks` and `headPose`, emotion detection for `expressions` and `blendshapes`).
362
370
  * @typedef {Object} FaceData
363
- * @property {number} id - id property
364
- * @property {Object} bounds - bounds property
365
- * @property {number} bounds.x - x property
366
- * @property {number} bounds.y - y property
367
- * @property {number} bounds.width - width property
368
- * @property {number} bounds.height - height property
369
- * @property {Object} center - center property
370
- * @property {number} center.x - x property
371
- * @property {number} center.y - y property
372
- * @property {number} confidence - confidence property
373
- * @property {Object} landmarks - landmarks property
374
- * @property {number} landmarks.x - x property
375
- * @property {number} landmarks.y - y property
376
- * @property {number} landmarks.z - z property
377
- * @property {Object} expressions - expressions property
378
- * @property {number} expressions.neutral - neutral property
379
- * @property {number} expressions.happy - happy property
380
- * @property {number} expressions.sad - sad property
381
- * @property {number} expressions.angry - angry property
382
- * @property {number} expressions.surprised - surprised property
383
- * @property {number} expressions.disgusted - disgusted property
384
- * @property {number} expressions.fearful - fearful property
385
- * @property {Object} headPose - headPose property
386
- * @property {number} headPose.pitch - pitch property
387
- * @property {number} headPose.yaw - yaw property
388
- * @property {number} headPose.roll - roll property
389
- * @property {FaceBlendshapes} blendshapes - blendshapes property
390
- */
391
-
392
- /**
393
- * FaceBlendshapes interface
371
+ * @property {number} id - Index-based face identifier (`0`, `1`, …). Stable within a single frame, may change between frames.
372
+ * @property {Object} bounds - Bounding box, all components normalized to 0..1 of the source frame.
373
+ * @property {number} bounds.x - Left edge of the face bounding box, normalized 0..1.
374
+ * @property {number} bounds.y - Top edge of the face bounding box, normalized 0..1.
375
+ * @property {number} bounds.width - Width of the face bounding box, normalized 0..1.
376
+ * @property {number} bounds.height - Height of the face bounding box, normalized 0..1.
377
+ * @property {Object} center - Center of the bounding box, normalized 0..1.
378
+ * @property {number} center.x - Horizontal center of the face, normalized 0..1.
379
+ * @property {number} center.y - Vertical center of the face, normalized 0..1.
380
+ * @property {number} confidence - Detection confidence in 0..1.
381
+ * @property {Object} landmarks - 468 normalized face mesh landmarks when face mesh is enabled; empty array otherwise.
382
+ * @property {number} landmarks.x - Horizontal landmark coordinate, normalized 0..1.
383
+ * @property {number} landmarks.y - Vertical landmark coordinate, normalized 0..1.
384
+ * @property {number} landmarks.z - Optional depth (relative).
385
+ * @property {Object} expressions - Seven expression confidence scores in 0..1. All zero unless emotion detection is enabled.
386
+ * @property {number} expressions.neutral - Confidence (0..1) of a neutral expression.
387
+ * @property {number} expressions.happy - Confidence (0..1) of a happy / smiling expression.
388
+ * @property {number} expressions.sad - Confidence (0..1) of a sad expression.
389
+ * @property {number} expressions.angry - Confidence (0..1) of an angry expression.
390
+ * @property {number} expressions.surprised - Confidence (0..1) of a surprised expression.
391
+ * @property {number} expressions.disgusted - Confidence (0..1) of a disgusted expression.
392
+ * @property {number} expressions.fearful - Confidence (0..1) of a fearful expression.
393
+ * @property {Object} headPose - Estimated head rotation. All zero unless face mesh is enabled.
394
+ * @property {number} headPose.pitch - Up/down rotation in degrees (-90..90).
395
+ * @property {number} headPose.yaw - Left/right rotation in degrees (-90..90).
396
+ * @property {number} headPose.roll - Tilt rotation in degrees (-180..180).
397
+ * @property {FaceBlendshapes} blendshapes - 52 ARKit-compatible facial muscle coefficients. All zero unless emotion detection is enabled.
398
+ */
399
+
400
+ /**
401
+ * 52 ARKit-compatible face blendshape coefficients (each in 0..1) derived from MediaPipe FaceLandmarker. All fields are `0` unless emotion detection is enabled. Available on each `FaceData.blendshapes`.
394
402
  * @typedef {Object} FaceBlendshapes
395
- * @property {number} browDownLeft - browDownLeft property
396
- * @property {number} browDownRight - browDownRight property
397
- * @property {number} browInnerUp - browInnerUp property
398
- * @property {number} browOuterUpLeft - browOuterUpLeft property
399
- * @property {number} browOuterUpRight - browOuterUpRight property
400
- * @property {number} cheekPuff - cheekPuff property
401
- * @property {number} cheekSquintLeft - cheekSquintLeft property
402
- * @property {number} cheekSquintRight - cheekSquintRight property
403
- * @property {number} eyeBlinkLeft - eyeBlinkLeft property
404
- * @property {number} eyeBlinkRight - eyeBlinkRight property
405
- * @property {number} eyeLookDownLeft - eyeLookDownLeft property
406
- * @property {number} eyeLookDownRight - eyeLookDownRight property
407
- * @property {number} eyeLookInLeft - eyeLookInLeft property
408
- * @property {number} eyeLookInRight - eyeLookInRight property
409
- * @property {number} eyeLookOutLeft - eyeLookOutLeft property
410
- * @property {number} eyeLookOutRight - eyeLookOutRight property
411
- * @property {number} eyeLookUpLeft - eyeLookUpLeft property
412
- * @property {number} eyeLookUpRight - eyeLookUpRight property
413
- * @property {number} eyeSquintLeft - eyeSquintLeft property
414
- * @property {number} eyeSquintRight - eyeSquintRight property
415
- * @property {number} eyeWideLeft - eyeWideLeft property
416
- * @property {number} eyeWideRight - eyeWideRight property
417
- * @property {number} jawForward - jawForward property
418
- * @property {number} jawLeft - jawLeft property
419
- * @property {number} jawOpen - jawOpen property
420
- * @property {number} jawRight - jawRight property
421
- * @property {number} mouthClose - mouthClose property
422
- * @property {number} mouthDimpleLeft - mouthDimpleLeft property
423
- * @property {number} mouthDimpleRight - mouthDimpleRight property
424
- * @property {number} mouthFrownLeft - mouthFrownLeft property
425
- * @property {number} mouthFrownRight - mouthFrownRight property
426
- * @property {number} mouthFunnel - mouthFunnel property
427
- * @property {number} mouthLeft - mouthLeft property
428
- * @property {number} mouthLowerDownLeft - mouthLowerDownLeft property
429
- * @property {number} mouthLowerDownRight - mouthLowerDownRight property
430
- * @property {number} mouthPressLeft - mouthPressLeft property
431
- * @property {number} mouthPressRight - mouthPressRight property
432
- * @property {number} mouthPucker - mouthPucker property
433
- * @property {number} mouthRight - mouthRight property
434
- * @property {number} mouthRollLower - mouthRollLower property
435
- * @property {number} mouthRollUpper - mouthRollUpper property
436
- * @property {number} mouthShrugLower - mouthShrugLower property
437
- * @property {number} mouthShrugUpper - mouthShrugUpper property
438
- * @property {number} mouthSmileLeft - mouthSmileLeft property
439
- * @property {number} mouthSmileRight - mouthSmileRight property
440
- * @property {number} mouthStretchLeft - mouthStretchLeft property
441
- * @property {number} mouthStretchRight - mouthStretchRight property
442
- * @property {number} mouthUpperUpLeft - mouthUpperUpLeft property
443
- * @property {number} mouthUpperUpRight - mouthUpperUpRight property
444
- * @property {number} noseSneerLeft - noseSneerLeft property
445
- * @property {number} noseSneerRight - noseSneerRight property
446
- * @property {number} tongueOut - tongueOut property
447
- */
448
-
449
- /**
450
- * HandData interface
403
+ * @property {number} browDownLeft - Coefficient (0..1) for the inner brow of the left eye pulling down.
404
+ * @property {number} browDownRight - Coefficient (0..1) for the inner brow of the right eye pulling down.
405
+ * @property {number} browInnerUp - Coefficient (0..1) for both inner brows pulling up.
406
+ * @property {number} browOuterUpLeft - Coefficient (0..1) for the outer left brow pulling up.
407
+ * @property {number} browOuterUpRight - Coefficient (0..1) for the outer right brow pulling up.
408
+ * @property {number} cheekPuff - Coefficient (0..1) for puffed cheeks.
409
+ * @property {number} cheekSquintLeft - Coefficient (0..1) for the left cheek squinting.
410
+ * @property {number} cheekSquintRight - Coefficient (0..1) for the right cheek squinting.
411
+ * @property {number} eyeBlinkLeft - Coefficient (0..1) for the left eyelid closing.
412
+ * @property {number} eyeBlinkRight - Coefficient (0..1) for the right eyelid closing.
413
+ * @property {number} eyeLookDownLeft - Coefficient (0..1) for the left eye looking down.
414
+ * @property {number} eyeLookDownRight - Coefficient (0..1) for the right eye looking down.
415
+ * @property {number} eyeLookInLeft - Coefficient (0..1) for the left eye looking inward (toward the nose).
416
+ * @property {number} eyeLookInRight - Coefficient (0..1) for the right eye looking inward.
417
+ * @property {number} eyeLookOutLeft - Coefficient (0..1) for the left eye looking outward (away from the nose).
418
+ * @property {number} eyeLookOutRight - Coefficient (0..1) for the right eye looking outward.
419
+ * @property {number} eyeLookUpLeft - Coefficient (0..1) for the left eye looking up.
420
+ * @property {number} eyeLookUpRight - Coefficient (0..1) for the right eye looking up.
421
+ * @property {number} eyeSquintLeft - Coefficient (0..1) for the left eye squinting.
422
+ * @property {number} eyeSquintRight - Coefficient (0..1) for the right eye squinting.
423
+ * @property {number} eyeWideLeft - Coefficient (0..1) for the left eye opening wide.
424
+ * @property {number} eyeWideRight - Coefficient (0..1) for the right eye opening wide.
425
+ * @property {number} jawForward - Coefficient (0..1) for the jaw moving forward.
426
+ * @property {number} jawLeft - Coefficient (0..1) for the jaw moving left.
427
+ * @property {number} jawOpen - Coefficient (0..1) for the jaw opening (mouth open).
428
+ * @property {number} jawRight - Coefficient (0..1) for the jaw moving right.
429
+ * @property {number} mouthClose - Coefficient (0..1) for the mouth closing tightly.
430
+ * @property {number} mouthDimpleLeft - Coefficient (0..1) for a dimple appearing on the left side.
431
+ * @property {number} mouthDimpleRight - Coefficient (0..1) for a dimple appearing on the right side.
432
+ * @property {number} mouthFrownLeft - Coefficient (0..1) for the left mouth corner pulling down (frown).
433
+ * @property {number} mouthFrownRight - Coefficient (0..1) for the right mouth corner pulling down (frown).
434
+ * @property {number} mouthFunnel - Coefficient (0..1) for funneling the lips outward.
435
+ * @property {number} mouthLeft - Coefficient (0..1) for the mouth shifting left.
436
+ * @property {number} mouthLowerDownLeft - Coefficient (0..1) for the lower lip on the left pulling down.
437
+ * @property {number} mouthLowerDownRight - Coefficient (0..1) for the lower lip on the right pulling down.
438
+ * @property {number} mouthPressLeft - Coefficient (0..1) for the left lip pressing inward.
439
+ * @property {number} mouthPressRight - Coefficient (0..1) for the right lip pressing inward.
440
+ * @property {number} mouthPucker - Coefficient (0..1) for puckered lips.
441
+ * @property {number} mouthRight - Coefficient (0..1) for the mouth shifting right.
442
+ * @property {number} mouthRollLower - Coefficient (0..1) for the lower lip rolling inward.
443
+ * @property {number} mouthRollUpper - Coefficient (0..1) for the upper lip rolling inward.
444
+ * @property {number} mouthShrugLower - Coefficient (0..1) for the lower lip shrugging.
445
+ * @property {number} mouthShrugUpper - Coefficient (0..1) for the upper lip shrugging.
446
+ * @property {number} mouthSmileLeft - Coefficient (0..1) for the left mouth corner pulling up (smile).
447
+ * @property {number} mouthSmileRight - Coefficient (0..1) for the right mouth corner pulling up (smile).
448
+ * @property {number} mouthStretchLeft - Coefficient (0..1) for the left mouth corner stretching outward.
449
+ * @property {number} mouthStretchRight - Coefficient (0..1) for the right mouth corner stretching outward.
450
+ * @property {number} mouthUpperUpLeft - Coefficient (0..1) for the upper lip on the left raising.
451
+ * @property {number} mouthUpperUpRight - Coefficient (0..1) for the upper lip on the right raising.
452
+ * @property {number} noseSneerLeft - Coefficient (0..1) for the left side of the nose sneering.
453
+ * @property {number} noseSneerRight - Coefficient (0..1) for the right side of the nose sneering.
454
+ * @property {number} tongueOut - Coefficient (0..1) for the tongue protruding.
455
+ */
456
+
457
+ /**
458
+ * One detected hand produced by the video CV pipeline. Carries 21 landmarks, a palm shortcut, and ML-classified gesture confidence scores. Up to two hands appear in `VideoAPI.hands` simultaneously.
451
459
  * @typedef {Object} HandData
452
- * @property {number} id - id property
453
- * @property {'left' | 'right'} handedness - handedness property
454
- * @property {number} confidence - confidence property
455
- * @property {Object} bounds - bounds property
456
- * @property {number} bounds.x - x property
457
- * @property {number} bounds.y - y property
458
- * @property {number} bounds.width - width property
459
- * @property {number} bounds.height - height property
460
- * @property {Object} landmarks - landmarks property
461
- * @property {number} landmarks.x - x property
462
- * @property {number} landmarks.y - y property
463
- * @property {number} landmarks.z - z property
464
- * @property {Object} palm - palm property
465
- * @property {number} palm.x - x property
466
- * @property {number} palm.y - y property
467
- * @property {number} palm.z - z property
468
- * @property {Object} gestures - gestures property
469
- * @property {number} gestures.fist - fist property
470
- * @property {number} gestures.openPalm - openPalm property
471
- * @property {number} gestures.peace - peace property
472
- * @property {number} gestures.thumbsUp - thumbsUp property
473
- * @property {number} gestures.thumbsDown - thumbsDown property
474
- * @property {number} gestures.pointing - pointing property
475
- * @property {number} gestures.iLoveYou - iLoveYou property
476
- */
477
-
478
- /**
479
- * PoseData interface
460
+ * @property {number} id - Index-based hand identifier (`0` or `1`).
461
+ * @property {'left' | 'right'} handedness - Which hand. Always lowercase.
462
+ * @property {number} confidence - Detection confidence in 0..1.
463
+ * @property {Object} bounds - Bounding box, all components normalized to 0..1 of the source frame.
464
+ * @property {number} bounds.x - Left edge of the hand bounding box, normalized 0..1.
465
+ * @property {number} bounds.y - Top edge of the hand bounding box, normalized 0..1.
466
+ * @property {number} bounds.width - Width of the hand bounding box, normalized 0..1.
467
+ * @property {number} bounds.height - Height of the hand bounding box, normalized 0..1.
468
+ * @property {Object} landmarks - 21 MediaPipe hand landmarks, normalized 0..1.
469
+ * @property {number} landmarks.x - Horizontal landmark coordinate, normalized 0..1.
470
+ * @property {number} landmarks.y - Vertical landmark coordinate, normalized 0..1.
471
+ * @property {number} landmarks.z - Depth (relative).
472
+ * @property {Object} palm - Palm center — equivalent to `landmarks[9]` (middle-finger MCP).
473
+ * @property {number} palm.x - Palm horizontal coordinate, normalized 0..1.
474
+ * @property {number} palm.y - Palm vertical coordinate, normalized 0..1.
475
+ * @property {number} palm.z - Palm depth (relative).
476
+ * @property {Object} gestures - ML-classified gesture confidences from MediaPipe GestureRecognizer. Each value is in 0..1; multiple gestures can fire simultaneously.
477
+ * @property {number} gestures.fist - Confidence (0..1) of a closed-fist gesture.
478
+ * @property {number} gestures.openPalm - Confidence (0..1) of an open-palm gesture.
479
+ * @property {number} gestures.peace - Confidence (0..1) of a peace / victory sign.
480
+ * @property {number} gestures.thumbsUp - Confidence (0..1) of a thumbs-up gesture.
481
+ * @property {number} gestures.thumbsDown - Confidence (0..1) of a thumbs-down gesture.
482
+ * @property {number} gestures.pointing - Confidence (0..1) of a pointing-up gesture.
483
+ * @property {number} gestures.iLoveYou - Confidence (0..1) of the ASL "I love you" sign.
484
+ */
485
+
486
+ /**
487
+ * Detected body pose from MediaPipe BlazePose. `landmarks` contains 33 points; the `face` / `torso` / `leftArm` / `rightArm` / `leftLeg` / `rightLeg` arrays are pre-grouped slices for convenience.
480
488
  * @typedef {Object} PoseData
481
- * @property {number} confidence - confidence property
482
- * @property {Object} landmarks - landmarks property
483
- * @property {number} landmarks.x - x property
484
- * @property {number} landmarks.y - y property
485
- * @property {number} landmarks.z - z property
486
- * @property {number} landmarks.visibility - visibility property
487
- * @property {Object} face - face property
488
- * @property {number} face.x - x property
489
- * @property {number} face.y - y property
490
- * @property {Object} torso - torso property
491
- * @property {number} torso.x - x property
492
- * @property {number} torso.y - y property
493
- * @property {Object} leftArm - leftArm property
494
- * @property {number} leftArm.x - x property
495
- * @property {number} leftArm.y - y property
496
- * @property {Object} rightArm - rightArm property
497
- * @property {number} rightArm.x - x property
498
- * @property {number} rightArm.y - y property
499
- * @property {Object} leftLeg - leftLeg property
500
- * @property {number} leftLeg.x - x property
501
- * @property {number} leftLeg.y - y property
502
- * @property {Object} rightLeg - rightLeg property
503
- * @property {number} rightLeg.x - x property
504
- * @property {number} rightLeg.y - y property
505
- */
506
-
507
- /**
508
- * SegmentationData interface
489
+ * @property {number} confidence - Average landmark visibility in 0..1.
490
+ * @property {Object} landmarks - 33 BlazePose landmarks, normalized 0..1. Each carries a per-point `visibility` score.
491
+ * @property {number} landmarks.x - Horizontal landmark coordinate, normalized 0..1.
492
+ * @property {number} landmarks.y - Vertical landmark coordinate, normalized 0..1.
493
+ * @property {number} landmarks.z - Depth (relative).
494
+ * @property {number} landmarks.visibility - Visibility / confidence of this individual landmark in 0..1.
495
+ * @property {Object} face - Face landmarks (BlazePose indices 0..10). Coordinates normalized 0..1.
496
+ * @property {number} face.x - Horizontal landmark coordinate, normalized 0..1.
497
+ * @property {number} face.y - Vertical landmark coordinate, normalized 0..1.
498
+ * @property {Object} torso - Torso landmarks (BlazePose indices 11, 12, 23, 24). Coordinates normalized 0..1.
499
+ * @property {number} torso.x - Horizontal landmark coordinate, normalized 0..1.
500
+ * @property {number} torso.y - Vertical landmark coordinate, normalized 0..1.
501
+ * @property {Object} leftArm - Left-arm landmarks (BlazePose indices 11, 13, 15). Coordinates normalized 0..1.
502
+ * @property {number} leftArm.x - Horizontal landmark coordinate, normalized 0..1.
503
+ * @property {number} leftArm.y - Vertical landmark coordinate, normalized 0..1.
504
+ * @property {Object} rightArm - Right-arm landmarks (BlazePose indices 12, 14, 16). Coordinates normalized 0..1.
505
+ * @property {number} rightArm.x - Horizontal landmark coordinate, normalized 0..1.
506
+ * @property {number} rightArm.y - Vertical landmark coordinate, normalized 0..1.
507
+ * @property {Object} leftLeg - Left-leg landmarks (BlazePose indices 23, 25, 27, 29, 31). Coordinates normalized 0..1.
508
+ * @property {number} leftLeg.x - Horizontal landmark coordinate, normalized 0..1.
509
+ * @property {number} leftLeg.y - Vertical landmark coordinate, normalized 0..1.
510
+ * @property {Object} rightLeg - Right-leg landmarks (BlazePose indices 24, 26, 28, 30, 32). Coordinates normalized 0..1.
511
+ * @property {number} rightLeg.x - Horizontal landmark coordinate, normalized 0..1.
512
+ * @property {number} rightLeg.y - Vertical landmark coordinate, normalized 0..1.
513
+ */
514
+
515
+ /**
516
+ * Per-pixel person/background segmentation mask. Mask dimensions reflect the ML model's output and may differ from the source video frame.
509
517
  * @typedef {Object} SegmentationData
510
- * @property {Uint8Array} mask - mask property
511
- * @property {number} width - width property
512
- * @property {number} height - height property
518
+ * @property {Uint8Array} mask - Flat per-pixel mask: each byte is `0` (background) or `1` (person). Length = `width * height`.
519
+ * @property {number} width - Mask width in pixels.
520
+ * @property {number} height - Mask height in pixels.
513
521
  */
514
522
 
515
523
  /**
516
- * DeviceMotionData interface
524
+ * Device motion data sourced from the platform's `DeviceMotionEvent`. All accelerations are in metres per second squared (m/s²); rotation rates are in degrees per second. Inner objects may be `null` when the underlying sensor is unavailable, and individual axes may be `null` on platforms that report only some components.
517
525
  * @typedef {Object} DeviceMotionData
518
- * @property {Object} acceleration - Acceleration without gravity (m/s²)
519
- * @property {number |null} acceleration.x - x property
520
- * @property {number |null} acceleration.y - y property
521
- * @property {number |null} acceleration.z - z property
522
- * @property {Object} accelerationIncludingGravity - Acceleration including gravity (m/s²)
523
- * @property {number |null} accelerationIncludingGravity.x - x property
524
- * @property {number |null} accelerationIncludingGravity.y - y property
525
- * @property {number |null} accelerationIncludingGravity.z - z property
526
- * @property {Object} rotationRate - Rotation rate (degrees/second)
527
- * @property {number |null} rotationRate.alpha - alpha property
528
- * @property {number |null} rotationRate.beta - beta property
529
- * @property {number |null} rotationRate.gamma - gamma property
530
- * @property {number} interval - Interval between updates (milliseconds)
531
- */
532
-
533
- /**
534
- * DeviceOrientationData interface
526
+ * @property {Object} acceleration - Acceleration without gravity (m/s²). `null` when no accelerometer is available.
527
+ * @property {number |null} acceleration.x - Acceleration along the device's X axis in m/s². `null` if unsupported.
528
+ * @property {number |null} acceleration.y - Acceleration along the device's Y axis in m/s². `null` if unsupported.
529
+ * @property {number |null} acceleration.z - Acceleration along the device's Z axis in m/s². `null` if unsupported.
530
+ * @property {Object} accelerationIncludingGravity - Acceleration including gravity (m/s²). `null` when no accelerometer is available.
531
+ * @property {number |null} accelerationIncludingGravity.x - Acceleration including gravity along the device's X axis in m/s². `null` if unsupported.
532
+ * @property {number |null} accelerationIncludingGravity.y - Acceleration including gravity along the device's Y axis in m/s². `null` if unsupported.
533
+ * @property {number |null} accelerationIncludingGravity.z - Acceleration including gravity along the device's Z axis in m/s². `null` if unsupported.
534
+ * @property {Object} rotationRate - Angular rotation rate (degrees per second). `null` when no gyroscope is available.
535
+ * @property {number |null} rotationRate.alpha - Rotation rate around the device's Z axis in deg/s. `null` if unsupported.
536
+ * @property {number |null} rotationRate.beta - Rotation rate around the device's X axis in deg/s. `null` if unsupported.
537
+ * @property {number |null} rotationRate.gamma - Rotation rate around the device's Y axis in deg/s. `null` if unsupported.
538
+ * @property {number} interval - Interval between sensor updates in milliseconds.
539
+ */
540
+
541
+ /**
542
+ * DeviceOrientationEvent data Matches native DeviceOrientationEvent structure
535
543
  * @typedef {Object} DeviceOrientationData
536
544
  * @property {number |null} alpha - Rotation around Z-axis (0-360 degrees, compass heading)
537
545
  * @property {number |null} beta - Rotation around X-axis (-180 to 180 degrees, front-to-back tilt)
@@ -540,14 +548,14 @@
540
548
  */
541
549
 
542
550
  /**
543
- * DeviceSensorState interface
551
+ * Sensor snapshot for the device running the scene. Both fields read `null` when the corresponding sensor is unavailable or has not delivered a sample yet.
544
552
  * @typedef {Object} DeviceSensorState
545
- * @property {DeviceMotionData |null} motion - motion property
546
- * @property {DeviceOrientationData |null} orientation - orientation property
553
+ * @property {DeviceMotionData |null} motion - Motion data (acceleration, rotation rate, sampling interval). `null` if no motion sensor is available.
554
+ * @property {DeviceOrientationData |null} orientation - Orientation data (alpha/beta/gamma in degrees). `null` if no orientation sensor is available.
547
555
  */
548
556
 
549
557
  /**
550
- * DeviceState interface
558
+ * External device state (includes id and name)
551
559
  * @typedef {Object} DeviceState
552
560
  * @property {string} id - Unique device identifier
553
561
  * @property {string} name - User-friendly device name