@viji-dev/core 0.3.39 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/artist-dts-p5.js +1 -1
- package/dist/artist-dts.js +1 -1
- package/dist/artist-global-p5.d.ts +836 -15
- package/dist/artist-global.d.ts +836 -15
- package/dist/artist-jsdoc.d.ts +449 -441
- package/dist/assets/{viji.worker-BoI8e3NI.js → viji.worker-4gGFik2A.js} +344 -27
- package/dist/assets/viji.worker-4gGFik2A.js.map +1 -0
- package/dist/docs-api.d.ts +7 -0
- package/dist/docs-api.js +6336 -650
- package/dist/{essentia-wasm.web-CdUmKTbm.js → essentia-wasm.web-CCpzl5zi.js} +2 -2
- package/dist/{essentia-wasm.web-CdUmKTbm.js.map → essentia-wasm.web-CCpzl5zi.js.map} +1 -1
- package/dist/{index-DZzYWg7c.js → index-G2N9Tgtd.js} +303 -22
- package/dist/index-G2N9Tgtd.js.map +1 -0
- package/dist/index.d.ts +839 -20
- package/dist/index.js +1 -1
- package/package.json +5 -1
- package/dist/assets/viji.worker-BoI8e3NI.js.map +0 -1
- package/dist/index-DZzYWg7c.js.map +0 -1
package/dist/artist-global.d.ts
CHANGED
|
@@ -4,180 +4,412 @@
|
|
|
4
4
|
// would otherwise be module-scoped, not global).
|
|
5
5
|
|
|
6
6
|
declare global {
|
|
7
|
+
/**
|
|
8
|
+
* Real-time audio analysis for the main audio stream (`viji.audio`).
|
|
9
|
+
* Provides volume, frequency bands, beat detection (with triggers, events, BPM),
|
|
10
|
+
* spectral features, and access to raw FFT / waveform data.
|
|
11
|
+
*
|
|
12
|
+
* All numeric outputs are zeroed when `isConnected` is `false`.
|
|
13
|
+
*/
|
|
7
14
|
interface AudioAPI {
|
|
15
|
+
/** `true` when an audio source is connected; otherwise all numeric fields read as `0` and helper methods return empty data. */
|
|
8
16
|
isConnected: boolean;
|
|
17
|
+
/** Overall loudness across the stream. */
|
|
9
18
|
volume: {
|
|
19
|
+
/** Instantaneous RMS volume in 0..1. */
|
|
10
20
|
current: number;
|
|
21
|
+
/** Instantaneous peak amplitude in 0..1. */
|
|
11
22
|
peak: number;
|
|
23
|
+
/** Volume with a 200ms decay envelope; rises fast, falls smoothly. Use for flicker-free animations. */
|
|
12
24
|
smoothed: number;
|
|
13
25
|
};
|
|
26
|
+
/** Energy in five fixed frequency bands, plus a smoothed companion (~150ms decay) for each. All in 0..1. */
|
|
14
27
|
bands: {
|
|
28
|
+
/** Instant low-band energy (20–120 Hz, bass/kick range). 0..1. */
|
|
15
29
|
low: number;
|
|
30
|
+
/** Instant low-mid energy (120–400 Hz). 0..1. */
|
|
16
31
|
lowMid: number;
|
|
32
|
+
/** Instant mid energy (400–1600 Hz, vocals/instruments). 0..1. */
|
|
17
33
|
mid: number;
|
|
34
|
+
/** Instant high-mid energy (1600–6000 Hz, cymbals/hi-hats). 0..1. */
|
|
18
35
|
highMid: number;
|
|
36
|
+
/** Instant high-band energy (6000–16000 Hz, air/brilliance). 0..1. */
|
|
19
37
|
high: number;
|
|
38
|
+
/** Smoothed `low` (~150ms decay). 0..1. */
|
|
20
39
|
lowSmoothed: number;
|
|
40
|
+
/** Smoothed `lowMid` (~150ms decay). 0..1. */
|
|
21
41
|
lowMidSmoothed: number;
|
|
42
|
+
/** Smoothed `mid` (~150ms decay). 0..1. */
|
|
22
43
|
midSmoothed: number;
|
|
44
|
+
/** Smoothed `highMid` (~150ms decay). 0..1. */
|
|
23
45
|
highMidSmoothed: number;
|
|
46
|
+
/** Smoothed `high` (~150ms decay). 0..1. */
|
|
24
47
|
highSmoothed: number;
|
|
25
48
|
};
|
|
49
|
+
/** Beat detection: fast/smoothed energy curves, single-frame boolean triggers, raw event list, and BPM tracking. */
|
|
26
50
|
beat: {
|
|
51
|
+
/** Kick energy curve in 0..1 with a 300ms decay. Peaks on each detected kick. */
|
|
27
52
|
kick: number;
|
|
53
|
+
/** Snare energy curve in 0..1 with a 300ms decay. */
|
|
28
54
|
snare: number;
|
|
55
|
+
/** Hi-hat energy curve in 0..1 with a 300ms decay. */
|
|
29
56
|
hat: number;
|
|
57
|
+
/** Energy curve for any beat type in 0..1 with a 300ms decay. */
|
|
30
58
|
any: number;
|
|
59
|
+
/** Smoothed `kick` curve in 0..1 with a 500ms decay. Use for ambient pulses. */
|
|
31
60
|
kickSmoothed: number;
|
|
61
|
+
/** Smoothed `snare` curve in 0..1 with a 500ms decay. */
|
|
32
62
|
snareSmoothed: number;
|
|
63
|
+
/** Smoothed `hat` curve in 0..1 with a 500ms decay. */
|
|
33
64
|
hatSmoothed: number;
|
|
65
|
+
/** Smoothed `any` curve in 0..1 with a 500ms decay. */
|
|
34
66
|
anySmoothed: number;
|
|
67
|
+
/** Single-frame boolean triggers OR-accumulated between frames, so no detected beat is lost. */
|
|
35
68
|
triggers: {
|
|
69
|
+
/** `true` for one frame when any beat is detected. */
|
|
36
70
|
any: boolean;
|
|
71
|
+
/** `true` for one frame when a kick is detected. */
|
|
37
72
|
kick: boolean;
|
|
73
|
+
/** `true` for one frame when a snare is detected. */
|
|
38
74
|
snare: boolean;
|
|
75
|
+
/** `true` for one frame when a hi-hat is detected. */
|
|
39
76
|
hat: boolean;
|
|
40
77
|
};
|
|
78
|
+
/** All beats detected since the last render frame (zero or more). Cleared each frame. */
|
|
41
79
|
events: Array<{
|
|
80
|
+
/** Which drum was detected. */
|
|
42
81
|
type: 'kick' | 'snare' | 'hat';
|
|
82
|
+
/** Timestamp of the detection in milliseconds. */
|
|
43
83
|
time: number;
|
|
84
|
+
/** Strength of the beat in 0..1. */
|
|
44
85
|
strength: number;
|
|
45
86
|
}>;
|
|
87
|
+
/** Currently detected tempo in beats per minute. Defaults to `120` when no audio. */
|
|
46
88
|
bpm: number;
|
|
89
|
+
/** Confidence of the BPM tracker in 0..1. */
|
|
47
90
|
confidence: number;
|
|
91
|
+
/** `true` when the BPM tracker has a stable lock on the current tempo. */
|
|
48
92
|
isLocked: boolean;
|
|
49
93
|
};
|
|
94
|
+
/** High-level spectral features derived from the FFT. */
|
|
50
95
|
spectral: {
|
|
96
|
+
/** Normalized spectral centroid in 0..1 — higher values mean brighter, more treble-heavy sound. */
|
|
51
97
|
brightness: number;
|
|
98
|
+
/** Normalized spectral flatness in 0..1 — higher values mean noisier (white-noise-like) sound; lower values mean tonal. */
|
|
52
99
|
flatness: number;
|
|
53
100
|
};
|
|
101
|
+
/**
|
|
102
|
+
* Returns the raw FFT magnitude spectrum as a `Uint8Array` (1024 bins, each 0..255).
|
|
103
|
+
* Bin `i` covers frequency `i × (sampleRate / fftSize)`. The returned array is a
|
|
104
|
+
* snapshot from the latest analysis tick — repeated calls in the same frame return the same data.
|
|
105
|
+
*
|
|
106
|
+
* @example
|
|
107
|
+
* const fft = viji.audio.getFrequencyData();
|
|
108
|
+
* for (let i = 0; i < fft.length; i++) drawBar(i, fft[i] / 255);
|
|
109
|
+
*/
|
|
54
110
|
getFrequencyData: () => Uint8Array;
|
|
111
|
+
/**
|
|
112
|
+
* Returns the raw time-domain waveform as a `Float32Array` (2048 PCM samples in -1..1).
|
|
113
|
+
* The returned array is a snapshot — repeated calls in the same frame return the same data.
|
|
114
|
+
*
|
|
115
|
+
* @example
|
|
116
|
+
* const wave = viji.audio.getWaveform();
|
|
117
|
+
* for (let i = 0; i < wave.length; i++) drawSample(i, wave[i]);
|
|
118
|
+
*/
|
|
55
119
|
getWaveform: () => Float32Array;
|
|
56
120
|
}
|
|
57
121
|
|
|
122
|
+
/**
|
|
123
|
+
* Lightweight audio analysis for additional or device audio streams. A strict
|
|
124
|
+
* subset of `AudioAPI` — exposes volume, frequency bands, spectral features,
|
|
125
|
+
* and raw FFT/waveform access, but **no beat detection** (no `beat`, no BPM,
|
|
126
|
+
* no triggers, no events).
|
|
127
|
+
*/
|
|
58
128
|
interface AudioStreamAPI {
|
|
129
|
+
/** `true` when this audio stream is connected; otherwise all numeric fields read as `0`. */
|
|
59
130
|
isConnected: boolean;
|
|
131
|
+
/** Overall loudness across the stream. */
|
|
60
132
|
volume: {
|
|
133
|
+
/** Instantaneous RMS volume in 0..1. */
|
|
61
134
|
current: number;
|
|
135
|
+
/** Instantaneous peak amplitude in 0..1. */
|
|
62
136
|
peak: number;
|
|
137
|
+
/** Volume with a 200ms decay envelope. */
|
|
63
138
|
smoothed: number;
|
|
64
139
|
};
|
|
140
|
+
/** Energy in five fixed frequency bands, plus a smoothed companion (~150ms decay) for each. All in 0..1. */
|
|
65
141
|
bands: {
|
|
142
|
+
/** Instant low-band energy (20–120 Hz). 0..1. */
|
|
66
143
|
low: number;
|
|
144
|
+
/** Instant low-mid energy (120–400 Hz). 0..1. */
|
|
67
145
|
lowMid: number;
|
|
146
|
+
/** Instant mid energy (400–1600 Hz). 0..1. */
|
|
68
147
|
mid: number;
|
|
148
|
+
/** Instant high-mid energy (1600–6000 Hz). 0..1. */
|
|
69
149
|
highMid: number;
|
|
150
|
+
/** Instant high-band energy (6000–16000 Hz). 0..1. */
|
|
70
151
|
high: number;
|
|
152
|
+
/** Smoothed `low` (~150ms decay). 0..1. */
|
|
71
153
|
lowSmoothed: number;
|
|
154
|
+
/** Smoothed `lowMid` (~150ms decay). 0..1. */
|
|
72
155
|
lowMidSmoothed: number;
|
|
156
|
+
/** Smoothed `mid` (~150ms decay). 0..1. */
|
|
73
157
|
midSmoothed: number;
|
|
158
|
+
/** Smoothed `highMid` (~150ms decay). 0..1. */
|
|
74
159
|
highMidSmoothed: number;
|
|
160
|
+
/** Smoothed `high` (~150ms decay). 0..1. */
|
|
75
161
|
highSmoothed: number;
|
|
76
162
|
};
|
|
163
|
+
/** High-level spectral features derived from the FFT. */
|
|
77
164
|
spectral: {
|
|
165
|
+
/** Normalized spectral centroid in 0..1. */
|
|
78
166
|
brightness: number;
|
|
167
|
+
/** Normalized spectral flatness in 0..1. */
|
|
79
168
|
flatness: number;
|
|
80
169
|
};
|
|
170
|
+
/**
|
|
171
|
+
* Returns the raw FFT magnitude spectrum for this stream as a `Uint8Array`
|
|
172
|
+
* (1024 bins, each 0..255). Snapshot semantics — see `AudioAPI.getFrequencyData`.
|
|
173
|
+
*/
|
|
81
174
|
getFrequencyData: () => Uint8Array;
|
|
175
|
+
/**
|
|
176
|
+
* Returns the raw time-domain waveform for this stream as a `Float32Array`
|
|
177
|
+
* (2048 samples, each -1..1). Snapshot semantics — see `AudioAPI.getWaveform`.
|
|
178
|
+
*/
|
|
82
179
|
getWaveform: () => Float32Array;
|
|
83
180
|
}
|
|
84
181
|
|
|
182
|
+
/**
|
|
183
|
+
* Configuration object passed to `viji.button()`. The host renders a clickable
|
|
184
|
+
* momentary button — the resulting `value` is `true` for one frame after press,
|
|
185
|
+
* then auto-resets to `false`.
|
|
186
|
+
*/
|
|
85
187
|
interface ButtonConfig {
|
|
188
|
+
/** Display name shown on the button in the parameter UI. Required. */
|
|
86
189
|
label: string;
|
|
190
|
+
/** Optional tooltip / help text shown next to the control. */
|
|
87
191
|
description?: string;
|
|
192
|
+
/** Group name for organizing related parameters under a shared heading. Default: `'general'`. */
|
|
88
193
|
group?: string;
|
|
194
|
+
/** Visibility category — the control hides when its capability is unavailable. Default: `'general'`. */
|
|
89
195
|
category?: ParameterCategory;
|
|
90
196
|
}
|
|
91
197
|
|
|
198
|
+
/**
|
|
199
|
+
* Reactive object returned by `viji.button()`. `value` is a momentary flag —
|
|
200
|
+
* `true` for exactly one frame after the user clicks, then auto-resets to
|
|
201
|
+
* `false`. Use it to trigger discrete events from `render()`.
|
|
202
|
+
*/
|
|
92
203
|
interface ButtonParameter {
|
|
204
|
+
/** `true` for the single frame after the user clicks the button, `false` otherwise. */
|
|
93
205
|
value: boolean;
|
|
206
|
+
/** Display name shown on the button in the parameter UI. */
|
|
94
207
|
label: string;
|
|
208
|
+
/** Tooltip / help text for the control (empty string when not configured). */
|
|
95
209
|
description?: string;
|
|
210
|
+
/** Group name under which the control appears in the UI. */
|
|
96
211
|
group: string;
|
|
212
|
+
/** Visibility category — controls when the parameter is shown. */
|
|
97
213
|
category: ParameterCategory;
|
|
98
214
|
}
|
|
99
215
|
|
|
216
|
+
/**
|
|
217
|
+
* Options accepted by `VijiCore.captureFrame()` for snapshotting the current
|
|
218
|
+
* scene. Controls output format, encoding, and target resolution / aspect.
|
|
219
|
+
*/
|
|
100
220
|
interface CaptureFrameOptions {
|
|
101
|
-
/** Output format: 'blob' for encoded image, 'bitmap' for GPU-friendly ImageBitmap */
|
|
221
|
+
/** Output format: `'blob'` for an encoded image, `'bitmap'` for a GPU-friendly `ImageBitmap`. Default: `'blob'`. */
|
|
102
222
|
format?: 'blob' | 'bitmap';
|
|
103
|
-
/** MIME type for blob output (ignored for bitmap)
|
|
223
|
+
/** MIME type for `'blob'` output (ignored for `'bitmap'`). Examples: `'image/png'`, `'image/jpeg'`, `'image/webp'`. */
|
|
104
224
|
type?: string;
|
|
105
225
|
/**
|
|
106
226
|
* Target resolution.
|
|
107
|
-
* - number
|
|
108
|
-
* -
|
|
109
|
-
* the source is center-cropped
|
|
227
|
+
* - `number`: scale factor relative to the current canvas size (e.g., `0.5` = 50%).
|
|
228
|
+
* - `Resolution`: exact output size. If the aspect ratio differs from the
|
|
229
|
+
* canvas, the source is center-cropped before scaling.
|
|
110
230
|
*/
|
|
111
|
-
resolution?: number |
|
|
112
|
-
width: number;
|
|
113
|
-
height: number;
|
|
114
|
-
};
|
|
231
|
+
resolution?: number | Resolution;
|
|
115
232
|
}
|
|
116
233
|
|
|
234
|
+
/**
|
|
235
|
+
* Configuration object passed to `viji.color()`. Defines the color picker's
|
|
236
|
+
* label and UI grouping/visibility.
|
|
237
|
+
*/
|
|
117
238
|
interface ColorConfig {
|
|
239
|
+
/** Display name shown next to the color swatch in the parameter UI. Required. */
|
|
118
240
|
label: string;
|
|
241
|
+
/** Optional tooltip / help text shown next to the control. */
|
|
119
242
|
description?: string;
|
|
243
|
+
/** Group name for organizing related parameters under a shared heading. Default: `'general'`. */
|
|
120
244
|
group?: string;
|
|
245
|
+
/** Visibility category — the control hides when its capability is unavailable. Default: `'general'`. */
|
|
121
246
|
category?: ParameterCategory;
|
|
122
247
|
}
|
|
123
248
|
|
|
249
|
+
/**
|
|
250
|
+
* Accepted input forms for color parameters. Internally normalized to canonical
|
|
251
|
+
* lowercase 6-digit hex `#rrggbb`. See `viji.color()` for full documentation.
|
|
252
|
+
*
|
|
253
|
+
* - `string`: hex (`#rrggbb`, `#rgb`), CSS `rgb(r, g, b)`, CSS `hsl(h, s%, l%)`,
|
|
254
|
+
* GLSL-style `vec3(r, g, b)` (0..1), or P5-style `hsb(h, s, b)` (0..360 / 0..100 / 0..100)
|
|
255
|
+
* - `{ r, g, b }`: RGB object with components in 0..255
|
|
256
|
+
* - `{ h, s, b }`: HSB object with `h` in 0..360, `s` and `b` in 0..100
|
|
257
|
+
*/
|
|
258
|
+
type ColorInput = string | {
|
|
259
|
+
/** Red component in 0..255. */
|
|
260
|
+
r: number;
|
|
261
|
+
/** Green component in 0..255. */
|
|
262
|
+
g: number;
|
|
263
|
+
/** Blue component in 0..255. */
|
|
264
|
+
b: number;
|
|
265
|
+
} | {
|
|
266
|
+
/** Hue in 0..360. */
|
|
267
|
+
h: number;
|
|
268
|
+
/** Saturation in 0..100. */
|
|
269
|
+
s: number;
|
|
270
|
+
/** Brightness in 0..100. */
|
|
271
|
+
b: number;
|
|
272
|
+
};
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* Reactive object returned by `viji.color()`. `value` is always a canonical
|
|
276
|
+
* lowercase 6-digit hex string; `rgb` and `hsb` are derived accessors that are
|
|
277
|
+
* recomputed (and frozen) every time the user changes the color.
|
|
278
|
+
*/
|
|
124
279
|
interface ColorParameter {
|
|
280
|
+
/** Canonical hex color string `#rrggbb` (lowercase). Updates in real-time. */
|
|
125
281
|
value: string;
|
|
282
|
+
/** RGB components as integers in 0..255 (frozen, recomputed when value changes). */
|
|
283
|
+
rgb: {
|
|
284
|
+
/** Red component, integer in 0..255. */
|
|
285
|
+
r: number;
|
|
286
|
+
/** Green component, integer in 0..255. */
|
|
287
|
+
g: number;
|
|
288
|
+
/** Blue component, integer in 0..255. */
|
|
289
|
+
b: number;
|
|
290
|
+
};
|
|
291
|
+
/** HSB components: `h` in 0..360, `s` and `b` in 0..100 (frozen, recomputed when value changes). */
|
|
292
|
+
hsb: {
|
|
293
|
+
/** Hue in 0..360. */
|
|
294
|
+
h: number;
|
|
295
|
+
/** Saturation in 0..100. */
|
|
296
|
+
s: number;
|
|
297
|
+
/** Brightness in 0..100. */
|
|
298
|
+
b: number;
|
|
299
|
+
};
|
|
300
|
+
/** Display name shown next to the swatch in the parameter UI. */
|
|
126
301
|
label: string;
|
|
302
|
+
/** Tooltip / help text for the control (empty string when not configured). */
|
|
127
303
|
description?: string;
|
|
304
|
+
/** Group name under which the control appears in the UI. */
|
|
128
305
|
group: string;
|
|
306
|
+
/** Visibility category — controls when the parameter is shown. */
|
|
129
307
|
category: ParameterCategory;
|
|
130
308
|
}
|
|
131
309
|
|
|
310
|
+
/**
|
|
311
|
+
* Configuration object passed to `viji.coordinate()`. The host renders a 2D
|
|
312
|
+
* draggable pad; both `x` and `y` are clamped to `-1..1`.
|
|
313
|
+
*/
|
|
132
314
|
interface CoordinateConfig {
|
|
315
|
+
/** Snap increment for both `x` and `y` components. Default: `0.01`. */
|
|
133
316
|
step?: number;
|
|
317
|
+
/** Display name shown next to the pad in the parameter UI. Required. */
|
|
134
318
|
label: string;
|
|
319
|
+
/** Optional tooltip / help text shown next to the control. */
|
|
135
320
|
description?: string;
|
|
321
|
+
/** Group name for organizing related parameters under a shared heading. Default: `'general'`. */
|
|
136
322
|
group?: string;
|
|
323
|
+
/** Visibility category — the control hides when its capability is unavailable. Default: `'general'`. */
|
|
137
324
|
category?: ParameterCategory;
|
|
138
325
|
}
|
|
139
326
|
|
|
327
|
+
/**
|
|
328
|
+
* Reactive object returned by `viji.coordinate()`. `value.x` and `value.y` are
|
|
329
|
+
* always in `-1..1`; the host UI renders a draggable 2D pad.
|
|
330
|
+
*/
|
|
140
331
|
interface CoordinateParameter {
|
|
332
|
+
/** Current `{ x, y }` position. Both components are in `-1..1`. Updates in real-time. */
|
|
141
333
|
value: CoordinateValue;
|
|
334
|
+
/** Snap increment for both `x` and `y` components. */
|
|
142
335
|
step: number;
|
|
336
|
+
/** Display name shown next to the pad in the parameter UI. */
|
|
143
337
|
label: string;
|
|
338
|
+
/** Tooltip / help text for the control (empty string when not configured). */
|
|
144
339
|
description?: string;
|
|
340
|
+
/** Group name under which the control appears in the UI. */
|
|
145
341
|
group: string;
|
|
342
|
+
/** Visibility category — controls when the parameter is shown. */
|
|
146
343
|
category: ParameterCategory;
|
|
147
344
|
}
|
|
148
345
|
|
|
346
|
+
/**
|
|
347
|
+
* Two-component coordinate, both axes in `-1..1` (centered on the canvas).
|
|
348
|
+
* Returned and consumed by `viji.coordinate()`.
|
|
349
|
+
*/
|
|
149
350
|
type CoordinateValue = {
|
|
351
|
+
/** Horizontal axis in `-1..1` (left to right, `0` = canvas center). */
|
|
150
352
|
x: number;
|
|
353
|
+
/** Vertical axis in `-1..1` (bottom to top, `0` = canvas center). */
|
|
151
354
|
y: number;
|
|
152
355
|
};
|
|
153
356
|
|
|
357
|
+
/**
|
|
358
|
+
* Identifier for an individual computer-vision feature pipeline. Returned by
|
|
359
|
+
* `VideoAPI.cv.getActiveFeatures()` and used internally to track state.
|
|
360
|
+
*/
|
|
154
361
|
type CVFeature = 'faceDetection' | 'faceMesh' | 'handTracking' | 'poseDetection' | 'bodySegmentation' | 'emotionDetection';
|
|
155
362
|
|
|
363
|
+
/**
|
|
364
|
+
* Computer-vision processing rate relative to the scene's render rate.
|
|
365
|
+
* `'full'` runs CV every frame, `'half'` every other frame, `'quarter'` every
|
|
366
|
+
* fourth, `'eighth'` every eighth — lower rates reduce CPU/GPU cost.
|
|
367
|
+
*/
|
|
156
368
|
type CVFrameRateMode = 'full' | 'half' | 'quarter' | 'eighth';
|
|
157
369
|
|
|
370
|
+
/**
|
|
371
|
+
* Device motion data sourced from the platform's `DeviceMotionEvent`. All
|
|
372
|
+
* accelerations are in metres per second squared (m/s²); rotation rates are
|
|
373
|
+
* in degrees per second. Inner objects may be `null` when the underlying
|
|
374
|
+
* sensor is unavailable, and individual axes may be `null` on platforms that
|
|
375
|
+
* report only some components.
|
|
376
|
+
*/
|
|
158
377
|
interface DeviceMotionData {
|
|
159
|
-
/** Acceleration without gravity (m/s²) */
|
|
378
|
+
/** Acceleration without gravity (m/s²). `null` when no accelerometer is available. */
|
|
160
379
|
acceleration: {
|
|
380
|
+
/** Acceleration along the device's X axis in m/s². `null` if unsupported. */
|
|
161
381
|
x: number | null;
|
|
382
|
+
/** Acceleration along the device's Y axis in m/s². `null` if unsupported. */
|
|
162
383
|
y: number | null;
|
|
384
|
+
/** Acceleration along the device's Z axis in m/s². `null` if unsupported. */
|
|
163
385
|
z: number | null;
|
|
164
386
|
} | null;
|
|
165
|
-
/** Acceleration including gravity (m/s²) */
|
|
387
|
+
/** Acceleration including gravity (m/s²). `null` when no accelerometer is available. */
|
|
166
388
|
accelerationIncludingGravity: {
|
|
389
|
+
/** Acceleration including gravity along the device's X axis in m/s². `null` if unsupported. */
|
|
167
390
|
x: number | null;
|
|
391
|
+
/** Acceleration including gravity along the device's Y axis in m/s². `null` if unsupported. */
|
|
168
392
|
y: number | null;
|
|
393
|
+
/** Acceleration including gravity along the device's Z axis in m/s². `null` if unsupported. */
|
|
169
394
|
z: number | null;
|
|
170
395
|
} | null;
|
|
171
|
-
/**
|
|
396
|
+
/** Angular rotation rate (degrees per second). `null` when no gyroscope is available. */
|
|
172
397
|
rotationRate: {
|
|
398
|
+
/** Rotation rate around the device's Z axis in deg/s. `null` if unsupported. */
|
|
173
399
|
alpha: number | null;
|
|
400
|
+
/** Rotation rate around the device's X axis in deg/s. `null` if unsupported. */
|
|
174
401
|
beta: number | null;
|
|
402
|
+
/** Rotation rate around the device's Y axis in deg/s. `null` if unsupported. */
|
|
175
403
|
gamma: number | null;
|
|
176
404
|
} | null;
|
|
177
|
-
/** Interval between updates
|
|
405
|
+
/** Interval between sensor updates in milliseconds. */
|
|
178
406
|
interval: number;
|
|
179
407
|
}
|
|
180
408
|
|
|
409
|
+
/**
|
|
410
|
+
* DeviceOrientationEvent data
|
|
411
|
+
* Matches native DeviceOrientationEvent structure
|
|
412
|
+
*/
|
|
181
413
|
interface DeviceOrientationData {
|
|
182
414
|
/** Rotation around Z-axis (0-360 degrees, compass heading) */
|
|
183
415
|
alpha: number | null;
|
|
@@ -189,11 +421,20 @@ declare global {
|
|
|
189
421
|
absolute: boolean;
|
|
190
422
|
}
|
|
191
423
|
|
|
424
|
+
/**
|
|
425
|
+
* Sensor snapshot for the device running the scene. Both fields read `null`
|
|
426
|
+
* when the corresponding sensor is unavailable or has not delivered a sample yet.
|
|
427
|
+
*/
|
|
192
428
|
interface DeviceSensorState {
|
|
429
|
+
/** Motion data (acceleration, rotation rate, sampling interval). `null` if no motion sensor is available. */
|
|
193
430
|
motion: DeviceMotionData | null;
|
|
431
|
+
/** Orientation data (alpha/beta/gamma in degrees). `null` if no orientation sensor is available. */
|
|
194
432
|
orientation: DeviceOrientationData | null;
|
|
195
433
|
}
|
|
196
434
|
|
|
435
|
+
/**
|
|
436
|
+
* External device state (includes id and name)
|
|
437
|
+
*/
|
|
197
438
|
interface DeviceState extends DeviceSensorState {
|
|
198
439
|
/** Unique device identifier */
|
|
199
440
|
id: string;
|
|
@@ -205,423 +446,1003 @@ declare global {
|
|
|
205
446
|
audio: AudioStreamAPI | null;
|
|
206
447
|
}
|
|
207
448
|
|
|
449
|
+
/**
|
|
450
|
+
* 52 ARKit-compatible face blendshape coefficients (each in 0..1) derived from
|
|
451
|
+
* MediaPipe FaceLandmarker. All fields are `0` unless emotion detection is
|
|
452
|
+
* enabled. Available on each `FaceData.blendshapes`.
|
|
453
|
+
*/
|
|
208
454
|
interface FaceBlendshapes {
|
|
455
|
+
/** Coefficient (0..1) for the inner brow of the left eye pulling down. */
|
|
209
456
|
browDownLeft: number;
|
|
457
|
+
/** Coefficient (0..1) for the inner brow of the right eye pulling down. */
|
|
210
458
|
browDownRight: number;
|
|
459
|
+
/** Coefficient (0..1) for both inner brows pulling up. */
|
|
211
460
|
browInnerUp: number;
|
|
461
|
+
/** Coefficient (0..1) for the outer left brow pulling up. */
|
|
212
462
|
browOuterUpLeft: number;
|
|
463
|
+
/** Coefficient (0..1) for the outer right brow pulling up. */
|
|
213
464
|
browOuterUpRight: number;
|
|
465
|
+
/** Coefficient (0..1) for puffed cheeks. */
|
|
214
466
|
cheekPuff: number;
|
|
467
|
+
/** Coefficient (0..1) for the left cheek squinting. */
|
|
215
468
|
cheekSquintLeft: number;
|
|
469
|
+
/** Coefficient (0..1) for the right cheek squinting. */
|
|
216
470
|
cheekSquintRight: number;
|
|
471
|
+
/** Coefficient (0..1) for the left eyelid closing. */
|
|
217
472
|
eyeBlinkLeft: number;
|
|
473
|
+
/** Coefficient (0..1) for the right eyelid closing. */
|
|
218
474
|
eyeBlinkRight: number;
|
|
475
|
+
/** Coefficient (0..1) for the left eye looking down. */
|
|
219
476
|
eyeLookDownLeft: number;
|
|
477
|
+
/** Coefficient (0..1) for the right eye looking down. */
|
|
220
478
|
eyeLookDownRight: number;
|
|
479
|
+
/** Coefficient (0..1) for the left eye looking inward (toward the nose). */
|
|
221
480
|
eyeLookInLeft: number;
|
|
481
|
+
/** Coefficient (0..1) for the right eye looking inward. */
|
|
222
482
|
eyeLookInRight: number;
|
|
483
|
+
/** Coefficient (0..1) for the left eye looking outward (away from the nose). */
|
|
223
484
|
eyeLookOutLeft: number;
|
|
485
|
+
/** Coefficient (0..1) for the right eye looking outward. */
|
|
224
486
|
eyeLookOutRight: number;
|
|
487
|
+
/** Coefficient (0..1) for the left eye looking up. */
|
|
225
488
|
eyeLookUpLeft: number;
|
|
489
|
+
/** Coefficient (0..1) for the right eye looking up. */
|
|
226
490
|
eyeLookUpRight: number;
|
|
491
|
+
/** Coefficient (0..1) for the left eye squinting. */
|
|
227
492
|
eyeSquintLeft: number;
|
|
493
|
+
/** Coefficient (0..1) for the right eye squinting. */
|
|
228
494
|
eyeSquintRight: number;
|
|
495
|
+
/** Coefficient (0..1) for the left eye opening wide. */
|
|
229
496
|
eyeWideLeft: number;
|
|
497
|
+
/** Coefficient (0..1) for the right eye opening wide. */
|
|
230
498
|
eyeWideRight: number;
|
|
499
|
+
/** Coefficient (0..1) for the jaw moving forward. */
|
|
231
500
|
jawForward: number;
|
|
501
|
+
/** Coefficient (0..1) for the jaw moving left. */
|
|
232
502
|
jawLeft: number;
|
|
503
|
+
/** Coefficient (0..1) for the jaw opening (mouth open). */
|
|
233
504
|
jawOpen: number;
|
|
505
|
+
/** Coefficient (0..1) for the jaw moving right. */
|
|
234
506
|
jawRight: number;
|
|
507
|
+
/** Coefficient (0..1) for the mouth closing tightly. */
|
|
235
508
|
mouthClose: number;
|
|
509
|
+
/** Coefficient (0..1) for a dimple appearing on the left side. */
|
|
236
510
|
mouthDimpleLeft: number;
|
|
511
|
+
/** Coefficient (0..1) for a dimple appearing on the right side. */
|
|
237
512
|
mouthDimpleRight: number;
|
|
513
|
+
/** Coefficient (0..1) for the left mouth corner pulling down (frown). */
|
|
238
514
|
mouthFrownLeft: number;
|
|
515
|
+
/** Coefficient (0..1) for the right mouth corner pulling down (frown). */
|
|
239
516
|
mouthFrownRight: number;
|
|
517
|
+
/** Coefficient (0..1) for funneling the lips outward. */
|
|
240
518
|
mouthFunnel: number;
|
|
519
|
+
/** Coefficient (0..1) for the mouth shifting left. */
|
|
241
520
|
mouthLeft: number;
|
|
521
|
+
/** Coefficient (0..1) for the lower lip on the left pulling down. */
|
|
242
522
|
mouthLowerDownLeft: number;
|
|
523
|
+
/** Coefficient (0..1) for the lower lip on the right pulling down. */
|
|
243
524
|
mouthLowerDownRight: number;
|
|
525
|
+
/** Coefficient (0..1) for the left lip pressing inward. */
|
|
244
526
|
mouthPressLeft: number;
|
|
527
|
+
/** Coefficient (0..1) for the right lip pressing inward. */
|
|
245
528
|
mouthPressRight: number;
|
|
529
|
+
/** Coefficient (0..1) for puckered lips. */
|
|
246
530
|
mouthPucker: number;
|
|
531
|
+
/** Coefficient (0..1) for the mouth shifting right. */
|
|
247
532
|
mouthRight: number;
|
|
533
|
+
/** Coefficient (0..1) for the lower lip rolling inward. */
|
|
248
534
|
mouthRollLower: number;
|
|
535
|
+
/** Coefficient (0..1) for the upper lip rolling inward. */
|
|
249
536
|
mouthRollUpper: number;
|
|
537
|
+
/** Coefficient (0..1) for the lower lip shrugging. */
|
|
250
538
|
mouthShrugLower: number;
|
|
539
|
+
/** Coefficient (0..1) for the upper lip shrugging. */
|
|
251
540
|
mouthShrugUpper: number;
|
|
541
|
+
/** Coefficient (0..1) for the left mouth corner pulling up (smile). */
|
|
252
542
|
mouthSmileLeft: number;
|
|
543
|
+
/** Coefficient (0..1) for the right mouth corner pulling up (smile). */
|
|
253
544
|
mouthSmileRight: number;
|
|
545
|
+
/** Coefficient (0..1) for the left mouth corner stretching outward. */
|
|
254
546
|
mouthStretchLeft: number;
|
|
547
|
+
/** Coefficient (0..1) for the right mouth corner stretching outward. */
|
|
255
548
|
mouthStretchRight: number;
|
|
549
|
+
/** Coefficient (0..1) for the upper lip on the left raising. */
|
|
256
550
|
mouthUpperUpLeft: number;
|
|
551
|
+
/** Coefficient (0..1) for the upper lip on the right raising. */
|
|
257
552
|
mouthUpperUpRight: number;
|
|
553
|
+
/** Coefficient (0..1) for the left side of the nose sneering. */
|
|
258
554
|
noseSneerLeft: number;
|
|
555
|
+
/** Coefficient (0..1) for the right side of the nose sneering. */
|
|
259
556
|
noseSneerRight: number;
|
|
557
|
+
/** Coefficient (0..1) for the tongue protruding. */
|
|
260
558
|
tongueOut: number;
|
|
261
559
|
}
|
|
262
560
|
|
|
561
|
+
/**
|
|
562
|
+
* One detected face produced by the video CV pipeline. Always carries `id`,
|
|
563
|
+
* `bounds`, `center`, and `confidence`; the rest is populated only when the
|
|
564
|
+
* matching CV feature is enabled (face mesh for `landmarks` and `headPose`,
|
|
565
|
+
* emotion detection for `expressions` and `blendshapes`).
|
|
566
|
+
*/
|
|
263
567
|
interface FaceData {
|
|
568
|
+
/** Index-based face identifier (`0`, `1`, …). Stable within a single frame, may change between frames. */
|
|
264
569
|
id: number;
|
|
570
|
+
/** Bounding box, all components normalized to 0..1 of the source frame. */
|
|
265
571
|
bounds: {
|
|
572
|
+
/** Left edge of the face bounding box, normalized 0..1. */
|
|
266
573
|
x: number;
|
|
574
|
+
/** Top edge of the face bounding box, normalized 0..1. */
|
|
267
575
|
y: number;
|
|
576
|
+
/** Width of the face bounding box, normalized 0..1. */
|
|
268
577
|
width: number;
|
|
578
|
+
/** Height of the face bounding box, normalized 0..1. */
|
|
269
579
|
height: number;
|
|
270
580
|
};
|
|
581
|
+
/** Center of the bounding box, normalized 0..1. */
|
|
271
582
|
center: {
|
|
583
|
+
/** Horizontal center of the face, normalized 0..1. */
|
|
272
584
|
x: number;
|
|
585
|
+
/** Vertical center of the face, normalized 0..1. */
|
|
273
586
|
y: number;
|
|
274
587
|
};
|
|
588
|
+
/** Detection confidence in 0..1. */
|
|
275
589
|
confidence: number;
|
|
590
|
+
/** 468 normalized face mesh landmarks when face mesh is enabled; empty array otherwise. */
|
|
276
591
|
landmarks: {
|
|
592
|
+
/** Horizontal landmark coordinate, normalized 0..1. */
|
|
277
593
|
x: number;
|
|
594
|
+
/** Vertical landmark coordinate, normalized 0..1. */
|
|
278
595
|
y: number;
|
|
596
|
+
/** Optional depth (relative). */
|
|
279
597
|
z?: number;
|
|
280
598
|
}[];
|
|
599
|
+
/** Seven expression confidence scores in 0..1. All zero unless emotion detection is enabled. */
|
|
281
600
|
expressions: {
|
|
601
|
+
/** Confidence (0..1) of a neutral expression. */
|
|
282
602
|
neutral: number;
|
|
603
|
+
/** Confidence (0..1) of a happy / smiling expression. */
|
|
283
604
|
happy: number;
|
|
605
|
+
/** Confidence (0..1) of a sad expression. */
|
|
284
606
|
sad: number;
|
|
607
|
+
/** Confidence (0..1) of an angry expression. */
|
|
285
608
|
angry: number;
|
|
609
|
+
/** Confidence (0..1) of a surprised expression. */
|
|
286
610
|
surprised: number;
|
|
611
|
+
/** Confidence (0..1) of a disgusted expression. */
|
|
287
612
|
disgusted: number;
|
|
613
|
+
/** Confidence (0..1) of a fearful expression. */
|
|
288
614
|
fearful: number;
|
|
289
615
|
};
|
|
616
|
+
/** Estimated head rotation. All zero unless face mesh is enabled. */
|
|
290
617
|
headPose: {
|
|
618
|
+
/** Up/down rotation in degrees (-90..90). */
|
|
291
619
|
pitch: number;
|
|
620
|
+
/** Left/right rotation in degrees (-90..90). */
|
|
292
621
|
yaw: number;
|
|
622
|
+
/** Tilt rotation in degrees (-180..180). */
|
|
293
623
|
roll: number;
|
|
294
624
|
};
|
|
625
|
+
/** 52 ARKit-compatible facial muscle coefficients. All zero unless emotion detection is enabled. */
|
|
295
626
|
blendshapes: FaceBlendshapes;
|
|
296
627
|
}
|
|
297
628
|
|
|
629
|
+
/**
|
|
630
|
+
* Render-loop pacing mode. `'full'` runs `render()` on every animation frame
|
|
631
|
+
* (~60 FPS). `'half'` runs every second frame (~30 FPS), saving CPU/GPU on
|
|
632
|
+
* lower-cost scenes.
|
|
633
|
+
*/
|
|
298
634
|
type FrameRateMode = 'full' | 'half';
|
|
299
635
|
|
|
636
|
+
/**
|
|
637
|
+
* One named frequency band reported by the audio analyzer
|
|
638
|
+
* (e.g., `{ name: 'low', min: 20, max: 120 }`). Frequencies are in Hertz.
|
|
639
|
+
*/
|
|
300
640
|
interface FrequencyBand {
|
|
641
|
+
/** Human-readable band name (e.g., `'low'`, `'mid'`, `'high'`). */
|
|
301
642
|
name: string;
|
|
643
|
+
/** Lower edge of the band in Hertz (inclusive). */
|
|
302
644
|
min: number;
|
|
645
|
+
/** Upper edge of the band in Hertz (exclusive). */
|
|
303
646
|
max: number;
|
|
304
647
|
}
|
|
305
648
|
|
|
649
|
+
/**
|
|
650
|
+
* One detected hand produced by the video CV pipeline. Carries 21 landmarks,
|
|
651
|
+
* a palm shortcut, and ML-classified gesture confidence scores. Up to two
|
|
652
|
+
* hands appear in `VideoAPI.hands` simultaneously.
|
|
653
|
+
*/
|
|
306
654
|
interface HandData {
|
|
655
|
+
/** Index-based hand identifier (`0` or `1`). */
|
|
307
656
|
id: number;
|
|
657
|
+
/** Which hand. Always lowercase. */
|
|
308
658
|
handedness: 'left' | 'right';
|
|
659
|
+
/** Detection confidence in 0..1. */
|
|
309
660
|
confidence: number;
|
|
661
|
+
/** Bounding box, all components normalized to 0..1 of the source frame. */
|
|
310
662
|
bounds: {
|
|
663
|
+
/** Left edge of the hand bounding box, normalized 0..1. */
|
|
311
664
|
x: number;
|
|
665
|
+
/** Top edge of the hand bounding box, normalized 0..1. */
|
|
312
666
|
y: number;
|
|
667
|
+
/** Width of the hand bounding box, normalized 0..1. */
|
|
313
668
|
width: number;
|
|
669
|
+
/** Height of the hand bounding box, normalized 0..1. */
|
|
314
670
|
height: number;
|
|
315
671
|
};
|
|
672
|
+
/** 21 MediaPipe hand landmarks, normalized 0..1. */
|
|
316
673
|
landmarks: {
|
|
674
|
+
/** Horizontal landmark coordinate, normalized 0..1. */
|
|
317
675
|
x: number;
|
|
676
|
+
/** Vertical landmark coordinate, normalized 0..1. */
|
|
318
677
|
y: number;
|
|
678
|
+
/** Depth (relative). */
|
|
319
679
|
z: number;
|
|
320
680
|
}[];
|
|
681
|
+
/** Palm center — equivalent to `landmarks[9]` (middle-finger MCP). */
|
|
321
682
|
palm: {
|
|
683
|
+
/** Palm horizontal coordinate, normalized 0..1. */
|
|
322
684
|
x: number;
|
|
685
|
+
/** Palm vertical coordinate, normalized 0..1. */
|
|
323
686
|
y: number;
|
|
687
|
+
/** Palm depth (relative). */
|
|
324
688
|
z: number;
|
|
325
689
|
};
|
|
690
|
+
/**
|
|
691
|
+
* ML-classified gesture confidences from MediaPipe GestureRecognizer.
|
|
692
|
+
* Each value is in 0..1; multiple gestures can fire simultaneously.
|
|
693
|
+
*/
|
|
326
694
|
gestures: {
|
|
695
|
+
/** Confidence (0..1) of a closed-fist gesture. */
|
|
327
696
|
fist: number;
|
|
697
|
+
/** Confidence (0..1) of an open-palm gesture. */
|
|
328
698
|
openPalm: number;
|
|
699
|
+
/** Confidence (0..1) of a peace / victory sign. */
|
|
329
700
|
peace: number;
|
|
701
|
+
/** Confidence (0..1) of a thumbs-up gesture. */
|
|
330
702
|
thumbsUp: number;
|
|
703
|
+
/** Confidence (0..1) of a thumbs-down gesture. */
|
|
331
704
|
thumbsDown: number;
|
|
705
|
+
/** Confidence (0..1) of a pointing-up gesture. */
|
|
332
706
|
pointing: number;
|
|
707
|
+
/** Confidence (0..1) of the ASL "I love you" sign. */
|
|
333
708
|
iLoveYou: number;
|
|
334
709
|
};
|
|
335
710
|
}
|
|
336
711
|
|
|
712
|
+
/**
|
|
713
|
+
* Configuration object passed to `viji.image()`. The host renders an upload
|
|
714
|
+
* field; the user-supplied image becomes available as an `ImageBitmap` (Native)
|
|
715
|
+
* or a P5 image wrapper (P5 renderer).
|
|
716
|
+
*/
|
|
337
717
|
interface ImageConfig {
|
|
718
|
+
/** Display name shown next to the upload field in the parameter UI. Required. */
|
|
338
719
|
label: string;
|
|
720
|
+
/** Optional tooltip / help text shown next to the control. */
|
|
339
721
|
description?: string;
|
|
722
|
+
/** Group name for organizing related parameters under a shared heading. Default: `'general'`. */
|
|
340
723
|
group?: string;
|
|
724
|
+
/** Visibility category — the control hides when its capability is unavailable. Default: `'general'`. */
|
|
341
725
|
category?: ParameterCategory;
|
|
342
726
|
}
|
|
343
727
|
|
|
728
|
+
/**
|
|
729
|
+
* Reactive object returned by `viji.image()`. `value` is `null` until the user
|
|
730
|
+
* uploads an image; in P5 scenes a `.p5` accessor is also added at runtime so
|
|
731
|
+
* the image works directly with `image()`, `texture()`, etc.
|
|
732
|
+
*/
|
|
344
733
|
interface ImageParameter {
|
|
734
|
+
/** Uploaded image as an `ImageBitmap`, or `null` until the user provides one. */
|
|
345
735
|
value: ImageBitmap | null;
|
|
346
736
|
/** P5-compatible image wrapper. Only available in the P5 renderer — added at runtime by the P5 adapter. */
|
|
347
737
|
readonly p5?: any;
|
|
738
|
+
/** Display name shown next to the upload field in the parameter UI. */
|
|
348
739
|
label: string;
|
|
740
|
+
/** Tooltip / help text for the control (empty string when not configured). */
|
|
349
741
|
description?: string;
|
|
742
|
+
/** Group name under which the control appears in the UI. */
|
|
350
743
|
group: string;
|
|
744
|
+
/** Visibility category — controls when the parameter is shown. */
|
|
351
745
|
category: ParameterCategory;
|
|
352
746
|
}
|
|
353
747
|
|
|
748
|
+
/**
|
|
749
|
+
* Keyboard input state for the current frame: per-key press / hold / release
|
|
750
|
+
* queries (case-insensitive), the live set of held keys, modifier flags, and
|
|
751
|
+
* a Shadertoy-compatible texture buffer for shader-side keyboard reads.
|
|
752
|
+
*
|
|
753
|
+
* Key names follow the browser's `event.key` standard
|
|
754
|
+
* (`'a'`, `'arrowup'`, `' '` for Space, `'enter'`, `'escape'`, etc.).
|
|
755
|
+
*/
|
|
354
756
|
interface KeyboardAPI {
|
|
757
|
+
/** Returns `true` while the named key is held. Case-insensitive. */
|
|
355
758
|
isPressed(key: string): boolean;
|
|
759
|
+
/** Returns `true` for exactly one frame on the rising edge of the named key (no key-repeat). Case-insensitive. */
|
|
356
760
|
wasPressed(key: string): boolean;
|
|
761
|
+
/** Returns `true` for exactly one frame on the falling edge of the named key. Case-insensitive. */
|
|
357
762
|
wasReleased(key: string): boolean;
|
|
763
|
+
/** Set of all currently held keys, lowercased. Persists across frames. */
|
|
358
764
|
activeKeys: Set<string>;
|
|
765
|
+
/** Set of keys whose key-down fired this frame, lowercased. Cleared each frame. */
|
|
359
766
|
pressedThisFrame: Set<string>;
|
|
767
|
+
/** Set of keys whose key-up fired this frame, lowercased. Cleared each frame. */
|
|
360
768
|
releasedThisFrame: Set<string>;
|
|
769
|
+
/** Most recently pressed key in its original case (e.g., `'A'` when Shift is held). */
|
|
361
770
|
lastKeyPressed: string;
|
|
771
|
+
/** Most recently released key in its original case. */
|
|
362
772
|
lastKeyReleased: string;
|
|
773
|
+
/** `true` while the Shift key is held. */
|
|
363
774
|
shift: boolean;
|
|
775
|
+
/** `true` while the Ctrl key is held. */
|
|
364
776
|
ctrl: boolean;
|
|
777
|
+
/** `true` while the Alt (or Option) key is held. */
|
|
365
778
|
alt: boolean;
|
|
779
|
+
/** `true` while the Meta (Windows / Cmd) key is held. */
|
|
366
780
|
meta: boolean;
|
|
781
|
+
/**
|
|
782
|
+
* Shadertoy-compatible 256×3 keyboard texture (256 keycodes × 3 rows).
|
|
783
|
+
* Row 0: held state (1 / 0). Row 1: pressed-this-frame (1 / 0). Row 2: per-key toggle.
|
|
784
|
+
* Bound to a uniform sampler in shader scenes; rarely needed in JS.
|
|
785
|
+
*/
|
|
367
786
|
textureData: Uint8Array;
|
|
368
787
|
}
|
|
369
788
|
|
|
789
|
+
/**
|
|
790
|
+
* Mouse input state for the current frame: position, individual button states,
|
|
791
|
+
* per-frame movement, scroll wheel, and frame-based press / release events.
|
|
792
|
+
* Coordinates are in canvas-space pixels with `(0, 0)` at the top-left corner.
|
|
793
|
+
*
|
|
794
|
+
* For interactions that should also work on touch devices, prefer `viji.pointer`.
|
|
795
|
+
*/
|
|
370
796
|
interface MouseAPI {
|
|
797
|
+
/** Canvas-space X position in pixels. */
|
|
371
798
|
x: number;
|
|
799
|
+
/** Canvas-space Y position in pixels. */
|
|
372
800
|
y: number;
|
|
801
|
+
/** `true` when the cursor is currently over the canvas. */
|
|
373
802
|
isInCanvas: boolean;
|
|
803
|
+
/** `true` if any mouse button is currently held. */
|
|
374
804
|
isPressed: boolean;
|
|
805
|
+
/** `true` while the left mouse button is held. */
|
|
375
806
|
leftButton: boolean;
|
|
807
|
+
/** `true` while the right mouse button is held. The browser context menu is suppressed on the canvas. */
|
|
376
808
|
rightButton: boolean;
|
|
809
|
+
/** `true` while the middle (wheel) mouse button is held. */
|
|
377
810
|
middleButton: boolean;
|
|
811
|
+
/** Horizontal movement this frame in pixels. Resets to `0` each frame. */
|
|
378
812
|
deltaX: number;
|
|
813
|
+
/** Vertical movement this frame in pixels. Resets to `0` each frame. */
|
|
379
814
|
deltaY: number;
|
|
815
|
+
/** Vertical scroll accumulated this frame (alias of `wheelY`). Resets to `0` each frame. */
|
|
380
816
|
wheelDelta: number;
|
|
817
|
+
/** Horizontal scroll accumulated this frame (e.g., trackpad gestures, tilt-wheel mice). Resets to `0` each frame. */
|
|
381
818
|
wheelX: number;
|
|
819
|
+
/** Vertical scroll accumulated this frame. Resets to `0` each frame. */
|
|
382
820
|
wheelY: number;
|
|
821
|
+
/** `true` for exactly one frame when any button is first pressed, then auto-resets. */
|
|
383
822
|
wasPressed: boolean;
|
|
823
|
+
/** `true` for exactly one frame when any button is released, then auto-resets. */
|
|
384
824
|
wasReleased: boolean;
|
|
825
|
+
/** `true` if the mouse moved this frame (any non-zero `deltaX` / `deltaY`). Resets each frame. */
|
|
385
826
|
wasMoved: boolean;
|
|
386
827
|
}
|
|
387
828
|
|
|
829
|
+
/**
|
|
830
|
+
* Configuration object passed to `viji.number()`. Defines the bounds, step,
|
|
831
|
+
* label, and UI grouping/visibility for a precise numeric input.
|
|
832
|
+
*/
|
|
388
833
|
interface NumberConfig {
|
|
834
|
+
/** Inclusive minimum value. Default: `0`. */
|
|
389
835
|
min?: number;
|
|
836
|
+
/** Inclusive maximum value. Default: `100`. */
|
|
390
837
|
max?: number;
|
|
838
|
+
/** Increment between values. Default: `1`. */
|
|
391
839
|
step?: number;
|
|
840
|
+
/** Display name shown next to the input in the parameter UI. Required. */
|
|
392
841
|
label: string;
|
|
842
|
+
/** Optional tooltip / help text shown next to the control. */
|
|
393
843
|
description?: string;
|
|
844
|
+
/** Group name for organizing related parameters under a shared heading. Default: `'general'`. */
|
|
394
845
|
group?: string;
|
|
846
|
+
/** Visibility category — the control hides when its capability is unavailable. Default: `'general'`. */
|
|
395
847
|
category?: ParameterCategory;
|
|
396
848
|
}
|
|
397
849
|
|
|
850
|
+
/**
|
|
851
|
+
* Reactive object returned by `viji.number()`. Like `SliderParameter` but the
|
|
852
|
+
* host UI renders a precise numeric input rather than a draggable slider.
|
|
853
|
+
*/
|
|
398
854
|
interface NumberParameter {
|
|
855
|
+
/** Current numeric value in the configured `min..max` range. Updates in real-time. */
|
|
399
856
|
value: number;
|
|
857
|
+
/** Inclusive minimum value. */
|
|
400
858
|
min: number;
|
|
859
|
+
/** Inclusive maximum value. */
|
|
401
860
|
max: number;
|
|
861
|
+
/** Increment between values. */
|
|
402
862
|
step: number;
|
|
863
|
+
/** Display name shown next to the input in the parameter UI. */
|
|
403
864
|
label: string;
|
|
865
|
+
/** Tooltip / help text for the control (empty string when not configured). */
|
|
404
866
|
description?: string;
|
|
867
|
+
/** Group name under which the control appears in the UI. */
|
|
405
868
|
group: string;
|
|
869
|
+
/** Visibility category — controls when the parameter is shown. */
|
|
406
870
|
category: ParameterCategory;
|
|
407
871
|
}
|
|
408
872
|
|
|
873
|
+
/**
|
|
874
|
+
* Visibility category for a parameter. The host hides parameters whose category
|
|
875
|
+
* is not currently available (e.g., `'audio'` controls hide when no audio
|
|
876
|
+
* source is connected). `'general'` parameters are always visible.
|
|
877
|
+
*/
|
|
409
878
|
type ParameterCategory = 'audio' | 'video' | 'interaction' | 'general';
|
|
410
879
|
|
|
880
|
+
/**
|
|
881
|
+
* Unified pointer that abstracts mouse and primary touch into a single input
|
|
882
|
+
* stream. Recommended for click / drag / position-tracking interactions that
|
|
883
|
+
* should work identically on desktop and mobile.
|
|
884
|
+
*
|
|
885
|
+
* Use `viji.mouse` for mouse-specific features (right click, middle click,
|
|
886
|
+
* scroll wheel); use `viji.touches` for multi-touch, pressure, or radius.
|
|
887
|
+
*/
|
|
411
888
|
interface PointerAPI {
|
|
889
|
+
/** Canvas-space X position in pixels. */
|
|
412
890
|
x: number;
|
|
891
|
+
/** Canvas-space Y position in pixels. */
|
|
413
892
|
y: number;
|
|
893
|
+
/** Horizontal movement since the last frame in pixels. */
|
|
414
894
|
deltaX: number;
|
|
895
|
+
/** Vertical movement since the last frame in pixels. */
|
|
415
896
|
deltaY: number;
|
|
897
|
+
/** `true` while the pointer is "down" — left mouse button held, or a touch active. */
|
|
416
898
|
isDown: boolean;
|
|
899
|
+
/** `true` for exactly one frame when the pointer becomes down, then auto-resets. */
|
|
417
900
|
wasPressed: boolean;
|
|
901
|
+
/** `true` for exactly one frame when the pointer is released, then auto-resets. */
|
|
418
902
|
wasReleased: boolean;
|
|
903
|
+
/** `true` while the pointer is within the canvas bounds. */
|
|
419
904
|
isInCanvas: boolean;
|
|
905
|
+
/** Currently active input source — `'touch'` when at least one touch is active, otherwise `'mouse'` (or `'none'` when the cursor is off-canvas). */
|
|
420
906
|
type: 'mouse' | 'touch' | 'none';
|
|
421
907
|
}
|
|
422
908
|
|
|
909
|
+
/**
|
|
910
|
+
* Detected body pose from MediaPipe BlazePose. `landmarks` contains 33 points;
|
|
911
|
+
* the `face` / `torso` / `leftArm` / `rightArm` / `leftLeg` / `rightLeg` arrays
|
|
912
|
+
* are pre-grouped slices for convenience.
|
|
913
|
+
*/
|
|
423
914
|
interface PoseData {
|
|
915
|
+
/** Average landmark visibility in 0..1. */
|
|
424
916
|
confidence: number;
|
|
917
|
+
/** 33 BlazePose landmarks, normalized 0..1. Each carries a per-point `visibility` score. */
|
|
425
918
|
landmarks: {
|
|
919
|
+
/** Horizontal landmark coordinate, normalized 0..1. */
|
|
426
920
|
x: number;
|
|
921
|
+
/** Vertical landmark coordinate, normalized 0..1. */
|
|
427
922
|
y: number;
|
|
923
|
+
/** Depth (relative). */
|
|
428
924
|
z: number;
|
|
925
|
+
/** Visibility / confidence of this individual landmark in 0..1. */
|
|
429
926
|
visibility: number;
|
|
430
927
|
}[];
|
|
928
|
+
/** Face landmarks (BlazePose indices 0..10). Coordinates normalized 0..1. */
|
|
431
929
|
face: {
|
|
930
|
+
/** Horizontal landmark coordinate, normalized 0..1. */
|
|
432
931
|
x: number;
|
|
932
|
+
/** Vertical landmark coordinate, normalized 0..1. */
|
|
433
933
|
y: number;
|
|
434
934
|
}[];
|
|
935
|
+
/** Torso landmarks (BlazePose indices 11, 12, 23, 24). Coordinates normalized 0..1. */
|
|
435
936
|
torso: {
|
|
937
|
+
/** Horizontal landmark coordinate, normalized 0..1. */
|
|
436
938
|
x: number;
|
|
939
|
+
/** Vertical landmark coordinate, normalized 0..1. */
|
|
437
940
|
y: number;
|
|
438
941
|
}[];
|
|
942
|
+
/** Left-arm landmarks (BlazePose indices 11, 13, 15). Coordinates normalized 0..1. */
|
|
439
943
|
leftArm: {
|
|
944
|
+
/** Horizontal landmark coordinate, normalized 0..1. */
|
|
440
945
|
x: number;
|
|
946
|
+
/** Vertical landmark coordinate, normalized 0..1. */
|
|
441
947
|
y: number;
|
|
442
948
|
}[];
|
|
949
|
+
/** Right-arm landmarks (BlazePose indices 12, 14, 16). Coordinates normalized 0..1. */
|
|
443
950
|
rightArm: {
|
|
951
|
+
/** Horizontal landmark coordinate, normalized 0..1. */
|
|
444
952
|
x: number;
|
|
953
|
+
/** Vertical landmark coordinate, normalized 0..1. */
|
|
445
954
|
y: number;
|
|
446
955
|
}[];
|
|
956
|
+
/** Left-leg landmarks (BlazePose indices 23, 25, 27, 29, 31). Coordinates normalized 0..1. */
|
|
447
957
|
leftLeg: {
|
|
958
|
+
/** Horizontal landmark coordinate, normalized 0..1. */
|
|
448
959
|
x: number;
|
|
960
|
+
/** Vertical landmark coordinate, normalized 0..1. */
|
|
449
961
|
y: number;
|
|
450
962
|
}[];
|
|
963
|
+
/** Right-leg landmarks (BlazePose indices 24, 26, 28, 30, 32). Coordinates normalized 0..1. */
|
|
451
964
|
rightLeg: {
|
|
965
|
+
/** Horizontal landmark coordinate, normalized 0..1. */
|
|
452
966
|
x: number;
|
|
967
|
+
/** Vertical landmark coordinate, normalized 0..1. */
|
|
453
968
|
y: number;
|
|
454
969
|
}[];
|
|
455
970
|
}
|
|
456
971
|
|
|
972
|
+
/**
|
|
973
|
+
* Discrete pixel dimensions for a canvas, frame capture, or render target.
|
|
974
|
+
*/
|
|
457
975
|
type Resolution = {
|
|
976
|
+
/** Width in pixels (positive integer). */
|
|
458
977
|
width: number;
|
|
978
|
+
/** Height in pixels (positive integer). */
|
|
459
979
|
height: number;
|
|
460
980
|
};
|
|
461
981
|
|
|
982
|
+
/**
|
|
983
|
+
* Per-pixel person/background segmentation mask. Mask dimensions reflect the
|
|
984
|
+
* ML model's output and may differ from the source video frame.
|
|
985
|
+
*/
|
|
462
986
|
interface SegmentationData {
|
|
987
|
+
/** Flat per-pixel mask: each byte is `0` (background) or `1` (person). Length = `width * height`. */
|
|
463
988
|
mask: Uint8Array;
|
|
989
|
+
/** Mask width in pixels. */
|
|
464
990
|
width: number;
|
|
991
|
+
/** Mask height in pixels. */
|
|
465
992
|
height: number;
|
|
466
993
|
}
|
|
467
994
|
|
|
995
|
+
/**
|
|
996
|
+
* Configuration object passed to `viji.select()`. The host renders the choices
|
|
997
|
+
* as a dropdown or segmented control.
|
|
998
|
+
*/
|
|
468
999
|
interface SelectConfig {
|
|
1000
|
+
/** Available choices. The element type (`string` or `number`) becomes the parameter's value type. Required. */
|
|
469
1001
|
options: string[] | number[];
|
|
1002
|
+
/** Display name shown next to the dropdown in the parameter UI. Required. */
|
|
470
1003
|
label: string;
|
|
1004
|
+
/** Optional tooltip / help text shown next to the control. */
|
|
471
1005
|
description?: string;
|
|
1006
|
+
/** Group name for organizing related parameters under a shared heading. Default: `'general'`. */
|
|
472
1007
|
group?: string;
|
|
1008
|
+
/** Visibility category — the control hides when its capability is unavailable. Default: `'general'`. */
|
|
473
1009
|
category?: ParameterCategory;
|
|
474
1010
|
}
|
|
475
1011
|
|
|
1012
|
+
/**
|
|
1013
|
+
* Reactive object returned by `viji.select()`. `value` matches the element type
|
|
1014
|
+
* of the configured `options` array (`string` or `number`).
|
|
1015
|
+
*/
|
|
476
1016
|
interface SelectParameter {
|
|
1017
|
+
/** Currently selected option. Updates in real-time when the user changes the selection. */
|
|
477
1018
|
value: string | number;
|
|
1019
|
+
/** The full list of choices originally configured. */
|
|
478
1020
|
options: string[] | number[];
|
|
1021
|
+
/** Display name shown next to the dropdown in the parameter UI. */
|
|
479
1022
|
label: string;
|
|
1023
|
+
/** Tooltip / help text for the control (empty string when not configured). */
|
|
480
1024
|
description?: string;
|
|
1025
|
+
/** Group name under which the control appears in the UI. */
|
|
481
1026
|
group: string;
|
|
1027
|
+
/** Visibility category — controls when the parameter is shown. */
|
|
482
1028
|
category: ParameterCategory;
|
|
483
1029
|
}
|
|
484
1030
|
|
|
1031
|
+
/**
|
|
1032
|
+
* Configuration object passed to `viji.slider()`. Defines the slider's bounds,
|
|
1033
|
+
* increment, label, and UI grouping/visibility.
|
|
1034
|
+
*/
|
|
485
1035
|
interface SliderConfig {
|
|
1036
|
+
/** Inclusive minimum value of the slider. Default: `0`. */
|
|
486
1037
|
min?: number;
|
|
1038
|
+
/** Inclusive maximum value of the slider. Default: `100`. */
|
|
487
1039
|
max?: number;
|
|
1040
|
+
/** Increment between values. Default: `1`. */
|
|
488
1041
|
step?: number;
|
|
1042
|
+
/** Display name shown next to the control in the parameter UI. Required. */
|
|
489
1043
|
label: string;
|
|
1044
|
+
/** Optional tooltip / help text shown next to the control. */
|
|
490
1045
|
description?: string;
|
|
1046
|
+
/** Group name for organizing related parameters under a shared heading. Default: `'general'`. */
|
|
491
1047
|
group?: string;
|
|
1048
|
+
/** Visibility category — the control hides when its capability is unavailable. Default: `'general'`. */
|
|
492
1049
|
category?: ParameterCategory;
|
|
493
1050
|
}
|
|
494
1051
|
|
|
1052
|
+
/**
|
|
1053
|
+
* Reactive object returned by `viji.slider()`. `value` updates in real-time as
|
|
1054
|
+
* the user drags the control; the remaining fields mirror the configuration.
|
|
1055
|
+
*/
|
|
495
1056
|
interface SliderParameter {
|
|
1057
|
+
/** Current slider value in the configured `min..max` range. Updates in real-time. */
|
|
496
1058
|
value: number;
|
|
1059
|
+
/** Inclusive minimum value of the slider. */
|
|
497
1060
|
min: number;
|
|
1061
|
+
/** Inclusive maximum value of the slider. */
|
|
498
1062
|
max: number;
|
|
1063
|
+
/** Increment between values. */
|
|
499
1064
|
step: number;
|
|
1065
|
+
/** Display name shown next to the control in the parameter UI. */
|
|
500
1066
|
label: string;
|
|
1067
|
+
/** Tooltip / help text for the control (empty string when not configured). */
|
|
501
1068
|
description?: string;
|
|
1069
|
+
/** Group name under which the control appears in the UI. */
|
|
502
1070
|
group: string;
|
|
1071
|
+
/** Visibility category — controls when the parameter is shown. */
|
|
503
1072
|
category: ParameterCategory;
|
|
504
1073
|
}
|
|
505
1074
|
|
|
1075
|
+
/**
|
|
1076
|
+
* Configuration object passed to `viji.text()`. The host renders a single-line
|
|
1077
|
+
* text input field.
|
|
1078
|
+
*/
|
|
506
1079
|
interface TextConfig {
|
|
1080
|
+
/** Display name shown next to the text field in the parameter UI. Required. */
|
|
507
1081
|
label: string;
|
|
1082
|
+
/** Optional tooltip / help text shown next to the control. */
|
|
508
1083
|
description?: string;
|
|
1084
|
+
/** Group name for organizing related parameters under a shared heading. Default: `'general'`. */
|
|
509
1085
|
group?: string;
|
|
1086
|
+
/** Visibility category — the control hides when its capability is unavailable. Default: `'general'`. */
|
|
510
1087
|
category?: ParameterCategory;
|
|
1088
|
+
/** Maximum character count enforced by the host UI. Default: `1000`. */
|
|
511
1089
|
maxLength?: number;
|
|
512
1090
|
}
|
|
513
1091
|
|
|
1092
|
+
/**
|
|
1093
|
+
* Reactive object returned by `viji.text()`. `value` updates as the user types
|
|
1094
|
+
* and is enforced to be no longer than `maxLength` by the host UI.
|
|
1095
|
+
*/
|
|
514
1096
|
interface TextParameter {
|
|
1097
|
+
/** Current text content. Updates in real-time as the user types. */
|
|
515
1098
|
value: string;
|
|
1099
|
+
/** Maximum character count enforced by the host UI. */
|
|
516
1100
|
maxLength?: number;
|
|
1101
|
+
/** Display name shown next to the input in the parameter UI. */
|
|
517
1102
|
label: string;
|
|
1103
|
+
/** Tooltip / help text for the control (empty string when not configured). */
|
|
518
1104
|
description?: string;
|
|
1105
|
+
/** Group name under which the control appears in the UI. */
|
|
519
1106
|
group: string;
|
|
1107
|
+
/** Visibility category — controls when the parameter is shown. */
|
|
520
1108
|
category: ParameterCategory;
|
|
521
1109
|
}
|
|
522
1110
|
|
|
1111
|
+
/**
|
|
1112
|
+
* Configuration object passed to `viji.toggle()`. Defines the on/off switch's
|
|
1113
|
+
* label and UI grouping/visibility.
|
|
1114
|
+
*/
|
|
523
1115
|
interface ToggleConfig {
|
|
1116
|
+
/** Display name shown next to the switch in the parameter UI. Required. */
|
|
524
1117
|
label: string;
|
|
1118
|
+
/** Optional tooltip / help text shown next to the control. */
|
|
525
1119
|
description?: string;
|
|
1120
|
+
/** Group name for organizing related parameters under a shared heading. Default: `'general'`. */
|
|
526
1121
|
group?: string;
|
|
1122
|
+
/** Visibility category — the control hides when its capability is unavailable. Default: `'general'`. */
|
|
527
1123
|
category?: ParameterCategory;
|
|
528
1124
|
}
|
|
529
1125
|
|
|
1126
|
+
/**
|
|
1127
|
+
* Reactive object returned by `viji.toggle()`. `value` flips between `true`
|
|
1128
|
+
* and `false` as the user interacts with the switch.
|
|
1129
|
+
*/
|
|
530
1130
|
interface ToggleParameter {
|
|
1131
|
+
/** Current state of the switch. Updates in real-time when the user toggles it. */
|
|
531
1132
|
value: boolean;
|
|
1133
|
+
/** Display name shown next to the switch in the parameter UI. */
|
|
532
1134
|
label: string;
|
|
1135
|
+
/** Tooltip / help text for the control (empty string when not configured). */
|
|
533
1136
|
description?: string;
|
|
1137
|
+
/** Group name under which the control appears in the UI. */
|
|
534
1138
|
group: string;
|
|
1139
|
+
/** Visibility category — controls when the parameter is shown. */
|
|
535
1140
|
category: ParameterCategory;
|
|
536
1141
|
}
|
|
537
1142
|
|
|
1143
|
+
/**
|
|
1144
|
+
* Multi-touch input state. Lists all active touches, plus per-frame
|
|
1145
|
+
* `started` / `moved` / `ended` arrays for frame-based gesture handling.
|
|
1146
|
+
*
|
|
1147
|
+
* For single-point interactions that should also work with a mouse, prefer
|
|
1148
|
+
* `viji.pointer` — it switches automatically between mouse and primary touch.
|
|
1149
|
+
*/
|
|
538
1150
|
interface TouchAPI {
|
|
1151
|
+
/** All currently active touch points. Order is stable but not ordered by id. */
|
|
539
1152
|
points: TouchPoint[];
|
|
1153
|
+
/** Number of currently active touches (equivalent to `points.length`). */
|
|
540
1154
|
count: number;
|
|
1155
|
+
/** Touches that started this frame (also visible in `points`). Cleared each frame. */
|
|
541
1156
|
started: TouchPoint[];
|
|
1157
|
+
/** Touches that moved this frame. Cleared each frame. */
|
|
542
1158
|
moved: TouchPoint[];
|
|
1159
|
+
/** Touches that ended this frame. Each entry has `isEnding === true`. Cleared each frame. */
|
|
543
1160
|
ended: TouchPoint[];
|
|
1161
|
+
/** Convenience shortcut to `points[0]`, or `null` when no touches are active. */
|
|
544
1162
|
primary: TouchPoint | null;
|
|
545
1163
|
}
|
|
546
1164
|
|
|
1165
|
+
/**
|
|
1166
|
+
* One active touch contact reported by `TouchAPI`. Position, pressure, radius,
|
|
1167
|
+
* rotation, per-frame movement, velocity, and lifecycle flags. Coordinates are
|
|
1168
|
+
* in canvas-space pixels.
|
|
1169
|
+
*
|
|
1170
|
+
* Some fields (notably `pressure` / `force`, `radiusX` / `radiusY`,
|
|
1171
|
+
* `rotationAngle`) are passed through from the device — values vary by
|
|
1172
|
+
* platform; many devices report `0` when unsupported.
|
|
1173
|
+
*/
|
|
547
1174
|
interface TouchPoint {
|
|
1175
|
+
/** Stable touch identifier across frames for the duration of this contact. */
|
|
548
1176
|
id: number;
|
|
1177
|
+
/** Canvas-space X position in pixels. */
|
|
549
1178
|
x: number;
|
|
1179
|
+
/** Canvas-space Y position in pixels. */
|
|
550
1180
|
y: number;
|
|
1181
|
+
/** Touch pressure in 0..1 (device-dependent; often `0` on devices without force support). */
|
|
551
1182
|
pressure: number;
|
|
1183
|
+
/** Contact radius — `Math.max(radiusX, radiusY)` in pixels. */
|
|
552
1184
|
radius: number;
|
|
1185
|
+
/** Horizontal contact radius in pixels (raw device value). */
|
|
553
1186
|
radiusX: number;
|
|
1187
|
+
/** Vertical contact radius in pixels (raw device value). */
|
|
554
1188
|
radiusY: number;
|
|
1189
|
+
/** Contact area rotation in radians (raw device value). */
|
|
555
1190
|
rotationAngle: number;
|
|
1191
|
+
/** Touch force in 0..1 — alias of `pressure`. */
|
|
556
1192
|
force: number;
|
|
1193
|
+
/** `true` while the touch position is within the canvas bounds. */
|
|
557
1194
|
isInCanvas: boolean;
|
|
1195
|
+
/** Horizontal movement since the last frame in pixels. */
|
|
558
1196
|
deltaX: number;
|
|
1197
|
+
/** Vertical movement since the last frame in pixels. */
|
|
559
1198
|
deltaY: number;
|
|
1199
|
+
/** Movement velocity in pixels per second, computed by Viji. */
|
|
560
1200
|
velocity: {
|
|
1201
|
+
/** Horizontal velocity component in pixels per second. */
|
|
561
1202
|
x: number;
|
|
1203
|
+
/** Vertical velocity component in pixels per second. */
|
|
562
1204
|
y: number;
|
|
563
1205
|
};
|
|
1206
|
+
/** `true` for exactly one frame when this touch starts, then auto-resets. */
|
|
564
1207
|
isNew: boolean;
|
|
1208
|
+
/** `true` while the touch is ongoing. */
|
|
565
1209
|
isActive: boolean;
|
|
1210
|
+
/** `true` for exactly one frame when this touch ends, then auto-resets. */
|
|
566
1211
|
isEnding: boolean;
|
|
567
1212
|
}
|
|
568
1213
|
|
|
569
|
-
const VERSION = "0.
|
|
1214
|
+
const VERSION = "0.4.0";
|
|
570
1215
|
|
|
1216
|
+
/**
|
|
1217
|
+
* Real-time video API: drawable frame, dimensions, and computer-vision results
|
|
1218
|
+
* (faces, hands, pose, segmentation). Activate individual CV features through `cv.*`.
|
|
1219
|
+
*
|
|
1220
|
+
* When `isConnected` is `false`, `currentFrame` is `null`, all dimensions are `0`,
|
|
1221
|
+
* and CV result fields hold their empty defaults.
|
|
1222
|
+
*/
|
|
571
1223
|
interface VideoAPI {
|
|
1224
|
+
/** `true` when a video stream is connected. When `false`, all other fields hold their default empty values. */
|
|
572
1225
|
isConnected: boolean;
|
|
1226
|
+
/** Latest video frame, drawable directly with `ctx.drawImage()`. `null` when disconnected. */
|
|
573
1227
|
currentFrame: OffscreenCanvas | ImageBitmap | null;
|
|
1228
|
+
/** Source video frame width in pixels. `0` when disconnected. */
|
|
574
1229
|
frameWidth: number;
|
|
1230
|
+
/** Source video frame height in pixels. `0` when disconnected. */
|
|
575
1231
|
frameHeight: number;
|
|
1232
|
+
/** Source video frame rate in Hz (frames per second). `0` when disconnected. */
|
|
576
1233
|
frameRate: number;
|
|
1234
|
+
/**
|
|
1235
|
+
* Returns the latest frame as raw RGBA `ImageData` for per-pixel CPU analysis,
|
|
1236
|
+
* or `null` when disconnected. Slow compared to drawing `currentFrame` directly —
|
|
1237
|
+
* use only when you need to read individual pixel values.
|
|
1238
|
+
*/
|
|
577
1239
|
getFrameData: () => ImageData | null;
|
|
1240
|
+
/** Detected faces (empty array when face detection is off or no faces are visible). */
|
|
578
1241
|
faces: FaceData[];
|
|
1242
|
+
/** Detected hands (up to 2; empty array when hand tracking is off or no hands are visible). */
|
|
579
1243
|
hands: HandData[];
|
|
1244
|
+
/** Detected body pose, or `null` when pose detection is off or no pose is visible. */
|
|
580
1245
|
pose: PoseData | null;
|
|
1246
|
+
/** Body segmentation mask, or `null` when segmentation is off. */
|
|
581
1247
|
segmentation: SegmentationData | null;
|
|
1248
|
+
/**
|
|
1249
|
+
* Computer-vision feature control. Each `enable*` method toggles a specific CV
|
|
1250
|
+
* pipeline; results land in the corresponding `faces` / `hands` / `pose` /
|
|
1251
|
+
* `segmentation` field on the next available frame. Multiple features can be
|
|
1252
|
+
* active simultaneously, but each adds CPU/GPU and WebGL-context cost.
|
|
1253
|
+
*/
|
|
582
1254
|
cv: {
|
|
1255
|
+
/**
|
|
1256
|
+
* Toggle face detection (bounding box, center, confidence, id).
|
|
1257
|
+
* @returns Resolves once the underlying pipeline has finished switching.
|
|
1258
|
+
*/
|
|
583
1259
|
enableFaceDetection(enabled: boolean): Promise<void>;
|
|
1260
|
+
/**
|
|
1261
|
+
* Toggle 468-point face mesh + head pose (`pitch`, `yaw`, `roll`). Implies
|
|
1262
|
+
* face detection.
|
|
1263
|
+
* @returns Resolves once the pipeline has finished switching.
|
|
1264
|
+
*/
|
|
584
1265
|
enableFaceMesh(enabled: boolean): Promise<void>;
|
|
1266
|
+
/**
|
|
1267
|
+
* Toggle 7 expressions + 52 ARKit blendshape coefficients on each face.
|
|
1268
|
+
* Implies face mesh.
|
|
1269
|
+
* @returns Resolves once the pipeline has finished switching.
|
|
1270
|
+
*/
|
|
585
1271
|
enableEmotionDetection(enabled: boolean): Promise<void>;
|
|
1272
|
+
/**
|
|
1273
|
+
* Toggle 21-point hand landmarks + ML-classified gesture confidences for up
|
|
1274
|
+
* to two hands.
|
|
1275
|
+
* @returns Resolves once the pipeline has finished switching.
|
|
1276
|
+
*/
|
|
586
1277
|
enableHandTracking(enabled: boolean): Promise<void>;
|
|
1278
|
+
/**
|
|
1279
|
+
* Toggle 33-point BlazePose body landmarks (face / torso / arms / legs).
|
|
1280
|
+
* @returns Resolves once the pipeline has finished switching.
|
|
1281
|
+
*/
|
|
587
1282
|
enablePoseDetection(enabled: boolean): Promise<void>;
|
|
1283
|
+
/**
|
|
1284
|
+
* Toggle per-pixel person/background segmentation mask.
|
|
1285
|
+
* @returns Resolves once the pipeline has finished switching.
|
|
1286
|
+
*/
|
|
588
1287
|
enableBodySegmentation(enabled: boolean): Promise<void>;
|
|
1288
|
+
/** Returns the list of CV features currently enabled on this stream. */
|
|
589
1289
|
getActiveFeatures(): CVFeature[];
|
|
1290
|
+
/** Returns `true` if the CV worker is actively processing frames. */
|
|
590
1291
|
isProcessing(): boolean;
|
|
591
1292
|
};
|
|
592
1293
|
}
|
|
593
1294
|
|
|
1295
|
+
/**
|
|
1296
|
+
* The Viji API surface available to artist code as the global `viji` object.
|
|
1297
|
+
* Groups canvas access, timing, connected media APIs, user interaction, device
|
|
1298
|
+
* sensors, parameter helpers, and the `useContext()` selector.
|
|
1299
|
+
*/
|
|
594
1300
|
interface VijiAPI {
|
|
1301
|
+
/** The `OffscreenCanvas` driving the scene. The host owns its lifecycle and dimensions; prefer `useContext()` over touching this directly. */
|
|
595
1302
|
canvas: OffscreenCanvas;
|
|
1303
|
+
/** 2D rendering context. `undefined` until `viji.useContext('2d')` is called; afterwards equals the cached context. */
|
|
596
1304
|
ctx?: OffscreenCanvasRenderingContext2D;
|
|
1305
|
+
/** WebGL rendering context. `undefined` until `viji.useContext('webgl')` or `'webgl2'` is called; afterwards equals the cached context. */
|
|
597
1306
|
gl?: WebGLRenderingContext | WebGL2RenderingContext;
|
|
1307
|
+
/** Current canvas width in pixels. Updates automatically when the host resizes the canvas — read every frame. */
|
|
598
1308
|
width: number;
|
|
1309
|
+
/** Current canvas height in pixels. Updates automatically when the host resizes the canvas — read every frame. */
|
|
599
1310
|
height: number;
|
|
1311
|
+
/** Seconds elapsed since the scene started (monotonically increasing float). Use for oscillations and absolute-time effects. */
|
|
600
1312
|
time: number;
|
|
1313
|
+
/** Seconds since the previous frame. Use for accumulation: movement, physics, fades — anything that should progress per second regardless of FPS. */
|
|
601
1314
|
deltaTime: number;
|
|
1315
|
+
/** Integer frame counter starting at `0` and incrementing by `1` each frame. */
|
|
602
1316
|
frameCount: number;
|
|
1317
|
+
/** Target frames-per-second for the host's current frame-rate mode (`60` for `'full'`, `30` for `'half'`). */
|
|
603
1318
|
fps: number;
|
|
1319
|
+
/** Real-time analysis of the main audio stream: volume, frequency bands, beat detection, spectral features. Fields are zeroed when no audio source is connected. */
|
|
604
1320
|
audio: AudioAPI;
|
|
1321
|
+
/** Main video stream API: pixel access, frame metadata, and computer-vision results (face, hands, pose, segmentation). */
|
|
605
1322
|
video: VideoAPI;
|
|
1323
|
+
/** Additional video streams provided by the host. These do not run CV processing — use them for raw pixel access only. */
|
|
606
1324
|
videoStreams: VideoAPI[];
|
|
1325
|
+
/** Additional audio streams (lightweight analysis: volume, bands, spectral). No beat detection — use the main `audio` for that. */
|
|
607
1326
|
audioStreams: AudioStreamAPI[];
|
|
1327
|
+
/** Mouse position, buttons, wheel, and per-frame movement state. Coordinates are in canvas pixels. */
|
|
608
1328
|
mouse: MouseAPI;
|
|
1329
|
+
/** Keyboard state: per-frame press/release queries, currently held keys, and modifier flags. */
|
|
609
1330
|
keyboard: KeyboardAPI;
|
|
1331
|
+
/** Multi-touch state: count, primary touch shortcut, and the per-frame `started` / `moved` / `ended` arrays. */
|
|
610
1332
|
touches: TouchAPI;
|
|
1333
|
+
/** Unified pointer (mouse + primary touch) — convenient single-point input that works on both desktop and mobile. */
|
|
611
1334
|
pointer: PointerAPI;
|
|
1335
|
+
/** Internal device sensors (motion + orientation) reported by the device running the scene. */
|
|
612
1336
|
device: DeviceSensorState;
|
|
1337
|
+
/** External connected devices reporting sensor / camera / audio. Each entry has its own `id`, `name`, and per-device API surface. */
|
|
613
1338
|
devices: DeviceState[];
|
|
1339
|
+
/**
|
|
1340
|
+
* Declares a numeric slider parameter. The host UI renders a draggable slider.
|
|
1341
|
+
* Must be called at the top level of the scene — never inside `render()`.
|
|
1342
|
+
*
|
|
1343
|
+
* @example
|
|
1344
|
+
* const radius = viji.slider(50, { min: 10, max: 200, step: 1, label: 'Radius' });
|
|
1345
|
+
* // ...
|
|
1346
|
+
* function render(viji) { drawCircle(radius.value); }
|
|
1347
|
+
*/
|
|
614
1348
|
slider: (defaultValue: number, config: SliderConfig) => SliderParameter;
|
|
615
|
-
|
|
1349
|
+
/**
|
|
1350
|
+
* Declares a color parameter. Accepts hex strings, CSS color functions, or RGB / HSB objects;
|
|
1351
|
+
* the value is always normalized to a canonical lowercase hex string and exposes derived `.rgb` / `.hsb` accessors.
|
|
1352
|
+
* Must be called at the top level of the scene — never inside `render()`.
|
|
1353
|
+
*
|
|
1354
|
+
* @example
|
|
1355
|
+
* const tint = viji.color('#ff6600', { label: 'Tint' });
|
|
1356
|
+
* // tint.value -> '#ff6600', tint.rgb -> { r: 255, g: 102, b: 0 }, tint.hsb -> { h: 24, s: 100, b: 100 }
|
|
1357
|
+
*/
|
|
1358
|
+
color: (defaultValue: ColorInput, config: ColorConfig) => ColorParameter;
|
|
1359
|
+
/**
|
|
1360
|
+
* Declares a boolean toggle parameter. The host UI renders an on/off switch.
|
|
1361
|
+
* Must be called at the top level of the scene — never inside `render()`.
|
|
1362
|
+
*
|
|
1363
|
+
* @example
|
|
1364
|
+
* const showTrail = viji.toggle(true, { label: 'Show Trail' });
|
|
1365
|
+
*/
|
|
616
1366
|
toggle: (defaultValue: boolean, config: ToggleConfig) => ToggleParameter;
|
|
1367
|
+
/**
|
|
1368
|
+
* Declares a dropdown selection parameter. The element type of `options`
|
|
1369
|
+
* (`string` or `number`) determines the parameter's value type.
|
|
1370
|
+
* Must be called at the top level of the scene — never inside `render()`.
|
|
1371
|
+
*
|
|
1372
|
+
* @example
|
|
1373
|
+
* const shape = viji.select('circle', { options: ['circle', 'square', 'triangle'], label: 'Shape' });
|
|
1374
|
+
*/
|
|
617
1375
|
select: (defaultValue: string | number, config: SelectConfig) => SelectParameter;
|
|
1376
|
+
/**
|
|
1377
|
+
* Declares a text input parameter. The host UI renders a single-line text field.
|
|
1378
|
+
* Must be called at the top level of the scene — never inside `render()`.
|
|
1379
|
+
*
|
|
1380
|
+
* @example
|
|
1381
|
+
* const caption = viji.text('Hello', { label: 'Caption', maxLength: 64 });
|
|
1382
|
+
*/
|
|
618
1383
|
text: (defaultValue: string, config: TextConfig) => TextParameter;
|
|
1384
|
+
/**
|
|
1385
|
+
* Declares a precise numeric input parameter. Like `slider()` but the host UI
|
|
1386
|
+
* shows a number field instead of a draggable handle.
|
|
1387
|
+
* Must be called at the top level of the scene — never inside `render()`.
|
|
1388
|
+
*
|
|
1389
|
+
* @example
|
|
1390
|
+
* const count = viji.number(12, { min: 1, max: 64, step: 1, label: 'Count' });
|
|
1391
|
+
*/
|
|
619
1392
|
number: (defaultValue: number, config: NumberConfig) => NumberParameter;
|
|
1393
|
+
/**
|
|
1394
|
+
* Declares an image upload parameter. The user-supplied image becomes available
|
|
1395
|
+
* as `value` (an `ImageBitmap`); in P5 scenes a `.p5` accessor is also added at runtime.
|
|
1396
|
+
* Must be called at the top level of the scene — never inside `render()`.
|
|
1397
|
+
*
|
|
1398
|
+
* @example
|
|
1399
|
+
* const tex = viji.image(null, { label: 'Texture' });
|
|
1400
|
+
* if (tex.value) ctx.drawImage(tex.value, 0, 0, viji.width, viji.height);
|
|
1401
|
+
*/
|
|
620
1402
|
image: (defaultValue: null, config: ImageConfig) => ImageParameter;
|
|
1403
|
+
/**
|
|
1404
|
+
* Declares a momentary button parameter. `value` is `true` for one frame after
|
|
1405
|
+
* the user clicks, then auto-resets — perfect for triggering discrete events.
|
|
1406
|
+
* Must be called at the top level of the scene — never inside `render()`.
|
|
1407
|
+
*
|
|
1408
|
+
* @example
|
|
1409
|
+
* const reset = viji.button({ label: 'Reset' });
|
|
1410
|
+
* function render(viji) { if (reset.value) state = initialState; }
|
|
1411
|
+
*/
|
|
621
1412
|
button: (config: ButtonConfig) => ButtonParameter;
|
|
1413
|
+
/**
|
|
1414
|
+
* Declares a 2D coordinate parameter. Both `value.x` and `value.y` are in `-1..1`.
|
|
1415
|
+
* Must be called at the top level of the scene — never inside `render()`.
|
|
1416
|
+
*
|
|
1417
|
+
* @example
|
|
1418
|
+
* const origin = viji.coordinate({ x: 0, y: 0 }, { label: 'Origin' });
|
|
1419
|
+
* const cx = viji.width / 2 + origin.value.x * viji.width * 0.4;
|
|
1420
|
+
*/
|
|
622
1421
|
coordinate: (defaultValue: CoordinateValue, config: CoordinateConfig) => CoordinateParameter;
|
|
1422
|
+
/**
|
|
1423
|
+
* Selects the 2D canvas rendering context. Returns the same instance on
|
|
1424
|
+
* subsequent calls and also stores it on `viji.ctx`. A canvas only supports
|
|
1425
|
+
* one context type — if a different context type was already requested,
|
|
1426
|
+
* the call returns `null`. Choose ONE type and use it for the entire scene.
|
|
1427
|
+
*
|
|
1428
|
+
* @example
|
|
1429
|
+
* const ctx = viji.useContext('2d');
|
|
1430
|
+
* ctx.fillRect(0, 0, viji.width, viji.height);
|
|
1431
|
+
*/
|
|
623
1432
|
useContext(type: '2d'): OffscreenCanvasRenderingContext2D;
|
|
1433
|
+
/**
|
|
1434
|
+
* Selects a WebGL 1 rendering context. Returns the same instance on subsequent
|
|
1435
|
+
* calls and also stores it on `viji.gl`. A canvas only supports one context
|
|
1436
|
+
* type — if a different context type was already requested, the call returns
|
|
1437
|
+
* `null`. Choose ONE type and use it for the entire scene.
|
|
1438
|
+
*/
|
|
624
1439
|
useContext(type: 'webgl'): WebGLRenderingContext;
|
|
1440
|
+
/**
|
|
1441
|
+
* Selects a WebGL 2 rendering context. Returns the same instance on subsequent
|
|
1442
|
+
* calls and also stores it on `viji.gl`. A canvas only supports one context
|
|
1443
|
+
* type — if a different context type was already requested, the call returns
|
|
1444
|
+
* `null`. Choose ONE type and use it for the entire scene.
|
|
1445
|
+
*/
|
|
625
1446
|
useContext(type: 'webgl2'): WebGL2RenderingContext;
|
|
626
1447
|
}
|
|
627
1448
|
|