@viji-dev/core 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/artist-dts-p5.js +1 -1
- package/dist/artist-dts.js +1 -1
- package/dist/artist-global-p5.d.ts +835 -14
- package/dist/artist-global.d.ts +835 -14
- package/dist/artist-jsdoc.d.ts +449 -441
- package/dist/assets/{viji.worker-BoI8e3NI.js → viji.worker-4gGFik2A.js} +344 -27
- package/dist/assets/viji.worker-4gGFik2A.js.map +1 -0
- package/dist/docs-api.js +137 -72
- package/dist/{essentia-wasm.web-CdUmKTbm.js → essentia-wasm.web-CCpzl5zi.js} +2 -2
- package/dist/{essentia-wasm.web-CdUmKTbm.js.map → essentia-wasm.web-CCpzl5zi.js.map} +1 -1
- package/dist/{index-DZzYWg7c.js → index-G2N9Tgtd.js} +303 -22
- package/dist/index-G2N9Tgtd.js.map +1 -0
- package/dist/index.d.ts +839 -20
- package/dist/index.js +1 -1
- package/package.json +1 -1
- package/dist/assets/viji.worker-BoI8e3NI.js.map +0 -1
- package/dist/index-DZzYWg7c.js.map +0 -1
|
@@ -4,180 +4,412 @@
|
|
|
4
4
|
// would otherwise be module-scoped, not global).
|
|
5
5
|
|
|
6
6
|
declare global {
|
|
7
|
+
/**
|
|
8
|
+
* Real-time audio analysis for the main audio stream (`viji.audio`).
|
|
9
|
+
* Provides volume, frequency bands, beat detection (with triggers, events, BPM),
|
|
10
|
+
* spectral features, and access to raw FFT / waveform data.
|
|
11
|
+
*
|
|
12
|
+
* All numeric outputs are zeroed when `isConnected` is `false`.
|
|
13
|
+
*/
|
|
7
14
|
interface AudioAPI {
|
|
15
|
+
/** `true` when an audio source is connected; otherwise all numeric fields read as `0` and helper methods return empty data. */
|
|
8
16
|
isConnected: boolean;
|
|
17
|
+
/** Overall loudness across the stream. */
|
|
9
18
|
volume: {
|
|
19
|
+
/** Instantaneous RMS volume in 0..1. */
|
|
10
20
|
current: number;
|
|
21
|
+
/** Instantaneous peak amplitude in 0..1. */
|
|
11
22
|
peak: number;
|
|
23
|
+
/** Volume with a 200ms decay envelope; rises fast, falls smoothly. Use for flicker-free animations. */
|
|
12
24
|
smoothed: number;
|
|
13
25
|
};
|
|
26
|
+
/** Energy in five fixed frequency bands, plus a smoothed companion (~150ms decay) for each. All in 0..1. */
|
|
14
27
|
bands: {
|
|
28
|
+
/** Instant low-band energy (20–120 Hz, bass/kick range). 0..1. */
|
|
15
29
|
low: number;
|
|
30
|
+
/** Instant low-mid energy (120–400 Hz). 0..1. */
|
|
16
31
|
lowMid: number;
|
|
32
|
+
/** Instant mid energy (400–1600 Hz, vocals/instruments). 0..1. */
|
|
17
33
|
mid: number;
|
|
34
|
+
/** Instant high-mid energy (1600–6000 Hz, cymbals/hi-hats). 0..1. */
|
|
18
35
|
highMid: number;
|
|
36
|
+
/** Instant high-band energy (6000–16000 Hz, air/brilliance). 0..1. */
|
|
19
37
|
high: number;
|
|
38
|
+
/** Smoothed `low` (~150ms decay). 0..1. */
|
|
20
39
|
lowSmoothed: number;
|
|
40
|
+
/** Smoothed `lowMid` (~150ms decay). 0..1. */
|
|
21
41
|
lowMidSmoothed: number;
|
|
42
|
+
/** Smoothed `mid` (~150ms decay). 0..1. */
|
|
22
43
|
midSmoothed: number;
|
|
44
|
+
/** Smoothed `highMid` (~150ms decay). 0..1. */
|
|
23
45
|
highMidSmoothed: number;
|
|
46
|
+
/** Smoothed `high` (~150ms decay). 0..1. */
|
|
24
47
|
highSmoothed: number;
|
|
25
48
|
};
|
|
49
|
+
/** Beat detection: fast/smoothed energy curves, single-frame boolean triggers, raw event list, and BPM tracking. */
|
|
26
50
|
beat: {
|
|
51
|
+
/** Kick energy curve in 0..1 with a 300ms decay. Peaks on each detected kick. */
|
|
27
52
|
kick: number;
|
|
53
|
+
/** Snare energy curve in 0..1 with a 300ms decay. */
|
|
28
54
|
snare: number;
|
|
55
|
+
/** Hi-hat energy curve in 0..1 with a 300ms decay. */
|
|
29
56
|
hat: number;
|
|
57
|
+
/** Energy curve for any beat type in 0..1 with a 300ms decay. */
|
|
30
58
|
any: number;
|
|
59
|
+
/** Smoothed `kick` curve in 0..1 with a 500ms decay. Use for ambient pulses. */
|
|
31
60
|
kickSmoothed: number;
|
|
61
|
+
/** Smoothed `snare` curve in 0..1 with a 500ms decay. */
|
|
32
62
|
snareSmoothed: number;
|
|
63
|
+
/** Smoothed `hat` curve in 0..1 with a 500ms decay. */
|
|
33
64
|
hatSmoothed: number;
|
|
65
|
+
/** Smoothed `any` curve in 0..1 with a 500ms decay. */
|
|
34
66
|
anySmoothed: number;
|
|
67
|
+
/** Single-frame boolean triggers OR-accumulated between frames, so no detected beat is lost. */
|
|
35
68
|
triggers: {
|
|
69
|
+
/** `true` for one frame when any beat is detected. */
|
|
36
70
|
any: boolean;
|
|
71
|
+
/** `true` for one frame when a kick is detected. */
|
|
37
72
|
kick: boolean;
|
|
73
|
+
/** `true` for one frame when a snare is detected. */
|
|
38
74
|
snare: boolean;
|
|
75
|
+
/** `true` for one frame when a hi-hat is detected. */
|
|
39
76
|
hat: boolean;
|
|
40
77
|
};
|
|
78
|
+
/** All beats detected since the last render frame (zero or more). Cleared each frame. */
|
|
41
79
|
events: Array<{
|
|
80
|
+
/** Which drum was detected. */
|
|
42
81
|
type: 'kick' | 'snare' | 'hat';
|
|
82
|
+
/** Timestamp of the detection in milliseconds. */
|
|
43
83
|
time: number;
|
|
84
|
+
/** Strength of the beat in 0..1. */
|
|
44
85
|
strength: number;
|
|
45
86
|
}>;
|
|
87
|
+
/** Currently detected tempo in beats per minute. Defaults to `120` when no audio. */
|
|
46
88
|
bpm: number;
|
|
89
|
+
/** Confidence of the BPM tracker in 0..1. */
|
|
47
90
|
confidence: number;
|
|
91
|
+
/** `true` when the BPM tracker has a stable lock on the current tempo. */
|
|
48
92
|
isLocked: boolean;
|
|
49
93
|
};
|
|
94
|
+
/** High-level spectral features derived from the FFT. */
|
|
50
95
|
spectral: {
|
|
96
|
+
/** Normalized spectral centroid in 0..1 — higher values mean brighter, more treble-heavy sound. */
|
|
51
97
|
brightness: number;
|
|
98
|
+
/** Normalized spectral flatness in 0..1 — higher values mean noisier (white-noise-like) sound; lower values mean tonal. */
|
|
52
99
|
flatness: number;
|
|
53
100
|
};
|
|
101
|
+
/**
|
|
102
|
+
* Returns the raw FFT magnitude spectrum as a `Uint8Array` (1024 bins, each 0..255).
|
|
103
|
+
* Bin `i` covers frequency `i × (sampleRate / fftSize)`. The returned array is a
|
|
104
|
+
* snapshot from the latest analysis tick — repeated calls in the same frame return the same data.
|
|
105
|
+
*
|
|
106
|
+
* @example
|
|
107
|
+
* const fft = viji.audio.getFrequencyData();
|
|
108
|
+
* for (let i = 0; i < fft.length; i++) drawBar(i, fft[i] / 255);
|
|
109
|
+
*/
|
|
54
110
|
getFrequencyData: () => Uint8Array;
|
|
111
|
+
/**
|
|
112
|
+
* Returns the raw time-domain waveform as a `Float32Array` (2048 PCM samples in -1..1).
|
|
113
|
+
* The returned array is a snapshot — repeated calls in the same frame return the same data.
|
|
114
|
+
*
|
|
115
|
+
* @example
|
|
116
|
+
* const wave = viji.audio.getWaveform();
|
|
117
|
+
* for (let i = 0; i < wave.length; i++) drawSample(i, wave[i]);
|
|
118
|
+
*/
|
|
55
119
|
getWaveform: () => Float32Array;
|
|
56
120
|
}
|
|
57
121
|
|
|
122
|
+
/**
|
|
123
|
+
* Lightweight audio analysis for additional or device audio streams. A strict
|
|
124
|
+
* subset of `AudioAPI` — exposes volume, frequency bands, spectral features,
|
|
125
|
+
* and raw FFT/waveform access, but **no beat detection** (no `beat`, no BPM,
|
|
126
|
+
* no triggers, no events).
|
|
127
|
+
*/
|
|
58
128
|
interface AudioStreamAPI {
|
|
129
|
+
/** `true` when this audio stream is connected; otherwise all numeric fields read as `0`. */
|
|
59
130
|
isConnected: boolean;
|
|
131
|
+
/** Overall loudness across the stream. */
|
|
60
132
|
volume: {
|
|
133
|
+
/** Instantaneous RMS volume in 0..1. */
|
|
61
134
|
current: number;
|
|
135
|
+
/** Instantaneous peak amplitude in 0..1. */
|
|
62
136
|
peak: number;
|
|
137
|
+
/** Volume with a 200ms decay envelope. */
|
|
63
138
|
smoothed: number;
|
|
64
139
|
};
|
|
140
|
+
/** Energy in five fixed frequency bands, plus a smoothed companion (~150ms decay) for each. All in 0..1. */
|
|
65
141
|
bands: {
|
|
142
|
+
/** Instant low-band energy (20–120 Hz). 0..1. */
|
|
66
143
|
low: number;
|
|
144
|
+
/** Instant low-mid energy (120–400 Hz). 0..1. */
|
|
67
145
|
lowMid: number;
|
|
146
|
+
/** Instant mid energy (400–1600 Hz). 0..1. */
|
|
68
147
|
mid: number;
|
|
148
|
+
/** Instant high-mid energy (1600–6000 Hz). 0..1. */
|
|
69
149
|
highMid: number;
|
|
150
|
+
/** Instant high-band energy (6000–16000 Hz). 0..1. */
|
|
70
151
|
high: number;
|
|
152
|
+
/** Smoothed `low` (~150ms decay). 0..1. */
|
|
71
153
|
lowSmoothed: number;
|
|
154
|
+
/** Smoothed `lowMid` (~150ms decay). 0..1. */
|
|
72
155
|
lowMidSmoothed: number;
|
|
156
|
+
/** Smoothed `mid` (~150ms decay). 0..1. */
|
|
73
157
|
midSmoothed: number;
|
|
158
|
+
/** Smoothed `highMid` (~150ms decay). 0..1. */
|
|
74
159
|
highMidSmoothed: number;
|
|
160
|
+
/** Smoothed `high` (~150ms decay). 0..1. */
|
|
75
161
|
highSmoothed: number;
|
|
76
162
|
};
|
|
163
|
+
/** High-level spectral features derived from the FFT. */
|
|
77
164
|
spectral: {
|
|
165
|
+
/** Normalized spectral centroid in 0..1. */
|
|
78
166
|
brightness: number;
|
|
167
|
+
/** Normalized spectral flatness in 0..1. */
|
|
79
168
|
flatness: number;
|
|
80
169
|
};
|
|
170
|
+
/**
|
|
171
|
+
* Returns the raw FFT magnitude spectrum for this stream as a `Uint8Array`
|
|
172
|
+
* (1024 bins, each 0..255). Snapshot semantics — see `AudioAPI.getFrequencyData`.
|
|
173
|
+
*/
|
|
81
174
|
getFrequencyData: () => Uint8Array;
|
|
175
|
+
/**
|
|
176
|
+
* Returns the raw time-domain waveform for this stream as a `Float32Array`
|
|
177
|
+
* (2048 samples, each -1..1). Snapshot semantics — see `AudioAPI.getWaveform`.
|
|
178
|
+
*/
|
|
82
179
|
getWaveform: () => Float32Array;
|
|
83
180
|
}
|
|
84
181
|
|
|
182
|
+
/**
|
|
183
|
+
* Configuration object passed to `viji.button()`. The host renders a clickable
|
|
184
|
+
* momentary button — the resulting `value` is `true` for one frame after press,
|
|
185
|
+
* then auto-resets to `false`.
|
|
186
|
+
*/
|
|
85
187
|
interface ButtonConfig {
|
|
188
|
+
/** Display name shown on the button in the parameter UI. Required. */
|
|
86
189
|
label: string;
|
|
190
|
+
/** Optional tooltip / help text shown next to the control. */
|
|
87
191
|
description?: string;
|
|
192
|
+
/** Group name for organizing related parameters under a shared heading. Default: `'general'`. */
|
|
88
193
|
group?: string;
|
|
194
|
+
/** Visibility category — the control hides when its capability is unavailable. Default: `'general'`. */
|
|
89
195
|
category?: ParameterCategory;
|
|
90
196
|
}
|
|
91
197
|
|
|
198
|
+
/**
|
|
199
|
+
* Reactive object returned by `viji.button()`. `value` is a momentary flag —
|
|
200
|
+
* `true` for exactly one frame after the user clicks, then auto-resets to
|
|
201
|
+
* `false`. Use it to trigger discrete events from `render()`.
|
|
202
|
+
*/
|
|
92
203
|
interface ButtonParameter {
|
|
204
|
+
/** `true` for the single frame after the user clicks the button, `false` otherwise. */
|
|
93
205
|
value: boolean;
|
|
206
|
+
/** Display name shown on the button in the parameter UI. */
|
|
94
207
|
label: string;
|
|
208
|
+
/** Tooltip / help text for the control (empty string when not configured). */
|
|
95
209
|
description?: string;
|
|
210
|
+
/** Group name under which the control appears in the UI. */
|
|
96
211
|
group: string;
|
|
212
|
+
/** Visibility category — controls when the parameter is shown. */
|
|
97
213
|
category: ParameterCategory;
|
|
98
214
|
}
|
|
99
215
|
|
|
216
|
+
/**
|
|
217
|
+
* Options accepted by `VijiCore.captureFrame()` for snapshotting the current
|
|
218
|
+
* scene. Controls output format, encoding, and target resolution / aspect.
|
|
219
|
+
*/
|
|
100
220
|
interface CaptureFrameOptions {
|
|
101
|
-
/** Output format: 'blob' for encoded image, 'bitmap' for GPU-friendly ImageBitmap */
|
|
221
|
+
/** Output format: `'blob'` for an encoded image, `'bitmap'` for a GPU-friendly `ImageBitmap`. Default: `'blob'`. */
|
|
102
222
|
format?: 'blob' | 'bitmap';
|
|
103
|
-
/** MIME type for blob output (ignored for bitmap)
|
|
223
|
+
/** MIME type for `'blob'` output (ignored for `'bitmap'`). Examples: `'image/png'`, `'image/jpeg'`, `'image/webp'`. */
|
|
104
224
|
type?: string;
|
|
105
225
|
/**
|
|
106
226
|
* Target resolution.
|
|
107
|
-
* - number
|
|
108
|
-
* -
|
|
109
|
-
* the source is center-cropped
|
|
227
|
+
* - `number`: scale factor relative to the current canvas size (e.g., `0.5` = 50%).
|
|
228
|
+
* - `Resolution`: exact output size. If the aspect ratio differs from the
|
|
229
|
+
* canvas, the source is center-cropped before scaling.
|
|
110
230
|
*/
|
|
111
|
-
resolution?: number |
|
|
112
|
-
width: number;
|
|
113
|
-
height: number;
|
|
114
|
-
};
|
|
231
|
+
resolution?: number | Resolution;
|
|
115
232
|
}
|
|
116
233
|
|
|
234
|
+
/**
|
|
235
|
+
* Configuration object passed to `viji.color()`. Defines the color picker's
|
|
236
|
+
* label and UI grouping/visibility.
|
|
237
|
+
*/
|
|
117
238
|
interface ColorConfig {
|
|
239
|
+
/** Display name shown next to the color swatch in the parameter UI. Required. */
|
|
118
240
|
label: string;
|
|
241
|
+
/** Optional tooltip / help text shown next to the control. */
|
|
119
242
|
description?: string;
|
|
243
|
+
/** Group name for organizing related parameters under a shared heading. Default: `'general'`. */
|
|
120
244
|
group?: string;
|
|
245
|
+
/** Visibility category — the control hides when its capability is unavailable. Default: `'general'`. */
|
|
121
246
|
category?: ParameterCategory;
|
|
122
247
|
}
|
|
123
248
|
|
|
249
|
+
/**
|
|
250
|
+
* Accepted input forms for color parameters. Internally normalized to canonical
|
|
251
|
+
* lowercase 6-digit hex `#rrggbb`. See `viji.color()` for full documentation.
|
|
252
|
+
*
|
|
253
|
+
* - `string`: hex (`#rrggbb`, `#rgb`), CSS `rgb(r, g, b)`, CSS `hsl(h, s%, l%)`,
|
|
254
|
+
* GLSL-style `vec3(r, g, b)` (0..1), or P5-style `hsb(h, s, b)` (0..360 / 0..100 / 0..100)
|
|
255
|
+
* - `{ r, g, b }`: RGB object with components in 0..255
|
|
256
|
+
* - `{ h, s, b }`: HSB object with `h` in 0..360, `s` and `b` in 0..100
|
|
257
|
+
*/
|
|
258
|
+
type ColorInput = string | {
|
|
259
|
+
/** Red component in 0..255. */
|
|
260
|
+
r: number;
|
|
261
|
+
/** Green component in 0..255. */
|
|
262
|
+
g: number;
|
|
263
|
+
/** Blue component in 0..255. */
|
|
264
|
+
b: number;
|
|
265
|
+
} | {
|
|
266
|
+
/** Hue in 0..360. */
|
|
267
|
+
h: number;
|
|
268
|
+
/** Saturation in 0..100. */
|
|
269
|
+
s: number;
|
|
270
|
+
/** Brightness in 0..100. */
|
|
271
|
+
b: number;
|
|
272
|
+
};
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* Reactive object returned by `viji.color()`. `value` is always a canonical
|
|
276
|
+
* lowercase 6-digit hex string; `rgb` and `hsb` are derived accessors that are
|
|
277
|
+
* recomputed (and frozen) every time the user changes the color.
|
|
278
|
+
*/
|
|
124
279
|
interface ColorParameter {
|
|
280
|
+
/** Canonical hex color string `#rrggbb` (lowercase). Updates in real-time. */
|
|
125
281
|
value: string;
|
|
282
|
+
/** RGB components as integers in 0..255 (frozen, recomputed when value changes). */
|
|
283
|
+
rgb: {
|
|
284
|
+
/** Red component, integer in 0..255. */
|
|
285
|
+
r: number;
|
|
286
|
+
/** Green component, integer in 0..255. */
|
|
287
|
+
g: number;
|
|
288
|
+
/** Blue component, integer in 0..255. */
|
|
289
|
+
b: number;
|
|
290
|
+
};
|
|
291
|
+
/** HSB components: `h` in 0..360, `s` and `b` in 0..100 (frozen, recomputed when value changes). */
|
|
292
|
+
hsb: {
|
|
293
|
+
/** Hue in 0..360. */
|
|
294
|
+
h: number;
|
|
295
|
+
/** Saturation in 0..100. */
|
|
296
|
+
s: number;
|
|
297
|
+
/** Brightness in 0..100. */
|
|
298
|
+
b: number;
|
|
299
|
+
};
|
|
300
|
+
/** Display name shown next to the swatch in the parameter UI. */
|
|
126
301
|
label: string;
|
|
302
|
+
/** Tooltip / help text for the control (empty string when not configured). */
|
|
127
303
|
description?: string;
|
|
304
|
+
/** Group name under which the control appears in the UI. */
|
|
128
305
|
group: string;
|
|
306
|
+
/** Visibility category — controls when the parameter is shown. */
|
|
129
307
|
category: ParameterCategory;
|
|
130
308
|
}
|
|
131
309
|
|
|
310
|
+
/**
|
|
311
|
+
* Configuration object passed to `viji.coordinate()`. The host renders a 2D
|
|
312
|
+
* draggable pad; both `x` and `y` are clamped to `-1..1`.
|
|
313
|
+
*/
|
|
132
314
|
interface CoordinateConfig {
|
|
315
|
+
/** Snap increment for both `x` and `y` components. Default: `0.01`. */
|
|
133
316
|
step?: number;
|
|
317
|
+
/** Display name shown next to the pad in the parameter UI. Required. */
|
|
134
318
|
label: string;
|
|
319
|
+
/** Optional tooltip / help text shown next to the control. */
|
|
135
320
|
description?: string;
|
|
321
|
+
/** Group name for organizing related parameters under a shared heading. Default: `'general'`. */
|
|
136
322
|
group?: string;
|
|
323
|
+
/** Visibility category — the control hides when its capability is unavailable. Default: `'general'`. */
|
|
137
324
|
category?: ParameterCategory;
|
|
138
325
|
}
|
|
139
326
|
|
|
327
|
+
/**
|
|
328
|
+
* Reactive object returned by `viji.coordinate()`. `value.x` and `value.y` are
|
|
329
|
+
* always in `-1..1`; the host UI renders a draggable 2D pad.
|
|
330
|
+
*/
|
|
140
331
|
interface CoordinateParameter {
|
|
332
|
+
/** Current `{ x, y }` position. Both components are in `-1..1`. Updates in real-time. */
|
|
141
333
|
value: CoordinateValue;
|
|
334
|
+
/** Snap increment for both `x` and `y` components. */
|
|
142
335
|
step: number;
|
|
336
|
+
/** Display name shown next to the pad in the parameter UI. */
|
|
143
337
|
label: string;
|
|
338
|
+
/** Tooltip / help text for the control (empty string when not configured). */
|
|
144
339
|
description?: string;
|
|
340
|
+
/** Group name under which the control appears in the UI. */
|
|
145
341
|
group: string;
|
|
342
|
+
/** Visibility category — controls when the parameter is shown. */
|
|
146
343
|
category: ParameterCategory;
|
|
147
344
|
}
|
|
148
345
|
|
|
346
|
+
/**
|
|
347
|
+
* Two-component coordinate, both axes in `-1..1` (centered on the canvas).
|
|
348
|
+
* Returned and consumed by `viji.coordinate()`.
|
|
349
|
+
*/
|
|
149
350
|
type CoordinateValue = {
|
|
351
|
+
/** Horizontal axis in `-1..1` (left to right, `0` = canvas center). */
|
|
150
352
|
x: number;
|
|
353
|
+
/** Vertical axis in `-1..1` (bottom to top, `0` = canvas center). */
|
|
151
354
|
y: number;
|
|
152
355
|
};
|
|
153
356
|
|
|
357
|
+
/**
|
|
358
|
+
* Identifier for an individual computer-vision feature pipeline. Returned by
|
|
359
|
+
* `VideoAPI.cv.getActiveFeatures()` and used internally to track state.
|
|
360
|
+
*/
|
|
154
361
|
type CVFeature = 'faceDetection' | 'faceMesh' | 'handTracking' | 'poseDetection' | 'bodySegmentation' | 'emotionDetection';
|
|
155
362
|
|
|
363
|
+
/**
|
|
364
|
+
* Computer-vision processing rate relative to the scene's render rate.
|
|
365
|
+
* `'full'` runs CV every frame, `'half'` every other frame, `'quarter'` every
|
|
366
|
+
* fourth, `'eighth'` every eighth — lower rates reduce CPU/GPU cost.
|
|
367
|
+
*/
|
|
156
368
|
type CVFrameRateMode = 'full' | 'half' | 'quarter' | 'eighth';
|
|
157
369
|
|
|
370
|
+
/**
|
|
371
|
+
* Device motion data sourced from the platform's `DeviceMotionEvent`. All
|
|
372
|
+
* accelerations are in metres per second squared (m/s²); rotation rates are
|
|
373
|
+
* in degrees per second. Inner objects may be `null` when the underlying
|
|
374
|
+
* sensor is unavailable, and individual axes may be `null` on platforms that
|
|
375
|
+
* report only some components.
|
|
376
|
+
*/
|
|
158
377
|
interface DeviceMotionData {
|
|
159
|
-
/** Acceleration without gravity (m/s²) */
|
|
378
|
+
/** Acceleration without gravity (m/s²). `null` when no accelerometer is available. */
|
|
160
379
|
acceleration: {
|
|
380
|
+
/** Acceleration along the device's X axis in m/s². `null` if unsupported. */
|
|
161
381
|
x: number | null;
|
|
382
|
+
/** Acceleration along the device's Y axis in m/s². `null` if unsupported. */
|
|
162
383
|
y: number | null;
|
|
384
|
+
/** Acceleration along the device's Z axis in m/s². `null` if unsupported. */
|
|
163
385
|
z: number | null;
|
|
164
386
|
} | null;
|
|
165
|
-
/** Acceleration including gravity (m/s²) */
|
|
387
|
+
/** Acceleration including gravity (m/s²). `null` when no accelerometer is available. */
|
|
166
388
|
accelerationIncludingGravity: {
|
|
389
|
+
/** Acceleration including gravity along the device's X axis in m/s². `null` if unsupported. */
|
|
167
390
|
x: number | null;
|
|
391
|
+
/** Acceleration including gravity along the device's Y axis in m/s². `null` if unsupported. */
|
|
168
392
|
y: number | null;
|
|
393
|
+
/** Acceleration including gravity along the device's Z axis in m/s². `null` if unsupported. */
|
|
169
394
|
z: number | null;
|
|
170
395
|
} | null;
|
|
171
|
-
/**
|
|
396
|
+
/** Angular rotation rate (degrees per second). `null` when no gyroscope is available. */
|
|
172
397
|
rotationRate: {
|
|
398
|
+
/** Rotation rate around the device's Z axis in deg/s. `null` if unsupported. */
|
|
173
399
|
alpha: number | null;
|
|
400
|
+
/** Rotation rate around the device's X axis in deg/s. `null` if unsupported. */
|
|
174
401
|
beta: number | null;
|
|
402
|
+
/** Rotation rate around the device's Y axis in deg/s. `null` if unsupported. */
|
|
175
403
|
gamma: number | null;
|
|
176
404
|
} | null;
|
|
177
|
-
/** Interval between updates
|
|
405
|
+
/** Interval between sensor updates in milliseconds. */
|
|
178
406
|
interval: number;
|
|
179
407
|
}
|
|
180
408
|
|
|
409
|
+
/**
|
|
410
|
+
* DeviceOrientationEvent data
|
|
411
|
+
* Matches native DeviceOrientationEvent structure
|
|
412
|
+
*/
|
|
181
413
|
interface DeviceOrientationData {
|
|
182
414
|
/** Rotation around Z-axis (0-360 degrees, compass heading) */
|
|
183
415
|
alpha: number | null;
|
|
@@ -189,11 +421,20 @@ declare global {
|
|
|
189
421
|
absolute: boolean;
|
|
190
422
|
}
|
|
191
423
|
|
|
424
|
+
/**
|
|
425
|
+
* Sensor snapshot for the device running the scene. Both fields read `null`
|
|
426
|
+
* when the corresponding sensor is unavailable or has not delivered a sample yet.
|
|
427
|
+
*/
|
|
192
428
|
interface DeviceSensorState {
|
|
429
|
+
/** Motion data (acceleration, rotation rate, sampling interval). `null` if no motion sensor is available. */
|
|
193
430
|
motion: DeviceMotionData | null;
|
|
431
|
+
/** Orientation data (alpha/beta/gamma in degrees). `null` if no orientation sensor is available. */
|
|
194
432
|
orientation: DeviceOrientationData | null;
|
|
195
433
|
}
|
|
196
434
|
|
|
435
|
+
/**
|
|
436
|
+
* External device state (includes id and name)
|
|
437
|
+
*/
|
|
197
438
|
interface DeviceState extends DeviceSensorState {
|
|
198
439
|
/** Unique device identifier */
|
|
199
440
|
id: string;
|
|
@@ -205,149 +446,302 @@ declare global {
|
|
|
205
446
|
audio: AudioStreamAPI | null;
|
|
206
447
|
}
|
|
207
448
|
|
|
449
|
+
/**
|
|
450
|
+
* 52 ARKit-compatible face blendshape coefficients (each in 0..1) derived from
|
|
451
|
+
* MediaPipe FaceLandmarker. All fields are `0` unless emotion detection is
|
|
452
|
+
* enabled. Available on each `FaceData.blendshapes`.
|
|
453
|
+
*/
|
|
208
454
|
interface FaceBlendshapes {
|
|
455
|
+
/** Coefficient (0..1) for the inner brow of the left eye pulling down. */
|
|
209
456
|
browDownLeft: number;
|
|
457
|
+
/** Coefficient (0..1) for the inner brow of the right eye pulling down. */
|
|
210
458
|
browDownRight: number;
|
|
459
|
+
/** Coefficient (0..1) for both inner brows pulling up. */
|
|
211
460
|
browInnerUp: number;
|
|
461
|
+
/** Coefficient (0..1) for the outer left brow pulling up. */
|
|
212
462
|
browOuterUpLeft: number;
|
|
463
|
+
/** Coefficient (0..1) for the outer right brow pulling up. */
|
|
213
464
|
browOuterUpRight: number;
|
|
465
|
+
/** Coefficient (0..1) for puffed cheeks. */
|
|
214
466
|
cheekPuff: number;
|
|
467
|
+
/** Coefficient (0..1) for the left cheek squinting. */
|
|
215
468
|
cheekSquintLeft: number;
|
|
469
|
+
/** Coefficient (0..1) for the right cheek squinting. */
|
|
216
470
|
cheekSquintRight: number;
|
|
471
|
+
/** Coefficient (0..1) for the left eyelid closing. */
|
|
217
472
|
eyeBlinkLeft: number;
|
|
473
|
+
/** Coefficient (0..1) for the right eyelid closing. */
|
|
218
474
|
eyeBlinkRight: number;
|
|
475
|
+
/** Coefficient (0..1) for the left eye looking down. */
|
|
219
476
|
eyeLookDownLeft: number;
|
|
477
|
+
/** Coefficient (0..1) for the right eye looking down. */
|
|
220
478
|
eyeLookDownRight: number;
|
|
479
|
+
/** Coefficient (0..1) for the left eye looking inward (toward the nose). */
|
|
221
480
|
eyeLookInLeft: number;
|
|
481
|
+
/** Coefficient (0..1) for the right eye looking inward. */
|
|
222
482
|
eyeLookInRight: number;
|
|
483
|
+
/** Coefficient (0..1) for the left eye looking outward (away from the nose). */
|
|
223
484
|
eyeLookOutLeft: number;
|
|
485
|
+
/** Coefficient (0..1) for the right eye looking outward. */
|
|
224
486
|
eyeLookOutRight: number;
|
|
487
|
+
/** Coefficient (0..1) for the left eye looking up. */
|
|
225
488
|
eyeLookUpLeft: number;
|
|
489
|
+
/** Coefficient (0..1) for the right eye looking up. */
|
|
226
490
|
eyeLookUpRight: number;
|
|
491
|
+
/** Coefficient (0..1) for the left eye squinting. */
|
|
227
492
|
eyeSquintLeft: number;
|
|
493
|
+
/** Coefficient (0..1) for the right eye squinting. */
|
|
228
494
|
eyeSquintRight: number;
|
|
495
|
+
/** Coefficient (0..1) for the left eye opening wide. */
|
|
229
496
|
eyeWideLeft: number;
|
|
497
|
+
/** Coefficient (0..1) for the right eye opening wide. */
|
|
230
498
|
eyeWideRight: number;
|
|
499
|
+
/** Coefficient (0..1) for the jaw moving forward. */
|
|
231
500
|
jawForward: number;
|
|
501
|
+
/** Coefficient (0..1) for the jaw moving left. */
|
|
232
502
|
jawLeft: number;
|
|
503
|
+
/** Coefficient (0..1) for the jaw opening (mouth open). */
|
|
233
504
|
jawOpen: number;
|
|
505
|
+
/** Coefficient (0..1) for the jaw moving right. */
|
|
234
506
|
jawRight: number;
|
|
507
|
+
/** Coefficient (0..1) for the mouth closing tightly. */
|
|
235
508
|
mouthClose: number;
|
|
509
|
+
/** Coefficient (0..1) for a dimple appearing on the left side. */
|
|
236
510
|
mouthDimpleLeft: number;
|
|
511
|
+
/** Coefficient (0..1) for a dimple appearing on the right side. */
|
|
237
512
|
mouthDimpleRight: number;
|
|
513
|
+
/** Coefficient (0..1) for the left mouth corner pulling down (frown). */
|
|
238
514
|
mouthFrownLeft: number;
|
|
515
|
+
/** Coefficient (0..1) for the right mouth corner pulling down (frown). */
|
|
239
516
|
mouthFrownRight: number;
|
|
517
|
+
/** Coefficient (0..1) for funneling the lips outward. */
|
|
240
518
|
mouthFunnel: number;
|
|
519
|
+
/** Coefficient (0..1) for the mouth shifting left. */
|
|
241
520
|
mouthLeft: number;
|
|
521
|
+
/** Coefficient (0..1) for the lower lip on the left pulling down. */
|
|
242
522
|
mouthLowerDownLeft: number;
|
|
523
|
+
/** Coefficient (0..1) for the lower lip on the right pulling down. */
|
|
243
524
|
mouthLowerDownRight: number;
|
|
525
|
+
/** Coefficient (0..1) for the left lip pressing inward. */
|
|
244
526
|
mouthPressLeft: number;
|
|
527
|
+
/** Coefficient (0..1) for the right lip pressing inward. */
|
|
245
528
|
mouthPressRight: number;
|
|
529
|
+
/** Coefficient (0..1) for puckered lips. */
|
|
246
530
|
mouthPucker: number;
|
|
531
|
+
/** Coefficient (0..1) for the mouth shifting right. */
|
|
247
532
|
mouthRight: number;
|
|
533
|
+
/** Coefficient (0..1) for the lower lip rolling inward. */
|
|
248
534
|
mouthRollLower: number;
|
|
535
|
+
/** Coefficient (0..1) for the upper lip rolling inward. */
|
|
249
536
|
mouthRollUpper: number;
|
|
537
|
+
/** Coefficient (0..1) for the lower lip shrugging. */
|
|
250
538
|
mouthShrugLower: number;
|
|
539
|
+
/** Coefficient (0..1) for the upper lip shrugging. */
|
|
251
540
|
mouthShrugUpper: number;
|
|
541
|
+
/** Coefficient (0..1) for the left mouth corner pulling up (smile). */
|
|
252
542
|
mouthSmileLeft: number;
|
|
543
|
+
/** Coefficient (0..1) for the right mouth corner pulling up (smile). */
|
|
253
544
|
mouthSmileRight: number;
|
|
545
|
+
/** Coefficient (0..1) for the left mouth corner stretching outward. */
|
|
254
546
|
mouthStretchLeft: number;
|
|
547
|
+
/** Coefficient (0..1) for the right mouth corner stretching outward. */
|
|
255
548
|
mouthStretchRight: number;
|
|
549
|
+
/** Coefficient (0..1) for the upper lip on the left raising. */
|
|
256
550
|
mouthUpperUpLeft: number;
|
|
551
|
+
/** Coefficient (0..1) for the upper lip on the right raising. */
|
|
257
552
|
mouthUpperUpRight: number;
|
|
553
|
+
/** Coefficient (0..1) for the left side of the nose sneering. */
|
|
258
554
|
noseSneerLeft: number;
|
|
555
|
+
/** Coefficient (0..1) for the right side of the nose sneering. */
|
|
259
556
|
noseSneerRight: number;
|
|
557
|
+
/** Coefficient (0..1) for the tongue protruding. */
|
|
260
558
|
tongueOut: number;
|
|
261
559
|
}
|
|
262
560
|
|
|
561
|
+
/**
|
|
562
|
+
* One detected face produced by the video CV pipeline. Always carries `id`,
|
|
563
|
+
* `bounds`, `center`, and `confidence`; the rest is populated only when the
|
|
564
|
+
* matching CV feature is enabled (face mesh for `landmarks` and `headPose`,
|
|
565
|
+
* emotion detection for `expressions` and `blendshapes`).
|
|
566
|
+
*/
|
|
263
567
|
interface FaceData {
|
|
568
|
+
/** Index-based face identifier (`0`, `1`, …). Stable within a single frame, may change between frames. */
|
|
264
569
|
id: number;
|
|
570
|
+
/** Bounding box, all components normalized to 0..1 of the source frame. */
|
|
265
571
|
bounds: {
|
|
572
|
+
/** Left edge of the face bounding box, normalized 0..1. */
|
|
266
573
|
x: number;
|
|
574
|
+
/** Top edge of the face bounding box, normalized 0..1. */
|
|
267
575
|
y: number;
|
|
576
|
+
/** Width of the face bounding box, normalized 0..1. */
|
|
268
577
|
width: number;
|
|
578
|
+
/** Height of the face bounding box, normalized 0..1. */
|
|
269
579
|
height: number;
|
|
270
580
|
};
|
|
581
|
+
/** Center of the bounding box, normalized 0..1. */
|
|
271
582
|
center: {
|
|
583
|
+
/** Horizontal center of the face, normalized 0..1. */
|
|
272
584
|
x: number;
|
|
585
|
+
/** Vertical center of the face, normalized 0..1. */
|
|
273
586
|
y: number;
|
|
274
587
|
};
|
|
588
|
+
/** Detection confidence in 0..1. */
|
|
275
589
|
confidence: number;
|
|
590
|
+
/** 468 normalized face mesh landmarks when face mesh is enabled; empty array otherwise. */
|
|
276
591
|
landmarks: {
|
|
592
|
+
/** Horizontal landmark coordinate, normalized 0..1. */
|
|
277
593
|
x: number;
|
|
594
|
+
/** Vertical landmark coordinate, normalized 0..1. */
|
|
278
595
|
y: number;
|
|
596
|
+
/** Optional depth (relative). */
|
|
279
597
|
z?: number;
|
|
280
598
|
}[];
|
|
599
|
+
/** Seven expression confidence scores in 0..1. All zero unless emotion detection is enabled. */
|
|
281
600
|
expressions: {
|
|
601
|
+
/** Confidence (0..1) of a neutral expression. */
|
|
282
602
|
neutral: number;
|
|
603
|
+
/** Confidence (0..1) of a happy / smiling expression. */
|
|
283
604
|
happy: number;
|
|
605
|
+
/** Confidence (0..1) of a sad expression. */
|
|
284
606
|
sad: number;
|
|
607
|
+
/** Confidence (0..1) of an angry expression. */
|
|
285
608
|
angry: number;
|
|
609
|
+
/** Confidence (0..1) of a surprised expression. */
|
|
286
610
|
surprised: number;
|
|
611
|
+
/** Confidence (0..1) of a disgusted expression. */
|
|
287
612
|
disgusted: number;
|
|
613
|
+
/** Confidence (0..1) of a fearful expression. */
|
|
288
614
|
fearful: number;
|
|
289
615
|
};
|
|
616
|
+
/** Estimated head rotation. All zero unless face mesh is enabled. */
|
|
290
617
|
headPose: {
|
|
618
|
+
/** Up/down rotation in degrees (-90..90). */
|
|
291
619
|
pitch: number;
|
|
620
|
+
/** Left/right rotation in degrees (-90..90). */
|
|
292
621
|
yaw: number;
|
|
622
|
+
/** Tilt rotation in degrees (-180..180). */
|
|
293
623
|
roll: number;
|
|
294
624
|
};
|
|
625
|
+
/** 52 ARKit-compatible facial muscle coefficients. All zero unless emotion detection is enabled. */
|
|
295
626
|
blendshapes: FaceBlendshapes;
|
|
296
627
|
}
|
|
297
628
|
|
|
629
|
+
/**
|
|
630
|
+
* Render-loop pacing mode. `'full'` runs `render()` on every animation frame
|
|
631
|
+
* (~60 FPS). `'half'` runs every second frame (~30 FPS), saving CPU/GPU on
|
|
632
|
+
* lower-cost scenes.
|
|
633
|
+
*/
|
|
298
634
|
type FrameRateMode = 'full' | 'half';
|
|
299
635
|
|
|
636
|
+
/**
|
|
637
|
+
* One named frequency band reported by the audio analyzer
|
|
638
|
+
* (e.g., `{ name: 'low', min: 20, max: 120 }`). Frequencies are in Hertz.
|
|
639
|
+
*/
|
|
300
640
|
interface FrequencyBand {
|
|
641
|
+
/** Human-readable band name (e.g., `'low'`, `'mid'`, `'high'`). */
|
|
301
642
|
name: string;
|
|
643
|
+
/** Lower edge of the band in Hertz (inclusive). */
|
|
302
644
|
min: number;
|
|
645
|
+
/** Upper edge of the band in Hertz (exclusive). */
|
|
303
646
|
max: number;
|
|
304
647
|
}
|
|
305
648
|
|
|
649
|
+
/**
|
|
650
|
+
* One detected hand produced by the video CV pipeline. Carries 21 landmarks,
|
|
651
|
+
* a palm shortcut, and ML-classified gesture confidence scores. Up to two
|
|
652
|
+
* hands appear in `VideoAPI.hands` simultaneously.
|
|
653
|
+
*/
|
|
306
654
|
interface HandData {
|
|
655
|
+
/** Index-based hand identifier (`0` or `1`). */
|
|
307
656
|
id: number;
|
|
657
|
+
/** Which hand. Always lowercase. */
|
|
308
658
|
handedness: 'left' | 'right';
|
|
659
|
+
/** Detection confidence in 0..1. */
|
|
309
660
|
confidence: number;
|
|
661
|
+
/** Bounding box, all components normalized to 0..1 of the source frame. */
|
|
310
662
|
bounds: {
|
|
663
|
+
/** Left edge of the hand bounding box, normalized 0..1. */
|
|
311
664
|
x: number;
|
|
665
|
+
/** Top edge of the hand bounding box, normalized 0..1. */
|
|
312
666
|
y: number;
|
|
667
|
+
/** Width of the hand bounding box, normalized 0..1. */
|
|
313
668
|
width: number;
|
|
669
|
+
/** Height of the hand bounding box, normalized 0..1. */
|
|
314
670
|
height: number;
|
|
315
671
|
};
|
|
672
|
+
/** 21 MediaPipe hand landmarks, normalized 0..1. */
|
|
316
673
|
landmarks: {
|
|
674
|
+
/** Horizontal landmark coordinate, normalized 0..1. */
|
|
317
675
|
x: number;
|
|
676
|
+
/** Vertical landmark coordinate, normalized 0..1. */
|
|
318
677
|
y: number;
|
|
678
|
+
/** Depth (relative). */
|
|
319
679
|
z: number;
|
|
320
680
|
}[];
|
|
681
|
+
/** Palm center — equivalent to `landmarks[9]` (middle-finger MCP). */
|
|
321
682
|
palm: {
|
|
683
|
+
/** Palm horizontal coordinate, normalized 0..1. */
|
|
322
684
|
x: number;
|
|
685
|
+
/** Palm vertical coordinate, normalized 0..1. */
|
|
323
686
|
y: number;
|
|
687
|
+
/** Palm depth (relative). */
|
|
324
688
|
z: number;
|
|
325
689
|
};
|
|
690
|
+
/**
|
|
691
|
+
* ML-classified gesture confidences from MediaPipe GestureRecognizer.
|
|
692
|
+
* Each value is in 0..1; multiple gestures can fire simultaneously.
|
|
693
|
+
*/
|
|
326
694
|
gestures: {
|
|
695
|
+
/** Confidence (0..1) of a closed-fist gesture. */
|
|
327
696
|
fist: number;
|
|
697
|
+
/** Confidence (0..1) of an open-palm gesture. */
|
|
328
698
|
openPalm: number;
|
|
699
|
+
/** Confidence (0..1) of a peace / victory sign. */
|
|
329
700
|
peace: number;
|
|
701
|
+
/** Confidence (0..1) of a thumbs-up gesture. */
|
|
330
702
|
thumbsUp: number;
|
|
703
|
+
/** Confidence (0..1) of a thumbs-down gesture. */
|
|
331
704
|
thumbsDown: number;
|
|
705
|
+
/** Confidence (0..1) of a pointing-up gesture. */
|
|
332
706
|
pointing: number;
|
|
707
|
+
/** Confidence (0..1) of the ASL "I love you" sign. */
|
|
333
708
|
iLoveYou: number;
|
|
334
709
|
};
|
|
335
710
|
}
|
|
336
711
|
|
|
712
|
+
/**
|
|
713
|
+
* Configuration object passed to `viji.image()`. The host renders an upload
|
|
714
|
+
* field; the user-supplied image becomes available as an `ImageBitmap` (Native)
|
|
715
|
+
* or a P5 image wrapper (P5 renderer).
|
|
716
|
+
*/
|
|
337
717
|
interface ImageConfig {
|
|
718
|
+
/** Display name shown next to the upload field in the parameter UI. Required. */
|
|
338
719
|
label: string;
|
|
720
|
+
/** Optional tooltip / help text shown next to the control. */
|
|
339
721
|
description?: string;
|
|
722
|
+
/** Group name for organizing related parameters under a shared heading. Default: `'general'`. */
|
|
340
723
|
group?: string;
|
|
724
|
+
/** Visibility category — the control hides when its capability is unavailable. Default: `'general'`. */
|
|
341
725
|
category?: ParameterCategory;
|
|
342
726
|
}
|
|
343
727
|
|
|
728
|
+
/**
|
|
729
|
+
* Reactive object returned by `viji.image()`. `value` is `null` until the user
|
|
730
|
+
* uploads an image; in P5 scenes a `.p5` accessor is also added at runtime so
|
|
731
|
+
* the image works directly with `image()`, `texture()`, etc.
|
|
732
|
+
*/
|
|
344
733
|
interface ImageParameter {
|
|
734
|
+
/** Uploaded image as an `ImageBitmap`, or `null` until the user provides one. */
|
|
345
735
|
value: ImageBitmap | null;
|
|
346
736
|
/** P5-compatible image wrapper. Only available in the P5 renderer — added at runtime by the P5 adapter. */
|
|
347
737
|
readonly p5?: any;
|
|
738
|
+
/** Display name shown next to the upload field in the parameter UI. */
|
|
348
739
|
label: string;
|
|
740
|
+
/** Tooltip / help text for the control (empty string when not configured). */
|
|
349
741
|
description?: string;
|
|
742
|
+
/** Group name under which the control appears in the UI. */
|
|
350
743
|
group: string;
|
|
744
|
+
/** Visibility category — controls when the parameter is shown. */
|
|
351
745
|
category: ParameterCategory;
|
|
352
746
|
/**
|
|
353
747
|
* P5-compatible image object (lazy-loaded, auto-converted from .value)
|
|
@@ -362,277 +756,704 @@ declare global {
|
|
|
362
756
|
};
|
|
363
757
|
}
|
|
364
758
|
|
|
759
|
+
/**
|
|
760
|
+
* Keyboard input state for the current frame: per-key press / hold / release
|
|
761
|
+
* queries (case-insensitive), the live set of held keys, modifier flags, and
|
|
762
|
+
* a Shadertoy-compatible texture buffer for shader-side keyboard reads.
|
|
763
|
+
*
|
|
764
|
+
* Key names follow the browser's `event.key` standard
|
|
765
|
+
* (`'a'`, `'arrowup'`, `' '` for Space, `'enter'`, `'escape'`, etc.).
|
|
766
|
+
*/
|
|
365
767
|
interface KeyboardAPI {
|
|
768
|
+
/** Returns `true` while the named key is held. Case-insensitive. */
|
|
366
769
|
isPressed(key: string): boolean;
|
|
770
|
+
/** Returns `true` for exactly one frame on the rising edge of the named key (no key-repeat). Case-insensitive. */
|
|
367
771
|
wasPressed(key: string): boolean;
|
|
772
|
+
/** Returns `true` for exactly one frame on the falling edge of the named key. Case-insensitive. */
|
|
368
773
|
wasReleased(key: string): boolean;
|
|
774
|
+
/** Set of all currently held keys, lowercased. Persists across frames. */
|
|
369
775
|
activeKeys: Set<string>;
|
|
776
|
+
/** Set of keys whose key-down fired this frame, lowercased. Cleared each frame. */
|
|
370
777
|
pressedThisFrame: Set<string>;
|
|
778
|
+
/** Set of keys whose key-up fired this frame, lowercased. Cleared each frame. */
|
|
371
779
|
releasedThisFrame: Set<string>;
|
|
780
|
+
/** Most recently pressed key in its original case (e.g., `'A'` when Shift is held). */
|
|
372
781
|
lastKeyPressed: string;
|
|
782
|
+
/** Most recently released key in its original case. */
|
|
373
783
|
lastKeyReleased: string;
|
|
784
|
+
/** `true` while the Shift key is held. */
|
|
374
785
|
shift: boolean;
|
|
786
|
+
/** `true` while the Ctrl key is held. */
|
|
375
787
|
ctrl: boolean;
|
|
788
|
+
/** `true` while the Alt (or Option) key is held. */
|
|
376
789
|
alt: boolean;
|
|
790
|
+
/** `true` while the Meta (Windows / Cmd) key is held. */
|
|
377
791
|
meta: boolean;
|
|
792
|
+
/**
|
|
793
|
+
* Shadertoy-compatible 256×3 keyboard texture (256 keycodes × 3 rows).
|
|
794
|
+
* Row 0: held state (1 / 0). Row 1: pressed-this-frame (1 / 0). Row 2: per-key toggle.
|
|
795
|
+
* Bound to a uniform sampler in shader scenes; rarely needed in JS.
|
|
796
|
+
*/
|
|
378
797
|
textureData: Uint8Array;
|
|
379
798
|
}
|
|
380
799
|
|
|
800
|
+
/**
|
|
801
|
+
* Mouse input state for the current frame: position, individual button states,
|
|
802
|
+
* per-frame movement, scroll wheel, and frame-based press / release events.
|
|
803
|
+
* Coordinates are in canvas-space pixels with `(0, 0)` at the top-left corner.
|
|
804
|
+
*
|
|
805
|
+
* For interactions that should also work on touch devices, prefer `viji.pointer`.
|
|
806
|
+
*/
|
|
381
807
|
interface MouseAPI {
|
|
808
|
+
/** Canvas-space X position in pixels. */
|
|
382
809
|
x: number;
|
|
810
|
+
/** Canvas-space Y position in pixels. */
|
|
383
811
|
y: number;
|
|
812
|
+
/** `true` when the cursor is currently over the canvas. */
|
|
384
813
|
isInCanvas: boolean;
|
|
814
|
+
/** `true` if any mouse button is currently held. */
|
|
385
815
|
isPressed: boolean;
|
|
816
|
+
/** `true` while the left mouse button is held. */
|
|
386
817
|
leftButton: boolean;
|
|
818
|
+
/** `true` while the right mouse button is held. The browser context menu is suppressed on the canvas. */
|
|
387
819
|
rightButton: boolean;
|
|
820
|
+
/** `true` while the middle (wheel) mouse button is held. */
|
|
388
821
|
middleButton: boolean;
|
|
822
|
+
/** Horizontal movement this frame in pixels. Resets to `0` each frame. */
|
|
389
823
|
deltaX: number;
|
|
824
|
+
/** Vertical movement this frame in pixels. Resets to `0` each frame. */
|
|
390
825
|
deltaY: number;
|
|
826
|
+
/** Vertical scroll accumulated this frame (alias of `wheelY`). Resets to `0` each frame. */
|
|
391
827
|
wheelDelta: number;
|
|
828
|
+
/** Horizontal scroll accumulated this frame (e.g., trackpad gestures, tilt-wheel mice). Resets to `0` each frame. */
|
|
392
829
|
wheelX: number;
|
|
830
|
+
/** Vertical scroll accumulated this frame. Resets to `0` each frame. */
|
|
393
831
|
wheelY: number;
|
|
832
|
+
/** `true` for exactly one frame when any button is first pressed, then auto-resets. */
|
|
394
833
|
wasPressed: boolean;
|
|
834
|
+
/** `true` for exactly one frame when any button is released, then auto-resets. */
|
|
395
835
|
wasReleased: boolean;
|
|
836
|
+
/** `true` if the mouse moved this frame (any non-zero `deltaX` / `deltaY`). Resets each frame. */
|
|
396
837
|
wasMoved: boolean;
|
|
397
838
|
}
|
|
398
839
|
|
|
840
|
+
/**
|
|
841
|
+
* Configuration object passed to `viji.number()`. Defines the bounds, step,
|
|
842
|
+
* label, and UI grouping/visibility for a precise numeric input.
|
|
843
|
+
*/
|
|
399
844
|
interface NumberConfig {
|
|
845
|
+
/** Inclusive minimum value. Default: `0`. */
|
|
400
846
|
min?: number;
|
|
847
|
+
/** Inclusive maximum value. Default: `100`. */
|
|
401
848
|
max?: number;
|
|
849
|
+
/** Increment between values. Default: `1`. */
|
|
402
850
|
step?: number;
|
|
851
|
+
/** Display name shown next to the input in the parameter UI. Required. */
|
|
403
852
|
label: string;
|
|
853
|
+
/** Optional tooltip / help text shown next to the control. */
|
|
404
854
|
description?: string;
|
|
855
|
+
/** Group name for organizing related parameters under a shared heading. Default: `'general'`. */
|
|
405
856
|
group?: string;
|
|
857
|
+
/** Visibility category — the control hides when its capability is unavailable. Default: `'general'`. */
|
|
406
858
|
category?: ParameterCategory;
|
|
407
859
|
}
|
|
408
860
|
|
|
861
|
+
/**
|
|
862
|
+
* Reactive object returned by `viji.number()`. Like `SliderParameter` but the
|
|
863
|
+
* host UI renders a precise numeric input rather than a draggable slider.
|
|
864
|
+
*/
|
|
409
865
|
interface NumberParameter {
|
|
866
|
+
/** Current numeric value in the configured `min..max` range. Updates in real-time. */
|
|
410
867
|
value: number;
|
|
868
|
+
/** Inclusive minimum value. */
|
|
411
869
|
min: number;
|
|
870
|
+
/** Inclusive maximum value. */
|
|
412
871
|
max: number;
|
|
872
|
+
/** Increment between values. */
|
|
413
873
|
step: number;
|
|
874
|
+
/** Display name shown next to the input in the parameter UI. */
|
|
414
875
|
label: string;
|
|
876
|
+
/** Tooltip / help text for the control (empty string when not configured). */
|
|
415
877
|
description?: string;
|
|
878
|
+
/** Group name under which the control appears in the UI. */
|
|
416
879
|
group: string;
|
|
880
|
+
/** Visibility category — controls when the parameter is shown. */
|
|
417
881
|
category: ParameterCategory;
|
|
418
882
|
}
|
|
419
883
|
|
|
884
|
+
/**
|
|
885
|
+
* Visibility category for a parameter. The host hides parameters whose category
|
|
886
|
+
* is not currently available (e.g., `'audio'` controls hide when no audio
|
|
887
|
+
* source is connected). `'general'` parameters are always visible.
|
|
888
|
+
*/
|
|
420
889
|
type ParameterCategory = 'audio' | 'video' | 'interaction' | 'general';
|
|
421
890
|
|
|
891
|
+
/**
|
|
892
|
+
* Unified pointer that abstracts mouse and primary touch into a single input
|
|
893
|
+
* stream. Recommended for click / drag / position-tracking interactions that
|
|
894
|
+
* should work identically on desktop and mobile.
|
|
895
|
+
*
|
|
896
|
+
* Use `viji.mouse` for mouse-specific features (right click, middle click,
|
|
897
|
+
* scroll wheel); use `viji.touches` for multi-touch, pressure, or radius.
|
|
898
|
+
*/
|
|
422
899
|
interface PointerAPI {
|
|
900
|
+
/** Canvas-space X position in pixels. */
|
|
423
901
|
x: number;
|
|
902
|
+
/** Canvas-space Y position in pixels. */
|
|
424
903
|
y: number;
|
|
904
|
+
/** Horizontal movement since the last frame in pixels. */
|
|
425
905
|
deltaX: number;
|
|
906
|
+
/** Vertical movement since the last frame in pixels. */
|
|
426
907
|
deltaY: number;
|
|
908
|
+
/** `true` while the pointer is "down" — left mouse button held, or a touch active. */
|
|
427
909
|
isDown: boolean;
|
|
910
|
+
/** `true` for exactly one frame when the pointer becomes down, then auto-resets. */
|
|
428
911
|
wasPressed: boolean;
|
|
912
|
+
/** `true` for exactly one frame when the pointer is released, then auto-resets. */
|
|
429
913
|
wasReleased: boolean;
|
|
914
|
+
/** `true` while the pointer is within the canvas bounds. */
|
|
430
915
|
isInCanvas: boolean;
|
|
916
|
+
/** Currently active input source — `'touch'` when at least one touch is active, otherwise `'mouse'` (or `'none'` when the cursor is off-canvas). */
|
|
431
917
|
type: 'mouse' | 'touch' | 'none';
|
|
432
918
|
}
|
|
433
919
|
|
|
920
|
+
/**
|
|
921
|
+
* Detected body pose from MediaPipe BlazePose. `landmarks` contains 33 points;
|
|
922
|
+
* the `face` / `torso` / `leftArm` / `rightArm` / `leftLeg` / `rightLeg` arrays
|
|
923
|
+
* are pre-grouped slices for convenience.
|
|
924
|
+
*/
|
|
434
925
|
interface PoseData {
|
|
926
|
+
/** Average landmark visibility in 0..1. */
|
|
435
927
|
confidence: number;
|
|
928
|
+
/** 33 BlazePose landmarks, normalized 0..1. Each carries a per-point `visibility` score. */
|
|
436
929
|
landmarks: {
|
|
930
|
+
/** Horizontal landmark coordinate, normalized 0..1. */
|
|
437
931
|
x: number;
|
|
932
|
+
/** Vertical landmark coordinate, normalized 0..1. */
|
|
438
933
|
y: number;
|
|
934
|
+
/** Depth (relative). */
|
|
439
935
|
z: number;
|
|
936
|
+
/** Visibility / confidence of this individual landmark in 0..1. */
|
|
440
937
|
visibility: number;
|
|
441
938
|
}[];
|
|
939
|
+
/** Face landmarks (BlazePose indices 0..10). Coordinates normalized 0..1. */
|
|
442
940
|
face: {
|
|
941
|
+
/** Horizontal landmark coordinate, normalized 0..1. */
|
|
443
942
|
x: number;
|
|
943
|
+
/** Vertical landmark coordinate, normalized 0..1. */
|
|
444
944
|
y: number;
|
|
445
945
|
}[];
|
|
946
|
+
/** Torso landmarks (BlazePose indices 11, 12, 23, 24). Coordinates normalized 0..1. */
|
|
446
947
|
torso: {
|
|
948
|
+
/** Horizontal landmark coordinate, normalized 0..1. */
|
|
447
949
|
x: number;
|
|
950
|
+
/** Vertical landmark coordinate, normalized 0..1. */
|
|
448
951
|
y: number;
|
|
449
952
|
}[];
|
|
953
|
+
/** Left-arm landmarks (BlazePose indices 11, 13, 15). Coordinates normalized 0..1. */
|
|
450
954
|
leftArm: {
|
|
955
|
+
/** Horizontal landmark coordinate, normalized 0..1. */
|
|
451
956
|
x: number;
|
|
957
|
+
/** Vertical landmark coordinate, normalized 0..1. */
|
|
452
958
|
y: number;
|
|
453
959
|
}[];
|
|
960
|
+
/** Right-arm landmarks (BlazePose indices 12, 14, 16). Coordinates normalized 0..1. */
|
|
454
961
|
rightArm: {
|
|
962
|
+
/** Horizontal landmark coordinate, normalized 0..1. */
|
|
455
963
|
x: number;
|
|
964
|
+
/** Vertical landmark coordinate, normalized 0..1. */
|
|
456
965
|
y: number;
|
|
457
966
|
}[];
|
|
967
|
+
/** Left-leg landmarks (BlazePose indices 23, 25, 27, 29, 31). Coordinates normalized 0..1. */
|
|
458
968
|
leftLeg: {
|
|
969
|
+
/** Horizontal landmark coordinate, normalized 0..1. */
|
|
459
970
|
x: number;
|
|
971
|
+
/** Vertical landmark coordinate, normalized 0..1. */
|
|
460
972
|
y: number;
|
|
461
973
|
}[];
|
|
974
|
+
/** Right-leg landmarks (BlazePose indices 24, 26, 28, 30, 32). Coordinates normalized 0..1. */
|
|
462
975
|
rightLeg: {
|
|
976
|
+
/** Horizontal landmark coordinate, normalized 0..1. */
|
|
463
977
|
x: number;
|
|
978
|
+
/** Vertical landmark coordinate, normalized 0..1. */
|
|
464
979
|
y: number;
|
|
465
980
|
}[];
|
|
466
981
|
}
|
|
467
982
|
|
|
983
|
+
/**
|
|
984
|
+
* Discrete pixel dimensions for a canvas, frame capture, or render target.
|
|
985
|
+
*/
|
|
468
986
|
type Resolution = {
|
|
987
|
+
/** Width in pixels (positive integer). */
|
|
469
988
|
width: number;
|
|
989
|
+
/** Height in pixels (positive integer). */
|
|
470
990
|
height: number;
|
|
471
991
|
};
|
|
472
992
|
|
|
993
|
+
/**
|
|
994
|
+
* Per-pixel person/background segmentation mask. Mask dimensions reflect the
|
|
995
|
+
* ML model's output and may differ from the source video frame.
|
|
996
|
+
*/
|
|
473
997
|
interface SegmentationData {
|
|
998
|
+
/** Flat per-pixel mask: each byte is `0` (background) or `1` (person). Length = `width * height`. */
|
|
474
999
|
mask: Uint8Array;
|
|
1000
|
+
/** Mask width in pixels. */
|
|
475
1001
|
width: number;
|
|
1002
|
+
/** Mask height in pixels. */
|
|
476
1003
|
height: number;
|
|
477
1004
|
}
|
|
478
1005
|
|
|
1006
|
+
/**
|
|
1007
|
+
* Configuration object passed to `viji.select()`. The host renders the choices
|
|
1008
|
+
* as a dropdown or segmented control.
|
|
1009
|
+
*/
|
|
479
1010
|
interface SelectConfig {
|
|
1011
|
+
/** Available choices. The element type (`string` or `number`) becomes the parameter's value type. Required. */
|
|
480
1012
|
options: string[] | number[];
|
|
1013
|
+
/** Display name shown next to the dropdown in the parameter UI. Required. */
|
|
481
1014
|
label: string;
|
|
1015
|
+
/** Optional tooltip / help text shown next to the control. */
|
|
482
1016
|
description?: string;
|
|
1017
|
+
/** Group name for organizing related parameters under a shared heading. Default: `'general'`. */
|
|
483
1018
|
group?: string;
|
|
1019
|
+
/** Visibility category — the control hides when its capability is unavailable. Default: `'general'`. */
|
|
484
1020
|
category?: ParameterCategory;
|
|
485
1021
|
}
|
|
486
1022
|
|
|
1023
|
+
/**
|
|
1024
|
+
* Reactive object returned by `viji.select()`. `value` matches the element type
|
|
1025
|
+
* of the configured `options` array (`string` or `number`).
|
|
1026
|
+
*/
|
|
487
1027
|
interface SelectParameter {
|
|
1028
|
+
/** Currently selected option. Updates in real-time when the user changes the selection. */
|
|
488
1029
|
value: string | number;
|
|
1030
|
+
/** The full list of choices originally configured. */
|
|
489
1031
|
options: string[] | number[];
|
|
1032
|
+
/** Display name shown next to the dropdown in the parameter UI. */
|
|
490
1033
|
label: string;
|
|
1034
|
+
/** Tooltip / help text for the control (empty string when not configured). */
|
|
491
1035
|
description?: string;
|
|
1036
|
+
/** Group name under which the control appears in the UI. */
|
|
492
1037
|
group: string;
|
|
1038
|
+
/** Visibility category — controls when the parameter is shown. */
|
|
493
1039
|
category: ParameterCategory;
|
|
494
1040
|
}
|
|
495
1041
|
|
|
1042
|
+
/**
|
|
1043
|
+
* Configuration object passed to `viji.slider()`. Defines the slider's bounds,
|
|
1044
|
+
* increment, label, and UI grouping/visibility.
|
|
1045
|
+
*/
|
|
496
1046
|
interface SliderConfig {
|
|
1047
|
+
/** Inclusive minimum value of the slider. Default: `0`. */
|
|
497
1048
|
min?: number;
|
|
1049
|
+
/** Inclusive maximum value of the slider. Default: `100`. */
|
|
498
1050
|
max?: number;
|
|
1051
|
+
/** Increment between values. Default: `1`. */
|
|
499
1052
|
step?: number;
|
|
1053
|
+
/** Display name shown next to the control in the parameter UI. Required. */
|
|
500
1054
|
label: string;
|
|
1055
|
+
/** Optional tooltip / help text shown next to the control. */
|
|
501
1056
|
description?: string;
|
|
1057
|
+
/** Group name for organizing related parameters under a shared heading. Default: `'general'`. */
|
|
502
1058
|
group?: string;
|
|
1059
|
+
/** Visibility category — the control hides when its capability is unavailable. Default: `'general'`. */
|
|
503
1060
|
category?: ParameterCategory;
|
|
504
1061
|
}
|
|
505
1062
|
|
|
1063
|
+
/**
|
|
1064
|
+
* Reactive object returned by `viji.slider()`. `value` updates in real-time as
|
|
1065
|
+
* the user drags the control; the remaining fields mirror the configuration.
|
|
1066
|
+
*/
|
|
506
1067
|
interface SliderParameter {
|
|
1068
|
+
/** Current slider value in the configured `min..max` range. Updates in real-time. */
|
|
507
1069
|
value: number;
|
|
1070
|
+
/** Inclusive minimum value of the slider. */
|
|
508
1071
|
min: number;
|
|
1072
|
+
/** Inclusive maximum value of the slider. */
|
|
509
1073
|
max: number;
|
|
1074
|
+
/** Increment between values. */
|
|
510
1075
|
step: number;
|
|
1076
|
+
/** Display name shown next to the control in the parameter UI. */
|
|
511
1077
|
label: string;
|
|
1078
|
+
/** Tooltip / help text for the control (empty string when not configured). */
|
|
512
1079
|
description?: string;
|
|
1080
|
+
/** Group name under which the control appears in the UI. */
|
|
513
1081
|
group: string;
|
|
1082
|
+
/** Visibility category — controls when the parameter is shown. */
|
|
514
1083
|
category: ParameterCategory;
|
|
515
1084
|
}
|
|
516
1085
|
|
|
1086
|
+
/**
|
|
1087
|
+
* Configuration object passed to `viji.text()`. The host renders a single-line
|
|
1088
|
+
* text input field.
|
|
1089
|
+
*/
|
|
517
1090
|
interface TextConfig {
|
|
1091
|
+
/** Display name shown next to the text field in the parameter UI. Required. */
|
|
518
1092
|
label: string;
|
|
1093
|
+
/** Optional tooltip / help text shown next to the control. */
|
|
519
1094
|
description?: string;
|
|
1095
|
+
/** Group name for organizing related parameters under a shared heading. Default: `'general'`. */
|
|
520
1096
|
group?: string;
|
|
1097
|
+
/** Visibility category — the control hides when its capability is unavailable. Default: `'general'`. */
|
|
521
1098
|
category?: ParameterCategory;
|
|
1099
|
+
/** Maximum character count enforced by the host UI. Default: `1000`. */
|
|
522
1100
|
maxLength?: number;
|
|
523
1101
|
}
|
|
524
1102
|
|
|
1103
|
+
/**
|
|
1104
|
+
* Reactive object returned by `viji.text()`. `value` updates as the user types
|
|
1105
|
+
* and is enforced to be no longer than `maxLength` by the host UI.
|
|
1106
|
+
*/
|
|
525
1107
|
interface TextParameter {
|
|
1108
|
+
/** Current text content. Updates in real-time as the user types. */
|
|
526
1109
|
value: string;
|
|
1110
|
+
/** Maximum character count enforced by the host UI. */
|
|
527
1111
|
maxLength?: number;
|
|
1112
|
+
/** Display name shown next to the input in the parameter UI. */
|
|
528
1113
|
label: string;
|
|
1114
|
+
/** Tooltip / help text for the control (empty string when not configured). */
|
|
529
1115
|
description?: string;
|
|
1116
|
+
/** Group name under which the control appears in the UI. */
|
|
530
1117
|
group: string;
|
|
1118
|
+
/** Visibility category — controls when the parameter is shown. */
|
|
531
1119
|
category: ParameterCategory;
|
|
532
1120
|
}
|
|
533
1121
|
|
|
1122
|
+
/**
|
|
1123
|
+
* Configuration object passed to `viji.toggle()`. Defines the on/off switch's
|
|
1124
|
+
* label and UI grouping/visibility.
|
|
1125
|
+
*/
|
|
534
1126
|
interface ToggleConfig {
|
|
1127
|
+
/** Display name shown next to the switch in the parameter UI. Required. */
|
|
535
1128
|
label: string;
|
|
1129
|
+
/** Optional tooltip / help text shown next to the control. */
|
|
536
1130
|
description?: string;
|
|
1131
|
+
/** Group name for organizing related parameters under a shared heading. Default: `'general'`. */
|
|
537
1132
|
group?: string;
|
|
1133
|
+
/** Visibility category — the control hides when its capability is unavailable. Default: `'general'`. */
|
|
538
1134
|
category?: ParameterCategory;
|
|
539
1135
|
}
|
|
540
1136
|
|
|
1137
|
+
/**
|
|
1138
|
+
* Reactive object returned by `viji.toggle()`. `value` flips between `true`
|
|
1139
|
+
* and `false` as the user interacts with the switch.
|
|
1140
|
+
*/
|
|
541
1141
|
interface ToggleParameter {
|
|
1142
|
+
/** Current state of the switch. Updates in real-time when the user toggles it. */
|
|
542
1143
|
value: boolean;
|
|
1144
|
+
/** Display name shown next to the switch in the parameter UI. */
|
|
543
1145
|
label: string;
|
|
1146
|
+
/** Tooltip / help text for the control (empty string when not configured). */
|
|
544
1147
|
description?: string;
|
|
1148
|
+
/** Group name under which the control appears in the UI. */
|
|
545
1149
|
group: string;
|
|
1150
|
+
/** Visibility category — controls when the parameter is shown. */
|
|
546
1151
|
category: ParameterCategory;
|
|
547
1152
|
}
|
|
548
1153
|
|
|
1154
|
+
/**
|
|
1155
|
+
* Multi-touch input state. Lists all active touches, plus per-frame
|
|
1156
|
+
* `started` / `moved` / `ended` arrays for frame-based gesture handling.
|
|
1157
|
+
*
|
|
1158
|
+
* For single-point interactions that should also work with a mouse, prefer
|
|
1159
|
+
* `viji.pointer` — it switches automatically between mouse and primary touch.
|
|
1160
|
+
*/
|
|
549
1161
|
interface TouchAPI {
|
|
1162
|
+
/** All currently active touch points. Order is stable but not ordered by id. */
|
|
550
1163
|
points: TouchPoint[];
|
|
1164
|
+
/** Number of currently active touches (equivalent to `points.length`). */
|
|
551
1165
|
count: number;
|
|
1166
|
+
/** Touches that started this frame (also visible in `points`). Cleared each frame. */
|
|
552
1167
|
started: TouchPoint[];
|
|
1168
|
+
/** Touches that moved this frame. Cleared each frame. */
|
|
553
1169
|
moved: TouchPoint[];
|
|
1170
|
+
/** Touches that ended this frame. Each entry has `isEnding === true`. Cleared each frame. */
|
|
554
1171
|
ended: TouchPoint[];
|
|
1172
|
+
/** Convenience shortcut to `points[0]`, or `null` when no touches are active. */
|
|
555
1173
|
primary: TouchPoint | null;
|
|
556
1174
|
}
|
|
557
1175
|
|
|
1176
|
+
/**
|
|
1177
|
+
* One active touch contact reported by `TouchAPI`. Position, pressure, radius,
|
|
1178
|
+
* rotation, per-frame movement, velocity, and lifecycle flags. Coordinates are
|
|
1179
|
+
* in canvas-space pixels.
|
|
1180
|
+
*
|
|
1181
|
+
* Some fields (notably `pressure` / `force`, `radiusX` / `radiusY`,
|
|
1182
|
+
* `rotationAngle`) are passed through from the device — values vary by
|
|
1183
|
+
* platform; many devices report `0` when unsupported.
|
|
1184
|
+
*/
|
|
558
1185
|
interface TouchPoint {
|
|
1186
|
+
/** Stable touch identifier across frames for the duration of this contact. */
|
|
559
1187
|
id: number;
|
|
1188
|
+
/** Canvas-space X position in pixels. */
|
|
560
1189
|
x: number;
|
|
1190
|
+
/** Canvas-space Y position in pixels. */
|
|
561
1191
|
y: number;
|
|
1192
|
+
/** Touch pressure in 0..1 (device-dependent; often `0` on devices without force support). */
|
|
562
1193
|
pressure: number;
|
|
1194
|
+
/** Contact radius — `Math.max(radiusX, radiusY)` in pixels. */
|
|
563
1195
|
radius: number;
|
|
1196
|
+
/** Horizontal contact radius in pixels (raw device value). */
|
|
564
1197
|
radiusX: number;
|
|
1198
|
+
/** Vertical contact radius in pixels (raw device value). */
|
|
565
1199
|
radiusY: number;
|
|
1200
|
+
/** Contact area rotation in radians (raw device value). */
|
|
566
1201
|
rotationAngle: number;
|
|
1202
|
+
/** Touch force in 0..1 — alias of `pressure`. */
|
|
567
1203
|
force: number;
|
|
1204
|
+
/** `true` while the touch position is within the canvas bounds. */
|
|
568
1205
|
isInCanvas: boolean;
|
|
1206
|
+
/** Horizontal movement since the last frame in pixels. */
|
|
569
1207
|
deltaX: number;
|
|
1208
|
+
/** Vertical movement since the last frame in pixels. */
|
|
570
1209
|
deltaY: number;
|
|
1210
|
+
/** Movement velocity in pixels per second, computed by Viji. */
|
|
571
1211
|
velocity: {
|
|
1212
|
+
/** Horizontal velocity component in pixels per second. */
|
|
572
1213
|
x: number;
|
|
1214
|
+
/** Vertical velocity component in pixels per second. */
|
|
573
1215
|
y: number;
|
|
574
1216
|
};
|
|
1217
|
+
/** `true` for exactly one frame when this touch starts, then auto-resets. */
|
|
575
1218
|
isNew: boolean;
|
|
1219
|
+
/** `true` while the touch is ongoing. */
|
|
576
1220
|
isActive: boolean;
|
|
1221
|
+
/** `true` for exactly one frame when this touch ends, then auto-resets. */
|
|
577
1222
|
isEnding: boolean;
|
|
578
1223
|
}
|
|
579
1224
|
|
|
580
1225
|
const VERSION = "0.4.0";
|
|
581
1226
|
|
|
1227
|
+
/**
|
|
1228
|
+
* Real-time video API: drawable frame, dimensions, and computer-vision results
|
|
1229
|
+
* (faces, hands, pose, segmentation). Activate individual CV features through `cv.*`.
|
|
1230
|
+
*
|
|
1231
|
+
* When `isConnected` is `false`, `currentFrame` is `null`, all dimensions are `0`,
|
|
1232
|
+
* and CV result fields hold their empty defaults.
|
|
1233
|
+
*/
|
|
582
1234
|
interface VideoAPI {
|
|
1235
|
+
/** `true` when a video stream is connected. When `false`, all other fields hold their default empty values. */
|
|
583
1236
|
isConnected: boolean;
|
|
1237
|
+
/** Latest video frame, drawable directly with `ctx.drawImage()`. `null` when disconnected. */
|
|
584
1238
|
currentFrame: OffscreenCanvas | ImageBitmap | null;
|
|
1239
|
+
/** Source video frame width in pixels. `0` when disconnected. */
|
|
585
1240
|
frameWidth: number;
|
|
1241
|
+
/** Source video frame height in pixels. `0` when disconnected. */
|
|
586
1242
|
frameHeight: number;
|
|
1243
|
+
/** Source video frame rate in Hz (frames per second). `0` when disconnected. */
|
|
587
1244
|
frameRate: number;
|
|
1245
|
+
/**
|
|
1246
|
+
* Returns the latest frame as raw RGBA `ImageData` for per-pixel CPU analysis,
|
|
1247
|
+
* or `null` when disconnected. Slow compared to drawing `currentFrame` directly —
|
|
1248
|
+
* use only when you need to read individual pixel values.
|
|
1249
|
+
*/
|
|
588
1250
|
getFrameData: () => ImageData | null;
|
|
1251
|
+
/** Detected faces (empty array when face detection is off or no faces are visible). */
|
|
589
1252
|
faces: FaceData[];
|
|
1253
|
+
/** Detected hands (up to 2; empty array when hand tracking is off or no hands are visible). */
|
|
590
1254
|
hands: HandData[];
|
|
1255
|
+
/** Detected body pose, or `null` when pose detection is off or no pose is visible. */
|
|
591
1256
|
pose: PoseData | null;
|
|
1257
|
+
/** Body segmentation mask, or `null` when segmentation is off. */
|
|
592
1258
|
segmentation: SegmentationData | null;
|
|
1259
|
+
/**
|
|
1260
|
+
* Computer-vision feature control. Each `enable*` method toggles a specific CV
|
|
1261
|
+
* pipeline; results land in the corresponding `faces` / `hands` / `pose` /
|
|
1262
|
+
* `segmentation` field on the next available frame. Multiple features can be
|
|
1263
|
+
* active simultaneously, but each adds CPU/GPU and WebGL-context cost.
|
|
1264
|
+
*/
|
|
593
1265
|
cv: {
|
|
1266
|
+
/**
|
|
1267
|
+
* Toggle face detection (bounding box, center, confidence, id).
|
|
1268
|
+
* @returns Resolves once the underlying pipeline has finished switching.
|
|
1269
|
+
*/
|
|
594
1270
|
enableFaceDetection(enabled: boolean): Promise<void>;
|
|
1271
|
+
/**
|
|
1272
|
+
* Toggle 468-point face mesh + head pose (`pitch`, `yaw`, `roll`). Implies
|
|
1273
|
+
* face detection.
|
|
1274
|
+
* @returns Resolves once the pipeline has finished switching.
|
|
1275
|
+
*/
|
|
595
1276
|
enableFaceMesh(enabled: boolean): Promise<void>;
|
|
1277
|
+
/**
|
|
1278
|
+
* Toggle 7 expressions + 52 ARKit blendshape coefficients on each face.
|
|
1279
|
+
* Implies face mesh.
|
|
1280
|
+
* @returns Resolves once the pipeline has finished switching.
|
|
1281
|
+
*/
|
|
596
1282
|
enableEmotionDetection(enabled: boolean): Promise<void>;
|
|
1283
|
+
/**
|
|
1284
|
+
* Toggle 21-point hand landmarks + ML-classified gesture confidences for up
|
|
1285
|
+
* to two hands.
|
|
1286
|
+
* @returns Resolves once the pipeline has finished switching.
|
|
1287
|
+
*/
|
|
597
1288
|
enableHandTracking(enabled: boolean): Promise<void>;
|
|
1289
|
+
/**
|
|
1290
|
+
* Toggle 33-point BlazePose body landmarks (face / torso / arms / legs).
|
|
1291
|
+
* @returns Resolves once the pipeline has finished switching.
|
|
1292
|
+
*/
|
|
598
1293
|
enablePoseDetection(enabled: boolean): Promise<void>;
|
|
1294
|
+
/**
|
|
1295
|
+
* Toggle per-pixel person/background segmentation mask.
|
|
1296
|
+
* @returns Resolves once the pipeline has finished switching.
|
|
1297
|
+
*/
|
|
599
1298
|
enableBodySegmentation(enabled: boolean): Promise<void>;
|
|
1299
|
+
/** Returns the list of CV features currently enabled on this stream. */
|
|
600
1300
|
getActiveFeatures(): CVFeature[];
|
|
1301
|
+
/** Returns `true` if the CV worker is actively processing frames. */
|
|
601
1302
|
isProcessing(): boolean;
|
|
602
1303
|
};
|
|
603
1304
|
}
|
|
604
1305
|
|
|
1306
|
+
/**
|
|
1307
|
+
* The Viji API surface available to artist code as the global `viji` object.
|
|
1308
|
+
* Groups canvas access, timing, connected media APIs, user interaction, device
|
|
1309
|
+
* sensors, parameter helpers, and the `useContext()` selector.
|
|
1310
|
+
*/
|
|
605
1311
|
interface VijiAPI {
|
|
1312
|
+
/** The `OffscreenCanvas` driving the scene. The host owns its lifecycle and dimensions; prefer `useContext()` over touching this directly. */
|
|
606
1313
|
canvas: OffscreenCanvas;
|
|
1314
|
+
/** 2D rendering context. `undefined` until `viji.useContext('2d')` is called; afterwards equals the cached context. */
|
|
607
1315
|
ctx?: OffscreenCanvasRenderingContext2D;
|
|
1316
|
+
/** WebGL rendering context. `undefined` until `viji.useContext('webgl')` or `'webgl2'` is called; afterwards equals the cached context. */
|
|
608
1317
|
gl?: WebGLRenderingContext | WebGL2RenderingContext;
|
|
1318
|
+
/** Current canvas width in pixels. Updates automatically when the host resizes the canvas — read every frame. */
|
|
609
1319
|
width: number;
|
|
1320
|
+
/** Current canvas height in pixels. Updates automatically when the host resizes the canvas — read every frame. */
|
|
610
1321
|
height: number;
|
|
1322
|
+
/** Seconds elapsed since the scene started (monotonically increasing float). Use for oscillations and absolute-time effects. */
|
|
611
1323
|
time: number;
|
|
1324
|
+
/** Seconds since the previous frame. Use for accumulation: movement, physics, fades — anything that should progress per second regardless of FPS. */
|
|
612
1325
|
deltaTime: number;
|
|
1326
|
+
/** Integer frame counter starting at `0` and incrementing by `1` each frame. */
|
|
613
1327
|
frameCount: number;
|
|
1328
|
+
/** Target frames-per-second for the host's current frame-rate mode (`60` for `'full'`, `30` for `'half'`). */
|
|
614
1329
|
fps: number;
|
|
1330
|
+
/** Real-time analysis of the main audio stream: volume, frequency bands, beat detection, spectral features. Fields are zeroed when no audio source is connected. */
|
|
615
1331
|
audio: AudioAPI;
|
|
1332
|
+
/** Main video stream API: pixel access, frame metadata, and computer-vision results (face, hands, pose, segmentation). */
|
|
616
1333
|
video: VideoAPI;
|
|
1334
|
+
/** Additional video streams provided by the host. These do not run CV processing — use them for raw pixel access only. */
|
|
617
1335
|
videoStreams: VideoAPI[];
|
|
1336
|
+
/** Additional audio streams (lightweight analysis: volume, bands, spectral). No beat detection — use the main `audio` for that. */
|
|
618
1337
|
audioStreams: AudioStreamAPI[];
|
|
1338
|
+
/** Mouse position, buttons, wheel, and per-frame movement state. Coordinates are in canvas pixels. */
|
|
619
1339
|
mouse: MouseAPI;
|
|
1340
|
+
/** Keyboard state: per-frame press/release queries, currently held keys, and modifier flags. */
|
|
620
1341
|
keyboard: KeyboardAPI;
|
|
1342
|
+
/** Multi-touch state: count, primary touch shortcut, and the per-frame `started` / `moved` / `ended` arrays. */
|
|
621
1343
|
touches: TouchAPI;
|
|
1344
|
+
/** Unified pointer (mouse + primary touch) — convenient single-point input that works on both desktop and mobile. */
|
|
622
1345
|
pointer: PointerAPI;
|
|
1346
|
+
/** Internal device sensors (motion + orientation) reported by the device running the scene. */
|
|
623
1347
|
device: DeviceSensorState;
|
|
1348
|
+
/** External connected devices reporting sensor / camera / audio. Each entry has its own `id`, `name`, and per-device API surface. */
|
|
624
1349
|
devices: DeviceState[];
|
|
1350
|
+
/**
|
|
1351
|
+
* Declares a numeric slider parameter. The host UI renders a draggable slider.
|
|
1352
|
+
* Must be called at the top level of the scene — never inside `render()`.
|
|
1353
|
+
*
|
|
1354
|
+
* @example
|
|
1355
|
+
* const radius = viji.slider(50, { min: 10, max: 200, step: 1, label: 'Radius' });
|
|
1356
|
+
* // ...
|
|
1357
|
+
* function render(viji) { drawCircle(radius.value); }
|
|
1358
|
+
*/
|
|
625
1359
|
slider: (defaultValue: number, config: SliderConfig) => SliderParameter;
|
|
626
|
-
|
|
1360
|
+
/**
|
|
1361
|
+
* Declares a color parameter. Accepts hex strings, CSS color functions, or RGB / HSB objects;
|
|
1362
|
+
* the value is always normalized to a canonical lowercase hex string and exposes derived `.rgb` / `.hsb` accessors.
|
|
1363
|
+
* Must be called at the top level of the scene — never inside `render()`.
|
|
1364
|
+
*
|
|
1365
|
+
* @example
|
|
1366
|
+
* const tint = viji.color('#ff6600', { label: 'Tint' });
|
|
1367
|
+
* // tint.value -> '#ff6600', tint.rgb -> { r: 255, g: 102, b: 0 }, tint.hsb -> { h: 24, s: 100, b: 100 }
|
|
1368
|
+
*/
|
|
1369
|
+
color: (defaultValue: ColorInput, config: ColorConfig) => ColorParameter;
|
|
1370
|
+
/**
|
|
1371
|
+
* Declares a boolean toggle parameter. The host UI renders an on/off switch.
|
|
1372
|
+
* Must be called at the top level of the scene — never inside `render()`.
|
|
1373
|
+
*
|
|
1374
|
+
* @example
|
|
1375
|
+
* const showTrail = viji.toggle(true, { label: 'Show Trail' });
|
|
1376
|
+
*/
|
|
627
1377
|
toggle: (defaultValue: boolean, config: ToggleConfig) => ToggleParameter;
|
|
1378
|
+
/**
|
|
1379
|
+
* Declares a dropdown selection parameter. The element type of `options`
|
|
1380
|
+
* (`string` or `number`) determines the parameter's value type.
|
|
1381
|
+
* Must be called at the top level of the scene — never inside `render()`.
|
|
1382
|
+
*
|
|
1383
|
+
* @example
|
|
1384
|
+
* const shape = viji.select('circle', { options: ['circle', 'square', 'triangle'], label: 'Shape' });
|
|
1385
|
+
*/
|
|
628
1386
|
select: (defaultValue: string | number, config: SelectConfig) => SelectParameter;
|
|
1387
|
+
/**
|
|
1388
|
+
* Declares a text input parameter. The host UI renders a single-line text field.
|
|
1389
|
+
* Must be called at the top level of the scene — never inside `render()`.
|
|
1390
|
+
*
|
|
1391
|
+
* @example
|
|
1392
|
+
* const caption = viji.text('Hello', { label: 'Caption', maxLength: 64 });
|
|
1393
|
+
*/
|
|
629
1394
|
text: (defaultValue: string, config: TextConfig) => TextParameter;
|
|
1395
|
+
/**
|
|
1396
|
+
* Declares a precise numeric input parameter. Like `slider()` but the host UI
|
|
1397
|
+
* shows a number field instead of a draggable handle.
|
|
1398
|
+
* Must be called at the top level of the scene — never inside `render()`.
|
|
1399
|
+
*
|
|
1400
|
+
* @example
|
|
1401
|
+
* const count = viji.number(12, { min: 1, max: 64, step: 1, label: 'Count' });
|
|
1402
|
+
*/
|
|
630
1403
|
number: (defaultValue: number, config: NumberConfig) => NumberParameter;
|
|
1404
|
+
/**
|
|
1405
|
+
* Declares an image upload parameter. The user-supplied image becomes available
|
|
1406
|
+
* as `value` (an `ImageBitmap`); in P5 scenes a `.p5` accessor is also added at runtime.
|
|
1407
|
+
* Must be called at the top level of the scene — never inside `render()`.
|
|
1408
|
+
*
|
|
1409
|
+
* @example
|
|
1410
|
+
* const tex = viji.image(null, { label: 'Texture' });
|
|
1411
|
+
* if (tex.value) ctx.drawImage(tex.value, 0, 0, viji.width, viji.height);
|
|
1412
|
+
*/
|
|
631
1413
|
image: (defaultValue: null, config: ImageConfig) => ImageParameter;
|
|
1414
|
+
/**
|
|
1415
|
+
* Declares a momentary button parameter. `value` is `true` for one frame after
|
|
1416
|
+
* the user clicks, then auto-resets — perfect for triggering discrete events.
|
|
1417
|
+
* Must be called at the top level of the scene — never inside `render()`.
|
|
1418
|
+
*
|
|
1419
|
+
* @example
|
|
1420
|
+
* const reset = viji.button({ label: 'Reset' });
|
|
1421
|
+
* function render(viji) { if (reset.value) state = initialState; }
|
|
1422
|
+
*/
|
|
632
1423
|
button: (config: ButtonConfig) => ButtonParameter;
|
|
1424
|
+
/**
|
|
1425
|
+
* Declares a 2D coordinate parameter. Both `value.x` and `value.y` are in `-1..1`.
|
|
1426
|
+
* Must be called at the top level of the scene — never inside `render()`.
|
|
1427
|
+
*
|
|
1428
|
+
* @example
|
|
1429
|
+
* const origin = viji.coordinate({ x: 0, y: 0 }, { label: 'Origin' });
|
|
1430
|
+
* const cx = viji.width / 2 + origin.value.x * viji.width * 0.4;
|
|
1431
|
+
*/
|
|
633
1432
|
coordinate: (defaultValue: CoordinateValue, config: CoordinateConfig) => CoordinateParameter;
|
|
1433
|
+
/**
|
|
1434
|
+
* Selects the 2D canvas rendering context. Returns the same instance on
|
|
1435
|
+
* subsequent calls and also stores it on `viji.ctx`. A canvas only supports
|
|
1436
|
+
* one context type — if a different context type was already requested,
|
|
1437
|
+
* the call returns `null`. Choose ONE type and use it for the entire scene.
|
|
1438
|
+
*
|
|
1439
|
+
* @example
|
|
1440
|
+
* const ctx = viji.useContext('2d');
|
|
1441
|
+
* ctx.fillRect(0, 0, viji.width, viji.height);
|
|
1442
|
+
*/
|
|
634
1443
|
useContext(type: '2d'): OffscreenCanvasRenderingContext2D;
|
|
1444
|
+
/**
|
|
1445
|
+
* Selects a WebGL 1 rendering context. Returns the same instance on subsequent
|
|
1446
|
+
* calls and also stores it on `viji.gl`. A canvas only supports one context
|
|
1447
|
+
* type — if a different context type was already requested, the call returns
|
|
1448
|
+
* `null`. Choose ONE type and use it for the entire scene.
|
|
1449
|
+
*/
|
|
635
1450
|
useContext(type: 'webgl'): WebGLRenderingContext;
|
|
1451
|
+
/**
|
|
1452
|
+
* Selects a WebGL 2 rendering context. Returns the same instance on subsequent
|
|
1453
|
+
* calls and also stores it on `viji.gl`. A canvas only supports one context
|
|
1454
|
+
* type — if a different context type was already requested, the call returns
|
|
1455
|
+
* `null`. Choose ONE type and use it for the entire scene.
|
|
1456
|
+
*/
|
|
636
1457
|
useContext(type: 'webgl2'): WebGL2RenderingContext;
|
|
637
1458
|
}
|
|
638
1459
|
|