@viji-dev/core 0.11.8 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/artist-dts-p5.js +1 -1
- package/dist/artist-dts.js +1 -1
- package/dist/artist-global-p5.d.ts +109 -28
- package/dist/artist-global.d.ts +109 -28
- package/dist/artist-jsdoc.d.ts +27 -18
- package/dist/assets/viji.worker-Clz2HjaH.js +587 -0
- package/dist/docs-api.js +12 -12
- package/dist/{essentia-wasm.web-B48G7Mzk.js → essentia-wasm.web-DGWIGRe8.js} +1 -2
- package/dist/essentia.js-core.es-CC_3Ap1i.js +0 -1
- package/dist/{index-fdus0y3o.js → index-CkmT_70R.js} +527 -561
- package/dist/index.d.ts +192 -737
- package/dist/index.js +5 -8
- package/dist/param-inference.d.ts +14 -0
- package/dist/param-inference.js +3235 -0
- package/dist/scene-analysis-DNN8jTaH.js +181 -0
- package/dist/scene-analysis.d.ts +83 -0
- package/dist/scene-analysis.js +4 -0
- package/package.json +17 -2
- package/dist/assets/viji.worker-B1ltcEP7.js +0 -24324
package/dist/index.d.ts
CHANGED
|
@@ -1,37 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Analyzes a scene's source code and returns the capabilities it declares,
|
|
3
|
-
* without constructing a core. Composes {@link SceneAnalyzer.detectRendererType}
|
|
4
|
-
* and {@link SceneAnalyzer.detectCapabilities} — the same two steps `VijiCore`
|
|
5
|
-
* performs at construction — so for a given core version the result is
|
|
6
|
-
* identical to a booted core's `getSceneData().capabilities`.
|
|
7
|
-
*
|
|
8
|
-
* Intended for out-of-engine consumers (e.g. server-side persistence of
|
|
9
|
-
* per-scene capabilities). The following properties are an intentional, kept
|
|
10
|
-
* contract:
|
|
11
|
-
* - **Pure and Node-safe** — static analysis only; the scene is never executed
|
|
12
|
-
* and no DOM, canvas, or worker API is touched. Callable in a bare Node
|
|
13
|
-
* process.
|
|
14
|
-
* - **Deterministic** — same code string, same result, within a core version.
|
|
15
|
-
* - **Inclusive-biased** — a capability the scene uses is never hidden; a
|
|
16
|
-
* rare ambiguous case may report an extra one.
|
|
17
|
-
* - **Safe on untrusted input** — linear-time scan and regexes with no throw
|
|
18
|
-
* paths on any string input.
|
|
19
|
-
*
|
|
20
|
-
* Note for consumers filtering on `video`: the flag means the scene reads the
|
|
21
|
-
* video *image* (pixels/texture). Shader scenes that consume only CV result
|
|
22
|
-
* uniforms report `cv.*` without `video`, so a "reacts to the camera" filter
|
|
23
|
-
* should test `video || any cv key`.
|
|
24
|
-
*
|
|
25
|
-
* @param sceneCode - The artist's scene code (native, p5, or shader)
|
|
26
|
-
* @returns The set of capabilities the scene declares it uses
|
|
27
|
-
* @example
|
|
28
|
-
* ```ts
|
|
29
|
-
* const caps = analyzeSceneCapabilities(scene.code);
|
|
30
|
-
* if (caps.audio) tags.push('audio');
|
|
31
|
-
* ```
|
|
32
|
-
*/
|
|
33
|
-
export declare function analyzeSceneCapabilities(sceneCode: string): SceneCapabilities;
|
|
34
|
-
|
|
35
1
|
declare interface AudioAnalysisState {
|
|
36
2
|
isConnected: boolean;
|
|
37
3
|
currentBPM: number;
|
|
@@ -220,618 +186,6 @@ declare interface AudioStreamAPI {
|
|
|
220
186
|
getWaveform: () => Float32Array;
|
|
221
187
|
}
|
|
222
188
|
|
|
223
|
-
/**
|
|
224
|
-
* Manages audio analysis and processing for the host side.
|
|
225
|
-
* Handles audio stream processing and provides real-time audio analysis.
|
|
226
|
-
*
|
|
227
|
-
* This system provides:
|
|
228
|
-
* - Real-time audio analysis (RMS, peak, frequency bands)
|
|
229
|
-
* - Beat detection with automatic BPM tracking
|
|
230
|
-
* - Onset detection for audio-reactive triggers
|
|
231
|
-
* - Smooth energy curves with envelope followers
|
|
232
|
-
* - Per-instrument onset tap patterns with automatic pattern recognition
|
|
233
|
-
* - Auto-gain normalization
|
|
234
|
-
* - Spectral feature extraction
|
|
235
|
-
* - AudioContext management and cleanup
|
|
236
|
-
* - Raw FFT data access for custom analysis
|
|
237
|
-
* - Analysis results sent to worker for artist API
|
|
238
|
-
*/
|
|
239
|
-
export declare class AudioSystem {
|
|
240
|
-
private audioContext;
|
|
241
|
-
private workletRegistered;
|
|
242
|
-
/**
|
|
243
|
-
* Persistent export capture tap (main channel only). Created lazily by
|
|
244
|
-
* `getAudioCaptureStream()`, lives for the AudioSystem lifetime, and is
|
|
245
|
-
* torn down only in `resetAudioState()`. Deliberately NOT severed by
|
|
246
|
-
* `disconnectMainStream()`: on mid-recording disconnect the tap stream
|
|
247
|
-
* goes silent while its tracks stay live, and `connectChannel()` re-wires
|
|
248
|
-
* each new main source node into it, so the exposed MediaStream stays
|
|
249
|
-
* stable across `setAudioStream()` swaps (a recorder never renegotiates).
|
|
250
|
-
*/
|
|
251
|
-
private captureDestination;
|
|
252
|
-
private mainChannel;
|
|
253
|
-
private additionalChannels;
|
|
254
|
-
private analysisInterval;
|
|
255
|
-
private analysisIntervalMs;
|
|
256
|
-
private stalenessTimer;
|
|
257
|
-
private static readonly STALENESS_THRESHOLD_MS;
|
|
258
|
-
/** Tracks the last non-zero BPM chosen for output to avoid dropping to 0 */
|
|
259
|
-
private lastNonZeroBpm;
|
|
260
|
-
/** Tracks which source provided the current BPM (pll | tempo | carry | default) */
|
|
261
|
-
private lastBpmSource;
|
|
262
|
-
private lastPhaseLogTime;
|
|
263
|
-
private onsetLogBuffer;
|
|
264
|
-
private debugMode;
|
|
265
|
-
/**
|
|
266
|
-
* One-way lifecycle flag flipped at the top of `resetAudioState()`.
|
|
267
|
-
* Gates re-entry into `ensureAudioContext()` and the public stream-mutation
|
|
268
|
-
* entry points so an in-flight async caller (e.g. a worker RPC firing
|
|
269
|
-
* mid-teardown) cannot resurrect a fresh `AudioContext` after destroy
|
|
270
|
-
* has already nulled the field. AudioSystem is single-use after reset.
|
|
271
|
-
*/
|
|
272
|
-
private destroyed;
|
|
273
|
-
private diagnosticLogger;
|
|
274
|
-
private onsetDetection;
|
|
275
|
-
private essentiaOnsetDetection;
|
|
276
|
-
private tempoInduction;
|
|
277
|
-
private pll;
|
|
278
|
-
private stateManager;
|
|
279
|
-
private lastKickDetected;
|
|
280
|
-
/** Raw band snapshot before auto-gain (for debug, main channel only) */
|
|
281
|
-
private rawBandsPreGain;
|
|
282
|
-
private readonly bandNames;
|
|
283
|
-
private essentiaBandHistories;
|
|
284
|
-
private readonly essentiaHistoryWindowMs;
|
|
285
|
-
private onsetTapManager;
|
|
286
|
-
private beatEnvelopeFollowers;
|
|
287
|
-
private beatDetectionEnabled;
|
|
288
|
-
private onsetDetectionEnabled;
|
|
289
|
-
private autoGainEnabled;
|
|
290
|
-
private idleTickerHandle;
|
|
291
|
-
private idleTickerLastTime;
|
|
292
|
-
/**
|
|
293
|
-
* Enable or disable comprehensive debug logging for all layers
|
|
294
|
-
* Enables enhanced logging in: MultiOnsetDetection, BeatStateManager
|
|
295
|
-
*/
|
|
296
|
-
setDebugMode(enabled: boolean): void;
|
|
297
|
-
/**
|
|
298
|
-
* Reset Essentia band onset histories (used to feed tempo induction when using the Essentia backend)
|
|
299
|
-
*/
|
|
300
|
-
private resetEssentiaBandHistories;
|
|
301
|
-
/**
|
|
302
|
-
* Update Essentia band onset histories and trim old entries
|
|
303
|
-
*/
|
|
304
|
-
private updateEssentiaBandHistories;
|
|
305
|
-
private getEssentiaBandHistory;
|
|
306
|
-
private getEssentiaAllHistories;
|
|
307
|
-
/**
|
|
308
|
-
* Handle frames pushed from AudioWorklet for main channel
|
|
309
|
-
*/
|
|
310
|
-
private handleMainWorkletFrame;
|
|
311
|
-
/**
|
|
312
|
-
* Handle frames pushed from AudioWorklet for an additional channel
|
|
313
|
-
*/
|
|
314
|
-
private handleAdditionalWorkletFrame;
|
|
315
|
-
/**
|
|
316
|
-
* Common analysis pipeline for any channel (FFT, volume, bands, auto-gain, spectral, smoothing).
|
|
317
|
-
* Returns raw band energies (pre-auto-gain) for optional beat processing by the caller.
|
|
318
|
-
*/
|
|
319
|
-
private analyzeChannelFrame;
|
|
320
|
-
/**
|
|
321
|
-
* Full analysis for the main channel (common analysis + beat pipeline).
|
|
322
|
-
*/
|
|
323
|
-
private analyzeMainFrame;
|
|
324
|
-
/**
|
|
325
|
-
* Lightweight analysis for additional/device channels (no beat detection).
|
|
326
|
-
*/
|
|
327
|
-
private analyzeAdditionalFrame;
|
|
328
|
-
/**
|
|
329
|
-
* Compute FFT on a specific channel's buffers
|
|
330
|
-
*/
|
|
331
|
-
private computeChannelFFT;
|
|
332
|
-
/**
|
|
333
|
-
* Calculate RMS/peak from float PCM frame (-1..1)
|
|
334
|
-
*/
|
|
335
|
-
private calculateVolumeMetrics;
|
|
336
|
-
/**
|
|
337
|
-
* Calculate perceptual/log-ish bands on a specific channel
|
|
338
|
-
*/
|
|
339
|
-
private calculateChannelBands;
|
|
340
|
-
/**
|
|
341
|
-
* Spectral features on a specific channel
|
|
342
|
-
*/
|
|
343
|
-
private calculateChannelSpectral;
|
|
344
|
-
/**
|
|
345
|
-
* Run onset + beat detection pipeline and return BeatState
|
|
346
|
-
*/
|
|
347
|
-
private runBeatPipeline;
|
|
348
|
-
/**
|
|
349
|
-
* Debug logging helper
|
|
350
|
-
*/
|
|
351
|
-
private debugLog;
|
|
352
|
-
private fftSize;
|
|
353
|
-
private audioStateBeat;
|
|
354
|
-
private analysisSinks;
|
|
355
|
-
private latestChannelResults;
|
|
356
|
-
private channelMeta;
|
|
357
|
-
constructor(sendAnalysisResultsCallback?: (data: any) => void);
|
|
358
|
-
/**
|
|
359
|
-
* Get the current audio analysis state (for host-side usage)
|
|
360
|
-
*/
|
|
361
|
-
getAudioState(): {
|
|
362
|
-
isConnected: boolean;
|
|
363
|
-
volume: {
|
|
364
|
-
current: number;
|
|
365
|
-
peak: number;
|
|
366
|
-
smoothed: number;
|
|
367
|
-
};
|
|
368
|
-
bands: {
|
|
369
|
-
low: number;
|
|
370
|
-
lowMid: number;
|
|
371
|
-
mid: number;
|
|
372
|
-
highMid: number;
|
|
373
|
-
high: number;
|
|
374
|
-
lowSmoothed: number;
|
|
375
|
-
lowMidSmoothed: number;
|
|
376
|
-
midSmoothed: number;
|
|
377
|
-
highMidSmoothed: number;
|
|
378
|
-
highSmoothed: number;
|
|
379
|
-
};
|
|
380
|
-
spectral: {
|
|
381
|
-
brightness: number;
|
|
382
|
-
flatness: number;
|
|
383
|
-
};
|
|
384
|
-
};
|
|
385
|
-
/**
|
|
386
|
-
* Get a recordable MediaStream carrying the main channel's audio, for
|
|
387
|
-
* host-side export recording. Main channel only — additional channels and
|
|
388
|
-
* source-fed device channels (analysis values, no PCM) are not included.
|
|
389
|
-
*
|
|
390
|
-
* The underlying destination node is created lazily on first call and then
|
|
391
|
-
* persists for the AudioSystem lifetime: on a source swap the stream keeps
|
|
392
|
-
* flowing with the new audio, and on disconnect it goes silent with tracks
|
|
393
|
-
* still live. Returns `null` when the main channel is not connected at
|
|
394
|
-
* call time (or after teardown).
|
|
395
|
-
*/
|
|
396
|
-
getAudioCaptureStream(): MediaStream | null;
|
|
397
|
-
/**
|
|
398
|
-
* Initialize Essentia.js (async WASM loading)
|
|
399
|
-
*/
|
|
400
|
-
private initializeEssentia;
|
|
401
|
-
/**
|
|
402
|
-
* Wait for Essentia.js to initialize (for testing/benchmarking)
|
|
403
|
-
* Returns immediately if already initialized or if initialization failed
|
|
404
|
-
*/
|
|
405
|
-
waitForEssentia(timeoutMs?: number): Promise<boolean>;
|
|
406
|
-
/**
|
|
407
|
-
* Handle audio stream update for main stream (called from VijiCore)
|
|
408
|
-
*/
|
|
409
|
-
handleAudioStreamUpdate(data: {
|
|
410
|
-
audioStream?: MediaStream | null;
|
|
411
|
-
timestamp: number;
|
|
412
|
-
}): void;
|
|
413
|
-
/**
|
|
414
|
-
* Ensure the shared AudioContext is created and resumed.
|
|
415
|
-
*/
|
|
416
|
-
private ensureAudioContext;
|
|
417
|
-
/**
|
|
418
|
-
* Connect a channel to Web Audio nodes (source, worklet/analyser).
|
|
419
|
-
* Used for both main and additional channels.
|
|
420
|
-
*
|
|
421
|
-
* Lifecycle ordering invariant: when wiring the main channel, the idle
|
|
422
|
-
* ticker MUST be stopped before the first `await` so that an in-flight
|
|
423
|
-
* tick cannot race the audio path coming online. The ticker callback
|
|
424
|
-
* additionally guards on `mainChannel.audioState.isConnected` for
|
|
425
|
-
* belt-and-braces.
|
|
426
|
-
*/
|
|
427
|
-
private connectChannel;
|
|
428
|
-
/**
|
|
429
|
-
* Set the main audio stream for analysis (preserves original public API surface).
|
|
430
|
-
*/
|
|
431
|
-
private setAudioStream;
|
|
432
|
-
/**
|
|
433
|
-
* Disconnect the main audio stream (does NOT close AudioContext -- additional channels may still be active).
|
|
434
|
-
*
|
|
435
|
-
* Lifecycle ordering invariant: tear down the audio nodes first, clear any
|
|
436
|
-
* stale frequency / waveform buffers, then restart the idle ticker. The
|
|
437
|
-
* ticker callback's defensive `isConnected` guard prevents any stale tick
|
|
438
|
-
* from observing a half-disconnected state.
|
|
439
|
-
*/
|
|
440
|
-
private disconnectMainStream;
|
|
441
|
-
/**
|
|
442
|
-
* Set up an AudioWorklet node for a specific channel.
|
|
443
|
-
* Registers the processor module once (shared), creates a new WorkletNode per channel.
|
|
444
|
-
*/
|
|
445
|
-
private setupChannelWorklet;
|
|
446
|
-
/**
|
|
447
|
-
* Ensure the shared analysis interval is running (for analyser-mode channels).
|
|
448
|
-
*/
|
|
449
|
-
private ensureAnalysisLoop;
|
|
450
|
-
/**
|
|
451
|
-
* Stop the shared analysis loop.
|
|
452
|
-
*/
|
|
453
|
-
private stopAnalysisLoop;
|
|
454
|
-
/**
|
|
455
|
-
* Shared staleness timer: checks all channels for stale data.
|
|
456
|
-
*/
|
|
457
|
-
private startStalenessTimer;
|
|
458
|
-
private stopStalenessTimer;
|
|
459
|
-
/**
|
|
460
|
-
* Pause audio analysis (for tests or temporary suspension)
|
|
461
|
-
*/
|
|
462
|
-
pauseAnalysis(): void;
|
|
463
|
-
/**
|
|
464
|
-
* Resume audio analysis after pause
|
|
465
|
-
*/
|
|
466
|
-
resumeAnalysis(): void;
|
|
467
|
-
/**
|
|
468
|
-
* Shared analysis loop callback: iterates main + additional channels in analyser mode.
|
|
469
|
-
*/
|
|
470
|
-
private performAnalysis;
|
|
471
|
-
private applyChannelAutoGain;
|
|
472
|
-
/**
|
|
473
|
-
* Update smooth versions of frequency bands on a specific channel
|
|
474
|
-
*/
|
|
475
|
-
private updateChannelSmoothBands;
|
|
476
|
-
/**
|
|
477
|
-
* Send analysis results for a specific channel to the worker.
|
|
478
|
-
* Tags with streamIndex and conditionally includes beat data (main channel only).
|
|
479
|
-
*/
|
|
480
|
-
private sendChannelResults;
|
|
481
|
-
/**
|
|
482
|
-
* Deliver an analysis message to one sink with PER-SINK typed-array
|
|
483
|
-
* copies. The host→worker relay transfers `frequencyData`/`waveformData`
|
|
484
|
-
* buffers (zero-copy receive, `collectTransferables` case
|
|
485
|
-
* 'audio-analysis-update') — posting the same message object to N sinks
|
|
486
|
-
* would let the first post DETACH the buffers and starve every other
|
|
487
|
-
* sink (and the replay cache). One copy per sink per analysis frame is
|
|
488
|
-
* the unavoidable cost of a transfer-per-receiver delivery.
|
|
489
|
-
*/
|
|
490
|
-
private deliverAnalysisMessage;
|
|
491
|
-
/**
|
|
492
|
-
* Register a result-delivery sink (one per attached core's worker) and
|
|
493
|
-
* replay current state to it in FIFO order: the latest main-channel
|
|
494
|
-
* analysis, then per non-main channel its `audio-stream-setup` identity
|
|
495
|
-
* followed by the latest cached analysis. A mid-stream attach is thereby
|
|
496
|
-
* indistinguishable from a fresh connect on the receiving worker.
|
|
497
|
-
*/
|
|
498
|
-
addSink(id: string, send: (message: {
|
|
499
|
-
type: string;
|
|
500
|
-
data: any;
|
|
501
|
-
}) => void): void;
|
|
502
|
-
/**
|
|
503
|
-
* Remove a previously registered result-delivery sink.
|
|
504
|
-
*/
|
|
505
|
-
removeSink(id: string): void;
|
|
506
|
-
private buildStreamSetupMessage;
|
|
507
|
-
private emitStreamSetup;
|
|
508
|
-
/**
|
|
509
|
-
* Reset audio values to defaults (main channel + beat state)
|
|
510
|
-
*/
|
|
511
|
-
private resetAudioValues;
|
|
512
|
-
/**
|
|
513
|
-
* Reset all audio state (called when destroying).
|
|
514
|
-
* Disconnects all channels, closes AudioContext, resets all modules.
|
|
515
|
-
*
|
|
516
|
-
* Returns a Promise that resolves once `AudioContext.close()` has fully
|
|
517
|
-
* transitioned the context to `closed`. Callers tearing the system down
|
|
518
|
-
* (e.g. `VijiCore.destroy()`) MUST await this; otherwise Blink keeps the
|
|
519
|
-
* context alive in `Pending activities` and leaks ~4 MB per cycle.
|
|
520
|
-
*/
|
|
521
|
-
resetAudioState(): Promise<void>;
|
|
522
|
-
private sourceFedIndices;
|
|
523
|
-
/**
|
|
524
|
-
* Register a source-fed device channel (no MediaStream). Emits the
|
|
525
|
-
* `audio-stream-setup` identity to all sinks and starts the staleness
|
|
526
|
-
* timer. Idempotent.
|
|
527
|
-
*/
|
|
528
|
-
addDeviceAnalysisChannel(streamIndex: number, deviceId: string): void;
|
|
529
|
-
/**
|
|
530
|
-
* Feed a controller analysis packet into a source-fed channel. Instant
|
|
531
|
-
* values pass through; the smoothed companions are derived host-side via
|
|
532
|
-
* the channel's envelope followers to absorb network packet bunching
|
|
533
|
-
* (§3.9 light smoothing). Missing `frequencyData`/`waveform` surface as
|
|
534
|
-
* empty arrays (the controller omits them unless the scene reads them).
|
|
535
|
-
*/
|
|
536
|
-
feedDeviceAnalysis(streamIndex: number, payload: {
|
|
537
|
-
volume?: {
|
|
538
|
-
current?: number;
|
|
539
|
-
peak?: number;
|
|
540
|
-
};
|
|
541
|
-
bands?: {
|
|
542
|
-
low?: number;
|
|
543
|
-
lowMid?: number;
|
|
544
|
-
mid?: number;
|
|
545
|
-
highMid?: number;
|
|
546
|
-
high?: number;
|
|
547
|
-
};
|
|
548
|
-
spectral?: {
|
|
549
|
-
brightness?: number;
|
|
550
|
-
flatness?: number;
|
|
551
|
-
};
|
|
552
|
-
frequencyData?: Uint8Array;
|
|
553
|
-
waveformData?: Float32Array;
|
|
554
|
-
}): void;
|
|
555
|
-
/** Remove a source-fed device channel (sends a disconnected state). */
|
|
556
|
-
removeDeviceAnalysisChannel(streamIndex: number): void;
|
|
557
|
-
/**
|
|
558
|
-
* Add an additional audio stream (lightweight analysis, no beat detection).
|
|
559
|
-
* When `meta` is provided, the stream identity is recorded and an
|
|
560
|
-
* `audio-stream-setup` message is emitted to every sink (and replayed to
|
|
561
|
-
* sinks attached later).
|
|
562
|
-
* @param streamIndex Global stream index (AUDIO_ADDITIONAL_BASE + n or AUDIO_DEVICE_BASE + n)
|
|
563
|
-
* @param stream The MediaStream to analyze
|
|
564
|
-
* @param meta Stream identity for the worker-side two-phase setup protocol
|
|
565
|
-
*/
|
|
566
|
-
addStream(streamIndex: number, stream: MediaStream, meta?: {
|
|
567
|
-
streamType: 'additional' | 'device';
|
|
568
|
-
deviceId?: string;
|
|
569
|
-
}): Promise<void>;
|
|
570
|
-
/**
|
|
571
|
-
* Remove an additional audio stream.
|
|
572
|
-
*/
|
|
573
|
-
removeStream(streamIndex: number): void;
|
|
574
|
-
/**
|
|
575
|
-
* Tear down all additional channels in the base range and rebuild with
|
|
576
|
-
* corrected indices, re-emitting `audio-stream-setup` for the re-indexed
|
|
577
|
-
* streams. Mirrors video's reinitializeAdditionalCoordinators pattern.
|
|
578
|
-
*/
|
|
579
|
-
reinitializeAdditionalChannels(streams: MediaStream[], baseIndex: number): Promise<void>;
|
|
580
|
-
/**
|
|
581
|
-
* Get the number of additional audio channels.
|
|
582
|
-
*/
|
|
583
|
-
getChannelCount(): number;
|
|
584
|
-
/**
|
|
585
|
-
* Record a tap for the specified instrument onset.
|
|
586
|
-
* `options.skipRecognition` opts the tap out of the recognition pipeline
|
|
587
|
-
* only (no IOI accumulation, no `tryRecognizePattern`, no `applyPattern`).
|
|
588
|
-
* Mode still flips `'auto' → 'tapping'` so the visual envelope and
|
|
589
|
-
* audio-event filter run; only the pattern-detection pipeline is bypassed.
|
|
590
|
-
* Used by host-side relay of forwarded tap messages where another core
|
|
591
|
-
* owns the authoritative recognition.
|
|
592
|
-
*/
|
|
593
|
-
tapOnset(instrument: InstrumentType, options?: {
|
|
594
|
-
skipRecognition?: boolean;
|
|
595
|
-
}): void;
|
|
596
|
-
/**
|
|
597
|
-
* Clear tap pattern for an instrument, restoring auto-detection
|
|
598
|
-
*/
|
|
599
|
-
clearOnsetTap(instrument: InstrumentType): void;
|
|
600
|
-
/**
|
|
601
|
-
* Get the current onset mode for an instrument
|
|
602
|
-
*/
|
|
603
|
-
getOnsetMode(instrument: InstrumentType): OnsetMode_2;
|
|
604
|
-
/**
|
|
605
|
-
* Get pattern info for an instrument (null if no pattern recognized)
|
|
606
|
-
*/
|
|
607
|
-
getOnsetPatternInfo(instrument: InstrumentType): PatternInfo | null;
|
|
608
|
-
/**
|
|
609
|
-
* Mute/unmute an instrument onset (suppresses output without destroying state)
|
|
610
|
-
*/
|
|
611
|
-
setOnsetMuted(instrument: InstrumentType, muted: boolean): void;
|
|
612
|
-
/**
|
|
613
|
-
* Check if an instrument onset is muted
|
|
614
|
-
*/
|
|
615
|
-
isOnsetMuted(instrument: InstrumentType): boolean;
|
|
616
|
-
onOnsetModeChange(listener: (ev: OnsetModeChangeEvent) => void): Unsubscribe;
|
|
617
|
-
onOnsetSessionEnd(listener: (ev: OnsetSessionEndEvent) => void): Unsubscribe;
|
|
618
|
-
onOnsetMuteChange(listener: (ev: OnsetMuteChangeEvent) => void): Unsubscribe;
|
|
619
|
-
exportOnsetSessionState(options?: {
|
|
620
|
-
instruments?: ReadonlyArray<InstrumentType>;
|
|
621
|
-
}): SerializedOnsetState;
|
|
622
|
-
importOnsetSessionState(state: SerializedOnsetState, clockOffset: number): void;
|
|
623
|
-
exportAudioAnalysisState(): SerializedAudioAnalysisState;
|
|
624
|
-
importAudioAnalysisState(state: SerializedAudioAnalysisState, clockOffset: number): void;
|
|
625
|
-
private startIdleTicker;
|
|
626
|
-
private stopIdleTicker;
|
|
627
|
-
private tickIdle;
|
|
628
|
-
private makeEmptyBeatState;
|
|
629
|
-
/**
|
|
630
|
-
* Get current BPM (manual or auto-detected)
|
|
631
|
-
*/
|
|
632
|
-
getCurrentBPM(): number;
|
|
633
|
-
/**
|
|
634
|
-
* Nudge beat phase
|
|
635
|
-
*/
|
|
636
|
-
nudgeBeatPhase(amount: number): void;
|
|
637
|
-
/**
|
|
638
|
-
* Reset beat phase to next beat
|
|
639
|
-
*/
|
|
640
|
-
resetBeatPhase(): void;
|
|
641
|
-
/**
|
|
642
|
-
* Set FFT size
|
|
643
|
-
*/
|
|
644
|
-
setFFTSize(size: 2048 | 4096 | 8192): void;
|
|
645
|
-
/**
|
|
646
|
-
* Enable/disable auto-gain
|
|
647
|
-
*/
|
|
648
|
-
setAutoGain(enabled: boolean): void;
|
|
649
|
-
/**
|
|
650
|
-
* Enable/disable beat detection
|
|
651
|
-
*/
|
|
652
|
-
setBeatDetection(enabled: boolean): void;
|
|
653
|
-
/**
|
|
654
|
-
* Enable/disable onset detection
|
|
655
|
-
*/
|
|
656
|
-
setOnsetDetection(enabled: boolean): void;
|
|
657
|
-
/**
|
|
658
|
-
* Get complete audio analysis state
|
|
659
|
-
*/
|
|
660
|
-
getState(): {
|
|
661
|
-
isConnected: boolean;
|
|
662
|
-
currentBPM: number;
|
|
663
|
-
confidence: number;
|
|
664
|
-
isLocked: boolean;
|
|
665
|
-
trackingState: string;
|
|
666
|
-
onsetModes: {
|
|
667
|
-
kick: OnsetMode_2;
|
|
668
|
-
snare: OnsetMode_2;
|
|
669
|
-
hat: OnsetMode_2;
|
|
670
|
-
};
|
|
671
|
-
};
|
|
672
|
-
/**
|
|
673
|
-
* Get current per-frame audio analysis data (for UI display)
|
|
674
|
-
* Returns a snapshot of the latest audio analysis values
|
|
675
|
-
*/
|
|
676
|
-
getCurrentAudioData(): {
|
|
677
|
-
isConnected: boolean;
|
|
678
|
-
volume: {
|
|
679
|
-
current: number;
|
|
680
|
-
peak: number;
|
|
681
|
-
smoothed: number;
|
|
682
|
-
};
|
|
683
|
-
bands: {
|
|
684
|
-
low: number;
|
|
685
|
-
lowMid: number;
|
|
686
|
-
mid: number;
|
|
687
|
-
highMid: number;
|
|
688
|
-
high: number;
|
|
689
|
-
lowSmoothed: number;
|
|
690
|
-
lowMidSmoothed: number;
|
|
691
|
-
midSmoothed: number;
|
|
692
|
-
highMidSmoothed: number;
|
|
693
|
-
highSmoothed: number;
|
|
694
|
-
};
|
|
695
|
-
beat: {
|
|
696
|
-
kick: number;
|
|
697
|
-
snare: number;
|
|
698
|
-
hat: number;
|
|
699
|
-
any: number;
|
|
700
|
-
kickSmoothed: number;
|
|
701
|
-
snareSmoothed: number;
|
|
702
|
-
hatSmoothed: number;
|
|
703
|
-
anySmoothed: number;
|
|
704
|
-
events: BeatEvent[];
|
|
705
|
-
bpm: number;
|
|
706
|
-
confidence: number;
|
|
707
|
-
isLocked: boolean;
|
|
708
|
-
};
|
|
709
|
-
spectral: {
|
|
710
|
-
brightness: number;
|
|
711
|
-
flatness: number;
|
|
712
|
-
};
|
|
713
|
-
};
|
|
714
|
-
/**
|
|
715
|
-
* Get enhanced debug snapshot from all layers
|
|
716
|
-
*/
|
|
717
|
-
getEnhancedDebugSnapshot(): {
|
|
718
|
-
onset: {
|
|
719
|
-
currentBPM: number;
|
|
720
|
-
debugMode: boolean;
|
|
721
|
-
totalOnsets: number;
|
|
722
|
-
currentCluster: {
|
|
723
|
-
primaryBands: string[];
|
|
724
|
-
ageMssinceStart: number;
|
|
725
|
-
} | null;
|
|
726
|
-
bandStates: Record<string, {
|
|
727
|
-
lastOnsetTime: number;
|
|
728
|
-
onsetCount: number;
|
|
729
|
-
cutoff: number;
|
|
730
|
-
minEnergy: number;
|
|
731
|
-
}>;
|
|
732
|
-
};
|
|
733
|
-
tempo: {
|
|
734
|
-
bpm: number;
|
|
735
|
-
confidence: number;
|
|
736
|
-
};
|
|
737
|
-
pll: PLLState;
|
|
738
|
-
state: {
|
|
739
|
-
trackingState: TrackingState;
|
|
740
|
-
isLocked: boolean;
|
|
741
|
-
};
|
|
742
|
-
};
|
|
743
|
-
/**
|
|
744
|
-
* Get comprehensive debug info for testing/debugging
|
|
745
|
-
* Exposes internal state of all analysis layers
|
|
746
|
-
*/
|
|
747
|
-
getDebugInfo(): {
|
|
748
|
-
onset: Record<BandName, {
|
|
749
|
-
energy: number;
|
|
750
|
-
prevEnergy: number;
|
|
751
|
-
prevEnergy2: number;
|
|
752
|
-
minEnergy: number;
|
|
753
|
-
flux: number;
|
|
754
|
-
cutoff: number;
|
|
755
|
-
minThreshold: number;
|
|
756
|
-
minFlux: number;
|
|
757
|
-
wouldTrigger: boolean;
|
|
758
|
-
reason: string;
|
|
759
|
-
debounceMs: number;
|
|
760
|
-
lastOnsetAgo: number;
|
|
761
|
-
baselineAlpha?: number;
|
|
762
|
-
dtMs?: number;
|
|
763
|
-
fluxPrevOnly?: number;
|
|
764
|
-
fluxBaseline?: number;
|
|
765
|
-
initialized?: boolean;
|
|
766
|
-
lastFluxStored?: number;
|
|
767
|
-
fluxCalcNow?: number;
|
|
768
|
-
}>;
|
|
769
|
-
tempo: {
|
|
770
|
-
currentBPM: number;
|
|
771
|
-
confidence: number;
|
|
772
|
-
method: string;
|
|
773
|
-
anchorBand: string | null;
|
|
774
|
-
methodAgreement: number;
|
|
775
|
-
bpmHistory: number[];
|
|
776
|
-
envelopeLength: number;
|
|
777
|
-
lastAutocorrBPM: number;
|
|
778
|
-
lastIOIBPM: number;
|
|
779
|
-
};
|
|
780
|
-
pll: {
|
|
781
|
-
bpm: number;
|
|
782
|
-
phase: number;
|
|
783
|
-
bar: number;
|
|
784
|
-
driftRate: number;
|
|
785
|
-
state: PLLState;
|
|
786
|
-
};
|
|
787
|
-
state: {
|
|
788
|
-
trackingState: TrackingState;
|
|
789
|
-
isLocked: boolean;
|
|
790
|
-
};
|
|
791
|
-
levels: {
|
|
792
|
-
low: number;
|
|
793
|
-
lowMid: number;
|
|
794
|
-
mid: number;
|
|
795
|
-
highMid: number;
|
|
796
|
-
high: number;
|
|
797
|
-
volume: number;
|
|
798
|
-
};
|
|
799
|
-
rawBands: {
|
|
800
|
-
low: number;
|
|
801
|
-
lowMid: number;
|
|
802
|
-
mid: number;
|
|
803
|
-
highMid: number;
|
|
804
|
-
high: number;
|
|
805
|
-
};
|
|
806
|
-
bandGains: {
|
|
807
|
-
low: number;
|
|
808
|
-
lowMid: number;
|
|
809
|
-
mid: number;
|
|
810
|
-
highMid: number;
|
|
811
|
-
high: number;
|
|
812
|
-
};
|
|
813
|
-
volumeGain: number;
|
|
814
|
-
dtMs: number;
|
|
815
|
-
analysisIntervalMs: number;
|
|
816
|
-
events: BeatEvent[];
|
|
817
|
-
};
|
|
818
|
-
}
|
|
819
|
-
|
|
820
|
-
/**
|
|
821
|
-
* Multi-band onset detection with hybrid peak-decay + spectral flux approach
|
|
822
|
-
*
|
|
823
|
-
* Industry-grade onset detection that:
|
|
824
|
-
* - Uses 5 frequency bands for precise instrument detection
|
|
825
|
-
* - Combines peak-decay threshold (proven reliable) with spectral flux (transient detection)
|
|
826
|
-
* - Maintains per-band onset history for tempo pattern analysis
|
|
827
|
-
* - Is frame-rate independent (dt-aware)
|
|
828
|
-
* - Prevents double-triggering with adaptive cooldown
|
|
829
|
-
*/
|
|
830
|
-
/**
|
|
831
|
-
* Band names for the 5-band analysis
|
|
832
|
-
*/
|
|
833
|
-
declare type BandName = 'low' | 'lowMid' | 'mid' | 'highMid' | 'high';
|
|
834
|
-
|
|
835
189
|
/**
|
|
836
190
|
* Beat event (discrete, timestamped)
|
|
837
191
|
*/
|
|
@@ -854,8 +208,17 @@ declare interface BeatEvent {
|
|
|
854
208
|
* then auto-resets to `false`.
|
|
855
209
|
*/
|
|
856
210
|
declare interface ButtonConfig {
|
|
857
|
-
/**
|
|
858
|
-
|
|
211
|
+
/**
|
|
212
|
+
* Advanced: explicit parameter identity. Normally you do NOT set this —
|
|
213
|
+
* the parameter's name is inferred from the variable or object key you
|
|
214
|
+
* assign it to (`const speed = viji.slider(...)` → `speed`). Set `name`
|
|
215
|
+
* only for parameters not assigned to a variable or key (created in an
|
|
216
|
+
* array, loop, or factory). Must be a valid identifier and scene-globally
|
|
217
|
+
* unique. When set, it overrides the inferred name.
|
|
218
|
+
*/
|
|
219
|
+
name?: string;
|
|
220
|
+
/** Display name shown on the button in the parameter UI. Optional — defaults to the parameter's name. */
|
|
221
|
+
label?: string;
|
|
859
222
|
/** Optional tooltip / help text shown next to the control. */
|
|
860
223
|
description?: string;
|
|
861
224
|
/** Group name for organizing related parameters under a shared heading. Default: `'general'`. */
|
|
@@ -873,7 +236,7 @@ declare interface ButtonParameter {
|
|
|
873
236
|
/** `true` for the single frame after the user clicks the button, `false` otherwise. */
|
|
874
237
|
value: boolean;
|
|
875
238
|
/** Display name shown on the button in the parameter UI. */
|
|
876
|
-
label
|
|
239
|
+
label?: string;
|
|
877
240
|
/** Tooltip / help text for the control (empty string when not configured). */
|
|
878
241
|
description?: string;
|
|
879
242
|
/** Group name under which the control appears in the UI. */
|
|
@@ -947,8 +310,17 @@ export declare interface CaptureFrameOptions {
|
|
|
947
310
|
* label and UI grouping/visibility.
|
|
948
311
|
*/
|
|
949
312
|
declare interface ColorConfig {
|
|
950
|
-
/**
|
|
951
|
-
|
|
313
|
+
/**
|
|
314
|
+
* Advanced: explicit parameter identity. Normally you do NOT set this —
|
|
315
|
+
* the parameter's name is inferred from the variable or object key you
|
|
316
|
+
* assign it to (`const speed = viji.slider(...)` → `speed`). Set `name`
|
|
317
|
+
* only for parameters not assigned to a variable or key (created in an
|
|
318
|
+
* array, loop, or factory). Must be a valid identifier and scene-globally
|
|
319
|
+
* unique. When set, it overrides the inferred name.
|
|
320
|
+
*/
|
|
321
|
+
name?: string;
|
|
322
|
+
/** Display name shown next to the color swatch in the parameter UI. Optional — defaults to the parameter's name. */
|
|
323
|
+
label?: string;
|
|
952
324
|
/** Optional tooltip / help text shown next to the control. */
|
|
953
325
|
description?: string;
|
|
954
326
|
/** Group name for organizing related parameters under a shared heading. Default: `'general'`. */
|
|
@@ -1009,7 +381,7 @@ declare interface ColorParameter {
|
|
|
1009
381
|
b: number;
|
|
1010
382
|
};
|
|
1011
383
|
/** Display name shown next to the swatch in the parameter UI. */
|
|
1012
|
-
label
|
|
384
|
+
label?: string;
|
|
1013
385
|
/** Tooltip / help text for the control (empty string when not configured). */
|
|
1014
386
|
description?: string;
|
|
1015
387
|
/** Group name under which the control appears in the UI. */
|
|
@@ -1023,10 +395,19 @@ declare interface ColorParameter {
|
|
|
1023
395
|
* draggable pad; both `x` and `y` are clamped to `-1..1`.
|
|
1024
396
|
*/
|
|
1025
397
|
declare interface CoordinateConfig {
|
|
398
|
+
/**
|
|
399
|
+
* Advanced: explicit parameter identity. Normally you do NOT set this —
|
|
400
|
+
* the parameter's name is inferred from the variable or object key you
|
|
401
|
+
* assign it to (`const speed = viji.slider(...)` → `speed`). Set `name`
|
|
402
|
+
* only for parameters not assigned to a variable or key (created in an
|
|
403
|
+
* array, loop, or factory). Must be a valid identifier and scene-globally
|
|
404
|
+
* unique. When set, it overrides the inferred name.
|
|
405
|
+
*/
|
|
406
|
+
name?: string;
|
|
1026
407
|
/** Snap increment for both `x` and `y` components. Default: `0.01`. */
|
|
1027
408
|
step?: number;
|
|
1028
|
-
/** Display name shown next to the pad in the parameter UI.
|
|
1029
|
-
label
|
|
409
|
+
/** Display name shown next to the pad in the parameter UI. Optional — defaults to the parameter's name. */
|
|
410
|
+
label?: string;
|
|
1030
411
|
/** Optional tooltip / help text shown next to the control. */
|
|
1031
412
|
description?: string;
|
|
1032
413
|
/** Group name for organizing related parameters under a shared heading. Default: `'general'`. */
|
|
@@ -1045,7 +426,7 @@ declare interface CoordinateParameter {
|
|
|
1045
426
|
/** Snap increment for both `x` and `y` components. */
|
|
1046
427
|
step: number;
|
|
1047
428
|
/** Display name shown next to the pad in the parameter UI. */
|
|
1048
|
-
label
|
|
429
|
+
label?: string;
|
|
1049
430
|
/** Tooltip / help text for the control (empty string when not configured). */
|
|
1050
431
|
description?: string;
|
|
1051
432
|
/** Group name under which the control appears in the UI. */
|
|
@@ -1591,8 +972,17 @@ export declare interface HandData {
|
|
|
1591
972
|
* or a P5 image wrapper (P5 renderer).
|
|
1592
973
|
*/
|
|
1593
974
|
declare interface ImageConfig {
|
|
1594
|
-
/**
|
|
1595
|
-
|
|
975
|
+
/**
|
|
976
|
+
* Advanced: explicit parameter identity. Normally you do NOT set this —
|
|
977
|
+
* the parameter's name is inferred from the variable or object key you
|
|
978
|
+
* assign it to (`const speed = viji.slider(...)` → `speed`). Set `name`
|
|
979
|
+
* only for parameters not assigned to a variable or key (created in an
|
|
980
|
+
* array, loop, or factory). Must be a valid identifier and scene-globally
|
|
981
|
+
* unique. When set, it overrides the inferred name.
|
|
982
|
+
*/
|
|
983
|
+
name?: string;
|
|
984
|
+
/** Display name shown next to the upload field in the parameter UI. Optional — defaults to the parameter's name. */
|
|
985
|
+
label?: string;
|
|
1596
986
|
/** Optional tooltip / help text shown next to the control. */
|
|
1597
987
|
description?: string;
|
|
1598
988
|
/** Group name for organizing related parameters under a shared heading. Default: `'general'`. */
|
|
@@ -1612,7 +1002,7 @@ declare interface ImageParameter {
|
|
|
1612
1002
|
/** P5-compatible image wrapper. Only available in the P5 renderer; added at runtime by the P5 adapter. */
|
|
1613
1003
|
readonly p5?: any;
|
|
1614
1004
|
/** Display name shown next to the upload field in the parameter UI. */
|
|
1615
|
-
label
|
|
1005
|
+
label?: string;
|
|
1616
1006
|
/** Tooltip / help text for the control (empty string when not configured). */
|
|
1617
1007
|
description?: string;
|
|
1618
1008
|
/** Group name under which the control appears in the UI. */
|
|
@@ -1652,8 +1042,6 @@ export declare interface InstrumentSerialized {
|
|
|
1652
1042
|
refinementCounts: number[];
|
|
1653
1043
|
}
|
|
1654
1044
|
|
|
1655
|
-
declare type InstrumentType = 'kick' | 'snare' | 'hat';
|
|
1656
|
-
|
|
1657
1045
|
/**
|
|
1658
1046
|
* Keyboard input state for the current frame: per-key press / hold / release
|
|
1659
1047
|
* queries (case-insensitive), the live set of held keys, modifier flags, and
|
|
@@ -1768,6 +1156,15 @@ export declare interface MouseEventData {
|
|
|
1768
1156
|
* label, and UI grouping/visibility for a precise numeric input.
|
|
1769
1157
|
*/
|
|
1770
1158
|
declare interface NumberConfig {
|
|
1159
|
+
/**
|
|
1160
|
+
* Advanced: explicit parameter identity. Normally you do NOT set this —
|
|
1161
|
+
* the parameter's name is inferred from the variable or object key you
|
|
1162
|
+
* assign it to (`const speed = viji.slider(...)` → `speed`). Set `name`
|
|
1163
|
+
* only for parameters not assigned to a variable or key (created in an
|
|
1164
|
+
* array, loop, or factory). Must be a valid identifier and scene-globally
|
|
1165
|
+
* unique. When set, it overrides the inferred name.
|
|
1166
|
+
*/
|
|
1167
|
+
name?: string;
|
|
1771
1168
|
/** Inclusive minimum value. Default: `0`. */
|
|
1772
1169
|
min?: number;
|
|
1773
1170
|
/** Inclusive maximum value. Default: `100`. */
|
|
@@ -1780,8 +1177,8 @@ declare interface NumberConfig {
|
|
|
1780
1177
|
* (`step: 0.001`).
|
|
1781
1178
|
*/
|
|
1782
1179
|
step?: number;
|
|
1783
|
-
/** Display name shown next to the input in the parameter UI.
|
|
1784
|
-
label
|
|
1180
|
+
/** Display name shown next to the input in the parameter UI. Optional — defaults to the parameter's name. */
|
|
1181
|
+
label?: string;
|
|
1785
1182
|
/** Optional tooltip / help text shown next to the control. */
|
|
1786
1183
|
description?: string;
|
|
1787
1184
|
/** Group name for organizing related parameters under a shared heading. Default: `'general'`. */
|
|
@@ -1804,7 +1201,7 @@ declare interface NumberParameter {
|
|
|
1804
1201
|
/** Increment between values. */
|
|
1805
1202
|
step: number;
|
|
1806
1203
|
/** Display name shown next to the input in the parameter UI. */
|
|
1807
|
-
label
|
|
1204
|
+
label?: string;
|
|
1808
1205
|
/** Tooltip / help text for the control (empty string when not configured). */
|
|
1809
1206
|
description?: string;
|
|
1810
1207
|
/** Group name under which the control appears in the UI. */
|
|
@@ -1817,8 +1214,6 @@ export declare type OnsetInstrument = 'kick' | 'snare' | 'hat';
|
|
|
1817
1214
|
|
|
1818
1215
|
declare type OnsetMode = 'auto' | 'tapping' | 'pattern';
|
|
1819
1216
|
|
|
1820
|
-
declare type OnsetMode_2 = 'auto' | 'tapping' | 'pattern';
|
|
1821
|
-
|
|
1822
1217
|
export declare interface OnsetModeChangeEvent {
|
|
1823
1218
|
instrument: OnsetInstrument;
|
|
1824
1219
|
prevMode: OnsetMode;
|
|
@@ -1880,8 +1275,19 @@ export declare interface ParameterConfig {
|
|
|
1880
1275
|
}
|
|
1881
1276
|
|
|
1882
1277
|
export declare interface ParameterDefinition {
|
|
1278
|
+
/**
|
|
1279
|
+
* Stable parameter identity. Always equals the Record key under which the
|
|
1280
|
+
* definition is registered in `groups[].parameters` — addressing
|
|
1281
|
+
* (`setParameter`), saved values, presets, locks, and the controller
|
|
1282
|
+
* protocol all resolve by `name`. For script scenes (native/p5) `name` is
|
|
1283
|
+
* the artist's own variable or object-key name, inferred at compile time;
|
|
1284
|
+
* for shader scenes it is the GLSL uniform name. `label` is display-only —
|
|
1285
|
+
* never key UI lists or lookups off `label`.
|
|
1286
|
+
*/
|
|
1287
|
+
name: string;
|
|
1883
1288
|
type: ParameterType;
|
|
1884
1289
|
defaultValue: ParameterValue;
|
|
1290
|
+
/** Display text shown in parameter UIs. Not an identity — see `name`. */
|
|
1885
1291
|
label?: string;
|
|
1886
1292
|
description?: string;
|
|
1887
1293
|
group?: string;
|
|
@@ -1907,12 +1313,6 @@ export declare type ParameterType = 'slider' | 'color' | 'toggle' | 'select' | '
|
|
|
1907
1313
|
|
|
1908
1314
|
export declare type ParameterValue = string | number | boolean | ImageBitmap | CoordinateValue | null;
|
|
1909
1315
|
|
|
1910
|
-
declare interface PatternInfo {
|
|
1911
|
-
length: number;
|
|
1912
|
-
intervals: number[];
|
|
1913
|
-
confidence: number;
|
|
1914
|
-
}
|
|
1915
|
-
|
|
1916
1316
|
export declare interface PerformanceStats {
|
|
1917
1317
|
resolution: Resolution;
|
|
1918
1318
|
scale: number;
|
|
@@ -1951,35 +1351,6 @@ export declare type PeriodCapabilityCounts = Record<InputCapabilityKey, CountPai
|
|
|
1951
1351
|
}>;
|
|
1952
1352
|
};
|
|
1953
1353
|
|
|
1954
|
-
/**
|
|
1955
|
-
* Phase-Locked Loop for stable beat phase tracking
|
|
1956
|
-
*
|
|
1957
|
-
* Industry-grade phase tracking that:
|
|
1958
|
-
* - Maintains expected beat times based on current BPM
|
|
1959
|
-
* - Compares actual onsets to expected times (phase error)
|
|
1960
|
-
* - Corrects phase gradually to avoid jumps
|
|
1961
|
-
* - Handles breakdowns by extrapolating phase
|
|
1962
|
-
* - Detects tempo drift in DJ sets
|
|
1963
|
-
* - Provides stable bar position (0-3 for 4/4 time)
|
|
1964
|
-
*/
|
|
1965
|
-
/**
|
|
1966
|
-
* PLL state for external access
|
|
1967
|
-
*/
|
|
1968
|
-
declare interface PLLState {
|
|
1969
|
-
/** Current phase (0-1, where 0 = beat) */
|
|
1970
|
-
phase: number;
|
|
1971
|
-
/** Current bar position (0-3 for 4/4 time) */
|
|
1972
|
-
bar: number;
|
|
1973
|
-
/** Current BPM being tracked */
|
|
1974
|
-
bpm: number;
|
|
1975
|
-
/** Time since last confirmed beat (ms) */
|
|
1976
|
-
timeSinceLastBeat: number;
|
|
1977
|
-
/** Whether tempo is drifting (DJ transition) */
|
|
1978
|
-
isDrifting: boolean;
|
|
1979
|
-
/** Drift rate (BPM per second, positive = speeding up) */
|
|
1980
|
-
driftRate: number;
|
|
1981
|
-
}
|
|
1982
|
-
|
|
1983
1354
|
/**
|
|
1984
1355
|
* Unified pointer that abstracts mouse and primary touch into a single input
|
|
1985
1356
|
* stream. Recommended for click / drag / position-tracking interactions that
|
|
@@ -2184,10 +1555,13 @@ export declare interface SceneConsoleMessage {
|
|
|
2184
1555
|
* Static metadata about a scene, derived without running it. Returned by
|
|
2185
1556
|
* `VijiCore.getSceneData()`.
|
|
2186
1557
|
*
|
|
2187
|
-
* `renderer`, `p5CanvasMode`, and `capabilities` are
|
|
2188
|
-
*
|
|
2189
|
-
*
|
|
2190
|
-
*
|
|
1558
|
+
* `renderer`, `p5CanvasMode`, and `capabilities` are **final when
|
|
1559
|
+
* `initialize()` resolves** — the worker analyzes the scene source and reports
|
|
1560
|
+
* the results in its `scene-analysis` ack, which `initialize()` awaits before
|
|
1561
|
+
* resolving. Reads before that return placeholder defaults. The parameter
|
|
1562
|
+
* counts populate once the scene has registered its parameters (mirroring how
|
|
1563
|
+
* parameter data has always surfaced), so read them after the core has
|
|
1564
|
+
* initialized the scene.
|
|
2191
1565
|
*/
|
|
2192
1566
|
export declare interface SceneData {
|
|
2193
1567
|
/** Detected renderer the scene targets. */
|
|
@@ -2222,9 +1596,19 @@ export declare interface SceneData {
|
|
|
2222
1596
|
* - `UNHANDLED_REJECTION`: caught by the worker's `unhandledrejection` handler.
|
|
2223
1597
|
* - `SHADER_COMPILE_ERROR`: GLSL compile failure (shader mode).
|
|
2224
1598
|
* - `P5_INIT_ERROR`: failure during P5 mode initialization (e.g. missing `render`).
|
|
1599
|
+
* - `SCENE_PARSE_ERROR`: the script scene could not be parsed by the load-time
|
|
1600
|
+
* parameter name-inference pass (a genuine syntax error — the scene would not
|
|
1601
|
+
* have executed either).
|
|
1602
|
+
* - `PARAM_NAME_MISSING`: a `viji.<type>(...)` call was not assigned to a
|
|
1603
|
+
* variable or object key and carries no explicit `name` — nothing to infer an
|
|
1604
|
+
* identity from. Assign it to a `const`, or pass `name:'...'`.
|
|
1605
|
+
* - `PARAM_NAME_DUPLICATE`: two parameters resolved to the same name. Names are
|
|
1606
|
+
* scene-global; rename one, or pass `name:'...'` to distinguish them.
|
|
1607
|
+
* - `PARAM_NAME_INVALID`: an explicit `name` is not a valid identifier or is a
|
|
1608
|
+
* reserved word (`__proto__`, `constructor`, `prototype`).
|
|
2225
1609
|
* - `ENGINE_INTERNAL`: engine path with no artist attribution.
|
|
2226
1610
|
*/
|
|
2227
|
-
export declare type SceneErrorCode = 'SCENE_CODE_ERROR' | 'RENDER_ERROR' | 'UNCAUGHT_ERROR' | 'UNHANDLED_REJECTION' | 'SHADER_COMPILE_ERROR' | 'P5_INIT_ERROR' | 'ENGINE_INTERNAL';
|
|
1611
|
+
export declare type SceneErrorCode = 'SCENE_CODE_ERROR' | 'RENDER_ERROR' | 'UNCAUGHT_ERROR' | 'UNHANDLED_REJECTION' | 'SHADER_COMPILE_ERROR' | 'P5_INIT_ERROR' | 'SCENE_PARSE_ERROR' | 'PARAM_NAME_MISSING' | 'PARAM_NAME_DUPLICATE' | 'PARAM_NAME_INVALID' | 'ENGINE_INTERNAL';
|
|
2228
1612
|
|
|
2229
1613
|
/**
|
|
2230
1614
|
* Structured scene runtime error envelope surfaced via `VijiCore.onSceneRuntimeError`.
|
|
@@ -2315,10 +1699,19 @@ declare interface SegmentationData {
|
|
|
2315
1699
|
* as a dropdown or segmented control.
|
|
2316
1700
|
*/
|
|
2317
1701
|
declare interface SelectConfig {
|
|
1702
|
+
/**
|
|
1703
|
+
* Advanced: explicit parameter identity. Normally you do NOT set this —
|
|
1704
|
+
* the parameter's name is inferred from the variable or object key you
|
|
1705
|
+
* assign it to (`const speed = viji.slider(...)` → `speed`). Set `name`
|
|
1706
|
+
* only for parameters not assigned to a variable or key (created in an
|
|
1707
|
+
* array, loop, or factory). Must be a valid identifier and scene-globally
|
|
1708
|
+
* unique. When set, it overrides the inferred name.
|
|
1709
|
+
*/
|
|
1710
|
+
name?: string;
|
|
2318
1711
|
/** Available choices. The element type (`string` or `number`) becomes the parameter's value type. Required. */
|
|
2319
1712
|
options: string[] | number[];
|
|
2320
|
-
/** Display name shown next to the dropdown in the parameter UI.
|
|
2321
|
-
label
|
|
1713
|
+
/** Display name shown next to the dropdown in the parameter UI. Optional — defaults to the parameter's name. */
|
|
1714
|
+
label?: string;
|
|
2322
1715
|
/** Optional tooltip / help text shown next to the control. */
|
|
2323
1716
|
description?: string;
|
|
2324
1717
|
/** Group name for organizing related parameters under a shared heading. Default: `'general'`. */
|
|
@@ -2337,7 +1730,7 @@ declare interface SelectParameter {
|
|
|
2337
1730
|
/** The full list of choices originally configured. */
|
|
2338
1731
|
options: string[] | number[];
|
|
2339
1732
|
/** Display name shown next to the dropdown in the parameter UI. */
|
|
2340
|
-
label
|
|
1733
|
+
label?: string;
|
|
2341
1734
|
/** Tooltip / help text for the control (empty string when not configured). */
|
|
2342
1735
|
description?: string;
|
|
2343
1736
|
/** Group name under which the control appears in the UI. */
|
|
@@ -2653,6 +2046,15 @@ export declare interface SerializedTempoInductionState {
|
|
|
2653
2046
|
* increment, label, and UI grouping/visibility.
|
|
2654
2047
|
*/
|
|
2655
2048
|
declare interface SliderConfig {
|
|
2049
|
+
/**
|
|
2050
|
+
* Advanced: explicit parameter identity. Normally you do NOT set this —
|
|
2051
|
+
* the parameter's name is inferred from the variable or object key you
|
|
2052
|
+
* assign it to (`const speed = viji.slider(...)` → `speed`). Set `name`
|
|
2053
|
+
* only for parameters not assigned to a variable or key (created in an
|
|
2054
|
+
* array, loop, or factory). Must be a valid identifier and scene-globally
|
|
2055
|
+
* unique. When set, it overrides the inferred name.
|
|
2056
|
+
*/
|
|
2057
|
+
name?: string;
|
|
2656
2058
|
/** Inclusive minimum value of the slider. Default: `0`. */
|
|
2657
2059
|
min?: number;
|
|
2658
2060
|
/** Inclusive maximum value of the slider. Default: `100`. */
|
|
@@ -2665,8 +2067,8 @@ declare interface SliderConfig {
|
|
|
2665
2067
|
* (`step: 1` on a range of `3..30`) or fine precision (`step: 0.001`).
|
|
2666
2068
|
*/
|
|
2667
2069
|
step?: number;
|
|
2668
|
-
/** Display name shown next to the control in the parameter UI.
|
|
2669
|
-
label
|
|
2070
|
+
/** Display name shown next to the control in the parameter UI. Optional — defaults to the parameter's name. */
|
|
2071
|
+
label?: string;
|
|
2670
2072
|
/** Optional tooltip / help text shown next to the control. */
|
|
2671
2073
|
description?: string;
|
|
2672
2074
|
/** Group name for organizing related parameters under a shared heading. Default: `'general'`. */
|
|
@@ -2689,7 +2091,7 @@ declare interface SliderParameter {
|
|
|
2689
2091
|
/** Increment between values. */
|
|
2690
2092
|
step: number;
|
|
2691
2093
|
/** Display name shown next to the control in the parameter UI. */
|
|
2692
|
-
label
|
|
2094
|
+
label?: string;
|
|
2693
2095
|
/** Tooltip / help text for the control (empty string when not configured). */
|
|
2694
2096
|
description?: string;
|
|
2695
2097
|
/** Group name under which the control appears in the UI. */
|
|
@@ -2714,8 +2116,17 @@ export declare interface StateImportError {
|
|
|
2714
2116
|
* text input field.
|
|
2715
2117
|
*/
|
|
2716
2118
|
declare interface TextConfig {
|
|
2717
|
-
/**
|
|
2718
|
-
|
|
2119
|
+
/**
|
|
2120
|
+
* Advanced: explicit parameter identity. Normally you do NOT set this —
|
|
2121
|
+
* the parameter's name is inferred from the variable or object key you
|
|
2122
|
+
* assign it to (`const speed = viji.slider(...)` → `speed`). Set `name`
|
|
2123
|
+
* only for parameters not assigned to a variable or key (created in an
|
|
2124
|
+
* array, loop, or factory). Must be a valid identifier and scene-globally
|
|
2125
|
+
* unique. When set, it overrides the inferred name.
|
|
2126
|
+
*/
|
|
2127
|
+
name?: string;
|
|
2128
|
+
/** Display name shown next to the text field in the parameter UI. Optional — defaults to the parameter's name. */
|
|
2129
|
+
label?: string;
|
|
2719
2130
|
/** Optional tooltip / help text shown next to the control. */
|
|
2720
2131
|
description?: string;
|
|
2721
2132
|
/** Group name for organizing related parameters under a shared heading. Default: `'general'`. */
|
|
@@ -2736,7 +2147,7 @@ declare interface TextParameter {
|
|
|
2736
2147
|
/** Maximum character count enforced by the host UI. */
|
|
2737
2148
|
maxLength?: number;
|
|
2738
2149
|
/** Display name shown next to the input in the parameter UI. */
|
|
2739
|
-
label
|
|
2150
|
+
label?: string;
|
|
2740
2151
|
/** Tooltip / help text for the control (empty string when not configured). */
|
|
2741
2152
|
description?: string;
|
|
2742
2153
|
/** Group name under which the control appears in the UI. */
|
|
@@ -2750,8 +2161,17 @@ declare interface TextParameter {
|
|
|
2750
2161
|
* label and UI grouping/visibility.
|
|
2751
2162
|
*/
|
|
2752
2163
|
declare interface ToggleConfig {
|
|
2753
|
-
/**
|
|
2754
|
-
|
|
2164
|
+
/**
|
|
2165
|
+
* Advanced: explicit parameter identity. Normally you do NOT set this —
|
|
2166
|
+
* the parameter's name is inferred from the variable or object key you
|
|
2167
|
+
* assign it to (`const speed = viji.slider(...)` → `speed`). Set `name`
|
|
2168
|
+
* only for parameters not assigned to a variable or key (created in an
|
|
2169
|
+
* array, loop, or factory). Must be a valid identifier and scene-globally
|
|
2170
|
+
* unique. When set, it overrides the inferred name.
|
|
2171
|
+
*/
|
|
2172
|
+
name?: string;
|
|
2173
|
+
/** Display name shown next to the switch in the parameter UI. Optional — defaults to the parameter's name. */
|
|
2174
|
+
label?: string;
|
|
2755
2175
|
/** Optional tooltip / help text shown next to the control. */
|
|
2756
2176
|
description?: string;
|
|
2757
2177
|
/** Group name for organizing related parameters under a shared heading. Default: `'general'`. */
|
|
@@ -2768,7 +2188,7 @@ declare interface ToggleParameter {
|
|
|
2768
2188
|
/** Current state of the switch. Updates in real-time when the user toggles it. */
|
|
2769
2189
|
value: boolean;
|
|
2770
2190
|
/** Display name shown next to the switch in the parameter UI. */
|
|
2771
|
-
label
|
|
2191
|
+
label?: string;
|
|
2772
2192
|
/** Tooltip / help text for the control (empty string when not configured). */
|
|
2773
2193
|
description?: string;
|
|
2774
2194
|
/** Group name under which the control appears in the UI. */
|
|
@@ -2862,16 +2282,10 @@ declare interface TouchPoint {
|
|
|
2862
2282
|
isEnding: boolean;
|
|
2863
2283
|
}
|
|
2864
2284
|
|
|
2865
|
-
/**
|
|
2866
|
-
* State machine states
|
|
2867
|
-
* NOTE: LEARNING state removed - system now uses continuous rolling window adaptation
|
|
2868
|
-
*/
|
|
2869
|
-
declare type TrackingState = 'TRACKING' | 'LOCKED' | 'BREAKDOWN' | 'LOST';
|
|
2870
|
-
|
|
2871
2285
|
/** Returned by `on*` listener registration calls; invoke to unsubscribe. */
|
|
2872
2286
|
export declare type Unsubscribe = () => void;
|
|
2873
2287
|
|
|
2874
|
-
export declare const VERSION = "0.
|
|
2288
|
+
export declare const VERSION = "0.13.0";
|
|
2875
2289
|
|
|
2876
2290
|
/**
|
|
2877
2291
|
* Real-time video API: drawable frame, dimensions, and the source-side
|
|
@@ -3057,15 +2471,6 @@ export declare interface VideoCVAPI {
|
|
|
3057
2471
|
isProcessing(): boolean;
|
|
3058
2472
|
}
|
|
3059
2473
|
|
|
3060
|
-
/**
|
|
3061
|
-
* Video stream role, derived from position in the index contract.
|
|
3062
|
-
* - main: primary video with CV (`viji.video`)
|
|
3063
|
-
* - additional: host-provided MediaStreams, no CV (`viji.videoStreams[]`)
|
|
3064
|
-
* - device: device camera streams (`viji.devices[].video`)
|
|
3065
|
-
* - directFrame: injected compositor frames (`viji.videoStreams[]`, after additional)
|
|
3066
|
-
*/
|
|
3067
|
-
export declare type VideoStreamType = 'main' | 'additional' | 'device' | 'directFrame';
|
|
3068
|
-
|
|
3069
2474
|
/**
|
|
3070
2475
|
* The Viji API surface available to artist code as the global `viji` object.
|
|
3071
2476
|
* Groups canvas access, timing, connected media APIs, user interaction, device
|
|
@@ -3234,6 +2639,7 @@ export declare class VijiCore {
|
|
|
3234
2639
|
private isInitialized;
|
|
3235
2640
|
private isDestroyed;
|
|
3236
2641
|
private isInitializing;
|
|
2642
|
+
private engineGroupReleased;
|
|
3237
2643
|
private instanceId;
|
|
3238
2644
|
private screenRefreshRate;
|
|
3239
2645
|
private refreshRateRafId;
|
|
@@ -3262,6 +2668,8 @@ export declare class VijiCore {
|
|
|
3262
2668
|
private sceneConsumesAnalysedFrame;
|
|
3263
2669
|
private sceneContextType?;
|
|
3264
2670
|
private parametersInitialized;
|
|
2671
|
+
private isAwaitingSceneAnalysis;
|
|
2672
|
+
private analysisWindowSceneError;
|
|
3265
2673
|
private parameterListeners;
|
|
3266
2674
|
private parameterDefinedListeners;
|
|
3267
2675
|
private parameterErrorListeners;
|
|
@@ -3359,9 +2767,36 @@ export declare class VijiCore {
|
|
|
3359
2767
|
*/
|
|
3360
2768
|
hide(): void;
|
|
3361
2769
|
/**
|
|
3362
|
-
* Initializes the core components in sequence
|
|
2770
|
+
* Initializes the core components in sequence. When this resolves the
|
|
2771
|
+
* scene is live and `getSceneData()`'s analysis fields are final.
|
|
2772
|
+
*
|
|
2773
|
+
* Rejection contract — three codes pass through unwrapped so consumers
|
|
2774
|
+
* can branch on them:
|
|
2775
|
+
* - `INITIALIZATION_CANCELLED` — `destroy()` landed while initializing.
|
|
2776
|
+
* Expected during rapid mount/unmount (hover previews, deck swaps);
|
|
2777
|
+
* not a failure.
|
|
2778
|
+
* - `INSTANCE_DESTROYED` — called on a destroyed core, or on one whose
|
|
2779
|
+
* previous `initialize()` failed and released its engine reference;
|
|
2780
|
+
* construct a fresh core instead.
|
|
2781
|
+
* - `SCENE_DECRYPT_ERROR` — an encrypted (`viji-enc:`) scene could not be
|
|
2782
|
+
* decrypted (wrong/missing key component, tampered or malformed
|
|
2783
|
+
* envelope, unsupported envelope version). Permanent for this payload:
|
|
2784
|
+
* the scene genuinely cannot run — surface a typed error state, do not
|
|
2785
|
+
* retry.
|
|
2786
|
+
*
|
|
2787
|
+
* Every other failure (including a `scene-analysis` ack failure or
|
|
2788
|
+
* timeout) rejects as `INITIALIZATION_ERROR` with the underlying error in
|
|
2789
|
+
* `details.error`. Note `details` no longer carries the config.
|
|
3363
2790
|
*/
|
|
3364
2791
|
initialize(): Promise<void>;
|
|
2792
|
+
/**
|
|
2793
|
+
* Release this core's engine-group reference after a failed initialize().
|
|
2794
|
+
* Idempotent; skipped when destroy() owns the teardown (it detaches and
|
|
2795
|
+
* releases itself). Detach + release are safe on partially-attached state:
|
|
2796
|
+
* `detachSink` deletes from maps and `release` is a member-set delete with
|
|
2797
|
+
* a `_destroyed`-guarded last-release dispose.
|
|
2798
|
+
*/
|
|
2799
|
+
private releaseEngineGroupOnFailure;
|
|
3365
2800
|
/**
|
|
3366
2801
|
* Sets up the interaction system for Phase 7.
|
|
3367
2802
|
*
|
|
@@ -3449,8 +2884,10 @@ export declare class VijiCore {
|
|
|
3449
2884
|
* Returns static metadata about the scene: its renderer, the capabilities its
|
|
3450
2885
|
* code declares it uses, and its parameter shape.
|
|
3451
2886
|
*
|
|
3452
|
-
* `renderer`, `p5CanvasMode`, and `capabilities` are
|
|
3453
|
-
*
|
|
2887
|
+
* `renderer`, `p5CanvasMode`, and `capabilities` are **final when
|
|
2888
|
+
* `initialize()` resolves** — they are assigned from the worker's
|
|
2889
|
+
* `scene-analysis` ack during initialization; reads before that return
|
|
2890
|
+
* placeholder defaults. The parameter counts populate once the scene has
|
|
3454
2891
|
* registered its parameters.
|
|
3455
2892
|
*
|
|
3456
2893
|
* Distinct from {@link getCapabilities}, which reports what the *host*
|
|
@@ -3495,7 +2932,13 @@ export declare class VijiCore {
|
|
|
3495
2932
|
*/
|
|
3496
2933
|
unlinkEventSource(): void;
|
|
3497
2934
|
/**
|
|
3498
|
-
* Link source cores that this core will pull frames from
|
|
2935
|
+
* Link source cores that this core will pull frames from.
|
|
2936
|
+
*
|
|
2937
|
+
* Requires an initialized core: the mixer's renderer type decides the
|
|
2938
|
+
* capture flip orientation, and `sceneRenderer` is final only when
|
|
2939
|
+
* `initialize()` resolves (Design B). A pre-ready call would silently read
|
|
2940
|
+
* the `'native'` default and composite shader mixers un-flipped — silently
|
|
2941
|
+
* wrong output, so it fails loud instead.
|
|
3499
2942
|
*/
|
|
3500
2943
|
linkFrameSources(...sourceCores: VijiCore[]): void;
|
|
3501
2944
|
/**
|
|
@@ -4424,6 +3867,18 @@ export declare interface VijiCoreConfig {
|
|
|
4424
3867
|
_sc?: string;
|
|
4425
3868
|
/** Initial render hint (logo variant/position), applied before the first frame. */
|
|
4426
3869
|
_rh?: number;
|
|
3870
|
+
/**
|
|
3871
|
+
* Per-scene key component for encrypted (`viji-enc:`) scene delivery —
|
|
3872
|
+
* an opaque base64url blob served next to the scene payload. Required to
|
|
3873
|
+
* decrypt an encrypted `sceneCode`; ignored for plaintext scenes. Routed
|
|
3874
|
+
* to the worker inside the `set-scene-code` payload on the direct control
|
|
3875
|
+
* port (never through the iframe bootstrap).
|
|
3876
|
+
*
|
|
3877
|
+
* Omitted from the `configuration` getter: serializing ciphertext and the
|
|
3878
|
+
* key component into one log/state dump would make the dump decryptable
|
|
3879
|
+
* with the shipped bundle constant — do not persist or log this value.
|
|
3880
|
+
*/
|
|
3881
|
+
_sk?: string;
|
|
4427
3882
|
/** Called when the scene is visually live (render loop running, branding applied) but before audio/video/WASM setup completes. */
|
|
4428
3883
|
onRenderStart?: () => void;
|
|
4429
3884
|
/**
|