@viji-dev/core 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/artist-dts-p5.js +1 -1
- package/dist/artist-dts.js +1 -1
- package/dist/artist-global-p5.d.ts +1 -1
- package/dist/artist-global.d.ts +1 -1
- package/dist/assets/{viji.worker-Clz2HjaH.js → viji.worker-ByIbGiBL.js} +14 -14
- package/dist/docs-api.js +8 -8
- package/dist/{essentia-wasm.web-7iahj_bF.js → essentia-wasm.web-0tQ5KtU1.js} +1 -1
- package/dist/{index-BcsAA44l.js → index-BYp7SdOD.js} +250 -428
- package/dist/index.d.ts +24 -701
- package/dist/index.js +5 -7
- package/dist/param-inference.d.ts +14 -0
- package/dist/param-inference.js +3235 -0
- package/dist/scene-analysis-CEQRlNUL.js +186 -0
- package/dist/scene-analysis.d.ts +127 -0
- package/dist/scene-analysis.js +4 -0
- package/package.json +11 -1
package/dist/index.d.ts
CHANGED
|
@@ -1,37 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Analyzes a scene's source code and returns the capabilities it declares,
|
|
3
|
-
* without constructing a core. Composes {@link SceneAnalyzer.detectRendererType}
|
|
4
|
-
* and {@link SceneAnalyzer.detectCapabilities} — the same two steps `VijiCore`
|
|
5
|
-
* performs at construction — so for a given core version the result is
|
|
6
|
-
* identical to a booted core's `getSceneData().capabilities`.
|
|
7
|
-
*
|
|
8
|
-
* Intended for out-of-engine consumers (e.g. server-side persistence of
|
|
9
|
-
* per-scene capabilities). The following properties are an intentional, kept
|
|
10
|
-
* contract:
|
|
11
|
-
* - **Pure and Node-safe** — static analysis only; the scene is never executed
|
|
12
|
-
* and no DOM, canvas, or worker API is touched. Callable in a bare Node
|
|
13
|
-
* process.
|
|
14
|
-
* - **Deterministic** — same code string, same result, within a core version.
|
|
15
|
-
* - **Inclusive-biased** — a capability the scene uses is never hidden; a
|
|
16
|
-
* rare ambiguous case may report an extra one.
|
|
17
|
-
* - **Safe on untrusted input** — linear-time scan and regexes with no throw
|
|
18
|
-
* paths on any string input.
|
|
19
|
-
*
|
|
20
|
-
* Note for consumers filtering on `video`: the flag means the scene reads the
|
|
21
|
-
* video *image* (pixels/texture). Shader scenes that consume only CV result
|
|
22
|
-
* uniforms report `cv.*` without `video`, so a "reacts to the camera" filter
|
|
23
|
-
* should test `video || any cv key`.
|
|
24
|
-
*
|
|
25
|
-
* @param sceneCode - The artist's scene code (native, p5, or shader)
|
|
26
|
-
* @returns The set of capabilities the scene declares it uses
|
|
27
|
-
* @example
|
|
28
|
-
* ```ts
|
|
29
|
-
* const caps = analyzeSceneCapabilities(scene.code);
|
|
30
|
-
* if (caps.audio) tags.push('audio');
|
|
31
|
-
* ```
|
|
32
|
-
*/
|
|
33
|
-
export declare function analyzeSceneCapabilities(sceneCode: string): SceneCapabilities;
|
|
34
|
-
|
|
35
1
|
declare interface AudioAnalysisState {
|
|
36
2
|
isConnected: boolean;
|
|
37
3
|
currentBPM: number;
|
|
@@ -220,618 +186,6 @@ declare interface AudioStreamAPI {
|
|
|
220
186
|
getWaveform: () => Float32Array;
|
|
221
187
|
}
|
|
222
188
|
|
|
223
|
-
/**
|
|
224
|
-
* Manages audio analysis and processing for the host side.
|
|
225
|
-
* Handles audio stream processing and provides real-time audio analysis.
|
|
226
|
-
*
|
|
227
|
-
* This system provides:
|
|
228
|
-
* - Real-time audio analysis (RMS, peak, frequency bands)
|
|
229
|
-
* - Beat detection with automatic BPM tracking
|
|
230
|
-
* - Onset detection for audio-reactive triggers
|
|
231
|
-
* - Smooth energy curves with envelope followers
|
|
232
|
-
* - Per-instrument onset tap patterns with automatic pattern recognition
|
|
233
|
-
* - Auto-gain normalization
|
|
234
|
-
* - Spectral feature extraction
|
|
235
|
-
* - AudioContext management and cleanup
|
|
236
|
-
* - Raw FFT data access for custom analysis
|
|
237
|
-
* - Analysis results sent to worker for artist API
|
|
238
|
-
*/
|
|
239
|
-
export declare class AudioSystem {
|
|
240
|
-
private audioContext;
|
|
241
|
-
private workletRegistered;
|
|
242
|
-
/**
|
|
243
|
-
* Persistent export capture tap (main channel only). Created lazily by
|
|
244
|
-
* `getAudioCaptureStream()`, lives for the AudioSystem lifetime, and is
|
|
245
|
-
* torn down only in `resetAudioState()`. Deliberately NOT severed by
|
|
246
|
-
* `disconnectMainStream()`: on mid-recording disconnect the tap stream
|
|
247
|
-
* goes silent while its tracks stay live, and `connectChannel()` re-wires
|
|
248
|
-
* each new main source node into it, so the exposed MediaStream stays
|
|
249
|
-
* stable across `setAudioStream()` swaps (a recorder never renegotiates).
|
|
250
|
-
*/
|
|
251
|
-
private captureDestination;
|
|
252
|
-
private mainChannel;
|
|
253
|
-
private additionalChannels;
|
|
254
|
-
private analysisInterval;
|
|
255
|
-
private analysisIntervalMs;
|
|
256
|
-
private stalenessTimer;
|
|
257
|
-
private static readonly STALENESS_THRESHOLD_MS;
|
|
258
|
-
/** Tracks the last non-zero BPM chosen for output to avoid dropping to 0 */
|
|
259
|
-
private lastNonZeroBpm;
|
|
260
|
-
/** Tracks which source provided the current BPM (pll | tempo | carry | default) */
|
|
261
|
-
private lastBpmSource;
|
|
262
|
-
private lastPhaseLogTime;
|
|
263
|
-
private onsetLogBuffer;
|
|
264
|
-
private debugMode;
|
|
265
|
-
/**
|
|
266
|
-
* One-way lifecycle flag flipped at the top of `resetAudioState()`.
|
|
267
|
-
* Gates re-entry into `ensureAudioContext()` and the public stream-mutation
|
|
268
|
-
* entry points so an in-flight async caller (e.g. a worker RPC firing
|
|
269
|
-
* mid-teardown) cannot resurrect a fresh `AudioContext` after destroy
|
|
270
|
-
* has already nulled the field. AudioSystem is single-use after reset.
|
|
271
|
-
*/
|
|
272
|
-
private destroyed;
|
|
273
|
-
private diagnosticLogger;
|
|
274
|
-
private onsetDetection;
|
|
275
|
-
private essentiaOnsetDetection;
|
|
276
|
-
private tempoInduction;
|
|
277
|
-
private pll;
|
|
278
|
-
private stateManager;
|
|
279
|
-
private lastKickDetected;
|
|
280
|
-
/** Raw band snapshot before auto-gain (for debug, main channel only) */
|
|
281
|
-
private rawBandsPreGain;
|
|
282
|
-
private readonly bandNames;
|
|
283
|
-
private essentiaBandHistories;
|
|
284
|
-
private readonly essentiaHistoryWindowMs;
|
|
285
|
-
private onsetTapManager;
|
|
286
|
-
private beatEnvelopeFollowers;
|
|
287
|
-
private beatDetectionEnabled;
|
|
288
|
-
private onsetDetectionEnabled;
|
|
289
|
-
private autoGainEnabled;
|
|
290
|
-
private idleTickerHandle;
|
|
291
|
-
private idleTickerLastTime;
|
|
292
|
-
/**
|
|
293
|
-
* Enable or disable comprehensive debug logging for all layers
|
|
294
|
-
* Enables enhanced logging in: MultiOnsetDetection, BeatStateManager
|
|
295
|
-
*/
|
|
296
|
-
setDebugMode(enabled: boolean): void;
|
|
297
|
-
/**
|
|
298
|
-
* Reset Essentia band onset histories (used to feed tempo induction when using the Essentia backend)
|
|
299
|
-
*/
|
|
300
|
-
private resetEssentiaBandHistories;
|
|
301
|
-
/**
|
|
302
|
-
* Update Essentia band onset histories and trim old entries
|
|
303
|
-
*/
|
|
304
|
-
private updateEssentiaBandHistories;
|
|
305
|
-
private getEssentiaBandHistory;
|
|
306
|
-
private getEssentiaAllHistories;
|
|
307
|
-
/**
|
|
308
|
-
* Handle frames pushed from AudioWorklet for main channel
|
|
309
|
-
*/
|
|
310
|
-
private handleMainWorkletFrame;
|
|
311
|
-
/**
|
|
312
|
-
* Handle frames pushed from AudioWorklet for an additional channel
|
|
313
|
-
*/
|
|
314
|
-
private handleAdditionalWorkletFrame;
|
|
315
|
-
/**
|
|
316
|
-
* Common analysis pipeline for any channel (FFT, volume, bands, auto-gain, spectral, smoothing).
|
|
317
|
-
* Returns raw band energies (pre-auto-gain) for optional beat processing by the caller.
|
|
318
|
-
*/
|
|
319
|
-
private analyzeChannelFrame;
|
|
320
|
-
/**
|
|
321
|
-
* Full analysis for the main channel (common analysis + beat pipeline).
|
|
322
|
-
*/
|
|
323
|
-
private analyzeMainFrame;
|
|
324
|
-
/**
|
|
325
|
-
* Lightweight analysis for additional/device channels (no beat detection).
|
|
326
|
-
*/
|
|
327
|
-
private analyzeAdditionalFrame;
|
|
328
|
-
/**
|
|
329
|
-
* Compute FFT on a specific channel's buffers
|
|
330
|
-
*/
|
|
331
|
-
private computeChannelFFT;
|
|
332
|
-
/**
|
|
333
|
-
* Calculate RMS/peak from float PCM frame (-1..1)
|
|
334
|
-
*/
|
|
335
|
-
private calculateVolumeMetrics;
|
|
336
|
-
/**
|
|
337
|
-
* Calculate perceptual/log-ish bands on a specific channel
|
|
338
|
-
*/
|
|
339
|
-
private calculateChannelBands;
|
|
340
|
-
/**
|
|
341
|
-
* Spectral features on a specific channel
|
|
342
|
-
*/
|
|
343
|
-
private calculateChannelSpectral;
|
|
344
|
-
/**
|
|
345
|
-
* Run onset + beat detection pipeline and return BeatState
|
|
346
|
-
*/
|
|
347
|
-
private runBeatPipeline;
|
|
348
|
-
/**
|
|
349
|
-
* Debug logging helper
|
|
350
|
-
*/
|
|
351
|
-
private debugLog;
|
|
352
|
-
private fftSize;
|
|
353
|
-
private audioStateBeat;
|
|
354
|
-
private analysisSinks;
|
|
355
|
-
private latestChannelResults;
|
|
356
|
-
private channelMeta;
|
|
357
|
-
constructor(sendAnalysisResultsCallback?: (data: any) => void);
|
|
358
|
-
/**
|
|
359
|
-
* Get the current audio analysis state (for host-side usage)
|
|
360
|
-
*/
|
|
361
|
-
getAudioState(): {
|
|
362
|
-
isConnected: boolean;
|
|
363
|
-
volume: {
|
|
364
|
-
current: number;
|
|
365
|
-
peak: number;
|
|
366
|
-
smoothed: number;
|
|
367
|
-
};
|
|
368
|
-
bands: {
|
|
369
|
-
low: number;
|
|
370
|
-
lowMid: number;
|
|
371
|
-
mid: number;
|
|
372
|
-
highMid: number;
|
|
373
|
-
high: number;
|
|
374
|
-
lowSmoothed: number;
|
|
375
|
-
lowMidSmoothed: number;
|
|
376
|
-
midSmoothed: number;
|
|
377
|
-
highMidSmoothed: number;
|
|
378
|
-
highSmoothed: number;
|
|
379
|
-
};
|
|
380
|
-
spectral: {
|
|
381
|
-
brightness: number;
|
|
382
|
-
flatness: number;
|
|
383
|
-
};
|
|
384
|
-
};
|
|
385
|
-
/**
|
|
386
|
-
* Get a recordable MediaStream carrying the main channel's audio, for
|
|
387
|
-
* host-side export recording. Main channel only — additional channels and
|
|
388
|
-
* source-fed device channels (analysis values, no PCM) are not included.
|
|
389
|
-
*
|
|
390
|
-
* The underlying destination node is created lazily on first call and then
|
|
391
|
-
* persists for the AudioSystem lifetime: on a source swap the stream keeps
|
|
392
|
-
* flowing with the new audio, and on disconnect it goes silent with tracks
|
|
393
|
-
* still live. Returns `null` when the main channel is not connected at
|
|
394
|
-
* call time (or after teardown).
|
|
395
|
-
*/
|
|
396
|
-
getAudioCaptureStream(): MediaStream | null;
|
|
397
|
-
/**
|
|
398
|
-
* Initialize Essentia.js (async WASM loading)
|
|
399
|
-
*/
|
|
400
|
-
private initializeEssentia;
|
|
401
|
-
/**
|
|
402
|
-
* Wait for Essentia.js to initialize (for testing/benchmarking)
|
|
403
|
-
* Returns immediately if already initialized or if initialization failed
|
|
404
|
-
*/
|
|
405
|
-
waitForEssentia(timeoutMs?: number): Promise<boolean>;
|
|
406
|
-
/**
|
|
407
|
-
* Handle audio stream update for main stream (called from VijiCore)
|
|
408
|
-
*/
|
|
409
|
-
handleAudioStreamUpdate(data: {
|
|
410
|
-
audioStream?: MediaStream | null;
|
|
411
|
-
timestamp: number;
|
|
412
|
-
}): void;
|
|
413
|
-
/**
|
|
414
|
-
* Ensure the shared AudioContext is created and resumed.
|
|
415
|
-
*/
|
|
416
|
-
private ensureAudioContext;
|
|
417
|
-
/**
|
|
418
|
-
* Connect a channel to Web Audio nodes (source, worklet/analyser).
|
|
419
|
-
* Used for both main and additional channels.
|
|
420
|
-
*
|
|
421
|
-
* Lifecycle ordering invariant: when wiring the main channel, the idle
|
|
422
|
-
* ticker MUST be stopped before the first `await` so that an in-flight
|
|
423
|
-
* tick cannot race the audio path coming online. The ticker callback
|
|
424
|
-
* additionally guards on `mainChannel.audioState.isConnected` for
|
|
425
|
-
* belt-and-braces.
|
|
426
|
-
*/
|
|
427
|
-
private connectChannel;
|
|
428
|
-
/**
|
|
429
|
-
* Set the main audio stream for analysis (preserves original public API surface).
|
|
430
|
-
*/
|
|
431
|
-
private setAudioStream;
|
|
432
|
-
/**
|
|
433
|
-
* Disconnect the main audio stream (does NOT close AudioContext -- additional channels may still be active).
|
|
434
|
-
*
|
|
435
|
-
* Lifecycle ordering invariant: tear down the audio nodes first, clear any
|
|
436
|
-
* stale frequency / waveform buffers, then restart the idle ticker. The
|
|
437
|
-
* ticker callback's defensive `isConnected` guard prevents any stale tick
|
|
438
|
-
* from observing a half-disconnected state.
|
|
439
|
-
*/
|
|
440
|
-
private disconnectMainStream;
|
|
441
|
-
/**
|
|
442
|
-
* Set up an AudioWorklet node for a specific channel.
|
|
443
|
-
* Registers the processor module once (shared), creates a new WorkletNode per channel.
|
|
444
|
-
*/
|
|
445
|
-
private setupChannelWorklet;
|
|
446
|
-
/**
|
|
447
|
-
* Ensure the shared analysis interval is running (for analyser-mode channels).
|
|
448
|
-
*/
|
|
449
|
-
private ensureAnalysisLoop;
|
|
450
|
-
/**
|
|
451
|
-
* Stop the shared analysis loop.
|
|
452
|
-
*/
|
|
453
|
-
private stopAnalysisLoop;
|
|
454
|
-
/**
|
|
455
|
-
* Shared staleness timer: checks all channels for stale data.
|
|
456
|
-
*/
|
|
457
|
-
private startStalenessTimer;
|
|
458
|
-
private stopStalenessTimer;
|
|
459
|
-
/**
|
|
460
|
-
* Pause audio analysis (for tests or temporary suspension)
|
|
461
|
-
*/
|
|
462
|
-
pauseAnalysis(): void;
|
|
463
|
-
/**
|
|
464
|
-
* Resume audio analysis after pause
|
|
465
|
-
*/
|
|
466
|
-
resumeAnalysis(): void;
|
|
467
|
-
/**
|
|
468
|
-
* Shared analysis loop callback: iterates main + additional channels in analyser mode.
|
|
469
|
-
*/
|
|
470
|
-
private performAnalysis;
|
|
471
|
-
private applyChannelAutoGain;
|
|
472
|
-
/**
|
|
473
|
-
* Update smooth versions of frequency bands on a specific channel
|
|
474
|
-
*/
|
|
475
|
-
private updateChannelSmoothBands;
|
|
476
|
-
/**
|
|
477
|
-
* Send analysis results for a specific channel to the worker.
|
|
478
|
-
* Tags with streamIndex and conditionally includes beat data (main channel only).
|
|
479
|
-
*/
|
|
480
|
-
private sendChannelResults;
|
|
481
|
-
/**
|
|
482
|
-
* Deliver an analysis message to one sink with PER-SINK typed-array
|
|
483
|
-
* copies. The host→worker relay transfers `frequencyData`/`waveformData`
|
|
484
|
-
* buffers (zero-copy receive, `collectTransferables` case
|
|
485
|
-
* 'audio-analysis-update') — posting the same message object to N sinks
|
|
486
|
-
* would let the first post DETACH the buffers and starve every other
|
|
487
|
-
* sink (and the replay cache). One copy per sink per analysis frame is
|
|
488
|
-
* the unavoidable cost of a transfer-per-receiver delivery.
|
|
489
|
-
*/
|
|
490
|
-
private deliverAnalysisMessage;
|
|
491
|
-
/**
|
|
492
|
-
* Register a result-delivery sink (one per attached core's worker) and
|
|
493
|
-
* replay current state to it in FIFO order: the latest main-channel
|
|
494
|
-
* analysis, then per non-main channel its `audio-stream-setup` identity
|
|
495
|
-
* followed by the latest cached analysis. A mid-stream attach is thereby
|
|
496
|
-
* indistinguishable from a fresh connect on the receiving worker.
|
|
497
|
-
*/
|
|
498
|
-
addSink(id: string, send: (message: {
|
|
499
|
-
type: string;
|
|
500
|
-
data: any;
|
|
501
|
-
}) => void): void;
|
|
502
|
-
/**
|
|
503
|
-
* Remove a previously registered result-delivery sink.
|
|
504
|
-
*/
|
|
505
|
-
removeSink(id: string): void;
|
|
506
|
-
private buildStreamSetupMessage;
|
|
507
|
-
private emitStreamSetup;
|
|
508
|
-
/**
|
|
509
|
-
* Reset audio values to defaults (main channel + beat state)
|
|
510
|
-
*/
|
|
511
|
-
private resetAudioValues;
|
|
512
|
-
/**
|
|
513
|
-
* Reset all audio state (called when destroying).
|
|
514
|
-
* Disconnects all channels, closes AudioContext, resets all modules.
|
|
515
|
-
*
|
|
516
|
-
* Returns a Promise that resolves once `AudioContext.close()` has fully
|
|
517
|
-
* transitioned the context to `closed`. Callers tearing the system down
|
|
518
|
-
* (e.g. `VijiCore.destroy()`) MUST await this; otherwise Blink keeps the
|
|
519
|
-
* context alive in `Pending activities` and leaks ~4 MB per cycle.
|
|
520
|
-
*/
|
|
521
|
-
resetAudioState(): Promise<void>;
|
|
522
|
-
private sourceFedIndices;
|
|
523
|
-
/**
|
|
524
|
-
* Register a source-fed device channel (no MediaStream). Emits the
|
|
525
|
-
* `audio-stream-setup` identity to all sinks and starts the staleness
|
|
526
|
-
* timer. Idempotent.
|
|
527
|
-
*/
|
|
528
|
-
addDeviceAnalysisChannel(streamIndex: number, deviceId: string): void;
|
|
529
|
-
/**
|
|
530
|
-
* Feed a controller analysis packet into a source-fed channel. Instant
|
|
531
|
-
* values pass through; the smoothed companions are derived host-side via
|
|
532
|
-
* the channel's envelope followers to absorb network packet bunching
|
|
533
|
-
* (§3.9 light smoothing). Missing `frequencyData`/`waveform` surface as
|
|
534
|
-
* empty arrays (the controller omits them unless the scene reads them).
|
|
535
|
-
*/
|
|
536
|
-
feedDeviceAnalysis(streamIndex: number, payload: {
|
|
537
|
-
volume?: {
|
|
538
|
-
current?: number;
|
|
539
|
-
peak?: number;
|
|
540
|
-
};
|
|
541
|
-
bands?: {
|
|
542
|
-
low?: number;
|
|
543
|
-
lowMid?: number;
|
|
544
|
-
mid?: number;
|
|
545
|
-
highMid?: number;
|
|
546
|
-
high?: number;
|
|
547
|
-
};
|
|
548
|
-
spectral?: {
|
|
549
|
-
brightness?: number;
|
|
550
|
-
flatness?: number;
|
|
551
|
-
};
|
|
552
|
-
frequencyData?: Uint8Array;
|
|
553
|
-
waveformData?: Float32Array;
|
|
554
|
-
}): void;
|
|
555
|
-
/** Remove a source-fed device channel (sends a disconnected state). */
|
|
556
|
-
removeDeviceAnalysisChannel(streamIndex: number): void;
|
|
557
|
-
/**
|
|
558
|
-
* Add an additional audio stream (lightweight analysis, no beat detection).
|
|
559
|
-
* When `meta` is provided, the stream identity is recorded and an
|
|
560
|
-
* `audio-stream-setup` message is emitted to every sink (and replayed to
|
|
561
|
-
* sinks attached later).
|
|
562
|
-
* @param streamIndex Global stream index (AUDIO_ADDITIONAL_BASE + n or AUDIO_DEVICE_BASE + n)
|
|
563
|
-
* @param stream The MediaStream to analyze
|
|
564
|
-
* @param meta Stream identity for the worker-side two-phase setup protocol
|
|
565
|
-
*/
|
|
566
|
-
addStream(streamIndex: number, stream: MediaStream, meta?: {
|
|
567
|
-
streamType: 'additional' | 'device';
|
|
568
|
-
deviceId?: string;
|
|
569
|
-
}): Promise<void>;
|
|
570
|
-
/**
|
|
571
|
-
* Remove an additional audio stream.
|
|
572
|
-
*/
|
|
573
|
-
removeStream(streamIndex: number): void;
|
|
574
|
-
/**
|
|
575
|
-
* Tear down all additional channels in the base range and rebuild with
|
|
576
|
-
* corrected indices, re-emitting `audio-stream-setup` for the re-indexed
|
|
577
|
-
* streams. Mirrors video's reinitializeAdditionalCoordinators pattern.
|
|
578
|
-
*/
|
|
579
|
-
reinitializeAdditionalChannels(streams: MediaStream[], baseIndex: number): Promise<void>;
|
|
580
|
-
/**
|
|
581
|
-
* Get the number of additional audio channels.
|
|
582
|
-
*/
|
|
583
|
-
getChannelCount(): number;
|
|
584
|
-
/**
|
|
585
|
-
* Record a tap for the specified instrument onset.
|
|
586
|
-
* `options.skipRecognition` opts the tap out of the recognition pipeline
|
|
587
|
-
* only (no IOI accumulation, no `tryRecognizePattern`, no `applyPattern`).
|
|
588
|
-
* Mode still flips `'auto' → 'tapping'` so the visual envelope and
|
|
589
|
-
* audio-event filter run; only the pattern-detection pipeline is bypassed.
|
|
590
|
-
* Used by host-side relay of forwarded tap messages where another core
|
|
591
|
-
* owns the authoritative recognition.
|
|
592
|
-
*/
|
|
593
|
-
tapOnset(instrument: InstrumentType, options?: {
|
|
594
|
-
skipRecognition?: boolean;
|
|
595
|
-
}): void;
|
|
596
|
-
/**
|
|
597
|
-
* Clear tap pattern for an instrument, restoring auto-detection
|
|
598
|
-
*/
|
|
599
|
-
clearOnsetTap(instrument: InstrumentType): void;
|
|
600
|
-
/**
|
|
601
|
-
* Get the current onset mode for an instrument
|
|
602
|
-
*/
|
|
603
|
-
getOnsetMode(instrument: InstrumentType): OnsetMode_2;
|
|
604
|
-
/**
|
|
605
|
-
* Get pattern info for an instrument (null if no pattern recognized)
|
|
606
|
-
*/
|
|
607
|
-
getOnsetPatternInfo(instrument: InstrumentType): PatternInfo | null;
|
|
608
|
-
/**
|
|
609
|
-
* Mute/unmute an instrument onset (suppresses output without destroying state)
|
|
610
|
-
*/
|
|
611
|
-
setOnsetMuted(instrument: InstrumentType, muted: boolean): void;
|
|
612
|
-
/**
|
|
613
|
-
* Check if an instrument onset is muted
|
|
614
|
-
*/
|
|
615
|
-
isOnsetMuted(instrument: InstrumentType): boolean;
|
|
616
|
-
onOnsetModeChange(listener: (ev: OnsetModeChangeEvent) => void): Unsubscribe;
|
|
617
|
-
onOnsetSessionEnd(listener: (ev: OnsetSessionEndEvent) => void): Unsubscribe;
|
|
618
|
-
onOnsetMuteChange(listener: (ev: OnsetMuteChangeEvent) => void): Unsubscribe;
|
|
619
|
-
exportOnsetSessionState(options?: {
|
|
620
|
-
instruments?: ReadonlyArray<InstrumentType>;
|
|
621
|
-
}): SerializedOnsetState;
|
|
622
|
-
importOnsetSessionState(state: SerializedOnsetState, clockOffset: number): void;
|
|
623
|
-
exportAudioAnalysisState(): SerializedAudioAnalysisState;
|
|
624
|
-
importAudioAnalysisState(state: SerializedAudioAnalysisState, clockOffset: number): void;
|
|
625
|
-
private startIdleTicker;
|
|
626
|
-
private stopIdleTicker;
|
|
627
|
-
private tickIdle;
|
|
628
|
-
private makeEmptyBeatState;
|
|
629
|
-
/**
|
|
630
|
-
* Get current BPM (manual or auto-detected)
|
|
631
|
-
*/
|
|
632
|
-
getCurrentBPM(): number;
|
|
633
|
-
/**
|
|
634
|
-
* Nudge beat phase
|
|
635
|
-
*/
|
|
636
|
-
nudgeBeatPhase(amount: number): void;
|
|
637
|
-
/**
|
|
638
|
-
* Reset beat phase to next beat
|
|
639
|
-
*/
|
|
640
|
-
resetBeatPhase(): void;
|
|
641
|
-
/**
|
|
642
|
-
* Set FFT size
|
|
643
|
-
*/
|
|
644
|
-
setFFTSize(size: 2048 | 4096 | 8192): void;
|
|
645
|
-
/**
|
|
646
|
-
* Enable/disable auto-gain
|
|
647
|
-
*/
|
|
648
|
-
setAutoGain(enabled: boolean): void;
|
|
649
|
-
/**
|
|
650
|
-
* Enable/disable beat detection
|
|
651
|
-
*/
|
|
652
|
-
setBeatDetection(enabled: boolean): void;
|
|
653
|
-
/**
|
|
654
|
-
* Enable/disable onset detection
|
|
655
|
-
*/
|
|
656
|
-
setOnsetDetection(enabled: boolean): void;
|
|
657
|
-
/**
|
|
658
|
-
* Get complete audio analysis state
|
|
659
|
-
*/
|
|
660
|
-
getState(): {
|
|
661
|
-
isConnected: boolean;
|
|
662
|
-
currentBPM: number;
|
|
663
|
-
confidence: number;
|
|
664
|
-
isLocked: boolean;
|
|
665
|
-
trackingState: string;
|
|
666
|
-
onsetModes: {
|
|
667
|
-
kick: OnsetMode_2;
|
|
668
|
-
snare: OnsetMode_2;
|
|
669
|
-
hat: OnsetMode_2;
|
|
670
|
-
};
|
|
671
|
-
};
|
|
672
|
-
/**
|
|
673
|
-
* Get current per-frame audio analysis data (for UI display)
|
|
674
|
-
* Returns a snapshot of the latest audio analysis values
|
|
675
|
-
*/
|
|
676
|
-
getCurrentAudioData(): {
|
|
677
|
-
isConnected: boolean;
|
|
678
|
-
volume: {
|
|
679
|
-
current: number;
|
|
680
|
-
peak: number;
|
|
681
|
-
smoothed: number;
|
|
682
|
-
};
|
|
683
|
-
bands: {
|
|
684
|
-
low: number;
|
|
685
|
-
lowMid: number;
|
|
686
|
-
mid: number;
|
|
687
|
-
highMid: number;
|
|
688
|
-
high: number;
|
|
689
|
-
lowSmoothed: number;
|
|
690
|
-
lowMidSmoothed: number;
|
|
691
|
-
midSmoothed: number;
|
|
692
|
-
highMidSmoothed: number;
|
|
693
|
-
highSmoothed: number;
|
|
694
|
-
};
|
|
695
|
-
beat: {
|
|
696
|
-
kick: number;
|
|
697
|
-
snare: number;
|
|
698
|
-
hat: number;
|
|
699
|
-
any: number;
|
|
700
|
-
kickSmoothed: number;
|
|
701
|
-
snareSmoothed: number;
|
|
702
|
-
hatSmoothed: number;
|
|
703
|
-
anySmoothed: number;
|
|
704
|
-
events: BeatEvent[];
|
|
705
|
-
bpm: number;
|
|
706
|
-
confidence: number;
|
|
707
|
-
isLocked: boolean;
|
|
708
|
-
};
|
|
709
|
-
spectral: {
|
|
710
|
-
brightness: number;
|
|
711
|
-
flatness: number;
|
|
712
|
-
};
|
|
713
|
-
};
|
|
714
|
-
/**
|
|
715
|
-
* Get enhanced debug snapshot from all layers
|
|
716
|
-
*/
|
|
717
|
-
getEnhancedDebugSnapshot(): {
|
|
718
|
-
onset: {
|
|
719
|
-
currentBPM: number;
|
|
720
|
-
debugMode: boolean;
|
|
721
|
-
totalOnsets: number;
|
|
722
|
-
currentCluster: {
|
|
723
|
-
primaryBands: string[];
|
|
724
|
-
ageMssinceStart: number;
|
|
725
|
-
} | null;
|
|
726
|
-
bandStates: Record<string, {
|
|
727
|
-
lastOnsetTime: number;
|
|
728
|
-
onsetCount: number;
|
|
729
|
-
cutoff: number;
|
|
730
|
-
minEnergy: number;
|
|
731
|
-
}>;
|
|
732
|
-
};
|
|
733
|
-
tempo: {
|
|
734
|
-
bpm: number;
|
|
735
|
-
confidence: number;
|
|
736
|
-
};
|
|
737
|
-
pll: PLLState;
|
|
738
|
-
state: {
|
|
739
|
-
trackingState: TrackingState;
|
|
740
|
-
isLocked: boolean;
|
|
741
|
-
};
|
|
742
|
-
};
|
|
743
|
-
/**
|
|
744
|
-
* Get comprehensive debug info for testing/debugging
|
|
745
|
-
* Exposes internal state of all analysis layers
|
|
746
|
-
*/
|
|
747
|
-
getDebugInfo(): {
|
|
748
|
-
onset: Record<BandName, {
|
|
749
|
-
energy: number;
|
|
750
|
-
prevEnergy: number;
|
|
751
|
-
prevEnergy2: number;
|
|
752
|
-
minEnergy: number;
|
|
753
|
-
flux: number;
|
|
754
|
-
cutoff: number;
|
|
755
|
-
minThreshold: number;
|
|
756
|
-
minFlux: number;
|
|
757
|
-
wouldTrigger: boolean;
|
|
758
|
-
reason: string;
|
|
759
|
-
debounceMs: number;
|
|
760
|
-
lastOnsetAgo: number;
|
|
761
|
-
baselineAlpha?: number;
|
|
762
|
-
dtMs?: number;
|
|
763
|
-
fluxPrevOnly?: number;
|
|
764
|
-
fluxBaseline?: number;
|
|
765
|
-
initialized?: boolean;
|
|
766
|
-
lastFluxStored?: number;
|
|
767
|
-
fluxCalcNow?: number;
|
|
768
|
-
}>;
|
|
769
|
-
tempo: {
|
|
770
|
-
currentBPM: number;
|
|
771
|
-
confidence: number;
|
|
772
|
-
method: string;
|
|
773
|
-
anchorBand: string | null;
|
|
774
|
-
methodAgreement: number;
|
|
775
|
-
bpmHistory: number[];
|
|
776
|
-
envelopeLength: number;
|
|
777
|
-
lastAutocorrBPM: number;
|
|
778
|
-
lastIOIBPM: number;
|
|
779
|
-
};
|
|
780
|
-
pll: {
|
|
781
|
-
bpm: number;
|
|
782
|
-
phase: number;
|
|
783
|
-
bar: number;
|
|
784
|
-
driftRate: number;
|
|
785
|
-
state: PLLState;
|
|
786
|
-
};
|
|
787
|
-
state: {
|
|
788
|
-
trackingState: TrackingState;
|
|
789
|
-
isLocked: boolean;
|
|
790
|
-
};
|
|
791
|
-
levels: {
|
|
792
|
-
low: number;
|
|
793
|
-
lowMid: number;
|
|
794
|
-
mid: number;
|
|
795
|
-
highMid: number;
|
|
796
|
-
high: number;
|
|
797
|
-
volume: number;
|
|
798
|
-
};
|
|
799
|
-
rawBands: {
|
|
800
|
-
low: number;
|
|
801
|
-
lowMid: number;
|
|
802
|
-
mid: number;
|
|
803
|
-
highMid: number;
|
|
804
|
-
high: number;
|
|
805
|
-
};
|
|
806
|
-
bandGains: {
|
|
807
|
-
low: number;
|
|
808
|
-
lowMid: number;
|
|
809
|
-
mid: number;
|
|
810
|
-
highMid: number;
|
|
811
|
-
high: number;
|
|
812
|
-
};
|
|
813
|
-
volumeGain: number;
|
|
814
|
-
dtMs: number;
|
|
815
|
-
analysisIntervalMs: number;
|
|
816
|
-
events: BeatEvent[];
|
|
817
|
-
};
|
|
818
|
-
}
|
|
819
|
-
|
|
820
|
-
/**
|
|
821
|
-
* Multi-band onset detection with hybrid peak-decay + spectral flux approach
|
|
822
|
-
*
|
|
823
|
-
* Industry-grade onset detection that:
|
|
824
|
-
* - Uses 5 frequency bands for precise instrument detection
|
|
825
|
-
* - Combines peak-decay threshold (proven reliable) with spectral flux (transient detection)
|
|
826
|
-
* - Maintains per-band onset history for tempo pattern analysis
|
|
827
|
-
* - Is frame-rate independent (dt-aware)
|
|
828
|
-
* - Prevents double-triggering with adaptive cooldown
|
|
829
|
-
*/
|
|
830
|
-
/**
|
|
831
|
-
* Band names for the 5-band analysis
|
|
832
|
-
*/
|
|
833
|
-
declare type BandName = 'low' | 'lowMid' | 'mid' | 'highMid' | 'high';
|
|
834
|
-
|
|
835
189
|
/**
|
|
836
190
|
* Beat event (discrete, timestamped)
|
|
837
191
|
*/
|
|
@@ -1688,8 +1042,6 @@ export declare interface InstrumentSerialized {
|
|
|
1688
1042
|
refinementCounts: number[];
|
|
1689
1043
|
}
|
|
1690
1044
|
|
|
1691
|
-
declare type InstrumentType = 'kick' | 'snare' | 'hat';
|
|
1692
|
-
|
|
1693
1045
|
/**
|
|
1694
1046
|
* Keyboard input state for the current frame: per-key press / hold / release
|
|
1695
1047
|
* queries (case-insensitive), the live set of held keys, modifier flags, and
|
|
@@ -1862,8 +1214,6 @@ export declare type OnsetInstrument = 'kick' | 'snare' | 'hat';
|
|
|
1862
1214
|
|
|
1863
1215
|
declare type OnsetMode = 'auto' | 'tapping' | 'pattern';
|
|
1864
1216
|
|
|
1865
|
-
declare type OnsetMode_2 = 'auto' | 'tapping' | 'pattern';
|
|
1866
|
-
|
|
1867
1217
|
export declare interface OnsetModeChangeEvent {
|
|
1868
1218
|
instrument: OnsetInstrument;
|
|
1869
1219
|
prevMode: OnsetMode;
|
|
@@ -1963,12 +1313,6 @@ export declare type ParameterType = 'slider' | 'color' | 'toggle' | 'select' | '
|
|
|
1963
1313
|
|
|
1964
1314
|
export declare type ParameterValue = string | number | boolean | ImageBitmap | CoordinateValue | null;
|
|
1965
1315
|
|
|
1966
|
-
declare interface PatternInfo {
|
|
1967
|
-
length: number;
|
|
1968
|
-
intervals: number[];
|
|
1969
|
-
confidence: number;
|
|
1970
|
-
}
|
|
1971
|
-
|
|
1972
1316
|
export declare interface PerformanceStats {
|
|
1973
1317
|
resolution: Resolution;
|
|
1974
1318
|
scale: number;
|
|
@@ -2007,35 +1351,6 @@ export declare type PeriodCapabilityCounts = Record<InputCapabilityKey, CountPai
|
|
|
2007
1351
|
}>;
|
|
2008
1352
|
};
|
|
2009
1353
|
|
|
2010
|
-
/**
|
|
2011
|
-
* Phase-Locked Loop for stable beat phase tracking
|
|
2012
|
-
*
|
|
2013
|
-
* Industry-grade phase tracking that:
|
|
2014
|
-
* - Maintains expected beat times based on current BPM
|
|
2015
|
-
* - Compares actual onsets to expected times (phase error)
|
|
2016
|
-
* - Corrects phase gradually to avoid jumps
|
|
2017
|
-
* - Handles breakdowns by extrapolating phase
|
|
2018
|
-
* - Detects tempo drift in DJ sets
|
|
2019
|
-
* - Provides stable bar position (0-3 for 4/4 time)
|
|
2020
|
-
*/
|
|
2021
|
-
/**
|
|
2022
|
-
* PLL state for external access
|
|
2023
|
-
*/
|
|
2024
|
-
declare interface PLLState {
|
|
2025
|
-
/** Current phase (0-1, where 0 = beat) */
|
|
2026
|
-
phase: number;
|
|
2027
|
-
/** Current bar position (0-3 for 4/4 time) */
|
|
2028
|
-
bar: number;
|
|
2029
|
-
/** Current BPM being tracked */
|
|
2030
|
-
bpm: number;
|
|
2031
|
-
/** Time since last confirmed beat (ms) */
|
|
2032
|
-
timeSinceLastBeat: number;
|
|
2033
|
-
/** Whether tempo is drifting (DJ transition) */
|
|
2034
|
-
isDrifting: boolean;
|
|
2035
|
-
/** Drift rate (BPM per second, positive = speeding up) */
|
|
2036
|
-
driftRate: number;
|
|
2037
|
-
}
|
|
2038
|
-
|
|
2039
1354
|
/**
|
|
2040
1355
|
* Unified pointer that abstracts mouse and primary touch into a single input
|
|
2041
1356
|
* stream. Recommended for click / drag / position-tracking interactions that
|
|
@@ -2140,6 +1455,29 @@ export declare type Resolution = {
|
|
|
2140
1455
|
height: number;
|
|
2141
1456
|
};
|
|
2142
1457
|
|
|
1458
|
+
/**
|
|
1459
|
+
* Payload of the worker's `scene-analysis` ack — the correlated response to
|
|
1460
|
+
* `set-scene-code`. The worker runs static analysis (renderer, capabilities,
|
|
1461
|
+
* p5 canvas mode, analysedFrame consumption) after receiving the scene code
|
|
1462
|
+
* and **before** compiling it, and the host's `initialize()` awaits this ack
|
|
1463
|
+
* to assign the four `SceneData` fields exactly once. Carries booleans/enums
|
|
1464
|
+
* only — never scene source.
|
|
1465
|
+
*
|
|
1466
|
+
* Public since 0.14.0: this is also the return type of `analyzeScene`
|
|
1467
|
+
* (`@viji-dev/core/scene-analysis`), which the worker itself calls to build
|
|
1468
|
+
* the ack — one shape, one producer, two transports.
|
|
1469
|
+
*/
|
|
1470
|
+
export declare interface SceneAnalysisData {
|
|
1471
|
+
/** Detected renderer the scene targets. */
|
|
1472
|
+
renderer: RendererType;
|
|
1473
|
+
/** P5 canvas mode; present only when `renderer` is `'p5'`. */
|
|
1474
|
+
p5CanvasMode?: P5CanvasMode;
|
|
1475
|
+
/** Capabilities the scene's code declares it uses (inclusive-biased). */
|
|
1476
|
+
capabilities: SceneCapabilities;
|
|
1477
|
+
/** Whether the scene consumes per-frame analysed-frame bitmaps (CV). */
|
|
1478
|
+
consumesAnalysedFrame: boolean;
|
|
1479
|
+
}
|
|
1480
|
+
|
|
2143
1481
|
/**
|
|
2144
1482
|
* The set of engine capabilities a scene's code declares it uses, derived by
|
|
2145
1483
|
* static analysis of the scene source (no execution). Detection is
|
|
@@ -2967,16 +2305,10 @@ declare interface TouchPoint {
|
|
|
2967
2305
|
isEnding: boolean;
|
|
2968
2306
|
}
|
|
2969
2307
|
|
|
2970
|
-
/**
|
|
2971
|
-
* State machine states
|
|
2972
|
-
* NOTE: LEARNING state removed - system now uses continuous rolling window adaptation
|
|
2973
|
-
*/
|
|
2974
|
-
declare type TrackingState = 'TRACKING' | 'LOCKED' | 'BREAKDOWN' | 'LOST';
|
|
2975
|
-
|
|
2976
2308
|
/** Returned by `on*` listener registration calls; invoke to unsubscribe. */
|
|
2977
2309
|
export declare type Unsubscribe = () => void;
|
|
2978
2310
|
|
|
2979
|
-
export declare const VERSION = "0.
|
|
2311
|
+
export declare const VERSION = "0.14.0";
|
|
2980
2312
|
|
|
2981
2313
|
/**
|
|
2982
2314
|
* Real-time video API: drawable frame, dimensions, and the source-side
|
|
@@ -3162,15 +2494,6 @@ export declare interface VideoCVAPI {
|
|
|
3162
2494
|
isProcessing(): boolean;
|
|
3163
2495
|
}
|
|
3164
2496
|
|
|
3165
|
-
/**
|
|
3166
|
-
* Video stream role, derived from position in the index contract.
|
|
3167
|
-
* - main: primary video with CV (`viji.video`)
|
|
3168
|
-
* - additional: host-provided MediaStreams, no CV (`viji.videoStreams[]`)
|
|
3169
|
-
* - device: device camera streams (`viji.devices[].video`)
|
|
3170
|
-
* - directFrame: injected compositor frames (`viji.videoStreams[]`, after additional)
|
|
3171
|
-
*/
|
|
3172
|
-
export declare type VideoStreamType = 'main' | 'additional' | 'device' | 'directFrame';
|
|
3173
|
-
|
|
3174
2497
|
/**
|
|
3175
2498
|
* The Viji API surface available to artist code as the global `viji` object.
|
|
3176
2499
|
* Groups canvas access, timing, connected media APIs, user interaction, device
|