@viji-dev/core 0.4.6 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/artist-dts-p5.js +1 -1
- package/dist/artist-dts.js +1 -1
- package/dist/artist-global-p5.d.ts +114 -48
- package/dist/artist-global.d.ts +114 -48
- package/dist/artist-jsdoc.d.ts +2 -6
- package/dist/assets/{viji.worker-BbzNOVhB.js → viji.worker-DwYMDyfQ.js} +539 -170
- package/dist/assets/viji.worker-DwYMDyfQ.js.map +1 -0
- package/dist/docs-api.js +548 -458
- package/dist/{essentia-wasm.web-qCUJ8zjr.js → essentia-wasm.web-x6zu4Vib.js} +2 -2
- package/dist/{essentia-wasm.web-qCUJ8zjr.js.map → essentia-wasm.web-x6zu4Vib.js.map} +1 -1
- package/dist/{index-Oj_9v8r4.js → index-Cqh1k_49.js} +999 -19
- package/dist/index-Cqh1k_49.js.map +1 -0
- package/dist/index.d.ts +571 -48
- package/dist/index.js +1 -1
- package/dist/shader-uniforms.js +10 -0
- package/package.json +1 -1
- package/dist/assets/viji.worker-BbzNOVhB.js.map +0 -1
- package/dist/index-Oj_9v8r4.js.map +0 -1
package/dist/index.d.ts
CHANGED
|
@@ -235,6 +235,8 @@ export declare class AudioSystem {
|
|
|
235
235
|
private beatDetectionEnabled;
|
|
236
236
|
private onsetDetectionEnabled;
|
|
237
237
|
private autoGainEnabled;
|
|
238
|
+
private idleTickerHandle;
|
|
239
|
+
private idleTickerLastTime;
|
|
238
240
|
/**
|
|
239
241
|
* Enable or disable comprehensive debug logging for all layers
|
|
240
242
|
* Enables enhanced logging in: MultiOnsetDetection, BeatStateManager
|
|
@@ -349,6 +351,12 @@ export declare class AudioSystem {
|
|
|
349
351
|
/**
|
|
350
352
|
* Connect a channel to Web Audio nodes (source, worklet/analyser).
|
|
351
353
|
* Used for both main and additional channels.
|
|
354
|
+
*
|
|
355
|
+
* Lifecycle ordering invariant: when wiring the main channel, the idle
|
|
356
|
+
* ticker MUST be stopped before the first `await` so that an in-flight
|
|
357
|
+
* tick cannot race the audio path coming online. The ticker callback
|
|
358
|
+
* additionally guards on `mainChannel.audioState.isConnected` for
|
|
359
|
+
* belt-and-braces.
|
|
352
360
|
*/
|
|
353
361
|
private connectChannel;
|
|
354
362
|
/**
|
|
@@ -357,6 +365,11 @@ export declare class AudioSystem {
|
|
|
357
365
|
private setAudioStream;
|
|
358
366
|
/**
|
|
359
367
|
* Disconnect the main audio stream (does NOT close AudioContext -- additional channels may still be active).
|
|
368
|
+
*
|
|
369
|
+
* Lifecycle ordering invariant: tear down the audio nodes first, clear any
|
|
370
|
+
* stale frequency / waveform buffers, then restart the idle ticker. The
|
|
371
|
+
* ticker callback's defensive `isConnected` guard prevents any stale tick
|
|
372
|
+
* from observing a half-disconnected state.
|
|
360
373
|
*/
|
|
361
374
|
private disconnectMainStream;
|
|
362
375
|
/**
|
|
@@ -451,6 +464,16 @@ export declare class AudioSystem {
|
|
|
451
464
|
* Check if an instrument onset is muted
|
|
452
465
|
*/
|
|
453
466
|
isOnsetMuted(instrument: InstrumentType): boolean;
|
|
467
|
+
onOnsetModeChange(listener: (ev: OnsetModeChangeEvent) => void): Unsubscribe;
|
|
468
|
+
onOnsetSessionEnd(listener: (ev: OnsetSessionEndEvent) => void): Unsubscribe;
|
|
469
|
+
exportOnsetSessionState(): SerializedOnsetState;
|
|
470
|
+
importOnsetSessionState(state: SerializedOnsetState, clockOffset: number): void;
|
|
471
|
+
exportAudioAnalysisState(): SerializedAudioAnalysisState;
|
|
472
|
+
importAudioAnalysisState(state: SerializedAudioAnalysisState, clockOffset: number): void;
|
|
473
|
+
private startIdleTicker;
|
|
474
|
+
private stopIdleTicker;
|
|
475
|
+
private tickIdle;
|
|
476
|
+
private makeEmptyBeatState;
|
|
454
477
|
/**
|
|
455
478
|
* Get current BPM (manual or auto-detected)
|
|
456
479
|
*/
|
|
@@ -1256,6 +1279,22 @@ declare interface ImageParameter {
|
|
|
1256
1279
|
category: ParameterCategory;
|
|
1257
1280
|
}
|
|
1258
1281
|
|
|
1282
|
+
export declare interface InstrumentSerialized {
|
|
1283
|
+
mode: OnsetMode;
|
|
1284
|
+
muted: boolean;
|
|
1285
|
+
pattern: number[] | null;
|
|
1286
|
+
/** Sender wall-clock; null if never set. */
|
|
1287
|
+
replayLastEventTime: number | null;
|
|
1288
|
+
replayIndex: number;
|
|
1289
|
+
tapIOIs: number[];
|
|
1290
|
+
/** Sender wall-clock; null if never set. */
|
|
1291
|
+
lastTapTime: number | null;
|
|
1292
|
+
/** Sender wall-clock; null if not muted. */
|
|
1293
|
+
mutedAt: number | null;
|
|
1294
|
+
refinementIndex: number;
|
|
1295
|
+
refinementCounts: number[];
|
|
1296
|
+
}
|
|
1297
|
+
|
|
1259
1298
|
declare type InstrumentType = 'kick' | 'snare' | 'hat';
|
|
1260
1299
|
|
|
1261
1300
|
/**
|
|
@@ -1404,10 +1443,29 @@ declare interface NumberParameter {
|
|
|
1404
1443
|
category: ParameterCategory;
|
|
1405
1444
|
}
|
|
1406
1445
|
|
|
1446
|
+
export declare type OnsetInstrument = 'kick' | 'snare' | 'hat';
|
|
1447
|
+
|
|
1407
1448
|
declare type OnsetMode = 'auto' | 'tapping' | 'pattern';
|
|
1408
1449
|
|
|
1409
1450
|
declare type OnsetMode_2 = 'auto' | 'tapping' | 'pattern';
|
|
1410
1451
|
|
|
1452
|
+
export declare interface OnsetModeChangeEvent {
|
|
1453
|
+
instrument: OnsetInstrument;
|
|
1454
|
+
prevMode: OnsetMode;
|
|
1455
|
+
newMode: OnsetMode;
|
|
1456
|
+
}
|
|
1457
|
+
|
|
1458
|
+
export declare interface OnsetSessionEndEvent {
|
|
1459
|
+
instrument: OnsetInstrument;
|
|
1460
|
+
/**
|
|
1461
|
+
* `'pattern'` — 500ms idle since last tap with instrument in `'pattern'` mode.
|
|
1462
|
+
* `'cleared'` — 5s idle in `'tapping'` mode without a recognized pattern;
|
|
1463
|
+
* instrument transitions to `'auto'`. Explicit `clear()` calls do NOT fire
|
|
1464
|
+
* this event (caller-initiated; caller already knows).
|
|
1465
|
+
*/
|
|
1466
|
+
outcome: 'pattern' | 'cleared';
|
|
1467
|
+
}
|
|
1468
|
+
|
|
1411
1469
|
export declare interface ParameterAPI {
|
|
1412
1470
|
define(groups: ParameterGroup[]): void;
|
|
1413
1471
|
[key: string]: any;
|
|
@@ -1660,6 +1718,308 @@ declare interface SelectParameter {
|
|
|
1660
1718
|
category: ParameterCategory;
|
|
1661
1719
|
}
|
|
1662
1720
|
|
|
1721
|
+
export declare interface SerializedAudioAnalysisState {
|
|
1722
|
+
/** Per-instrument onset state (mirrors SerializedOnsetState.instruments shape). */
|
|
1723
|
+
onset: SerializedOnsetState['instruments'];
|
|
1724
|
+
bpmTracker: SerializedTempoInductionState;
|
|
1725
|
+
pll: SerializedPLLState;
|
|
1726
|
+
beatState: SerializedBeatStateManagerState;
|
|
1727
|
+
}
|
|
1728
|
+
|
|
1729
|
+
/**
|
|
1730
|
+
* BeatStateManager continuity-relevant state. Treat as opaque on the consumer
|
|
1731
|
+
* side. Envelope follower state is omitted by design — receiver re-triggers
|
|
1732
|
+
* envelopes naturally as detected events flow.
|
|
1733
|
+
*/
|
|
1734
|
+
export declare interface SerializedBeatStateManagerState {
|
|
1735
|
+
state: 'TRACKING' | 'LOCKED' | 'BREAKDOWN' | 'LOST';
|
|
1736
|
+
/** Sender wall-clock; 0 if never set. */
|
|
1737
|
+
stateEnteredTime: number;
|
|
1738
|
+
kickProfile: {
|
|
1739
|
+
avgEnergy: number;
|
|
1740
|
+
bassRatio: number;
|
|
1741
|
+
midRatio: number;
|
|
1742
|
+
trebleRatio: number;
|
|
1743
|
+
sampleCount: number;
|
|
1744
|
+
};
|
|
1745
|
+
snareProfile: {
|
|
1746
|
+
avgEnergy: number;
|
|
1747
|
+
bassRatio: number;
|
|
1748
|
+
midRatio: number;
|
|
1749
|
+
trebleRatio: number;
|
|
1750
|
+
sampleCount: number;
|
|
1751
|
+
};
|
|
1752
|
+
hatProfile: {
|
|
1753
|
+
avgEnergy: number;
|
|
1754
|
+
bassRatio: number;
|
|
1755
|
+
midRatio: number;
|
|
1756
|
+
trebleRatio: number;
|
|
1757
|
+
sampleCount: number;
|
|
1758
|
+
};
|
|
1759
|
+
recentOnsetStrengths: number[];
|
|
1760
|
+
averageOnsetStrength: number;
|
|
1761
|
+
tempoMethodAgreement: number;
|
|
1762
|
+
gridScore: number;
|
|
1763
|
+
consistencyScore: number;
|
|
1764
|
+
anchorClarity: number;
|
|
1765
|
+
recentGridScores: number[];
|
|
1766
|
+
lockedBPM: number;
|
|
1767
|
+
/** Sender wall-clock; 0 if never set. */
|
|
1768
|
+
lastOnsetTime: number;
|
|
1769
|
+
/** Sender wall-clock; 0 if never set. */
|
|
1770
|
+
lastKickTime: number;
|
|
1771
|
+
kickIntervals: number[];
|
|
1772
|
+
bpmHistory: number[];
|
|
1773
|
+
/** Sender wall-clocks in nested `time` fields. */
|
|
1774
|
+
adaptiveProfiles: {
|
|
1775
|
+
kick: {
|
|
1776
|
+
samples: Array<{
|
|
1777
|
+
midRatio: number;
|
|
1778
|
+
midToBass: number;
|
|
1779
|
+
time: number;
|
|
1780
|
+
}>;
|
|
1781
|
+
};
|
|
1782
|
+
snareIndependent: {
|
|
1783
|
+
samples: Array<{
|
|
1784
|
+
strength: number;
|
|
1785
|
+
midRatio: number;
|
|
1786
|
+
trebleRatio: number;
|
|
1787
|
+
sharpness: number;
|
|
1788
|
+
time: number;
|
|
1789
|
+
}>;
|
|
1790
|
+
};
|
|
1791
|
+
snareLayered: {
|
|
1792
|
+
samples: Array<{
|
|
1793
|
+
strength: number;
|
|
1794
|
+
midRatio: number;
|
|
1795
|
+
energy: number;
|
|
1796
|
+
sharpness: number;
|
|
1797
|
+
time: number;
|
|
1798
|
+
}>;
|
|
1799
|
+
};
|
|
1800
|
+
hatIndependent: {
|
|
1801
|
+
samples: Array<{
|
|
1802
|
+
strength: number;
|
|
1803
|
+
trebleRatio: number;
|
|
1804
|
+
energy: number;
|
|
1805
|
+
sharpness: number;
|
|
1806
|
+
time: number;
|
|
1807
|
+
}>;
|
|
1808
|
+
};
|
|
1809
|
+
hatLayered: {
|
|
1810
|
+
samples: Array<{
|
|
1811
|
+
strength: number;
|
|
1812
|
+
trebleRatio: number;
|
|
1813
|
+
energy: number;
|
|
1814
|
+
sharpness: number;
|
|
1815
|
+
time: number;
|
|
1816
|
+
}>;
|
|
1817
|
+
};
|
|
1818
|
+
};
|
|
1819
|
+
adaptiveThresholds: {
|
|
1820
|
+
snareMinMidRatio: number;
|
|
1821
|
+
snareMinMidToBass: number;
|
|
1822
|
+
kickMaxMidRatio: number;
|
|
1823
|
+
snareIndependent: {
|
|
1824
|
+
minStrength: number;
|
|
1825
|
+
minMidRatio: number;
|
|
1826
|
+
minTrebleRatio: number;
|
|
1827
|
+
minSharpness: number;
|
|
1828
|
+
};
|
|
1829
|
+
snareLayered: {
|
|
1830
|
+
minStrength: number;
|
|
1831
|
+
minEnergy: number;
|
|
1832
|
+
minSharpness: number;
|
|
1833
|
+
};
|
|
1834
|
+
hatIndependent: {
|
|
1835
|
+
minStrength: number;
|
|
1836
|
+
minTrebleRatio: number;
|
|
1837
|
+
minSharpness: number;
|
|
1838
|
+
};
|
|
1839
|
+
hatLayered: {
|
|
1840
|
+
minStrength: number;
|
|
1841
|
+
minTrebleRatio: number;
|
|
1842
|
+
minSharpness: number;
|
|
1843
|
+
};
|
|
1844
|
+
};
|
|
1845
|
+
/** Sender wall-clocks in nested `lastOnsetTime`. */
|
|
1846
|
+
ioiTrackers: {
|
|
1847
|
+
kick: {
|
|
1848
|
+
intervals: number[];
|
|
1849
|
+
lastOnsetTime: number;
|
|
1850
|
+
};
|
|
1851
|
+
snare: {
|
|
1852
|
+
intervals: number[];
|
|
1853
|
+
lastOnsetTime: number;
|
|
1854
|
+
};
|
|
1855
|
+
hat: {
|
|
1856
|
+
intervals: number[];
|
|
1857
|
+
lastOnsetTime: number;
|
|
1858
|
+
};
|
|
1859
|
+
};
|
|
1860
|
+
/** Each event has a `time` field (sender wall-clock); `expiresAt` is also wall-clock. */
|
|
1861
|
+
eventBuffer: Array<{
|
|
1862
|
+
event: {
|
|
1863
|
+
type: 'kick' | 'snare' | 'hat';
|
|
1864
|
+
time: number;
|
|
1865
|
+
strength: number;
|
|
1866
|
+
isPredicted?: boolean;
|
|
1867
|
+
bpm?: number;
|
|
1868
|
+
};
|
|
1869
|
+
expiresAt: number;
|
|
1870
|
+
}>;
|
|
1871
|
+
/** Per-band history with sender wall-clocks. */
|
|
1872
|
+
bandEnergyHistory: {
|
|
1873
|
+
low: Array<{
|
|
1874
|
+
time: number;
|
|
1875
|
+
value: number;
|
|
1876
|
+
}>;
|
|
1877
|
+
mid: Array<{
|
|
1878
|
+
time: number;
|
|
1879
|
+
value: number;
|
|
1880
|
+
}>;
|
|
1881
|
+
high: Array<{
|
|
1882
|
+
time: number;
|
|
1883
|
+
value: number;
|
|
1884
|
+
}>;
|
|
1885
|
+
};
|
|
1886
|
+
/** Per-class arrays of sender wall-clocks (rate-cap enforcement). */
|
|
1887
|
+
eventTimestamps: {
|
|
1888
|
+
kick: number[];
|
|
1889
|
+
snare: number[];
|
|
1890
|
+
hat: number[];
|
|
1891
|
+
};
|
|
1892
|
+
/** Per-class last event time (cooldown enforcement); 0 = never fired. */
|
|
1893
|
+
lastEventTime: {
|
|
1894
|
+
kick: number;
|
|
1895
|
+
snare: number;
|
|
1896
|
+
hat: number;
|
|
1897
|
+
};
|
|
1898
|
+
}
|
|
1899
|
+
|
|
1900
|
+
export declare interface SerializedCoreState {
|
|
1901
|
+
version: 1;
|
|
1902
|
+
/** Sender wall-clock at serialize time. */
|
|
1903
|
+
senderTime: number;
|
|
1904
|
+
/** Onset state only — cross-device-safe, no audio analysis carry-over. */
|
|
1905
|
+
onset?: SerializedOnsetState['instruments'];
|
|
1906
|
+
/**
|
|
1907
|
+
* Full audio analysis state — same-process scene-switch only. Cross-device
|
|
1908
|
+
* transfer should omit this block (sender's audio analysis doesn't apply
|
|
1909
|
+
* to receiver's audio source). Robust to gaps under ~1 second; longer
|
|
1910
|
+
* scene-loads (>5s) may cause stale-timestamp drift in fields like
|
|
1911
|
+
* `gridLockMemoryTime` / `warmupStartTimeMs` — caller should treat as a
|
|
1912
|
+
* degraded path and skip the audio block on stale exports.
|
|
1913
|
+
*/
|
|
1914
|
+
audio?: SerializedAudioAnalysisState;
|
|
1915
|
+
}
|
|
1916
|
+
|
|
1917
|
+
export declare interface SerializedOnsetState {
|
|
1918
|
+
version: 1;
|
|
1919
|
+
/** Sender wall-clock at serialize time. */
|
|
1920
|
+
senderTime: number;
|
|
1921
|
+
/** Per-instrument payload — only included instruments are imported. */
|
|
1922
|
+
instruments: {
|
|
1923
|
+
kick?: InstrumentSerialized;
|
|
1924
|
+
snare?: InstrumentSerialized;
|
|
1925
|
+
hat?: InstrumentSerialized;
|
|
1926
|
+
};
|
|
1927
|
+
}
|
|
1928
|
+
|
|
1929
|
+
/**
|
|
1930
|
+
* PhaseLockedLoop continuity-relevant state. Treat as opaque on the consumer
|
|
1931
|
+
* side.
|
|
1932
|
+
*/
|
|
1933
|
+
export declare interface SerializedPLLState {
|
|
1934
|
+
phase: number;
|
|
1935
|
+
/** Sender wall-clock; 0 if never set. */
|
|
1936
|
+
lastBeatTime: number;
|
|
1937
|
+
periodMs: number;
|
|
1938
|
+
trackedBPM: number;
|
|
1939
|
+
beatCounter: number;
|
|
1940
|
+
/** Sender wall-clock; 0 if never set. */
|
|
1941
|
+
lastOnsetTime: number;
|
|
1942
|
+
/** Sender wall-clocks in `time`. */
|
|
1943
|
+
bpmHistory: Array<{
|
|
1944
|
+
time: number;
|
|
1945
|
+
bpm: number;
|
|
1946
|
+
}>;
|
|
1947
|
+
driftRate: number;
|
|
1948
|
+
inBreakdown: boolean;
|
|
1949
|
+
tempoConfidence: number;
|
|
1950
|
+
inTransition: boolean;
|
|
1951
|
+
currentGain: number;
|
|
1952
|
+
/** Sender wall-clock; 0 if never set. */
|
|
1953
|
+
lastBarAdvanceTime: number;
|
|
1954
|
+
pendingBarAdvance: boolean;
|
|
1955
|
+
/** Sender wall-clock; 0 if never set. */
|
|
1956
|
+
lastPhaseWrapTime: number;
|
|
1957
|
+
consecutiveAlignedKicks: number;
|
|
1958
|
+
/** Sender wall-clock; 0 if never set. */
|
|
1959
|
+
lastHardSyncTime: number;
|
|
1960
|
+
/** Sender wall-clock; 0 if never set. */
|
|
1961
|
+
trackingStartTime: number;
|
|
1962
|
+
kickPhaseHistory: number[];
|
|
1963
|
+
phaseOffsetCalibrated: number;
|
|
1964
|
+
/** Sender wall-clock; 0 if never set. */
|
|
1965
|
+
lastCalibrationTime: number;
|
|
1966
|
+
lastPhaseError: number;
|
|
1967
|
+
}
|
|
1968
|
+
|
|
1969
|
+
/**
|
|
1970
|
+
* TempoInduction continuity-relevant state. Treat as opaque on the consumer
|
|
1971
|
+
* side — the field set is internal and may evolve in future versions
|
|
1972
|
+
* (gated by `version`).
|
|
1973
|
+
*/
|
|
1974
|
+
export declare interface SerializedTempoInductionState {
|
|
1975
|
+
onsetEnvelope: number[];
|
|
1976
|
+
lowBandEnvelope: number[];
|
|
1977
|
+
midBandEnvelope: number[];
|
|
1978
|
+
highBandEnvelope: number[];
|
|
1979
|
+
fpsEstimate: number;
|
|
1980
|
+
currentBPM: number;
|
|
1981
|
+
confidence: number;
|
|
1982
|
+
method: 'autocorr' | 'ioi' | 'combined';
|
|
1983
|
+
anchorBand: string | null;
|
|
1984
|
+
methodAgreement: number;
|
|
1985
|
+
bpmHistory: number[];
|
|
1986
|
+
/** Sender wall-clock; null if never set. */
|
|
1987
|
+
lastUpdateTime: number | null;
|
|
1988
|
+
hypotheses: Array<{
|
|
1989
|
+
bpm: number;
|
|
1990
|
+
likelihood: number;
|
|
1991
|
+
age: number;
|
|
1992
|
+
/** Sender wall-clock. */
|
|
1993
|
+
lastEvidence: number;
|
|
1994
|
+
type: 'main' | 'half' | 'double' | 'other';
|
|
1995
|
+
}>;
|
|
1996
|
+
inTransition: boolean;
|
|
1997
|
+
/** Sender wall-clocks. */
|
|
1998
|
+
bpmDriftHistory: Array<{
|
|
1999
|
+
time: number;
|
|
2000
|
+
bpm: number;
|
|
2001
|
+
}>;
|
|
2002
|
+
driftRate: number;
|
|
2003
|
+
tempoChangeConfirmCount: number;
|
|
2004
|
+
/** Sender wall-clocks. */
|
|
2005
|
+
onsetHistory: Array<{
|
|
2006
|
+
time: number;
|
|
2007
|
+
strength: number;
|
|
2008
|
+
type: string;
|
|
2009
|
+
}>;
|
|
2010
|
+
gridLockBPM: number | null;
|
|
2011
|
+
gridLockScore: number;
|
|
2012
|
+
/** Sender wall-clock; 0 if not locked. */
|
|
2013
|
+
gridLockTime: number;
|
|
2014
|
+
gridLockMemoryBPM: number | null;
|
|
2015
|
+
/** Sender wall-clock; 0 if no memory. */
|
|
2016
|
+
gridLockMemoryTime: number;
|
|
2017
|
+
syncopationLevel: number;
|
|
2018
|
+
/** Sender wall-clock; 0 if not started. */
|
|
2019
|
+
warmupStartTimeMs: number;
|
|
2020
|
+
warmupComplete: boolean;
|
|
2021
|
+
}
|
|
2022
|
+
|
|
1663
2023
|
/**
|
|
1664
2024
|
* Configuration object passed to `viji.slider()`. Defines the slider's bounds,
|
|
1665
2025
|
* increment, label, and UI grouping/visibility.
|
|
@@ -1704,6 +2064,17 @@ declare interface SliderParameter {
|
|
|
1704
2064
|
category: ParameterCategory;
|
|
1705
2065
|
}
|
|
1706
2066
|
|
|
2067
|
+
export declare interface StateImportError {
|
|
2068
|
+
code: 'version-mismatch' | 'malformed' | 'invalid-field';
|
|
2069
|
+
details: string;
|
|
2070
|
+
/** Present for `'version-mismatch'`. */
|
|
2071
|
+
payloadVersion?: number;
|
|
2072
|
+
/** Present for `'version-mismatch'`. */
|
|
2073
|
+
expectedVersion?: number;
|
|
2074
|
+
/** Present for `'invalid-field'`. */
|
|
2075
|
+
field?: string;
|
|
2076
|
+
}
|
|
2077
|
+
|
|
1707
2078
|
/**
|
|
1708
2079
|
* Configuration object passed to `viji.text()`. The host renders a single-line
|
|
1709
2080
|
* text input field.
|
|
@@ -1863,14 +2234,19 @@ declare interface TouchPoint {
|
|
|
1863
2234
|
*/
|
|
1864
2235
|
declare type TrackingState = 'TRACKING' | 'LOCKED' | 'BREAKDOWN' | 'LOST';
|
|
1865
2236
|
|
|
1866
|
-
|
|
2237
|
+
/** Returned by `on*` listener registration calls; invoke to unsubscribe. */
|
|
2238
|
+
export declare type Unsubscribe = () => void;
|
|
2239
|
+
|
|
2240
|
+
export declare const VERSION = "0.5.1";
|
|
1867
2241
|
|
|
1868
2242
|
/**
|
|
1869
|
-
* Real-time video API: drawable frame, dimensions, and
|
|
1870
|
-
*
|
|
2243
|
+
* Real-time video API: drawable frame, dimensions, and the source-side
|
|
2244
|
+
* properties of the connected stream. All computer-vision data and controls
|
|
2245
|
+
* (faces, hands, pose, segmentation, the analysed frame, and feature
|
|
2246
|
+
* enable/disable verbs) live on `viji.video.cv`.
|
|
1871
2247
|
*
|
|
1872
|
-
* When `isConnected` is `false`, `currentFrame` is `null`, all dimensions are
|
|
1873
|
-
* and
|
|
2248
|
+
* When `isConnected` is `false`, `currentFrame` is `null`, all dimensions are
|
|
2249
|
+
* `0`, and every member of `cv` holds its empty default.
|
|
1874
2250
|
*/
|
|
1875
2251
|
export declare interface VideoAPI {
|
|
1876
2252
|
/** `true` when a video stream is connected. When `false`, all other fields hold their default empty values. */
|
|
@@ -1889,6 +2265,78 @@ export declare interface VideoAPI {
|
|
|
1889
2265
|
* use only when you need to read individual pixel values.
|
|
1890
2266
|
*/
|
|
1891
2267
|
getFrameData: () => ImageData | null;
|
|
2268
|
+
/**
|
|
2269
|
+
* Computer-vision surface for this video stream. Holds both the data
|
|
2270
|
+
* outputs (`faces`, `hands`, `pose`, `segmentation`, `analysedFrame`,
|
|
2271
|
+
* `getAnalysedFrameData`) and the verbs that activate them
|
|
2272
|
+
* (`enable*`/`disable*`/`getActiveFeatures`/`isProcessing`).
|
|
2273
|
+
*
|
|
2274
|
+
* Multiple features can be active simultaneously, but each adds CPU/GPU
|
|
2275
|
+
* and WebGL-context cost; toggle only what you need.
|
|
2276
|
+
*
|
|
2277
|
+
* `analysedFrame` and the result fields refresh together once per host
|
|
2278
|
+
* frame as an atomic snapshot: within a single `render()` call they are
|
|
2279
|
+
* always paired.
|
|
2280
|
+
*/
|
|
2281
|
+
cv: VideoCVAPI;
|
|
2282
|
+
}
|
|
2283
|
+
|
|
2284
|
+
/**
|
|
2285
|
+
* Computer-vision surface attached to a `VideoAPI`. Combines paired CV data
|
|
2286
|
+
* outputs (the analysed frame and its detection / segmentation results) with
|
|
2287
|
+
* the verbs that activate the underlying MediaPipe pipelines.
|
|
2288
|
+
*
|
|
2289
|
+
* Available only on the main video stream (`viji.video.cv`). Additional
|
|
2290
|
+
* streams (`viji.videoStreams[i].cv`) and device videos
|
|
2291
|
+
* (`viji.devices[i].video.cv`) do not run CV; their `cv` surface exists for
|
|
2292
|
+
* API consistency but the data fields stay at their empty defaults and the
|
|
2293
|
+
* `enable*` verbs are no-ops.
|
|
2294
|
+
*/
|
|
2295
|
+
export declare interface VideoCVAPI {
|
|
2296
|
+
/**
|
|
2297
|
+
* The exact video frame that produced the current `faces` / `hands` /
|
|
2298
|
+
* `pose` / `segmentation` results: the frame MediaPipe actually analysed.
|
|
2299
|
+
*
|
|
2300
|
+
* Choose between `analysedFrame` and `viji.video.currentFrame` by asking
|
|
2301
|
+
* whether the effect reads pixels from the displayed frame at CV-derived
|
|
2302
|
+
* positions. Reach for `analysedFrame` when the answer is yes
|
|
2303
|
+
* (compositing the segmentation mask onto the body, sampling skin tone
|
|
2304
|
+
* under a face landmark, warping the face along its mesh, texture-mapped
|
|
2305
|
+
* face filters): pairing the displayed frame with the CV results is
|
|
2306
|
+
* required for those effects. Stay on `currentFrame` when the answer is
|
|
2307
|
+
* no (drawing landmark dots, particles, debug overlays, geometry driven
|
|
2308
|
+
* by CV positions): `analysedFrame` only refreshes when MediaPipe
|
|
2309
|
+
* completes an inference, which is not every frame, so reaching for it
|
|
2310
|
+
* without a reason makes the displayed video stutter or hold between
|
|
2311
|
+
* inferences.
|
|
2312
|
+
*
|
|
2313
|
+
* Pairing guarantee: within a single `render()` call, `analysedFrame`
|
|
2314
|
+
* is paired with the values of `faces`, `hands`, `pose`, `segmentation`
|
|
2315
|
+
* read in the same call.
|
|
2316
|
+
*
|
|
2317
|
+
* `null` until the first CV inference completes after a feature is
|
|
2318
|
+
* enabled, after the video disconnects, or after a CV-feature toggle
|
|
2319
|
+
* clears state. A common pattern is
|
|
2320
|
+
* `viji.video.cv.analysedFrame ?? viji.video.currentFrame` to fall back
|
|
2321
|
+
* to the live feed during the brief startup window; expect a small
|
|
2322
|
+
* visual hitch when the source switches as the first inference lands.
|
|
2323
|
+
*
|
|
2324
|
+
* Read-only: this `OffscreenCanvas` is owned by the engine. Do not
|
|
2325
|
+
* mutate it (no `getContext`-and-paint, no `transferToImageBitmap`).
|
|
2326
|
+
* Drawing it as a source (`ctx.drawImage`, `gl.texImage2D`,
|
|
2327
|
+
* `p5.image(...)`) is the only supported usage.
|
|
2328
|
+
*/
|
|
2329
|
+
analysedFrame: OffscreenCanvas | null;
|
|
2330
|
+
/**
|
|
2331
|
+
* Returns `analysedFrame` as raw RGBA `ImageData` for per-pixel CPU
|
|
2332
|
+
* analysis, or `null` when no CV result has landed yet (or the stream is
|
|
2333
|
+
* disconnected). Cached: re-extracted only when a new CV result arrives,
|
|
2334
|
+
* so multiple readers within a render share one allocation. Slow
|
|
2335
|
+
* compared to drawing `analysedFrame` directly. Use only when you need
|
|
2336
|
+
* pixel values aligned with CV landmark positions (e.g. sampling skin
|
|
2337
|
+
* colour at a face landmark).
|
|
2338
|
+
*/
|
|
2339
|
+
getAnalysedFrameData: () => ImageData | null;
|
|
1892
2340
|
/** Detected faces (empty array when face detection is off or no faces are visible). */
|
|
1893
2341
|
faces: FaceData[];
|
|
1894
2342
|
/** Detected hands (up to 2; empty array when hand tracking is off or no hands are visible). */
|
|
@@ -1898,50 +2346,42 @@ export declare interface VideoAPI {
|
|
|
1898
2346
|
/** Body segmentation mask, or `null` when segmentation is off. */
|
|
1899
2347
|
segmentation: SegmentationData | null;
|
|
1900
2348
|
/**
|
|
1901
|
-
*
|
|
1902
|
-
*
|
|
1903
|
-
* `segmentation` field on the next available frame. Multiple features can be
|
|
1904
|
-
* active simultaneously, but each adds CPU/GPU and WebGL-context cost.
|
|
2349
|
+
* Toggle face detection (bounding box, center, confidence, id).
|
|
2350
|
+
* @returns Resolves once the underlying pipeline has finished switching.
|
|
1905
2351
|
*/
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
enableBodySegmentation(enabled: boolean): Promise<void>;
|
|
1940
|
-
/** Returns the list of CV features currently enabled on this stream. */
|
|
1941
|
-
getActiveFeatures(): CVFeature[];
|
|
1942
|
-
/** Returns `true` if the CV worker is actively processing frames. */
|
|
1943
|
-
isProcessing(): boolean;
|
|
1944
|
-
};
|
|
2352
|
+
enableFaceDetection(enabled: boolean): Promise<void>;
|
|
2353
|
+
/**
|
|
2354
|
+
* Toggle 468-point face mesh + head pose (`pitch`, `yaw`, `roll`). Implies
|
|
2355
|
+
* face detection.
|
|
2356
|
+
* @returns Resolves once the pipeline has finished switching.
|
|
2357
|
+
*/
|
|
2358
|
+
enableFaceMesh(enabled: boolean): Promise<void>;
|
|
2359
|
+
/**
|
|
2360
|
+
* Toggle 7 expressions + 52 ARKit blendshape coefficients on each face.
|
|
2361
|
+
* Implies face mesh.
|
|
2362
|
+
* @returns Resolves once the pipeline has finished switching.
|
|
2363
|
+
*/
|
|
2364
|
+
enableEmotionDetection(enabled: boolean): Promise<void>;
|
|
2365
|
+
/**
|
|
2366
|
+
* Toggle 21-point hand landmarks + ML-classified gesture confidences for up
|
|
2367
|
+
* to two hands.
|
|
2368
|
+
* @returns Resolves once the pipeline has finished switching.
|
|
2369
|
+
*/
|
|
2370
|
+
enableHandTracking(enabled: boolean): Promise<void>;
|
|
2371
|
+
/**
|
|
2372
|
+
* Toggle 33-point BlazePose body landmarks (face / torso / arms / legs).
|
|
2373
|
+
* @returns Resolves once the pipeline has finished switching.
|
|
2374
|
+
*/
|
|
2375
|
+
enablePoseDetection(enabled: boolean): Promise<void>;
|
|
2376
|
+
/**
|
|
2377
|
+
* Toggle per-pixel person/background segmentation mask.
|
|
2378
|
+
* @returns Resolves once the pipeline has finished switching.
|
|
2379
|
+
*/
|
|
2380
|
+
enableBodySegmentation(enabled: boolean): Promise<void>;
|
|
2381
|
+
/** Returns the list of CV features currently enabled on this stream. */
|
|
2382
|
+
getActiveFeatures(): CVFeature[];
|
|
2383
|
+
/** Returns `true` if the CV worker is actively processing frames. */
|
|
2384
|
+
isProcessing(): boolean;
|
|
1945
2385
|
}
|
|
1946
2386
|
|
|
1947
2387
|
/**
|
|
@@ -2162,6 +2602,7 @@ export declare class VijiCore {
|
|
|
2162
2602
|
private parameterDefinedListeners;
|
|
2163
2603
|
private parameterErrorListeners;
|
|
2164
2604
|
private capabilitiesChangeListeners;
|
|
2605
|
+
private stateImportErrorListeners;
|
|
2165
2606
|
private stats;
|
|
2166
2607
|
constructor(config: VijiCoreConfig);
|
|
2167
2608
|
/**
|
|
@@ -2415,6 +2856,44 @@ export declare class VijiCore {
|
|
|
2415
2856
|
message: string;
|
|
2416
2857
|
code: string;
|
|
2417
2858
|
}) => void): void;
|
|
2859
|
+
/**
|
|
2860
|
+
* Snapshot the full audio analysis + onset state for cross-instance
|
|
2861
|
+
* transfer. Use for **same-process scene-switch** continuity (a fresh
|
|
2862
|
+
* `VijiCore` can resume detection where the previous one left off).
|
|
2863
|
+
*
|
|
2864
|
+
* For **cross-device** transfer, prefer `audio.onset.exportSessionState`
|
|
2865
|
+
* — the controller's audio analysis state doesn't apply to the host's
|
|
2866
|
+
* audio source and would corrupt detection.
|
|
2867
|
+
*
|
|
2868
|
+
* Wall-clock fields are in this instance's `performance.now()` clock space.
|
|
2869
|
+
* Receiver applies `clockOffset` on import (use `0` for same-process).
|
|
2870
|
+
*
|
|
2871
|
+
* **Staleness caveat**: the audio block is robust to scene-load gaps under
|
|
2872
|
+
* ~1 second. Longer gaps (>5s, e.g. cold cache + slow network) may cause
|
|
2873
|
+
* stale-timestamp drift in `gridLockMemoryTime` / `warmupStartTimeMs`;
|
|
2874
|
+
* callers should treat that as a degraded path and skip the audio block.
|
|
2875
|
+
*/
|
|
2876
|
+
exportFullState(): SerializedCoreState;
|
|
2877
|
+
/**
|
|
2878
|
+
* Replace the audio analysis + onset state from a serialized payload.
|
|
2879
|
+
* `clockOffset` is added to all sender-clocked fields (`0` for same-process,
|
|
2880
|
+
* NTP-derived for cross-device). Mutation is synchronous; no `onModeChange`
|
|
2881
|
+
* or `onSessionEnd` events are emitted (state replacement is not a transition).
|
|
2882
|
+
*
|
|
2883
|
+
* Validates `version`; on mismatch, fires `onStateImportError` and leaves
|
|
2884
|
+
* existing state intact.
|
|
2885
|
+
*/
|
|
2886
|
+
importFullState(state: SerializedCoreState, clockOffset: number): void;
|
|
2887
|
+
/**
|
|
2888
|
+
* Listen for state-import validation errors. Fires when `importFullState`
|
|
2889
|
+
* or `audio.onset.importSessionState` rejects a payload (version mismatch,
|
|
2890
|
+
* malformed shape, invalid field). Existing state is left intact when
|
|
2891
|
+
* an error fires.
|
|
2892
|
+
*
|
|
2893
|
+
* @returns Unsubscribe function. Call to remove the listener.
|
|
2894
|
+
*/
|
|
2895
|
+
onStateImportError(listener: (error: StateImportError) => void): Unsubscribe;
|
|
2896
|
+
private fireStateImportError;
|
|
2418
2897
|
/**
|
|
2419
2898
|
* Notify parameter change listeners
|
|
2420
2899
|
*/
|
|
@@ -2553,6 +3032,50 @@ export declare class VijiCore {
|
|
|
2553
3032
|
* Check if an instrument onset is muted
|
|
2554
3033
|
*/
|
|
2555
3034
|
isMuted: (instrument: "kick" | "snare" | "hat") => boolean;
|
|
3035
|
+
/**
|
|
3036
|
+
* Listen for instrument mode transitions (`'auto' | 'tapping' | 'pattern'`).
|
|
3037
|
+
* Fires on every transition including the first tap (`'auto' → 'tapping'`)
|
|
3038
|
+
* and pattern recognition (`'tapping' → 'pattern'`). Imported state does
|
|
3039
|
+
* NOT fire mode-change events — state replacement is not a transition.
|
|
3040
|
+
*
|
|
3041
|
+
* @returns Unsubscribe function. Call to remove the listener.
|
|
3042
|
+
*/
|
|
3043
|
+
onModeChange: (listener: (ev: OnsetModeChangeEvent) => void) => Unsubscribe;
|
|
3044
|
+
/**
|
|
3045
|
+
* Listen for natural session-end events. Fires when:
|
|
3046
|
+
* - 500ms elapse since last tap with instrument in `'pattern'` mode
|
|
3047
|
+
* (outcome `'pattern'`), or
|
|
3048
|
+
* - 5s elapse in `'tapping'` mode without a recognized pattern
|
|
3049
|
+
* (outcome `'cleared'`; instrument transitions to `'auto'`).
|
|
3050
|
+
*
|
|
3051
|
+
* Explicit `clear()` calls do NOT fire this event (caller-initiated;
|
|
3052
|
+
* caller already knows). Imported state does NOT fire either.
|
|
3053
|
+
*
|
|
3054
|
+
* @returns Unsubscribe function. Call to remove the listener.
|
|
3055
|
+
*/
|
|
3056
|
+
onSessionEnd: (listener: (ev: OnsetSessionEndEvent) => void) => Unsubscribe;
|
|
3057
|
+
/**
|
|
3058
|
+
* Snapshot per-instrument onset state for cross-instance transfer.
|
|
3059
|
+
* Cross-device-safe (no audio analysis state included). Pair with
|
|
3060
|
+
* `importSessionState` on a receiver. Wall-clock fields are in this
|
|
3061
|
+
* instance's `performance.now()` clock space.
|
|
3062
|
+
*/
|
|
3063
|
+
exportSessionState: () => SerializedOnsetState;
|
|
3064
|
+
/**
|
|
3065
|
+
* Replace per-instrument onset state from a serialized payload.
|
|
3066
|
+
* `clockOffset` is added to all sender-clocked fields during import
|
|
3067
|
+
* (use `0` for same-process transfer; NTP-derived offset for cross-device).
|
|
3068
|
+
* Mutation is synchronous; no events are emitted.
|
|
3069
|
+
*
|
|
3070
|
+
* Patterns are rebased forward by whole pattern cycles to eliminate
|
|
3071
|
+
* the catch-up burst that would otherwise occur on stale payloads
|
|
3072
|
+
* (phase-preserving — events still land on the original beat positions
|
|
3073
|
+
* modulo pattern length).
|
|
3074
|
+
*
|
|
3075
|
+
* Validates `version`; on mismatch, fires `onStateImportError` and
|
|
3076
|
+
* leaves existing state intact.
|
|
3077
|
+
*/
|
|
3078
|
+
importSessionState: (state: SerializedOnsetState, clockOffset: number) => void;
|
|
2556
3079
|
};
|
|
2557
3080
|
/**
|
|
2558
3081
|
* Advanced audio controls
|