@viji-dev/core 0.3.24 → 0.3.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/artist-dts-p5.js +1 -1
- package/dist/artist-dts.js +1 -1
- package/dist/artist-global.d.ts +2 -42
- package/dist/artist-js-ambient.d.ts +2 -35
- package/dist/artist-jsdoc.d.ts +406 -13
- package/dist/assets/cv-tasks.worker.js +108 -32
- package/dist/assets/{viji.worker-Zg128woJ.js → viji.worker-PAf0oIec.js} +240 -144
- package/dist/assets/viji.worker-PAf0oIec.js.map +1 -0
- package/dist/docs-api.d.ts +7 -0
- package/dist/docs-api.js +1893 -267
- package/dist/{essentia-wasm.web-Dq7KVAx1.js → essentia-wasm.web-1yXYrWJ5.js} +2 -2
- package/dist/{essentia-wasm.web-Dq7KVAx1.js.map → essentia-wasm.web-1yXYrWJ5.js.map} +1 -1
- package/dist/{index-C6fMmKQj.js → index-B8BYfP9z.js} +3 -3
- package/dist/index-B8BYfP9z.js.map +1 -0
- package/dist/index.d.ts +2 -42
- package/dist/index.js +1 -1
- package/dist/shader-uniforms.js +126 -11
- package/package.json +1 -1
- package/dist/assets/viji.worker-Zg128woJ.js.map +0 -1
- package/dist/index-C6fMmKQj.js.map +0 -1
|
@@ -1185,21 +1185,21 @@ class CVSystem {
|
|
|
1185
1185
|
setTimeout(() => reject(new Error("CV processing timeout")), 500);
|
|
1186
1186
|
});
|
|
1187
1187
|
const results = await Promise.race([processPromise, timeoutPromise]);
|
|
1188
|
-
if (
|
|
1188
|
+
if ("faces" in results && (this.activeFeatures.has("faceDetection") || this.activeFeatures.has("faceMesh") || this.activeFeatures.has("emotionDetection"))) {
|
|
1189
1189
|
this.results.faces = results.faces;
|
|
1190
1190
|
this.debugLog(`📥 Received ${results.faces.length} face results`);
|
|
1191
1191
|
}
|
|
1192
|
-
if (
|
|
1192
|
+
if ("hands" in results && this.activeFeatures.has("handTracking")) {
|
|
1193
1193
|
this.results.hands = results.hands;
|
|
1194
1194
|
this.debugLog(`📥 Received ${results.hands.length} hand results`);
|
|
1195
1195
|
}
|
|
1196
|
-
if (
|
|
1196
|
+
if ("pose" in results && this.activeFeatures.has("poseDetection")) {
|
|
1197
1197
|
this.results.pose = results.pose;
|
|
1198
|
-
this.debugLog(`📥 Received pose results
|
|
1198
|
+
this.debugLog(`📥 Received pose results: ${results.pose ? results.pose.landmarks.length + " landmarks" : "none"}`);
|
|
1199
1199
|
}
|
|
1200
|
-
if (
|
|
1200
|
+
if ("segmentation" in results && this.activeFeatures.has("bodySegmentation")) {
|
|
1201
1201
|
this.results.segmentation = results.segmentation;
|
|
1202
|
-
this.debugLog(`📥 Received segmentation
|
|
1202
|
+
this.debugLog(`📥 Received segmentation: ${results.segmentation ? results.segmentation.width + "x" + results.segmentation.height : "none"}`);
|
|
1203
1203
|
}
|
|
1204
1204
|
const processingTime = performance.now() - this.processingStartTime;
|
|
1205
1205
|
this.processingTimes.push(processingTime);
|
|
@@ -2642,11 +2642,11 @@ uniform float u_audioPeak; // Peak amplitude (0-1)
|
|
|
2642
2642
|
uniform float u_audioVolumeSmoothed; // Smoothed volume for animation (200ms decay)
|
|
2643
2643
|
|
|
2644
2644
|
// Audio - Frequency Bands (instant)
|
|
2645
|
-
uniform float u_audioLow; // Low 20-
|
|
2646
|
-
uniform float u_audioLowMid; // Low-mid
|
|
2647
|
-
uniform float u_audioMid; // Mid 400-
|
|
2648
|
-
uniform float u_audioHighMid; // High-mid
|
|
2649
|
-
uniform float u_audioHigh; // High
|
|
2645
|
+
uniform float u_audioLow; // Low 20-120 Hz (0-1)
|
|
2646
|
+
uniform float u_audioLowMid; // Low-mid 120-400 Hz (0-1)
|
|
2647
|
+
uniform float u_audioMid; // Mid 400-1600 Hz (0-1)
|
|
2648
|
+
uniform float u_audioHighMid; // High-mid 1600-6000 Hz (0-1)
|
|
2649
|
+
uniform float u_audioHigh; // High 6000-16000 Hz (0-1)
|
|
2650
2650
|
|
|
2651
2651
|
// Audio - Frequency Bands (smoothed, 150ms decay)
|
|
2652
2652
|
uniform float u_audioLowSmoothed; // Smoothed low band (0-1)
|
|
@@ -2683,7 +2683,7 @@ uniform float u_audioBrightness; // Spectral brightness / centroid (0-1)
|
|
|
2683
2683
|
uniform float u_audioFlatness; // Spectral flatness / noisiness (0-1)
|
|
2684
2684
|
|
|
2685
2685
|
// Audio - Textures
|
|
2686
|
-
uniform sampler2D u_audioFFT; // FFT frequency spectrum (
|
|
2686
|
+
uniform sampler2D u_audioFFT; // FFT frequency spectrum (1024 bins, 0-255)
|
|
2687
2687
|
uniform sampler2D u_audioWaveform; // Time-domain waveform (fftSize samples, -1 to 1)
|
|
2688
2688
|
|
|
2689
2689
|
// Video
|
|
@@ -2823,18 +2823,41 @@ uniform float u_face0TongueOut;
|
|
|
2823
2823
|
uniform int u_handCount; // Number of detected hands (0-2)
|
|
2824
2824
|
uniform vec3 u_leftHandPalm; // Left hand palm position (x, y, z)
|
|
2825
2825
|
uniform vec3 u_rightHandPalm; // Right hand palm position (x, y, z)
|
|
2826
|
+
uniform float u_leftHandConfidence; // Left hand detection confidence (0-1)
|
|
2827
|
+
uniform float u_rightHandConfidence; // Right hand detection confidence (0-1)
|
|
2828
|
+
uniform vec4 u_leftHandBounds; // Left hand bounding box (x, y, width, height) normalized 0-1
|
|
2829
|
+
uniform vec4 u_rightHandBounds; // Right hand bounding box (x, y, width, height) normalized 0-1
|
|
2826
2830
|
uniform float u_leftHandFist; // Left hand fist gesture confidence (0-1)
|
|
2827
2831
|
uniform float u_leftHandOpen; // Left hand open palm gesture confidence (0-1)
|
|
2832
|
+
uniform float u_leftHandPeace; // Left hand peace/victory gesture confidence (0-1)
|
|
2833
|
+
uniform float u_leftHandThumbsUp; // Left hand thumbs up gesture confidence (0-1)
|
|
2834
|
+
uniform float u_leftHandThumbsDown; // Left hand thumbs down gesture confidence (0-1)
|
|
2835
|
+
uniform float u_leftHandPointing; // Left hand pointing gesture confidence (0-1)
|
|
2836
|
+
uniform float u_leftHandILoveYou; // Left hand I-love-you gesture confidence (0-1)
|
|
2828
2837
|
uniform float u_rightHandFist; // Right hand fist gesture confidence (0-1)
|
|
2829
2838
|
uniform float u_rightHandOpen; // Right hand open palm gesture confidence (0-1)
|
|
2839
|
+
uniform float u_rightHandPeace; // Right hand peace/victory gesture confidence (0-1)
|
|
2840
|
+
uniform float u_rightHandThumbsUp; // Right hand thumbs up gesture confidence (0-1)
|
|
2841
|
+
uniform float u_rightHandThumbsDown; // Right hand thumbs down gesture confidence (0-1)
|
|
2842
|
+
uniform float u_rightHandPointing; // Right hand pointing gesture confidence (0-1)
|
|
2843
|
+
uniform float u_rightHandILoveYou; // Right hand I-love-you gesture confidence (0-1)
|
|
2830
2844
|
|
|
2831
2845
|
// CV - Pose Detection
|
|
2832
2846
|
uniform bool u_poseDetected; // True if a pose is currently detected
|
|
2833
|
-
uniform
|
|
2834
|
-
uniform vec2
|
|
2835
|
-
uniform vec2
|
|
2836
|
-
uniform vec2
|
|
2837
|
-
uniform vec2
|
|
2847
|
+
uniform float u_poseConfidence; // Pose detection confidence (0-1)
|
|
2848
|
+
uniform vec2 u_nosePosition; // Nose landmark position (normalized 0-1)
|
|
2849
|
+
uniform vec2 u_leftShoulderPosition; // Left shoulder position (normalized 0-1)
|
|
2850
|
+
uniform vec2 u_rightShoulderPosition; // Right shoulder position (normalized 0-1)
|
|
2851
|
+
uniform vec2 u_leftElbowPosition; // Left elbow position (normalized 0-1)
|
|
2852
|
+
uniform vec2 u_rightElbowPosition; // Right elbow position (normalized 0-1)
|
|
2853
|
+
uniform vec2 u_leftWristPosition; // Left wrist position (normalized 0-1)
|
|
2854
|
+
uniform vec2 u_rightWristPosition; // Right wrist position (normalized 0-1)
|
|
2855
|
+
uniform vec2 u_leftHipPosition; // Left hip position (normalized 0-1)
|
|
2856
|
+
uniform vec2 u_rightHipPosition; // Right hip position (normalized 0-1)
|
|
2857
|
+
uniform vec2 u_leftKneePosition; // Left knee position (normalized 0-1)
|
|
2858
|
+
uniform vec2 u_rightKneePosition; // Right knee position (normalized 0-1)
|
|
2859
|
+
uniform vec2 u_leftAnklePosition; // Left ankle position (normalized 0-1)
|
|
2860
|
+
uniform vec2 u_rightAnklePosition; // Right ankle position (normalized 0-1)
|
|
2838
2861
|
|
|
2839
2862
|
// CV - Segmentation
|
|
2840
2863
|
uniform sampler2D u_segmentationMask; // Body segmentation mask texture (0=background, 1=person)
|
|
@@ -3268,40 +3291,88 @@ ${error}`);
|
|
|
3268
3291
|
const leftHand = hands.find((h) => h.handedness === "left");
|
|
3269
3292
|
const rightHand = hands.find((h) => h.handedness === "right");
|
|
3270
3293
|
if (leftHand) {
|
|
3271
|
-
|
|
3272
|
-
|
|
3273
|
-
|
|
3294
|
+
const palm = leftHand.palm || leftHand.landmarks?.[9];
|
|
3295
|
+
const g = leftHand.gestures || {};
|
|
3296
|
+
const b = leftHand.bounds;
|
|
3297
|
+
this.setUniform("u_leftHandPalm", "vec3", [palm?.x || 0, palm?.y || 0, palm?.z || 0]);
|
|
3298
|
+
this.setUniform("u_leftHandConfidence", "float", leftHand.confidence || 0);
|
|
3299
|
+
this.setUniform("u_leftHandBounds", "vec4", [b?.x || 0, b?.y || 0, b?.width || 0, b?.height || 0]);
|
|
3300
|
+
this.setUniform("u_leftHandFist", "float", g.fist || 0);
|
|
3301
|
+
this.setUniform("u_leftHandOpen", "float", g.openPalm || 0);
|
|
3302
|
+
this.setUniform("u_leftHandPeace", "float", g.peace || 0);
|
|
3303
|
+
this.setUniform("u_leftHandThumbsUp", "float", g.thumbsUp || 0);
|
|
3304
|
+
this.setUniform("u_leftHandThumbsDown", "float", g.thumbsDown || 0);
|
|
3305
|
+
this.setUniform("u_leftHandPointing", "float", g.pointing || 0);
|
|
3306
|
+
this.setUniform("u_leftHandILoveYou", "float", g.iLoveYou || 0);
|
|
3274
3307
|
} else {
|
|
3275
3308
|
this.setUniform("u_leftHandPalm", "vec3", [0, 0, 0]);
|
|
3309
|
+
this.setUniform("u_leftHandConfidence", "float", 0);
|
|
3310
|
+
this.setUniform("u_leftHandBounds", "vec4", [0, 0, 0, 0]);
|
|
3276
3311
|
this.setUniform("u_leftHandFist", "float", 0);
|
|
3277
3312
|
this.setUniform("u_leftHandOpen", "float", 0);
|
|
3313
|
+
this.setUniform("u_leftHandPeace", "float", 0);
|
|
3314
|
+
this.setUniform("u_leftHandThumbsUp", "float", 0);
|
|
3315
|
+
this.setUniform("u_leftHandThumbsDown", "float", 0);
|
|
3316
|
+
this.setUniform("u_leftHandPointing", "float", 0);
|
|
3317
|
+
this.setUniform("u_leftHandILoveYou", "float", 0);
|
|
3278
3318
|
}
|
|
3279
3319
|
if (rightHand) {
|
|
3280
|
-
|
|
3281
|
-
|
|
3282
|
-
|
|
3320
|
+
const palm = rightHand.palm || rightHand.landmarks?.[9];
|
|
3321
|
+
const g = rightHand.gestures || {};
|
|
3322
|
+
const b = rightHand.bounds;
|
|
3323
|
+
this.setUniform("u_rightHandPalm", "vec3", [palm?.x || 0, palm?.y || 0, palm?.z || 0]);
|
|
3324
|
+
this.setUniform("u_rightHandConfidence", "float", rightHand.confidence || 0);
|
|
3325
|
+
this.setUniform("u_rightHandBounds", "vec4", [b?.x || 0, b?.y || 0, b?.width || 0, b?.height || 0]);
|
|
3326
|
+
this.setUniform("u_rightHandFist", "float", g.fist || 0);
|
|
3327
|
+
this.setUniform("u_rightHandOpen", "float", g.openPalm || 0);
|
|
3328
|
+
this.setUniform("u_rightHandPeace", "float", g.peace || 0);
|
|
3329
|
+
this.setUniform("u_rightHandThumbsUp", "float", g.thumbsUp || 0);
|
|
3330
|
+
this.setUniform("u_rightHandThumbsDown", "float", g.thumbsDown || 0);
|
|
3331
|
+
this.setUniform("u_rightHandPointing", "float", g.pointing || 0);
|
|
3332
|
+
this.setUniform("u_rightHandILoveYou", "float", g.iLoveYou || 0);
|
|
3283
3333
|
} else {
|
|
3284
3334
|
this.setUniform("u_rightHandPalm", "vec3", [0, 0, 0]);
|
|
3335
|
+
this.setUniform("u_rightHandConfidence", "float", 0);
|
|
3336
|
+
this.setUniform("u_rightHandBounds", "vec4", [0, 0, 0, 0]);
|
|
3285
3337
|
this.setUniform("u_rightHandFist", "float", 0);
|
|
3286
3338
|
this.setUniform("u_rightHandOpen", "float", 0);
|
|
3339
|
+
this.setUniform("u_rightHandPeace", "float", 0);
|
|
3340
|
+
this.setUniform("u_rightHandThumbsUp", "float", 0);
|
|
3341
|
+
this.setUniform("u_rightHandThumbsDown", "float", 0);
|
|
3342
|
+
this.setUniform("u_rightHandPointing", "float", 0);
|
|
3343
|
+
this.setUniform("u_rightHandILoveYou", "float", 0);
|
|
3287
3344
|
}
|
|
3288
3345
|
const pose = video.pose;
|
|
3289
3346
|
this.setUniform("u_poseDetected", "bool", pose !== null);
|
|
3290
|
-
if (pose) {
|
|
3291
|
-
const
|
|
3292
|
-
|
|
3293
|
-
|
|
3294
|
-
|
|
3295
|
-
|
|
3296
|
-
this.setUniform("
|
|
3297
|
-
this.setUniform("
|
|
3298
|
-
this.setUniform("
|
|
3299
|
-
this.setUniform("
|
|
3300
|
-
this.setUniform("
|
|
3347
|
+
if (pose && pose.landmarks) {
|
|
3348
|
+
const lm = pose.landmarks;
|
|
3349
|
+
this.setUniform("u_poseConfidence", "float", pose.confidence || 0);
|
|
3350
|
+
this.setUniform("u_nosePosition", "vec2", [lm[0]?.x || 0, lm[0]?.y || 0]);
|
|
3351
|
+
this.setUniform("u_leftShoulderPosition", "vec2", [lm[11]?.x || 0, lm[11]?.y || 0]);
|
|
3352
|
+
this.setUniform("u_rightShoulderPosition", "vec2", [lm[12]?.x || 0, lm[12]?.y || 0]);
|
|
3353
|
+
this.setUniform("u_leftElbowPosition", "vec2", [lm[13]?.x || 0, lm[13]?.y || 0]);
|
|
3354
|
+
this.setUniform("u_rightElbowPosition", "vec2", [lm[14]?.x || 0, lm[14]?.y || 0]);
|
|
3355
|
+
this.setUniform("u_leftWristPosition", "vec2", [lm[15]?.x || 0, lm[15]?.y || 0]);
|
|
3356
|
+
this.setUniform("u_rightWristPosition", "vec2", [lm[16]?.x || 0, lm[16]?.y || 0]);
|
|
3357
|
+
this.setUniform("u_leftHipPosition", "vec2", [lm[23]?.x || 0, lm[23]?.y || 0]);
|
|
3358
|
+
this.setUniform("u_rightHipPosition", "vec2", [lm[24]?.x || 0, lm[24]?.y || 0]);
|
|
3359
|
+
this.setUniform("u_leftKneePosition", "vec2", [lm[25]?.x || 0, lm[25]?.y || 0]);
|
|
3360
|
+
this.setUniform("u_rightKneePosition", "vec2", [lm[26]?.x || 0, lm[26]?.y || 0]);
|
|
3361
|
+
this.setUniform("u_leftAnklePosition", "vec2", [lm[27]?.x || 0, lm[27]?.y || 0]);
|
|
3362
|
+
this.setUniform("u_rightAnklePosition", "vec2", [lm[28]?.x || 0, lm[28]?.y || 0]);
|
|
3301
3363
|
} else {
|
|
3364
|
+
this.setUniform("u_poseConfidence", "float", 0);
|
|
3302
3365
|
this.setUniform("u_nosePosition", "vec2", [0, 0]);
|
|
3366
|
+
this.setUniform("u_leftShoulderPosition", "vec2", [0, 0]);
|
|
3367
|
+
this.setUniform("u_rightShoulderPosition", "vec2", [0, 0]);
|
|
3368
|
+
this.setUniform("u_leftElbowPosition", "vec2", [0, 0]);
|
|
3369
|
+
this.setUniform("u_rightElbowPosition", "vec2", [0, 0]);
|
|
3303
3370
|
this.setUniform("u_leftWristPosition", "vec2", [0, 0]);
|
|
3304
3371
|
this.setUniform("u_rightWristPosition", "vec2", [0, 0]);
|
|
3372
|
+
this.setUniform("u_leftHipPosition", "vec2", [0, 0]);
|
|
3373
|
+
this.setUniform("u_rightHipPosition", "vec2", [0, 0]);
|
|
3374
|
+
this.setUniform("u_leftKneePosition", "vec2", [0, 0]);
|
|
3375
|
+
this.setUniform("u_rightKneePosition", "vec2", [0, 0]);
|
|
3305
3376
|
this.setUniform("u_leftAnklePosition", "vec2", [0, 0]);
|
|
3306
3377
|
this.setUniform("u_rightAnklePosition", "vec2", [0, 0]);
|
|
3307
3378
|
}
|
|
@@ -3943,43 +4014,9 @@ class VijiWorkerRuntime {
|
|
|
3943
4014
|
autoCaptureEnabled = false;
|
|
3944
4015
|
autoCaptureFormat = { flipY: true };
|
|
3945
4016
|
// Default: flip for WebGL compatibility
|
|
3946
|
-
// Audio
|
|
3947
|
-
|
|
3948
|
-
|
|
3949
|
-
volume: { current: 0, peak: 0, smoothed: 0 },
|
|
3950
|
-
bands: {
|
|
3951
|
-
low: 0,
|
|
3952
|
-
lowMid: 0,
|
|
3953
|
-
mid: 0,
|
|
3954
|
-
highMid: 0,
|
|
3955
|
-
high: 0,
|
|
3956
|
-
lowSmoothed: 0,
|
|
3957
|
-
lowMidSmoothed: 0,
|
|
3958
|
-
midSmoothed: 0,
|
|
3959
|
-
highMidSmoothed: 0,
|
|
3960
|
-
highSmoothed: 0
|
|
3961
|
-
},
|
|
3962
|
-
beat: {
|
|
3963
|
-
kick: 0,
|
|
3964
|
-
snare: 0,
|
|
3965
|
-
hat: 0,
|
|
3966
|
-
any: 0,
|
|
3967
|
-
kickSmoothed: 0,
|
|
3968
|
-
snareSmoothed: 0,
|
|
3969
|
-
hatSmoothed: 0,
|
|
3970
|
-
anySmoothed: 0,
|
|
3971
|
-
triggers: { any: false, kick: false, snare: false, hat: false },
|
|
3972
|
-
bpm: 120,
|
|
3973
|
-
confidence: 0,
|
|
3974
|
-
isLocked: false
|
|
3975
|
-
},
|
|
3976
|
-
spectral: {
|
|
3977
|
-
brightness: 0,
|
|
3978
|
-
flatness: 0
|
|
3979
|
-
},
|
|
3980
|
-
frequencyData: new Uint8Array(0),
|
|
3981
|
-
waveformData: new Float32Array(0)
|
|
3982
|
-
};
|
|
4017
|
+
// Audio raw data buffers (referenced by closures in viji.audio)
|
|
4018
|
+
audioFrequencyData = new Uint8Array(0);
|
|
4019
|
+
audioWaveformData = new Float32Array(0);
|
|
3983
4020
|
// Device sensor state (internal device + external devices)
|
|
3984
4021
|
deviceState = {
|
|
3985
4022
|
device: {
|
|
@@ -4004,8 +4041,41 @@ class VijiWorkerRuntime {
|
|
|
4004
4041
|
deltaTime: 0,
|
|
4005
4042
|
frameCount: 0,
|
|
4006
4043
|
fps: 60,
|
|
4007
|
-
// Audio API (Phase 5)
|
|
4008
|
-
audio: {
|
|
4044
|
+
// Audio API (Phase 5) — stable reference, mutated in place
|
|
4045
|
+
audio: {
|
|
4046
|
+
isConnected: false,
|
|
4047
|
+
volume: { current: 0, peak: 0, smoothed: 0 },
|
|
4048
|
+
bands: {
|
|
4049
|
+
low: 0,
|
|
4050
|
+
lowMid: 0,
|
|
4051
|
+
mid: 0,
|
|
4052
|
+
highMid: 0,
|
|
4053
|
+
high: 0,
|
|
4054
|
+
lowSmoothed: 0,
|
|
4055
|
+
lowMidSmoothed: 0,
|
|
4056
|
+
midSmoothed: 0,
|
|
4057
|
+
highMidSmoothed: 0,
|
|
4058
|
+
highSmoothed: 0
|
|
4059
|
+
},
|
|
4060
|
+
beat: {
|
|
4061
|
+
kick: 0,
|
|
4062
|
+
snare: 0,
|
|
4063
|
+
hat: 0,
|
|
4064
|
+
any: 0,
|
|
4065
|
+
kickSmoothed: 0,
|
|
4066
|
+
snareSmoothed: 0,
|
|
4067
|
+
hatSmoothed: 0,
|
|
4068
|
+
anySmoothed: 0,
|
|
4069
|
+
triggers: { any: false, kick: false, snare: false, hat: false },
|
|
4070
|
+
events: [],
|
|
4071
|
+
bpm: 120,
|
|
4072
|
+
confidence: 0,
|
|
4073
|
+
isLocked: false
|
|
4074
|
+
},
|
|
4075
|
+
spectral: { brightness: 0, flatness: 0 },
|
|
4076
|
+
getFrequencyData: () => new Uint8Array(0),
|
|
4077
|
+
getWaveform: () => new Float32Array(0)
|
|
4078
|
+
},
|
|
4009
4079
|
// Main video stream (index 0, CV enabled)
|
|
4010
4080
|
video: {
|
|
4011
4081
|
isConnected: false,
|
|
@@ -4117,11 +4187,8 @@ class VijiWorkerRuntime {
|
|
|
4117
4187
|
});
|
|
4118
4188
|
this.interactionSystem = new InteractionSystem();
|
|
4119
4189
|
Object.assign(this.viji, this.interactionSystem.getInteractionAPIs());
|
|
4120
|
-
this.viji.audio =
|
|
4121
|
-
|
|
4122
|
-
getFrequencyData: () => this.audioState.frequencyData,
|
|
4123
|
-
getWaveform: () => this.audioState.waveformData
|
|
4124
|
-
};
|
|
4190
|
+
this.viji.audio.getFrequencyData = () => this.audioFrequencyData;
|
|
4191
|
+
this.viji.audio.getWaveform = () => this.audioWaveformData;
|
|
4125
4192
|
this.setupMessageHandling();
|
|
4126
4193
|
}
|
|
4127
4194
|
/**
|
|
@@ -4224,47 +4291,7 @@ class VijiWorkerRuntime {
|
|
|
4224
4291
|
resetParameterState() {
|
|
4225
4292
|
this.parameterSystem.resetParameterState();
|
|
4226
4293
|
this.interactionSystem.resetInteractionState();
|
|
4227
|
-
this.
|
|
4228
|
-
isConnected: false,
|
|
4229
|
-
volume: { current: 0, peak: 0, smoothed: 0 },
|
|
4230
|
-
bands: {
|
|
4231
|
-
low: 0,
|
|
4232
|
-
lowMid: 0,
|
|
4233
|
-
mid: 0,
|
|
4234
|
-
highMid: 0,
|
|
4235
|
-
high: 0,
|
|
4236
|
-
lowSmoothed: 0,
|
|
4237
|
-
lowMidSmoothed: 0,
|
|
4238
|
-
midSmoothed: 0,
|
|
4239
|
-
highMidSmoothed: 0,
|
|
4240
|
-
highSmoothed: 0
|
|
4241
|
-
},
|
|
4242
|
-
beat: {
|
|
4243
|
-
kick: 0,
|
|
4244
|
-
snare: 0,
|
|
4245
|
-
hat: 0,
|
|
4246
|
-
any: 0,
|
|
4247
|
-
kickSmoothed: 0,
|
|
4248
|
-
snareSmoothed: 0,
|
|
4249
|
-
hatSmoothed: 0,
|
|
4250
|
-
anySmoothed: 0,
|
|
4251
|
-
triggers: { any: false, kick: false, snare: false, hat: false },
|
|
4252
|
-
bpm: 120,
|
|
4253
|
-
confidence: 0,
|
|
4254
|
-
isLocked: false
|
|
4255
|
-
},
|
|
4256
|
-
spectral: {
|
|
4257
|
-
brightness: 0,
|
|
4258
|
-
flatness: 0
|
|
4259
|
-
},
|
|
4260
|
-
frequencyData: new Uint8Array(0),
|
|
4261
|
-
waveformData: new Float32Array(0)
|
|
4262
|
-
};
|
|
4263
|
-
this.viji.audio = {
|
|
4264
|
-
...this.audioState,
|
|
4265
|
-
getFrequencyData: () => this.audioState.frequencyData,
|
|
4266
|
-
getWaveform: () => this.audioState.waveformData
|
|
4267
|
-
};
|
|
4294
|
+
this.resetAudioState();
|
|
4268
4295
|
this.videoSystems.forEach((vs) => vs?.resetVideoState());
|
|
4269
4296
|
if (this.videoSystems[0]) {
|
|
4270
4297
|
Object.assign(this.viji.video, this.videoSystems[0].getVideoAPI());
|
|
@@ -4494,30 +4521,98 @@ class VijiWorkerRuntime {
|
|
|
4494
4521
|
this.debugLog("Stream update:", message.data);
|
|
4495
4522
|
}
|
|
4496
4523
|
handleAudioAnalysisUpdate(message) {
|
|
4497
|
-
const
|
|
4498
|
-
const
|
|
4499
|
-
|
|
4500
|
-
|
|
4501
|
-
|
|
4502
|
-
|
|
4503
|
-
|
|
4504
|
-
|
|
4505
|
-
|
|
4506
|
-
|
|
4507
|
-
|
|
4508
|
-
|
|
4509
|
-
|
|
4510
|
-
|
|
4511
|
-
|
|
4512
|
-
|
|
4513
|
-
|
|
4514
|
-
|
|
4515
|
-
|
|
4516
|
-
|
|
4517
|
-
|
|
4518
|
-
|
|
4519
|
-
|
|
4520
|
-
|
|
4524
|
+
const d = message.data;
|
|
4525
|
+
const audio = this.viji.audio;
|
|
4526
|
+
audio.isConnected = d.isConnected;
|
|
4527
|
+
audio.volume.current = d.volume.current;
|
|
4528
|
+
audio.volume.peak = d.volume.peak;
|
|
4529
|
+
audio.volume.smoothed = d.volume.smoothed;
|
|
4530
|
+
audio.bands.low = d.bands.low;
|
|
4531
|
+
audio.bands.lowMid = d.bands.lowMid;
|
|
4532
|
+
audio.bands.mid = d.bands.mid;
|
|
4533
|
+
audio.bands.highMid = d.bands.highMid;
|
|
4534
|
+
audio.bands.high = d.bands.high;
|
|
4535
|
+
audio.bands.lowSmoothed = d.bands.lowSmoothed;
|
|
4536
|
+
audio.bands.lowMidSmoothed = d.bands.lowMidSmoothed;
|
|
4537
|
+
audio.bands.midSmoothed = d.bands.midSmoothed;
|
|
4538
|
+
audio.bands.highMidSmoothed = d.bands.highMidSmoothed;
|
|
4539
|
+
audio.bands.highSmoothed = d.bands.highSmoothed;
|
|
4540
|
+
audio.beat.kick = d.beat.kick;
|
|
4541
|
+
audio.beat.snare = d.beat.snare;
|
|
4542
|
+
audio.beat.hat = d.beat.hat;
|
|
4543
|
+
audio.beat.any = d.beat.any;
|
|
4544
|
+
audio.beat.kickSmoothed = d.beat.kickSmoothed;
|
|
4545
|
+
audio.beat.snareSmoothed = d.beat.snareSmoothed;
|
|
4546
|
+
audio.beat.hatSmoothed = d.beat.hatSmoothed;
|
|
4547
|
+
audio.beat.anySmoothed = d.beat.anySmoothed;
|
|
4548
|
+
const events = d.beat.events || [];
|
|
4549
|
+
if (events.length > 0) {
|
|
4550
|
+
audio.beat.triggers.kick = audio.beat.triggers.kick || events.some((e) => e.type === "kick");
|
|
4551
|
+
audio.beat.triggers.snare = audio.beat.triggers.snare || events.some((e) => e.type === "snare");
|
|
4552
|
+
audio.beat.triggers.hat = audio.beat.triggers.hat || events.some((e) => e.type === "hat");
|
|
4553
|
+
audio.beat.triggers.any = true;
|
|
4554
|
+
audio.beat.events.push(...events);
|
|
4555
|
+
}
|
|
4556
|
+
audio.beat.bpm = d.beat.bpm;
|
|
4557
|
+
audio.beat.confidence = d.beat.confidence;
|
|
4558
|
+
audio.beat.isLocked = d.beat.isLocked;
|
|
4559
|
+
audio.spectral.brightness = d.spectral.brightness;
|
|
4560
|
+
audio.spectral.flatness = d.spectral.flatness;
|
|
4561
|
+
this.audioFrequencyData = new Uint8Array(d.frequencyData);
|
|
4562
|
+
this.audioWaveformData = d.waveformData ? new Float32Array(d.waveformData) : new Float32Array(0);
|
|
4563
|
+
}
|
|
4564
|
+
/**
|
|
4565
|
+
* Reset frame-scoped audio events (triggers + events array).
|
|
4566
|
+
* Called after each render, mirroring interactionSystem.frameStart().
|
|
4567
|
+
*/
|
|
4568
|
+
resetAudioFrameEvents() {
|
|
4569
|
+
const t = this.viji.audio.beat.triggers;
|
|
4570
|
+
t.kick = false;
|
|
4571
|
+
t.snare = false;
|
|
4572
|
+
t.hat = false;
|
|
4573
|
+
t.any = false;
|
|
4574
|
+
this.viji.audio.beat.events.length = 0;
|
|
4575
|
+
}
|
|
4576
|
+
/**
|
|
4577
|
+
* Full audio state reset (scene reload / disconnect).
|
|
4578
|
+
* Zeroes everything without replacing the stable viji.audio reference.
|
|
4579
|
+
*/
|
|
4580
|
+
resetAudioState() {
|
|
4581
|
+
const audio = this.viji.audio;
|
|
4582
|
+
audio.isConnected = false;
|
|
4583
|
+
audio.volume.current = 0;
|
|
4584
|
+
audio.volume.peak = 0;
|
|
4585
|
+
audio.volume.smoothed = 0;
|
|
4586
|
+
audio.bands.low = 0;
|
|
4587
|
+
audio.bands.lowMid = 0;
|
|
4588
|
+
audio.bands.mid = 0;
|
|
4589
|
+
audio.bands.highMid = 0;
|
|
4590
|
+
audio.bands.high = 0;
|
|
4591
|
+
audio.bands.lowSmoothed = 0;
|
|
4592
|
+
audio.bands.lowMidSmoothed = 0;
|
|
4593
|
+
audio.bands.midSmoothed = 0;
|
|
4594
|
+
audio.bands.highMidSmoothed = 0;
|
|
4595
|
+
audio.bands.highSmoothed = 0;
|
|
4596
|
+
audio.beat.kick = 0;
|
|
4597
|
+
audio.beat.snare = 0;
|
|
4598
|
+
audio.beat.hat = 0;
|
|
4599
|
+
audio.beat.any = 0;
|
|
4600
|
+
audio.beat.kickSmoothed = 0;
|
|
4601
|
+
audio.beat.snareSmoothed = 0;
|
|
4602
|
+
audio.beat.hatSmoothed = 0;
|
|
4603
|
+
audio.beat.anySmoothed = 0;
|
|
4604
|
+
audio.beat.triggers.kick = false;
|
|
4605
|
+
audio.beat.triggers.snare = false;
|
|
4606
|
+
audio.beat.triggers.hat = false;
|
|
4607
|
+
audio.beat.triggers.any = false;
|
|
4608
|
+
audio.beat.events.length = 0;
|
|
4609
|
+
audio.beat.bpm = 120;
|
|
4610
|
+
audio.beat.confidence = 0;
|
|
4611
|
+
audio.beat.isLocked = false;
|
|
4612
|
+
audio.spectral.brightness = 0;
|
|
4613
|
+
audio.spectral.flatness = 0;
|
|
4614
|
+
this.audioFrequencyData = new Uint8Array(0);
|
|
4615
|
+
this.audioWaveformData = new Float32Array(0);
|
|
4521
4616
|
}
|
|
4522
4617
|
handleVideoCanvasSetup(message) {
|
|
4523
4618
|
const { streamIndex, streamType, deviceId } = message.data;
|
|
@@ -4828,6 +4923,7 @@ class VijiWorkerRuntime {
|
|
|
4828
4923
|
}
|
|
4829
4924
|
this.reportPerformanceStats(currentTime);
|
|
4830
4925
|
this.interactionSystem.frameStart();
|
|
4926
|
+
this.resetAudioFrameEvents();
|
|
4831
4927
|
requestAnimationFrame(() => this.renderFrame());
|
|
4832
4928
|
}
|
|
4833
4929
|
postMessage(type, data) {
|
|
@@ -26249,4 +26345,4 @@ async function setSceneCode(sceneCode) {
|
|
|
26249
26345
|
}
|
|
26250
26346
|
}
|
|
26251
26347
|
self.setSceneCode = setSceneCode;
|
|
26252
|
-
//# sourceMappingURL=viji.worker-
|
|
26348
|
+
//# sourceMappingURL=viji.worker-PAf0oIec.js.map
|