@viji-dev/core 0.3.24 → 0.3.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1185,21 +1185,21 @@ class CVSystem {
1185
1185
  setTimeout(() => reject(new Error("CV processing timeout")), 500);
1186
1186
  });
1187
1187
  const results = await Promise.race([processPromise, timeoutPromise]);
1188
- if (results.faces && (this.activeFeatures.has("faceDetection") || this.activeFeatures.has("faceMesh") || this.activeFeatures.has("emotionDetection"))) {
1188
+ if ("faces" in results && (this.activeFeatures.has("faceDetection") || this.activeFeatures.has("faceMesh") || this.activeFeatures.has("emotionDetection"))) {
1189
1189
  this.results.faces = results.faces;
1190
1190
  this.debugLog(`📥 Received ${results.faces.length} face results`);
1191
1191
  }
1192
- if (results.hands && this.activeFeatures.has("handTracking")) {
1192
+ if ("hands" in results && this.activeFeatures.has("handTracking")) {
1193
1193
  this.results.hands = results.hands;
1194
1194
  this.debugLog(`📥 Received ${results.hands.length} hand results`);
1195
1195
  }
1196
- if (results.pose && this.activeFeatures.has("poseDetection")) {
1196
+ if ("pose" in results && this.activeFeatures.has("poseDetection")) {
1197
1197
  this.results.pose = results.pose;
1198
- this.debugLog(`📥 Received pose results with ${results.pose.landmarks.length} landmarks`);
1198
+ this.debugLog(`📥 Received pose results: ${results.pose ? results.pose.landmarks.length + " landmarks" : "none"}`);
1199
1199
  }
1200
- if (results.segmentation && this.activeFeatures.has("bodySegmentation")) {
1200
+ if ("segmentation" in results && this.activeFeatures.has("bodySegmentation")) {
1201
1201
  this.results.segmentation = results.segmentation;
1202
- this.debugLog(`📥 Received segmentation results ${results.segmentation.width}x${results.segmentation.height}`);
1202
+ this.debugLog(`📥 Received segmentation: ${results.segmentation ? results.segmentation.width + "x" + results.segmentation.height : "none"}`);
1203
1203
  }
1204
1204
  const processingTime = performance.now() - this.processingStartTime;
1205
1205
  this.processingTimes.push(processingTime);
@@ -2642,11 +2642,11 @@ uniform float u_audioPeak; // Peak amplitude (0-1)
2642
2642
  uniform float u_audioVolumeSmoothed; // Smoothed volume for animation (200ms decay)
2643
2643
 
2644
2644
  // Audio - Frequency Bands (instant)
2645
- uniform float u_audioLow; // Low 20-150 Hz (0-1)
2646
- uniform float u_audioLowMid; // Low-mid 150-400 Hz (0-1)
2647
- uniform float u_audioMid; // Mid 400-2500 Hz (0-1)
2648
- uniform float u_audioHighMid; // High-mid 2500-8000 Hz (0-1)
2649
- uniform float u_audioHigh; // High 8000-20000 Hz (0-1)
2645
+ uniform float u_audioLow; // Low 20-120 Hz (0-1)
2646
+ uniform float u_audioLowMid; // Low-mid 120-400 Hz (0-1)
2647
+ uniform float u_audioMid; // Mid 400-1600 Hz (0-1)
2648
+ uniform float u_audioHighMid; // High-mid 1600-6000 Hz (0-1)
2649
+ uniform float u_audioHigh; // High 6000-16000 Hz (0-1)
2650
2650
 
2651
2651
  // Audio - Frequency Bands (smoothed, 150ms decay)
2652
2652
  uniform float u_audioLowSmoothed; // Smoothed low band (0-1)
@@ -2683,7 +2683,7 @@ uniform float u_audioBrightness; // Spectral brightness / centroid (0-1)
2683
2683
  uniform float u_audioFlatness; // Spectral flatness / noisiness (0-1)
2684
2684
 
2685
2685
  // Audio - Textures
2686
- uniform sampler2D u_audioFFT; // FFT frequency spectrum (512 bins, 0-255)
2686
+ uniform sampler2D u_audioFFT; // FFT frequency spectrum (1024 bins, 0-255)
2687
2687
  uniform sampler2D u_audioWaveform; // Time-domain waveform (fftSize samples, -1 to 1)
2688
2688
 
2689
2689
  // Video
@@ -2823,18 +2823,41 @@ uniform float u_face0TongueOut;
2823
2823
  uniform int u_handCount; // Number of detected hands (0-2)
2824
2824
  uniform vec3 u_leftHandPalm; // Left hand palm position (x, y, z)
2825
2825
  uniform vec3 u_rightHandPalm; // Right hand palm position (x, y, z)
2826
+ uniform float u_leftHandConfidence; // Left hand detection confidence (0-1)
2827
+ uniform float u_rightHandConfidence; // Right hand detection confidence (0-1)
2828
+ uniform vec4 u_leftHandBounds; // Left hand bounding box (x, y, width, height) normalized 0-1
2829
+ uniform vec4 u_rightHandBounds; // Right hand bounding box (x, y, width, height) normalized 0-1
2826
2830
  uniform float u_leftHandFist; // Left hand fist gesture confidence (0-1)
2827
2831
  uniform float u_leftHandOpen; // Left hand open palm gesture confidence (0-1)
2832
+ uniform float u_leftHandPeace; // Left hand peace/victory gesture confidence (0-1)
2833
+ uniform float u_leftHandThumbsUp; // Left hand thumbs up gesture confidence (0-1)
2834
+ uniform float u_leftHandThumbsDown; // Left hand thumbs down gesture confidence (0-1)
2835
+ uniform float u_leftHandPointing; // Left hand pointing gesture confidence (0-1)
2836
+ uniform float u_leftHandILoveYou; // Left hand I-love-you gesture confidence (0-1)
2828
2837
  uniform float u_rightHandFist; // Right hand fist gesture confidence (0-1)
2829
2838
  uniform float u_rightHandOpen; // Right hand open palm gesture confidence (0-1)
2839
+ uniform float u_rightHandPeace; // Right hand peace/victory gesture confidence (0-1)
2840
+ uniform float u_rightHandThumbsUp; // Right hand thumbs up gesture confidence (0-1)
2841
+ uniform float u_rightHandThumbsDown; // Right hand thumbs down gesture confidence (0-1)
2842
+ uniform float u_rightHandPointing; // Right hand pointing gesture confidence (0-1)
2843
+ uniform float u_rightHandILoveYou; // Right hand I-love-you gesture confidence (0-1)
2830
2844
 
2831
2845
  // CV - Pose Detection
2832
2846
  uniform bool u_poseDetected; // True if a pose is currently detected
2833
- uniform vec2 u_nosePosition; // Nose landmark position in pixels
2834
- uniform vec2 u_leftWristPosition; // Left wrist landmark position in pixels
2835
- uniform vec2 u_rightWristPosition; // Right wrist landmark position in pixels
2836
- uniform vec2 u_leftAnklePosition; // Left ankle landmark position in pixels
2837
- uniform vec2 u_rightAnklePosition; // Right ankle landmark position in pixels
2847
+ uniform float u_poseConfidence; // Pose detection confidence (0-1)
2848
+ uniform vec2 u_nosePosition; // Nose landmark position (normalized 0-1)
2849
+ uniform vec2 u_leftShoulderPosition; // Left shoulder position (normalized 0-1)
2850
+ uniform vec2 u_rightShoulderPosition; // Right shoulder position (normalized 0-1)
2851
+ uniform vec2 u_leftElbowPosition; // Left elbow position (normalized 0-1)
2852
+ uniform vec2 u_rightElbowPosition; // Right elbow position (normalized 0-1)
2853
+ uniform vec2 u_leftWristPosition; // Left wrist position (normalized 0-1)
2854
+ uniform vec2 u_rightWristPosition; // Right wrist position (normalized 0-1)
2855
+ uniform vec2 u_leftHipPosition; // Left hip position (normalized 0-1)
2856
+ uniform vec2 u_rightHipPosition; // Right hip position (normalized 0-1)
2857
+ uniform vec2 u_leftKneePosition; // Left knee position (normalized 0-1)
2858
+ uniform vec2 u_rightKneePosition; // Right knee position (normalized 0-1)
2859
+ uniform vec2 u_leftAnklePosition; // Left ankle position (normalized 0-1)
2860
+ uniform vec2 u_rightAnklePosition; // Right ankle position (normalized 0-1)
2838
2861
 
2839
2862
  // CV - Segmentation
2840
2863
  uniform sampler2D u_segmentationMask; // Body segmentation mask texture (0=background, 1=person)
@@ -3268,40 +3291,88 @@ ${error}`);
3268
3291
  const leftHand = hands.find((h) => h.handedness === "left");
3269
3292
  const rightHand = hands.find((h) => h.handedness === "right");
3270
3293
  if (leftHand) {
3271
- this.setUniform("u_leftHandPalm", "vec3", [leftHand.palm.x, leftHand.palm.y, leftHand.palm.z]);
3272
- this.setUniform("u_leftHandFist", "float", leftHand.gestures?.fist || 0);
3273
- this.setUniform("u_leftHandOpen", "float", leftHand.gestures?.openPalm || 0);
3294
+ const palm = leftHand.palm || leftHand.landmarks?.[9];
3295
+ const g = leftHand.gestures || {};
3296
+ const b = leftHand.bounds;
3297
+ this.setUniform("u_leftHandPalm", "vec3", [palm?.x || 0, palm?.y || 0, palm?.z || 0]);
3298
+ this.setUniform("u_leftHandConfidence", "float", leftHand.confidence || 0);
3299
+ this.setUniform("u_leftHandBounds", "vec4", [b?.x || 0, b?.y || 0, b?.width || 0, b?.height || 0]);
3300
+ this.setUniform("u_leftHandFist", "float", g.fist || 0);
3301
+ this.setUniform("u_leftHandOpen", "float", g.openPalm || 0);
3302
+ this.setUniform("u_leftHandPeace", "float", g.peace || 0);
3303
+ this.setUniform("u_leftHandThumbsUp", "float", g.thumbsUp || 0);
3304
+ this.setUniform("u_leftHandThumbsDown", "float", g.thumbsDown || 0);
3305
+ this.setUniform("u_leftHandPointing", "float", g.pointing || 0);
3306
+ this.setUniform("u_leftHandILoveYou", "float", g.iLoveYou || 0);
3274
3307
  } else {
3275
3308
  this.setUniform("u_leftHandPalm", "vec3", [0, 0, 0]);
3309
+ this.setUniform("u_leftHandConfidence", "float", 0);
3310
+ this.setUniform("u_leftHandBounds", "vec4", [0, 0, 0, 0]);
3276
3311
  this.setUniform("u_leftHandFist", "float", 0);
3277
3312
  this.setUniform("u_leftHandOpen", "float", 0);
3313
+ this.setUniform("u_leftHandPeace", "float", 0);
3314
+ this.setUniform("u_leftHandThumbsUp", "float", 0);
3315
+ this.setUniform("u_leftHandThumbsDown", "float", 0);
3316
+ this.setUniform("u_leftHandPointing", "float", 0);
3317
+ this.setUniform("u_leftHandILoveYou", "float", 0);
3278
3318
  }
3279
3319
  if (rightHand) {
3280
- this.setUniform("u_rightHandPalm", "vec3", [rightHand.palm.x, rightHand.palm.y, rightHand.palm.z]);
3281
- this.setUniform("u_rightHandFist", "float", rightHand.gestures?.fist || 0);
3282
- this.setUniform("u_rightHandOpen", "float", rightHand.gestures?.openPalm || 0);
3320
+ const palm = rightHand.palm || rightHand.landmarks?.[9];
3321
+ const g = rightHand.gestures || {};
3322
+ const b = rightHand.bounds;
3323
+ this.setUniform("u_rightHandPalm", "vec3", [palm?.x || 0, palm?.y || 0, palm?.z || 0]);
3324
+ this.setUniform("u_rightHandConfidence", "float", rightHand.confidence || 0);
3325
+ this.setUniform("u_rightHandBounds", "vec4", [b?.x || 0, b?.y || 0, b?.width || 0, b?.height || 0]);
3326
+ this.setUniform("u_rightHandFist", "float", g.fist || 0);
3327
+ this.setUniform("u_rightHandOpen", "float", g.openPalm || 0);
3328
+ this.setUniform("u_rightHandPeace", "float", g.peace || 0);
3329
+ this.setUniform("u_rightHandThumbsUp", "float", g.thumbsUp || 0);
3330
+ this.setUniform("u_rightHandThumbsDown", "float", g.thumbsDown || 0);
3331
+ this.setUniform("u_rightHandPointing", "float", g.pointing || 0);
3332
+ this.setUniform("u_rightHandILoveYou", "float", g.iLoveYou || 0);
3283
3333
  } else {
3284
3334
  this.setUniform("u_rightHandPalm", "vec3", [0, 0, 0]);
3335
+ this.setUniform("u_rightHandConfidence", "float", 0);
3336
+ this.setUniform("u_rightHandBounds", "vec4", [0, 0, 0, 0]);
3285
3337
  this.setUniform("u_rightHandFist", "float", 0);
3286
3338
  this.setUniform("u_rightHandOpen", "float", 0);
3339
+ this.setUniform("u_rightHandPeace", "float", 0);
3340
+ this.setUniform("u_rightHandThumbsUp", "float", 0);
3341
+ this.setUniform("u_rightHandThumbsDown", "float", 0);
3342
+ this.setUniform("u_rightHandPointing", "float", 0);
3343
+ this.setUniform("u_rightHandILoveYou", "float", 0);
3287
3344
  }
3288
3345
  const pose = video.pose;
3289
3346
  this.setUniform("u_poseDetected", "bool", pose !== null);
3290
- if (pose) {
3291
- const nose = pose.landmarks[0];
3292
- const leftWrist = pose.landmarks[15];
3293
- const rightWrist = pose.landmarks[16];
3294
- const leftAnkle = pose.landmarks[27];
3295
- const rightAnkle = pose.landmarks[28];
3296
- this.setUniform("u_nosePosition", "vec2", [nose?.x || 0, nose?.y || 0]);
3297
- this.setUniform("u_leftWristPosition", "vec2", [leftWrist?.x || 0, leftWrist?.y || 0]);
3298
- this.setUniform("u_rightWristPosition", "vec2", [rightWrist?.x || 0, rightWrist?.y || 0]);
3299
- this.setUniform("u_leftAnklePosition", "vec2", [leftAnkle?.x || 0, leftAnkle?.y || 0]);
3300
- this.setUniform("u_rightAnklePosition", "vec2", [rightAnkle?.x || 0, rightAnkle?.y || 0]);
3347
+ if (pose && pose.landmarks) {
3348
+ const lm = pose.landmarks;
3349
+ this.setUniform("u_poseConfidence", "float", pose.confidence || 0);
3350
+ this.setUniform("u_nosePosition", "vec2", [lm[0]?.x || 0, lm[0]?.y || 0]);
3351
+ this.setUniform("u_leftShoulderPosition", "vec2", [lm[11]?.x || 0, lm[11]?.y || 0]);
3352
+ this.setUniform("u_rightShoulderPosition", "vec2", [lm[12]?.x || 0, lm[12]?.y || 0]);
3353
+ this.setUniform("u_leftElbowPosition", "vec2", [lm[13]?.x || 0, lm[13]?.y || 0]);
3354
+ this.setUniform("u_rightElbowPosition", "vec2", [lm[14]?.x || 0, lm[14]?.y || 0]);
3355
+ this.setUniform("u_leftWristPosition", "vec2", [lm[15]?.x || 0, lm[15]?.y || 0]);
3356
+ this.setUniform("u_rightWristPosition", "vec2", [lm[16]?.x || 0, lm[16]?.y || 0]);
3357
+ this.setUniform("u_leftHipPosition", "vec2", [lm[23]?.x || 0, lm[23]?.y || 0]);
3358
+ this.setUniform("u_rightHipPosition", "vec2", [lm[24]?.x || 0, lm[24]?.y || 0]);
3359
+ this.setUniform("u_leftKneePosition", "vec2", [lm[25]?.x || 0, lm[25]?.y || 0]);
3360
+ this.setUniform("u_rightKneePosition", "vec2", [lm[26]?.x || 0, lm[26]?.y || 0]);
3361
+ this.setUniform("u_leftAnklePosition", "vec2", [lm[27]?.x || 0, lm[27]?.y || 0]);
3362
+ this.setUniform("u_rightAnklePosition", "vec2", [lm[28]?.x || 0, lm[28]?.y || 0]);
3301
3363
  } else {
3364
+ this.setUniform("u_poseConfidence", "float", 0);
3302
3365
  this.setUniform("u_nosePosition", "vec2", [0, 0]);
3366
+ this.setUniform("u_leftShoulderPosition", "vec2", [0, 0]);
3367
+ this.setUniform("u_rightShoulderPosition", "vec2", [0, 0]);
3368
+ this.setUniform("u_leftElbowPosition", "vec2", [0, 0]);
3369
+ this.setUniform("u_rightElbowPosition", "vec2", [0, 0]);
3303
3370
  this.setUniform("u_leftWristPosition", "vec2", [0, 0]);
3304
3371
  this.setUniform("u_rightWristPosition", "vec2", [0, 0]);
3372
+ this.setUniform("u_leftHipPosition", "vec2", [0, 0]);
3373
+ this.setUniform("u_rightHipPosition", "vec2", [0, 0]);
3374
+ this.setUniform("u_leftKneePosition", "vec2", [0, 0]);
3375
+ this.setUniform("u_rightKneePosition", "vec2", [0, 0]);
3305
3376
  this.setUniform("u_leftAnklePosition", "vec2", [0, 0]);
3306
3377
  this.setUniform("u_rightAnklePosition", "vec2", [0, 0]);
3307
3378
  }
@@ -3943,43 +4014,9 @@ class VijiWorkerRuntime {
3943
4014
  autoCaptureEnabled = false;
3944
4015
  autoCaptureFormat = { flipY: true };
3945
4016
  // Default: flip for WebGL compatibility
3946
- // Audio state (Phase 5) - receives analysis results from host
3947
- audioState = {
3948
- isConnected: false,
3949
- volume: { current: 0, peak: 0, smoothed: 0 },
3950
- bands: {
3951
- low: 0,
3952
- lowMid: 0,
3953
- mid: 0,
3954
- highMid: 0,
3955
- high: 0,
3956
- lowSmoothed: 0,
3957
- lowMidSmoothed: 0,
3958
- midSmoothed: 0,
3959
- highMidSmoothed: 0,
3960
- highSmoothed: 0
3961
- },
3962
- beat: {
3963
- kick: 0,
3964
- snare: 0,
3965
- hat: 0,
3966
- any: 0,
3967
- kickSmoothed: 0,
3968
- snareSmoothed: 0,
3969
- hatSmoothed: 0,
3970
- anySmoothed: 0,
3971
- triggers: { any: false, kick: false, snare: false, hat: false },
3972
- bpm: 120,
3973
- confidence: 0,
3974
- isLocked: false
3975
- },
3976
- spectral: {
3977
- brightness: 0,
3978
- flatness: 0
3979
- },
3980
- frequencyData: new Uint8Array(0),
3981
- waveformData: new Float32Array(0)
3982
- };
4017
+ // Audio raw data buffers (referenced by closures in viji.audio)
4018
+ audioFrequencyData = new Uint8Array(0);
4019
+ audioWaveformData = new Float32Array(0);
3983
4020
  // Device sensor state (internal device + external devices)
3984
4021
  deviceState = {
3985
4022
  device: {
@@ -4004,8 +4041,41 @@ class VijiWorkerRuntime {
4004
4041
  deltaTime: 0,
4005
4042
  frameCount: 0,
4006
4043
  fps: 60,
4007
- // Audio API (Phase 5) - will be set in constructor
4008
- audio: {},
4044
+ // Audio API (Phase 5) stable reference, mutated in place
4045
+ audio: {
4046
+ isConnected: false,
4047
+ volume: { current: 0, peak: 0, smoothed: 0 },
4048
+ bands: {
4049
+ low: 0,
4050
+ lowMid: 0,
4051
+ mid: 0,
4052
+ highMid: 0,
4053
+ high: 0,
4054
+ lowSmoothed: 0,
4055
+ lowMidSmoothed: 0,
4056
+ midSmoothed: 0,
4057
+ highMidSmoothed: 0,
4058
+ highSmoothed: 0
4059
+ },
4060
+ beat: {
4061
+ kick: 0,
4062
+ snare: 0,
4063
+ hat: 0,
4064
+ any: 0,
4065
+ kickSmoothed: 0,
4066
+ snareSmoothed: 0,
4067
+ hatSmoothed: 0,
4068
+ anySmoothed: 0,
4069
+ triggers: { any: false, kick: false, snare: false, hat: false },
4070
+ events: [],
4071
+ bpm: 120,
4072
+ confidence: 0,
4073
+ isLocked: false
4074
+ },
4075
+ spectral: { brightness: 0, flatness: 0 },
4076
+ getFrequencyData: () => new Uint8Array(0),
4077
+ getWaveform: () => new Float32Array(0)
4078
+ },
4009
4079
  // Main video stream (index 0, CV enabled)
4010
4080
  video: {
4011
4081
  isConnected: false,
@@ -4117,11 +4187,8 @@ class VijiWorkerRuntime {
4117
4187
  });
4118
4188
  this.interactionSystem = new InteractionSystem();
4119
4189
  Object.assign(this.viji, this.interactionSystem.getInteractionAPIs());
4120
- this.viji.audio = {
4121
- ...this.audioState,
4122
- getFrequencyData: () => this.audioState.frequencyData,
4123
- getWaveform: () => this.audioState.waveformData
4124
- };
4190
+ this.viji.audio.getFrequencyData = () => this.audioFrequencyData;
4191
+ this.viji.audio.getWaveform = () => this.audioWaveformData;
4125
4192
  this.setupMessageHandling();
4126
4193
  }
4127
4194
  /**
@@ -4224,47 +4291,7 @@ class VijiWorkerRuntime {
4224
4291
  resetParameterState() {
4225
4292
  this.parameterSystem.resetParameterState();
4226
4293
  this.interactionSystem.resetInteractionState();
4227
- this.audioState = {
4228
- isConnected: false,
4229
- volume: { current: 0, peak: 0, smoothed: 0 },
4230
- bands: {
4231
- low: 0,
4232
- lowMid: 0,
4233
- mid: 0,
4234
- highMid: 0,
4235
- high: 0,
4236
- lowSmoothed: 0,
4237
- lowMidSmoothed: 0,
4238
- midSmoothed: 0,
4239
- highMidSmoothed: 0,
4240
- highSmoothed: 0
4241
- },
4242
- beat: {
4243
- kick: 0,
4244
- snare: 0,
4245
- hat: 0,
4246
- any: 0,
4247
- kickSmoothed: 0,
4248
- snareSmoothed: 0,
4249
- hatSmoothed: 0,
4250
- anySmoothed: 0,
4251
- triggers: { any: false, kick: false, snare: false, hat: false },
4252
- bpm: 120,
4253
- confidence: 0,
4254
- isLocked: false
4255
- },
4256
- spectral: {
4257
- brightness: 0,
4258
- flatness: 0
4259
- },
4260
- frequencyData: new Uint8Array(0),
4261
- waveformData: new Float32Array(0)
4262
- };
4263
- this.viji.audio = {
4264
- ...this.audioState,
4265
- getFrequencyData: () => this.audioState.frequencyData,
4266
- getWaveform: () => this.audioState.waveformData
4267
- };
4294
+ this.resetAudioState();
4268
4295
  this.videoSystems.forEach((vs) => vs?.resetVideoState());
4269
4296
  if (this.videoSystems[0]) {
4270
4297
  Object.assign(this.viji.video, this.videoSystems[0].getVideoAPI());
@@ -4494,30 +4521,98 @@ class VijiWorkerRuntime {
4494
4521
  this.debugLog("Stream update:", message.data);
4495
4522
  }
4496
4523
  handleAudioAnalysisUpdate(message) {
4497
- const events = message.data.beat.events || [];
4498
- const triggers = {
4499
- kick: events.some((e) => e.type === "kick"),
4500
- snare: events.some((e) => e.type === "snare"),
4501
- hat: events.some((e) => e.type === "hat"),
4502
- any: events.length > 0
4503
- };
4504
- this.audioState = {
4505
- isConnected: message.data.isConnected,
4506
- volume: message.data.volume,
4507
- bands: message.data.bands,
4508
- beat: {
4509
- ...message.data.beat,
4510
- triggers
4511
- },
4512
- spectral: message.data.spectral,
4513
- frequencyData: new Uint8Array(message.data.frequencyData),
4514
- waveformData: message.data.waveformData ? new Float32Array(message.data.waveformData) : new Float32Array(0)
4515
- };
4516
- this.viji.audio = {
4517
- ...this.audioState,
4518
- getFrequencyData: () => this.audioState.frequencyData,
4519
- getWaveform: () => this.audioState.waveformData
4520
- };
4524
+ const d = message.data;
4525
+ const audio = this.viji.audio;
4526
+ audio.isConnected = d.isConnected;
4527
+ audio.volume.current = d.volume.current;
4528
+ audio.volume.peak = d.volume.peak;
4529
+ audio.volume.smoothed = d.volume.smoothed;
4530
+ audio.bands.low = d.bands.low;
4531
+ audio.bands.lowMid = d.bands.lowMid;
4532
+ audio.bands.mid = d.bands.mid;
4533
+ audio.bands.highMid = d.bands.highMid;
4534
+ audio.bands.high = d.bands.high;
4535
+ audio.bands.lowSmoothed = d.bands.lowSmoothed;
4536
+ audio.bands.lowMidSmoothed = d.bands.lowMidSmoothed;
4537
+ audio.bands.midSmoothed = d.bands.midSmoothed;
4538
+ audio.bands.highMidSmoothed = d.bands.highMidSmoothed;
4539
+ audio.bands.highSmoothed = d.bands.highSmoothed;
4540
+ audio.beat.kick = d.beat.kick;
4541
+ audio.beat.snare = d.beat.snare;
4542
+ audio.beat.hat = d.beat.hat;
4543
+ audio.beat.any = d.beat.any;
4544
+ audio.beat.kickSmoothed = d.beat.kickSmoothed;
4545
+ audio.beat.snareSmoothed = d.beat.snareSmoothed;
4546
+ audio.beat.hatSmoothed = d.beat.hatSmoothed;
4547
+ audio.beat.anySmoothed = d.beat.anySmoothed;
4548
+ const events = d.beat.events || [];
4549
+ if (events.length > 0) {
4550
+ audio.beat.triggers.kick = audio.beat.triggers.kick || events.some((e) => e.type === "kick");
4551
+ audio.beat.triggers.snare = audio.beat.triggers.snare || events.some((e) => e.type === "snare");
4552
+ audio.beat.triggers.hat = audio.beat.triggers.hat || events.some((e) => e.type === "hat");
4553
+ audio.beat.triggers.any = true;
4554
+ audio.beat.events.push(...events);
4555
+ }
4556
+ audio.beat.bpm = d.beat.bpm;
4557
+ audio.beat.confidence = d.beat.confidence;
4558
+ audio.beat.isLocked = d.beat.isLocked;
4559
+ audio.spectral.brightness = d.spectral.brightness;
4560
+ audio.spectral.flatness = d.spectral.flatness;
4561
+ this.audioFrequencyData = new Uint8Array(d.frequencyData);
4562
+ this.audioWaveformData = d.waveformData ? new Float32Array(d.waveformData) : new Float32Array(0);
4563
+ }
4564
+ /**
4565
+ * Reset frame-scoped audio events (triggers + events array).
4566
+ * Called after each render, mirroring interactionSystem.frameStart().
4567
+ */
4568
+ resetAudioFrameEvents() {
4569
+ const t = this.viji.audio.beat.triggers;
4570
+ t.kick = false;
4571
+ t.snare = false;
4572
+ t.hat = false;
4573
+ t.any = false;
4574
+ this.viji.audio.beat.events.length = 0;
4575
+ }
4576
+ /**
4577
+ * Full audio state reset (scene reload / disconnect).
4578
+ * Zeroes everything without replacing the stable viji.audio reference.
4579
+ */
4580
+ resetAudioState() {
4581
+ const audio = this.viji.audio;
4582
+ audio.isConnected = false;
4583
+ audio.volume.current = 0;
4584
+ audio.volume.peak = 0;
4585
+ audio.volume.smoothed = 0;
4586
+ audio.bands.low = 0;
4587
+ audio.bands.lowMid = 0;
4588
+ audio.bands.mid = 0;
4589
+ audio.bands.highMid = 0;
4590
+ audio.bands.high = 0;
4591
+ audio.bands.lowSmoothed = 0;
4592
+ audio.bands.lowMidSmoothed = 0;
4593
+ audio.bands.midSmoothed = 0;
4594
+ audio.bands.highMidSmoothed = 0;
4595
+ audio.bands.highSmoothed = 0;
4596
+ audio.beat.kick = 0;
4597
+ audio.beat.snare = 0;
4598
+ audio.beat.hat = 0;
4599
+ audio.beat.any = 0;
4600
+ audio.beat.kickSmoothed = 0;
4601
+ audio.beat.snareSmoothed = 0;
4602
+ audio.beat.hatSmoothed = 0;
4603
+ audio.beat.anySmoothed = 0;
4604
+ audio.beat.triggers.kick = false;
4605
+ audio.beat.triggers.snare = false;
4606
+ audio.beat.triggers.hat = false;
4607
+ audio.beat.triggers.any = false;
4608
+ audio.beat.events.length = 0;
4609
+ audio.beat.bpm = 120;
4610
+ audio.beat.confidence = 0;
4611
+ audio.beat.isLocked = false;
4612
+ audio.spectral.brightness = 0;
4613
+ audio.spectral.flatness = 0;
4614
+ this.audioFrequencyData = new Uint8Array(0);
4615
+ this.audioWaveformData = new Float32Array(0);
4521
4616
  }
4522
4617
  handleVideoCanvasSetup(message) {
4523
4618
  const { streamIndex, streamType, deviceId } = message.data;
@@ -4828,6 +4923,7 @@ class VijiWorkerRuntime {
4828
4923
  }
4829
4924
  this.reportPerformanceStats(currentTime);
4830
4925
  this.interactionSystem.frameStart();
4926
+ this.resetAudioFrameEvents();
4831
4927
  requestAnimationFrame(() => this.renderFrame());
4832
4928
  }
4833
4929
  postMessage(type, data) {
@@ -26249,4 +26345,4 @@ async function setSceneCode(sceneCode) {
26249
26345
  }
26250
26346
  }
26251
26347
  self.setSceneCode = setSceneCode;
26252
- //# sourceMappingURL=viji.worker-Zg128woJ.js.map
26348
+ //# sourceMappingURL=viji.worker-PAf0oIec.js.map