@needle-tools/engine 5.0.0-canary.3048027 → 5.0.0-canary.663b7ae

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -11,7 +11,7 @@ import type { INetworkConnection } from "./engine_networking_types.js";
11
11
  import type { IModel } from "./engine_networking_types.js";
12
12
  import { SendQueue } from "./engine_networking_types.js";
13
13
  import { Context } from "./engine_setup.js"
14
- import type { IComponent as Component, IGameObject as GameObject, IContext } from "./engine_types.js"
14
+ import type { IComponent as Component, IContext,IGameObject as GameObject } from "./engine_types.js"
15
15
  import type { UIDProvider } from "./engine_types.js";
16
16
  import * as utils from "./engine_utils.js"
17
17
 
@@ -512,11 +512,11 @@ function instantiateSeeded(obj: GameObject, opts: IInstantiateOptions | null): {
512
512
  return { seed: seed, instance: instance };
513
513
  }
514
514
 
515
- export { Prefabs, type PrefabProviderCallback } from "./engine_networking_prefabs.js";
515
+ export { type PrefabProviderCallback,Prefabs } from "./engine_networking_prefabs.js";
516
516
  import { Prefabs } from "./engine_networking_prefabs.js";
517
517
 
518
518
  /**
519
- * @deprecated Use `Prefabs.register()` instead.
519
+ * Register a prefab provider. Forwards to {@link Prefabs.register}.
520
520
  * @category Networking
521
521
  */
522
522
  export function registerPrefabProvider(key: string, fn: (guid: string) => Promise<Object3D | null>) {
@@ -524,7 +524,7 @@ export function registerPrefabProvider(key: string, fn: (guid: string) => Promis
524
524
  }
525
525
 
526
526
  /**
527
- * @deprecated Use `Prefabs.unregister()` instead.
527
+ * Unregister a prefab provider. Forwards to {@link Prefabs.unregister}.
528
528
  * @category Networking
529
529
  */
530
530
  export function unregisterPrefabProvider(key: string) {
@@ -15,10 +15,10 @@ import { DeviceUtilities, getParam } from "../engine/engine_utils.js";
15
15
  import { NeedleEngineWebComponent } from "../engine/webcomponents/needle-engine.js";
16
16
  import { Camera } from "./Camera.js";
17
17
  import { Behaviour, GameObject } from "./Component.js";
18
+ import { LookAtConstraint } from "./LookAtConstraint.js";
18
19
  import { SyncedTransform } from "./SyncedTransform.js";
19
20
  import { type AfterHandleInputEvent, EventSystem, EventSystemEvents } from "./ui/EventSystem.js";
20
21
  import { tryGetUIComponent } from "./ui/Utils.js";
21
- import { LookAtConstraint } from "./LookAtConstraint.js";
22
22
 
23
23
 
24
24
  const debug = getParam("debugorbit");
@@ -3,10 +3,10 @@ import { Application } from "../engine/engine_application.js";
3
3
  import { RoomEvents } from "../engine/engine_networking.js";
4
4
  import { disposeStream, NetworkedStreamEvents, NetworkedStreams, StreamEndedEvent, StreamReceivedEvent } from "../engine/engine_networking_streams.js"
5
5
  import { serializable } from "../engine/engine_serialization_decorator.js";
6
- import { DeviceUtilities, getParam } from "../engine/engine_utils.js";
7
- import { delay } from "../engine/engine_utils.js";
6
+ import { delay, DeviceUtilities, getParam } from "../engine/engine_utils.js";
8
7
  import { getIconElement } from "../engine/webcomponents/icons.js";
9
8
  import { Behaviour } from "./Component.js";
9
+ import { EventList } from "./EventList.js";
10
10
 
11
11
  export const noVoip = "noVoip";
12
12
  const debugParam = getParam("debugvoip");
@@ -75,6 +75,66 @@ export class Voip extends Behaviour {
75
75
  */
76
76
  debug: boolean = false;
77
77
 
78
+ private _volume: number = 1;
79
+
80
+ /**
81
+ * Volume for incoming audio streams (0 = silent, 1 = full volume).
82
+ * Changes apply immediately to all active incoming streams.
83
+ */
84
+ @serializable()
85
+ get volume(): number { return this._volume; }
86
+ set volume(val: number) {
87
+ this._volume = val;
88
+ for (const audio of this._incomingStreams.values()) {
89
+ audio.volume = val;
90
+ }
91
+ }
92
+
93
+ /**
94
+ * Get the incoming audio element for a specific user.
95
+ * Use this to route audio through the Web Audio API for spatial audio, effects, or analysis.
96
+ * @param userId The connection ID of the remote user
97
+ * @returns The HTMLAudioElement for this user's stream, or undefined if not connected
98
+ * @example
99
+ * ```ts
100
+ * const audioEl = voip.getAudioElement(userId);
101
+ * if (audioEl) {
102
+ * const audioCtx = new AudioContext();
103
+ * const source = audioCtx.createMediaElementSource(audioEl);
104
+ * const panner = audioCtx.createPanner();
105
+ * source.connect(panner).connect(audioCtx.destination);
106
+ * }
107
+ * ```
108
+ */
109
+ getAudioElement(userId: string): HTMLAudioElement | undefined {
110
+ return this._incomingStreams.get(userId);
111
+ }
112
+
113
+ /**
114
+ * Get all active incoming audio streams as a Map of userId → HTMLAudioElement.
115
+ * Useful for iterating over all connected voice users.
116
+ */
117
+ get incomingStreams(): ReadonlyMap<string, HTMLAudioElement> {
118
+ return this._incomingStreams;
119
+ }
120
+
121
+ /**
122
+ * Threshold for speaking detection (0–255). When a user's audio amplitude exceeds this,
123
+ * they are considered "speaking". Default is 30.
124
+ */
125
+ @serializable()
126
+ speakingThreshold: number = 30;
127
+
128
+ /**
129
+ * Event fired when a user's speaking state changes.
130
+ * Passes `{ userId: string, isSpeaking: boolean, volume: number }`.
131
+ */
132
+ @serializable(EventList)
133
+ onSpeakingChanged: EventList = new EventList();
134
+
135
+ private _speakingStates = new Map<string, boolean>();
136
+ private _analysers = new Map<string, { analyser: AnalyserNode, data: Uint8Array, context: AudioContext }>();
137
+
78
138
  private _net?: NetworkedStreams;
79
139
  private _menubutton?: HTMLElement;
80
140
 
@@ -141,12 +201,25 @@ export class Voip extends Behaviour {
141
201
  this.onEnabledChanged();
142
202
  this.updateButton();
143
203
  window.removeEventListener("visibilitychange", this.onVisibilityChanged);
204
+ // Clean up analysers
205
+ for (const userId of [...this._analysers.keys()]) {
206
+ this.cleanupAnalyser(userId);
207
+ }
144
208
  }
145
209
 
146
210
  /** @internal */
147
211
  onDestroy(): void {
148
212
  this._menubutton?.remove();
149
213
  this._menubutton = undefined;
214
+ // Clean up all streams and analysers
215
+ for (const userId of [...this._analysers.keys()]) {
216
+ this.cleanupAnalyser(userId);
217
+ }
218
+ for (const incoming of this._incomingStreams.values()) {
219
+ disposeStream(incoming.srcObject as MediaStream);
220
+ }
221
+ this._incomingStreams.clear();
222
+ this._speakingStates.clear();
150
223
  }
151
224
 
152
225
  /** Set via the mic button (e.g. when the websocket connection closes and rejoins but the user was muted before we don't want to enable VOIP again automatically) */
@@ -347,10 +420,61 @@ export class Voip extends Behaviour {
347
420
  disposeStream(incoming.srcObject as MediaStream);
348
421
  }
349
422
  this._incomingStreams.clear();
423
+ for (const userId of this._analysers.keys()) {
424
+ this.cleanupAnalyser(userId);
425
+ }
350
426
  }
351
427
 
352
428
  private _incomingStreams: Map<string, HTMLAudioElement> = new Map();
353
429
 
430
+ /** @internal */
431
+ update() {
432
+ // Only run speaking detection if someone is listening
433
+ if (!this.onSpeakingChanged || this.onSpeakingChanged.listenerCount <= 0) return;
434
+
435
+ for (const [userId, info] of this._analysers) {
436
+ info.analyser.getByteFrequencyData(info.data as Uint8Array<ArrayBuffer>);
437
+ // Average amplitude
438
+ let sum = 0;
439
+ for (let i = 0; i < info.data.length; i++) sum += info.data[i];
440
+ const avg = sum / info.data.length;
441
+
442
+ const wasSpeaking = this._speakingStates.get(userId) ?? false;
443
+ const isSpeaking = avg > this.speakingThreshold;
444
+
445
+ if (isSpeaking !== wasSpeaking) {
446
+ this._speakingStates.set(userId, isSpeaking);
447
+ this.onSpeakingChanged.invoke({ userId, isSpeaking, volume: avg / 255 });
448
+ }
449
+ }
450
+ }
451
+
452
+ private setupAnalyser(userId: string, _audioElement: HTMLAudioElement, stream: MediaStream) {
453
+ // Only set up if someone is listening or might listen
454
+ if (this._analysers.has(userId)) return;
455
+ try {
456
+ const audioCtx = new AudioContext();
457
+ const source = audioCtx.createMediaStreamSource(stream);
458
+ const analyser = audioCtx.createAnalyser();
459
+ analyser.fftSize = 256;
460
+ source.connect(analyser);
461
+ const data = new Uint8Array(analyser.frequencyBinCount);
462
+ this._analysers.set(userId, { analyser, data, context: audioCtx });
463
+ }
464
+ catch (err) {
465
+ if (this.debug) console.warn("VOIP: Failed to create analyser for", userId, err);
466
+ }
467
+ }
468
+
469
+ private cleanupAnalyser(userId: string) {
470
+ const info = this._analysers.get(userId);
471
+ if (info) {
472
+ info.context.close();
473
+ this._analysers.delete(userId);
474
+ }
475
+ this._speakingStates.delete(userId);
476
+ }
477
+
354
478
  private onReceiveStream = (evt: StreamReceivedEvent) => {
355
479
  const userId = evt.target.userId;
356
480
  const stream = evt.stream;
@@ -361,7 +485,9 @@ export class Voip extends Behaviour {
361
485
  this._incomingStreams.set(userId, audioElement);
362
486
  }
363
487
  audioElement.srcObject = stream;
488
+ audioElement.volume = this._volume;
364
489
  audioElement.setAttribute("autoplay", "true");
490
+ this.setupAnalyser(userId, audioElement, stream);
365
491
  // for mobile we need to wait for user interaction to play audio. Auto play doesnt work on android when the page is refreshed
366
492
  Application.registerWaitForInteraction(() => {
367
493
  audioElement?.play().catch((err) => {
@@ -374,6 +500,7 @@ export class Voip extends Behaviour {
374
500
  const existing = this._incomingStreams.get(evt.userId);
375
501
  disposeStream(existing?.srcObject as MediaStream);
376
502
  this._incomingStreams.delete(evt.userId);
503
+ this.cleanupAnalyser(evt.userId);
377
504
  }
378
505
 
379
506
  private onEnabledChanged = () => {