@needle-tools/engine 5.0.0-canary.3048027 → 5.0.0-canary.663b7ae
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/components.needle.json +1 -1
- package/dist/{needle-engine.bundle-Bdpuafrd.umd.cjs → needle-engine.bundle-B2scYiA2.umd.cjs} +114 -114
- package/dist/{needle-engine.bundle-BpPyvMoK.min.js → needle-engine.bundle-BFEYJdZs.min.js} +105 -105
- package/dist/{needle-engine.bundle-BgtOdVUL.js → needle-engine.bundle-CYxZeAfg.js} +4137 -4054
- package/dist/needle-engine.d.ts +46 -2
- package/dist/needle-engine.js +2 -2
- package/dist/needle-engine.min.js +1 -1
- package/dist/needle-engine.umd.cjs +1 -1
- package/lib/engine/engine_addressables.js +3 -0
- package/lib/engine/engine_addressables.js.map +1 -1
- package/lib/engine/engine_gameobject.js.map +1 -1
- package/lib/engine/engine_networking_instantiate.d.ts +4 -4
- package/lib/engine/engine_networking_instantiate.js +2 -2
- package/lib/engine/engine_networking_instantiate.js.map +1 -1
- package/lib/engine-components/OrbitControls.js +1 -1
- package/lib/engine-components/OrbitControls.js.map +1 -1
- package/lib/engine-components/Voip.d.ts +46 -0
- package/lib/engine-components/Voip.js +126 -2
- package/lib/engine-components/Voip.js.map +1 -1
- package/package.json +1 -1
- package/src/engine/engine_addressables.ts +3 -0
- package/src/engine/engine_gameobject.ts +1 -1
- package/src/engine/engine_networking_instantiate.ts +4 -4
- package/src/engine-components/OrbitControls.ts +1 -1
- package/src/engine-components/Voip.ts +129 -2
|
@@ -11,7 +11,7 @@ import type { INetworkConnection } from "./engine_networking_types.js";
|
|
|
11
11
|
import type { IModel } from "./engine_networking_types.js";
|
|
12
12
|
import { SendQueue } from "./engine_networking_types.js";
|
|
13
13
|
import { Context } from "./engine_setup.js"
|
|
14
|
-
import type { IComponent as Component, IGameObject as GameObject
|
|
14
|
+
import type { IComponent as Component, IContext,IGameObject as GameObject } from "./engine_types.js"
|
|
15
15
|
import type { UIDProvider } from "./engine_types.js";
|
|
16
16
|
import * as utils from "./engine_utils.js"
|
|
17
17
|
|
|
@@ -512,11 +512,11 @@ function instantiateSeeded(obj: GameObject, opts: IInstantiateOptions | null): {
|
|
|
512
512
|
return { seed: seed, instance: instance };
|
|
513
513
|
}
|
|
514
514
|
|
|
515
|
-
export {
|
|
515
|
+
export { type PrefabProviderCallback,Prefabs } from "./engine_networking_prefabs.js";
|
|
516
516
|
import { Prefabs } from "./engine_networking_prefabs.js";
|
|
517
517
|
|
|
518
518
|
/**
|
|
519
|
-
* @
|
|
519
|
+
* Register a prefab provider. Forwards to {@link Prefabs.register}.
|
|
520
520
|
* @category Networking
|
|
521
521
|
*/
|
|
522
522
|
export function registerPrefabProvider(key: string, fn: (guid: string) => Promise<Object3D | null>) {
|
|
@@ -524,7 +524,7 @@ export function registerPrefabProvider(key: string, fn: (guid: string) => Promis
|
|
|
524
524
|
}
|
|
525
525
|
|
|
526
526
|
/**
|
|
527
|
-
* @
|
|
527
|
+
* Unregister a prefab provider. Forwards to {@link Prefabs.unregister}.
|
|
528
528
|
* @category Networking
|
|
529
529
|
*/
|
|
530
530
|
export function unregisterPrefabProvider(key: string) {
|
|
@@ -15,10 +15,10 @@ import { DeviceUtilities, getParam } from "../engine/engine_utils.js";
|
|
|
15
15
|
import { NeedleEngineWebComponent } from "../engine/webcomponents/needle-engine.js";
|
|
16
16
|
import { Camera } from "./Camera.js";
|
|
17
17
|
import { Behaviour, GameObject } from "./Component.js";
|
|
18
|
+
import { LookAtConstraint } from "./LookAtConstraint.js";
|
|
18
19
|
import { SyncedTransform } from "./SyncedTransform.js";
|
|
19
20
|
import { type AfterHandleInputEvent, EventSystem, EventSystemEvents } from "./ui/EventSystem.js";
|
|
20
21
|
import { tryGetUIComponent } from "./ui/Utils.js";
|
|
21
|
-
import { LookAtConstraint } from "./LookAtConstraint.js";
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
const debug = getParam("debugorbit");
|
|
@@ -3,10 +3,10 @@ import { Application } from "../engine/engine_application.js";
|
|
|
3
3
|
import { RoomEvents } from "../engine/engine_networking.js";
|
|
4
4
|
import { disposeStream, NetworkedStreamEvents, NetworkedStreams, StreamEndedEvent, StreamReceivedEvent } from "../engine/engine_networking_streams.js"
|
|
5
5
|
import { serializable } from "../engine/engine_serialization_decorator.js";
|
|
6
|
-
import { DeviceUtilities, getParam } from "../engine/engine_utils.js";
|
|
7
|
-
import { delay } from "../engine/engine_utils.js";
|
|
6
|
+
import { delay, DeviceUtilities, getParam } from "../engine/engine_utils.js";
|
|
8
7
|
import { getIconElement } from "../engine/webcomponents/icons.js";
|
|
9
8
|
import { Behaviour } from "./Component.js";
|
|
9
|
+
import { EventList } from "./EventList.js";
|
|
10
10
|
|
|
11
11
|
export const noVoip = "noVoip";
|
|
12
12
|
const debugParam = getParam("debugvoip");
|
|
@@ -75,6 +75,66 @@ export class Voip extends Behaviour {
|
|
|
75
75
|
*/
|
|
76
76
|
debug: boolean = false;
|
|
77
77
|
|
|
78
|
+
private _volume: number = 1;
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Volume for incoming audio streams (0 = silent, 1 = full volume).
|
|
82
|
+
* Changes apply immediately to all active incoming streams.
|
|
83
|
+
*/
|
|
84
|
+
@serializable()
|
|
85
|
+
get volume(): number { return this._volume; }
|
|
86
|
+
set volume(val: number) {
|
|
87
|
+
this._volume = val;
|
|
88
|
+
for (const audio of this._incomingStreams.values()) {
|
|
89
|
+
audio.volume = val;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Get the incoming audio element for a specific user.
|
|
95
|
+
* Use this to route audio through the Web Audio API for spatial audio, effects, or analysis.
|
|
96
|
+
* @param userId The connection ID of the remote user
|
|
97
|
+
* @returns The HTMLAudioElement for this user's stream, or undefined if not connected
|
|
98
|
+
* @example
|
|
99
|
+
* ```ts
|
|
100
|
+
* const audioEl = voip.getAudioElement(userId);
|
|
101
|
+
* if (audioEl) {
|
|
102
|
+
* const audioCtx = new AudioContext();
|
|
103
|
+
* const source = audioCtx.createMediaElementSource(audioEl);
|
|
104
|
+
* const panner = audioCtx.createPanner();
|
|
105
|
+
* source.connect(panner).connect(audioCtx.destination);
|
|
106
|
+
* }
|
|
107
|
+
* ```
|
|
108
|
+
*/
|
|
109
|
+
getAudioElement(userId: string): HTMLAudioElement | undefined {
|
|
110
|
+
return this._incomingStreams.get(userId);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Get all active incoming audio streams as a Map of userId → HTMLAudioElement.
|
|
115
|
+
* Useful for iterating over all connected voice users.
|
|
116
|
+
*/
|
|
117
|
+
get incomingStreams(): ReadonlyMap<string, HTMLAudioElement> {
|
|
118
|
+
return this._incomingStreams;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Threshold for speaking detection (0–255). When a user's audio amplitude exceeds this,
|
|
123
|
+
* they are considered "speaking". Default is 30.
|
|
124
|
+
*/
|
|
125
|
+
@serializable()
|
|
126
|
+
speakingThreshold: number = 30;
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Event fired when a user's speaking state changes.
|
|
130
|
+
* Passes `{ userId: string, isSpeaking: boolean, volume: number }`.
|
|
131
|
+
*/
|
|
132
|
+
@serializable(EventList)
|
|
133
|
+
onSpeakingChanged: EventList = new EventList();
|
|
134
|
+
|
|
135
|
+
private _speakingStates = new Map<string, boolean>();
|
|
136
|
+
private _analysers = new Map<string, { analyser: AnalyserNode, data: Uint8Array, context: AudioContext }>();
|
|
137
|
+
|
|
78
138
|
private _net?: NetworkedStreams;
|
|
79
139
|
private _menubutton?: HTMLElement;
|
|
80
140
|
|
|
@@ -141,12 +201,25 @@ export class Voip extends Behaviour {
|
|
|
141
201
|
this.onEnabledChanged();
|
|
142
202
|
this.updateButton();
|
|
143
203
|
window.removeEventListener("visibilitychange", this.onVisibilityChanged);
|
|
204
|
+
// Clean up analysers
|
|
205
|
+
for (const userId of [...this._analysers.keys()]) {
|
|
206
|
+
this.cleanupAnalyser(userId);
|
|
207
|
+
}
|
|
144
208
|
}
|
|
145
209
|
|
|
146
210
|
/** @internal */
|
|
147
211
|
onDestroy(): void {
|
|
148
212
|
this._menubutton?.remove();
|
|
149
213
|
this._menubutton = undefined;
|
|
214
|
+
// Clean up all streams and analysers
|
|
215
|
+
for (const userId of [...this._analysers.keys()]) {
|
|
216
|
+
this.cleanupAnalyser(userId);
|
|
217
|
+
}
|
|
218
|
+
for (const incoming of this._incomingStreams.values()) {
|
|
219
|
+
disposeStream(incoming.srcObject as MediaStream);
|
|
220
|
+
}
|
|
221
|
+
this._incomingStreams.clear();
|
|
222
|
+
this._speakingStates.clear();
|
|
150
223
|
}
|
|
151
224
|
|
|
152
225
|
/** Set via the mic button (e.g. when the websocket connection closes and rejoins but the user was muted before we don't want to enable VOIP again automatically) */
|
|
@@ -347,10 +420,61 @@ export class Voip extends Behaviour {
|
|
|
347
420
|
disposeStream(incoming.srcObject as MediaStream);
|
|
348
421
|
}
|
|
349
422
|
this._incomingStreams.clear();
|
|
423
|
+
for (const userId of this._analysers.keys()) {
|
|
424
|
+
this.cleanupAnalyser(userId);
|
|
425
|
+
}
|
|
350
426
|
}
|
|
351
427
|
|
|
352
428
|
private _incomingStreams: Map<string, HTMLAudioElement> = new Map();
|
|
353
429
|
|
|
430
|
+
/** @internal */
|
|
431
|
+
update() {
|
|
432
|
+
// Only run speaking detection if someone is listening
|
|
433
|
+
if (!this.onSpeakingChanged || this.onSpeakingChanged.listenerCount <= 0) return;
|
|
434
|
+
|
|
435
|
+
for (const [userId, info] of this._analysers) {
|
|
436
|
+
info.analyser.getByteFrequencyData(info.data as Uint8Array<ArrayBuffer>);
|
|
437
|
+
// Average amplitude
|
|
438
|
+
let sum = 0;
|
|
439
|
+
for (let i = 0; i < info.data.length; i++) sum += info.data[i];
|
|
440
|
+
const avg = sum / info.data.length;
|
|
441
|
+
|
|
442
|
+
const wasSpeaking = this._speakingStates.get(userId) ?? false;
|
|
443
|
+
const isSpeaking = avg > this.speakingThreshold;
|
|
444
|
+
|
|
445
|
+
if (isSpeaking !== wasSpeaking) {
|
|
446
|
+
this._speakingStates.set(userId, isSpeaking);
|
|
447
|
+
this.onSpeakingChanged.invoke({ userId, isSpeaking, volume: avg / 255 });
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
private setupAnalyser(userId: string, _audioElement: HTMLAudioElement, stream: MediaStream) {
|
|
453
|
+
// Only set up if someone is listening or might listen
|
|
454
|
+
if (this._analysers.has(userId)) return;
|
|
455
|
+
try {
|
|
456
|
+
const audioCtx = new AudioContext();
|
|
457
|
+
const source = audioCtx.createMediaStreamSource(stream);
|
|
458
|
+
const analyser = audioCtx.createAnalyser();
|
|
459
|
+
analyser.fftSize = 256;
|
|
460
|
+
source.connect(analyser);
|
|
461
|
+
const data = new Uint8Array(analyser.frequencyBinCount);
|
|
462
|
+
this._analysers.set(userId, { analyser, data, context: audioCtx });
|
|
463
|
+
}
|
|
464
|
+
catch (err) {
|
|
465
|
+
if (this.debug) console.warn("VOIP: Failed to create analyser for", userId, err);
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
private cleanupAnalyser(userId: string) {
|
|
470
|
+
const info = this._analysers.get(userId);
|
|
471
|
+
if (info) {
|
|
472
|
+
info.context.close();
|
|
473
|
+
this._analysers.delete(userId);
|
|
474
|
+
}
|
|
475
|
+
this._speakingStates.delete(userId);
|
|
476
|
+
}
|
|
477
|
+
|
|
354
478
|
private onReceiveStream = (evt: StreamReceivedEvent) => {
|
|
355
479
|
const userId = evt.target.userId;
|
|
356
480
|
const stream = evt.stream;
|
|
@@ -361,7 +485,9 @@ export class Voip extends Behaviour {
|
|
|
361
485
|
this._incomingStreams.set(userId, audioElement);
|
|
362
486
|
}
|
|
363
487
|
audioElement.srcObject = stream;
|
|
488
|
+
audioElement.volume = this._volume;
|
|
364
489
|
audioElement.setAttribute("autoplay", "true");
|
|
490
|
+
this.setupAnalyser(userId, audioElement, stream);
|
|
365
491
|
// for mobile we need to wait for user interaction to play audio. Auto play doesnt work on android when the page is refreshed
|
|
366
492
|
Application.registerWaitForInteraction(() => {
|
|
367
493
|
audioElement?.play().catch((err) => {
|
|
@@ -374,6 +500,7 @@ export class Voip extends Behaviour {
|
|
|
374
500
|
const existing = this._incomingStreams.get(evt.userId);
|
|
375
501
|
disposeStream(existing?.srcObject as MediaStream);
|
|
376
502
|
this._incomingStreams.delete(evt.userId);
|
|
503
|
+
this.cleanupAnalyser(evt.userId);
|
|
377
504
|
}
|
|
378
505
|
|
|
379
506
|
private onEnabledChanged = () => {
|