@hypersoniclabs/helix-sdk 0.1.1 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +954 -2
- package/dist/action-surface.d.ts +182 -0
- package/dist/action-surface.js +1032 -0
- package/dist/action-surface.js.map +1 -0
- package/dist/camera.d.ts +78 -0
- package/dist/camera.js +640 -0
- package/dist/camera.js.map +1 -0
- package/dist/dev-shell/criteria.d.ts +46 -0
- package/dist/dev-shell/criteria.js +98 -0
- package/dist/dev-shell/criteria.js.map +1 -0
- package/dist/dev-shell/index.d.ts +189 -0
- package/dist/dev-shell/index.js +287 -0
- package/dist/dev-shell/index.js.map +1 -0
- package/dist/dev-shell/keys.d.ts +14 -0
- package/dist/dev-shell/keys.js +78 -0
- package/dist/dev-shell/keys.js.map +1 -0
- package/dist/dev-shell/simulator.d.ts +436 -0
- package/dist/dev-shell/simulator.js +1634 -0
- package/dist/dev-shell/simulator.js.map +1 -0
- package/dist/dev-shell/state.d.ts +148 -0
- package/dist/dev-shell/state.js +196 -0
- package/dist/dev-shell/state.js.map +1 -0
- package/dist/device-definition-resolver.d.ts +34 -0
- package/dist/device-definition-resolver.js +172 -0
- package/dist/device-definition-resolver.js.map +1 -0
- package/dist/device-trial.d.ts +3 -0
- package/dist/device-trial.js +132 -0
- package/dist/device-trial.js.map +1 -0
- package/dist/devices.d.ts +228 -0
- package/dist/devices.js +198 -0
- package/dist/devices.js.map +1 -0
- package/dist/dreamerAudio.d.ts +126 -0
- package/dist/dreamerAudio.js +135 -0
- package/dist/dreamerAudio.js.map +1 -0
- package/dist/generated/runtime-contract.d.ts +15529 -0
- package/dist/generated/runtime-contract.js +20470 -0
- package/dist/generated/runtime-contract.js.map +1 -0
- package/dist/index.d.ts +364 -3
- package/dist/index.js +1954 -13
- package/dist/index.js.map +1 -1
- package/dist/interaction-mode.d.ts +172 -0
- package/dist/interaction-mode.js +553 -0
- package/dist/interaction-mode.js.map +1 -0
- package/dist/internal/avatar-pose.d.ts +25 -0
- package/dist/internal/avatar-pose.js +46 -0
- package/dist/internal/avatar-pose.js.map +1 -0
- package/dist/internal/keyed-collection.d.ts +2 -0
- package/dist/internal/keyed-collection.js +12 -0
- package/dist/internal/keyed-collection.js.map +1 -0
- package/dist/internal/vehicle-detach-action.d.ts +2 -0
- package/dist/internal/vehicle-detach-action.js +6 -0
- package/dist/internal/vehicle-detach-action.js.map +1 -0
- package/dist/matchmaking.d.ts +433 -0
- package/dist/matchmaking.js +1487 -0
- package/dist/matchmaking.js.map +1 -0
- package/dist/materials.d.ts +122 -0
- package/dist/materials.js +217 -0
- package/dist/materials.js.map +1 -0
- package/dist/multiplayer-accessors.d.ts +100 -0
- package/dist/multiplayer-accessors.js +200 -0
- package/dist/multiplayer-accessors.js.map +1 -0
- package/dist/multiplayer-contract/avatar-pose.d.ts +112 -0
- package/dist/multiplayer-contract/avatar-pose.js +250 -0
- package/dist/multiplayer-contract/avatar-pose.js.map +1 -0
- package/dist/multiplayer-contract/credential.d.ts +80 -0
- package/dist/multiplayer-contract/credential.js +5 -0
- package/dist/multiplayer-contract/credential.js.map +1 -0
- package/dist/multiplayer-contract/final-rest.d.ts +66 -0
- package/dist/multiplayer-contract/final-rest.js +38 -0
- package/dist/multiplayer-contract/final-rest.js.map +1 -0
- package/dist/multiplayer-contract/index.d.ts +20 -0
- package/dist/multiplayer-contract/index.js +41 -0
- package/dist/multiplayer-contract/index.js.map +1 -0
- package/dist/multiplayer-contract/messages.d.ts +731 -0
- package/dist/multiplayer-contract/messages.js +237 -0
- package/dist/multiplayer-contract/messages.js.map +1 -0
- package/dist/multiplayer-contract/room.d.ts +177 -0
- package/dist/multiplayer-contract/room.js +17 -0
- package/dist/multiplayer-contract/room.js.map +1 -0
- package/dist/multiplayer-contract/state.d.ts +241 -0
- package/dist/multiplayer-contract/state.js +15 -0
- package/dist/multiplayer-contract/state.js.map +1 -0
- package/dist/multiplayer-final-rest.d.ts +32 -0
- package/dist/multiplayer-final-rest.js +217 -0
- package/dist/multiplayer-final-rest.js.map +1 -0
- package/dist/multiplayer.d.ts +624 -0
- package/dist/multiplayer.js +2503 -0
- package/dist/multiplayer.js.map +1 -0
- package/dist/package.json +3 -0
- package/dist/protocol.d.ts +859 -1
- package/dist/protocol.js +117 -5
- package/dist/protocol.js.map +1 -1
- package/dist/runtime-contract/index.d.ts +15 -0
- package/dist/runtime-contract/index.js +12 -0
- package/dist/runtime-contract/index.js.map +1 -0
- package/dist/server-clock.d.ts +25 -0
- package/dist/server-clock.js +88 -0
- package/dist/server-clock.js.map +1 -0
- package/dist/tsconfig.build.tsbuildinfo +1 -1
- package/dist/universal-vehicles.d.ts +111 -0
- package/dist/universal-vehicles.js +2 -0
- package/dist/universal-vehicles.js.map +1 -0
- package/dist/vault.d.ts +562 -0
- package/dist/vault.js +365 -0
- package/dist/vault.js.map +1 -0
- package/dist/vehicle-specialty-placement.d.ts +28 -0
- package/dist/vehicle-specialty-placement.js +154 -0
- package/dist/vehicle-specialty-placement.js.map +1 -0
- package/dist/vehicle-specialty-transport.d.ts +78 -0
- package/dist/vehicle-specialty-transport.js +183 -0
- package/dist/vehicle-specialty-transport.js.map +1 -0
- package/dist/voice.d.ts +229 -0
- package/dist/voice.js +1348 -0
- package/dist/voice.js.map +1 -0
- package/generated/runtime-contract/helix-runtime-contract.json +13332 -0
- package/generated/runtime-contract/known-answer.json +20471 -0
- package/generated/runtime-contract/known-answer.schema.json +630 -0
- package/generated/runtime-contract/runtime-contract.schema.json +607 -0
- package/generated/runtime-contract/unreal/HelixRuntimeContract.generated.h +908 -0
- package/package.json +44 -5
package/dist/voice.js
ADDED
|
@@ -0,0 +1,1348 @@
|
|
|
1
|
+
// Helix.voice — the client half of HELIX voice chat (LiveKit). Wraps livekit-client the same way
|
|
2
|
+
// multiplayer.ts wraps @colyseus/sdk: worlds never import it directly, and the heavy client is
|
|
3
|
+
// dynamically imported only when a world actually joins voice — no-voice bundles never pull it in
|
|
4
|
+
// (the golden rule). Flow: POST /api/v1/instant-worlds/:worldId/voice-token { roomId } with the
|
|
5
|
+
// world_session token → { livekitUrl, token } → Room.connect → mic capture per the shell-owned mic
|
|
6
|
+
// mode (ptt / open / muted). Voice is a PARALLEL media plane: the Colyseus room is never involved;
|
|
7
|
+
// the server pairs the voice room to the joined room instance (v:{worldId}:{roomId}), so voice maps
|
|
8
|
+
// 1:1 to game instances and the LiveKit identity is the platform userId (= PlayerState.userId, v18).
|
|
9
|
+
//
|
|
10
|
+
// CHANNELS: every participant is in exactly ONE voice space — the ambient world or a
|
|
11
|
+
// named channel (EXCLUSIVE semantics; radios/phone calls are world-code patterns over setChannel).
|
|
12
|
+
// Membership rides LiveKit participant attributes (CHANNEL_ATTR), so it needs no game-state sync and
|
|
13
|
+
// works with dynamic ids; delivery from out-of-channel participants is SFU-paused exactly like a local
|
|
14
|
+
// mute. Render policy (proximity vs flat) is the engine's job (CharacterVoice reads channelOf) — this
|
|
15
|
+
// module only gates WHO is audible, never how loud.
|
|
16
|
+
//
|
|
17
|
+
// SPATIAL: opt-in directional rendering (manifest voice.spatial) — the engine pushes speaker positions
|
|
18
|
+
// (setParticipantPosition) and the camera pose (setListenerPose); each position inserts an HRTF PannerNode
|
|
19
|
+
// into that track's playback chain via setWebAudioPlugins (@internal/@experimental in livekit-client —
|
|
20
|
+
// feature-detected; absent ⇒ today's non-spatial render). Direction ONLY (rolloffFactor 0): loudness keeps
|
|
21
|
+
// flowing through setVolume/outputVolume untouched. Channels never spatialize (the engine passes null).
|
|
22
|
+
// The LiveKit participant attribute carrying the exclusive channel id ('' / absent = the ambient
|
|
23
|
+
// space). Free-form ids — declared channels get a render policy from the manifest voice block;
|
|
24
|
+
// dynamic ids (e.g. a per-call channel) render flat by convention.
|
|
25
|
+
const CHANNEL_ATTR = 'helix:channel';
|
|
26
|
+
/** Platform default for KeyboardEvent.key while push-to-talk is active. */
|
|
27
|
+
export const DEFAULT_PUSH_TO_TALK_KEY = 'n';
|
|
28
|
+
// Remote playback gain composes two factors: the per-participant base (0..1 proximity, driven
|
|
29
|
+
// per-frame by the engine from synced positions) × the player's master outputVolume (0..2, tablet
|
|
30
|
+
// slider). The product is capped at MAX_VOLUME; boosting above 1 needs the Room's webAudioMix.
|
|
31
|
+
const MAX_VOLUME = 4;
|
|
32
|
+
// Mic pipeline cadences: the analyser samples every LEVEL_TICK_MS (drives the gate), the shell meter
|
|
33
|
+
// gets every other sample (~10 Hz). GATE_HOLD_MS keeps the gate open through word gaps so a manual
|
|
34
|
+
// sensitivity cut doesn't chop mid-sentence. RMS_METER_SCALE maps speech RMS (~0.05–0.25) onto a
|
|
35
|
+
// readable 0..1 meter — the sensitivity threshold shares these units, so slider and meter line up.
|
|
36
|
+
const LEVEL_TICK_MS = 50;
|
|
37
|
+
const GATE_HOLD_MS = 400;
|
|
38
|
+
const RMS_METER_SCALE = 4;
|
|
39
|
+
// Receiver-side face animation gets a deliberately tiny, privacy-preserving signal: one bounded
|
|
40
|
+
// RMS scalar per audible participant. Raw samples never leave the WebAudio graph, are never stored,
|
|
41
|
+
// and are never sent over Colyseus/LiveKit. One shared ~30 Hz timer keeps onset below a video frame
|
|
42
|
+
// plus one analysis tick without creating per-track callback storms.
|
|
43
|
+
const REMOTE_LEVEL_TICK_MS = 33;
|
|
44
|
+
const REMOTE_LEVEL_FFT_SIZE = 256;
|
|
45
|
+
const REMOTE_LEVEL_MAX_TRACKS = 32;
|
|
46
|
+
/** Speech band edges in Hz, closed at the top: <350, 350-700, 700-1400, 1400-2800, 2800-5600. */
|
|
47
|
+
const SPEECH_BAND_EDGES = [0, 350, 700, 1400, 2800, 5600];
|
|
48
|
+
const SILENT_FEATURES = { rms: 0, zeroCrossingRate: 0, bands: [0, 0, 0, 0, 0] };
|
|
49
|
+
/**
|
|
50
|
+
* Time-domain RMS plus zero-crossing rate over one analyser buffer. Shared by the playback lane and
|
|
51
|
+
* the microphone lane so a viseme solver sees the same units from either side.
|
|
52
|
+
*/
|
|
53
|
+
function timeDomainEnergy(samples) {
|
|
54
|
+
let sum = 0;
|
|
55
|
+
let crossings = 0;
|
|
56
|
+
for (let i = 0; i < samples.length; i++) {
|
|
57
|
+
const sample = samples[i];
|
|
58
|
+
sum += sample * sample;
|
|
59
|
+
if (i > 0 && (sample >= 0) !== (samples[i - 1] >= 0))
|
|
60
|
+
crossings++;
|
|
61
|
+
}
|
|
62
|
+
return {
|
|
63
|
+
rms: Math.min(1, Math.sqrt(sum / samples.length) * RMS_METER_SCALE),
|
|
64
|
+
zeroCrossingRate: crossings / Math.max(1, samples.length - 1),
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
/** Mean magnitude per fixed speech band, normalized to 0..1. The bin width follows the analyser. */
|
|
68
|
+
function speechBands(spectrum, sampleRate, fftSize) {
|
|
69
|
+
const hzPerBin = sampleRate / fftSize;
|
|
70
|
+
return SPEECH_BAND_EDGES.slice(0, 5).map((lowHz, index) => {
|
|
71
|
+
const start = Math.max(0, Math.floor(lowHz / hzPerBin));
|
|
72
|
+
const end = Math.min(spectrum.length, Math.max(start + 1, Math.ceil(SPEECH_BAND_EDGES[index + 1] / hzPerBin)));
|
|
73
|
+
let total = 0;
|
|
74
|
+
for (let bin = start; bin < end; bin++)
|
|
75
|
+
total += spectrum[bin];
|
|
76
|
+
return Math.min(1, total / Math.max(1, end - start) / 255);
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
export class HelixVoice {
|
|
80
|
+
host;
|
|
81
|
+
room = null;
|
|
82
|
+
events = null;
|
|
83
|
+
connected = false;
|
|
84
|
+
micMode = 'ptt';
|
|
85
|
+
captured = false; // the mic pipeline exists (published; may be gain-muted)
|
|
86
|
+
capturePromise = null;
|
|
87
|
+
permission = 'prompt';
|
|
88
|
+
requestedDeviceId = null;
|
|
89
|
+
requestedDeviceLabel = null; // the portable identifier — ids are origin-scoped, labels aren't
|
|
90
|
+
activeDeviceId = null;
|
|
91
|
+
// The outgoing mic pipeline: source → volumeGain (mic volume) → compressor (loudness leveling) →
|
|
92
|
+
// analyser (meter; PRE-gate so the meter moves even while muted/gated) → gateGain (sensitivity cut)
|
|
93
|
+
// → muteGain (mode/PTT, instant) → destination (the ONE track we publish). Device switches swap the
|
|
94
|
+
// source node; nothing republishes.
|
|
95
|
+
audioCtx = null;
|
|
96
|
+
micStream = null;
|
|
97
|
+
micSource = null;
|
|
98
|
+
volumeGain = null;
|
|
99
|
+
compressor = null;
|
|
100
|
+
analyser = null;
|
|
101
|
+
analyserBuf = null;
|
|
102
|
+
analyserSpectrum = null;
|
|
103
|
+
gateGain = null;
|
|
104
|
+
muteGain = null;
|
|
105
|
+
levelTimer = null;
|
|
106
|
+
levelTick = 0;
|
|
107
|
+
micLevel = 0;
|
|
108
|
+
gateOpenUntil = 0;
|
|
109
|
+
micVolume = 1;
|
|
110
|
+
autoSensitivity = true;
|
|
111
|
+
sensitivity = 0.15;
|
|
112
|
+
outputVolume = 1;
|
|
113
|
+
participantBase = new Map(); // last base (proximity) gain per userId
|
|
114
|
+
channelId = null; // the exclusive voice channel (null = ambient); survives teardown like mic prefs
|
|
115
|
+
// How the LIVE voice room was joined — kept so a multiplayer room hop can re-pair voice with the same
|
|
116
|
+
// per-join options (see detachForRoomSwitch). Null when not joined. `followsRoom` is false when the world
|
|
117
|
+
// pinned an explicit roomId: that voice room is not paired to the multiplayer room and must not follow it.
|
|
118
|
+
pairing = null;
|
|
119
|
+
// A join still in flight (token fetch, module import, connect, mic capture) and its generation. join() is
|
|
120
|
+
// several awaits long and resolves the room id at its very first line, so a room hop that lands mid-join
|
|
121
|
+
// would otherwise let it finish against the room being LEFT. Bumping joinSeq supersedes it: every await in
|
|
122
|
+
// join() re-checks its generation and bails out without touching state a newer join may now own.
|
|
123
|
+
pendingJoin = null;
|
|
124
|
+
joinSeq = 0;
|
|
125
|
+
// Spatial (directional) rendering. The mix AudioContext is OURS — handed to LiveKit via webAudioMix —
|
|
126
|
+
// so per-track PannerNodes live on the same graph LiveKit plays through. A panner exists iff the
|
|
127
|
+
// engine pushed a position for that user; all of this is session state (cleared on teardown).
|
|
128
|
+
mixCtx = null;
|
|
129
|
+
spatialTracks = new Map(); // subscribed remote audio track per userId
|
|
130
|
+
panners = new Map();
|
|
131
|
+
remoteAnalysers = new Map();
|
|
132
|
+
remoteLevelListeners = new Set();
|
|
133
|
+
remoteFeatureListeners = new Set();
|
|
134
|
+
localLevelListeners = new Set();
|
|
135
|
+
localFeatureListeners = new Set();
|
|
136
|
+
localWasAudible = false;
|
|
137
|
+
remoteLevelTimer = null;
|
|
138
|
+
positions = new Map(); // last engine-pushed position per userId
|
|
139
|
+
listenerPose = null;
|
|
140
|
+
devices = [];
|
|
141
|
+
muted = new Set(); // locally muted platform userIds (LiveKit identities)
|
|
142
|
+
speaking = new Set();
|
|
143
|
+
audioBlocked = false;
|
|
144
|
+
error = null;
|
|
145
|
+
available = true;
|
|
146
|
+
pttKey = DEFAULT_PUSH_TO_TALK_KEY;
|
|
147
|
+
pttHeld = false;
|
|
148
|
+
stateSig = null;
|
|
149
|
+
audioRoot = null; // hidden host for attached <audio> elements
|
|
150
|
+
unlockInstalled = false;
|
|
151
|
+
speakingListeners = new Set();
|
|
152
|
+
participantListeners = new Set();
|
|
153
|
+
constructor(host) {
|
|
154
|
+
this.host = host;
|
|
155
|
+
}
|
|
156
|
+
// Join this session's voice room. Requires a joined multiplayer room (the voice room pairs to its
|
|
157
|
+
// instance id). Resolves false — after publishing available:false to the shell — when the platform
|
|
158
|
+
// has voice off for this world/environment (/voice-token 404, fail-soft); throws on real errors.
|
|
159
|
+
async join(options = {}) {
|
|
160
|
+
if (!this.host.isInitialized())
|
|
161
|
+
throw new Error('Helix: call Helix.init() before Helix.voice.join()');
|
|
162
|
+
if (this.room)
|
|
163
|
+
throw new Error('Helix.voice: already in a voice room — call leave() first');
|
|
164
|
+
const token = this.host.getToken();
|
|
165
|
+
if (!token)
|
|
166
|
+
throw new Error('Helix.voice: voice requires login (no session)');
|
|
167
|
+
const worldId = this.host.getWorldId();
|
|
168
|
+
if (!worldId)
|
|
169
|
+
throw new Error('Helix.voice: no world id — is this world running outside a HELIX shell?');
|
|
170
|
+
const roomId = options.roomId ?? this.host.getRoomId();
|
|
171
|
+
if (!roomId)
|
|
172
|
+
throw new Error('Helix.voice: no room to pair with — call Helix.multiplayer.joinRoom() first');
|
|
173
|
+
const apiBase = options.apiBaseUrl?.replace(/\/$/, '') ?? this.host.resolveApiBase(token);
|
|
174
|
+
if (!apiBase)
|
|
175
|
+
throw new Error('Helix.voice: cannot resolve the API base URL');
|
|
176
|
+
this.pttKey = options.pttKey?.toLowerCase() || DEFAULT_PUSH_TO_TALK_KEY;
|
|
177
|
+
if (options.channel !== undefined)
|
|
178
|
+
this.channelId = options.channel;
|
|
179
|
+
const seq = ++this.joinSeq;
|
|
180
|
+
const pairing = {
|
|
181
|
+
followsRoom: options.roomId === undefined,
|
|
182
|
+
options: {
|
|
183
|
+
...(options.apiBaseUrl !== undefined ? { apiBaseUrl: options.apiBaseUrl } : {}),
|
|
184
|
+
...(options.tokenPath !== undefined ? { tokenPath: options.tokenPath } : {}),
|
|
185
|
+
...(options.pttKey !== undefined ? { pttKey: options.pttKey } : {}),
|
|
186
|
+
},
|
|
187
|
+
};
|
|
188
|
+
this.pendingJoin = pairing;
|
|
189
|
+
try {
|
|
190
|
+
return await this.connectVoice(options, { apiBase, worldId, token, roomId }, seq, pairing);
|
|
191
|
+
}
|
|
192
|
+
finally {
|
|
193
|
+
if (this.pendingJoin === pairing)
|
|
194
|
+
this.pendingJoin = null;
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
// The awaiting half of join(). `superseded()` is re-checked after EVERY await: a leave() or a room hop
|
|
198
|
+
// (detachForRoomSwitch) in the meantime means this join must end quietly — resolving false, the same "voice is
|
|
199
|
+
// not on" answer as an unconfigured environment — and must not write to state a newer join may already own.
|
|
200
|
+
async connectVoice(options, target, seq, pairing) {
|
|
201
|
+
const { apiBase, worldId, token, roomId } = target;
|
|
202
|
+
const superseded = () => seq !== this.joinSeq;
|
|
203
|
+
// ORDERING RULE for everything below: after an await, check superseded() BEFORE writing to `this` — in the
|
|
204
|
+
// straight path and in every catch. A superseded join's failure is about a room nobody wants any more; it
|
|
205
|
+
// must neither surface as this join's error nor overwrite state the join that replaced it now owns.
|
|
206
|
+
let grant;
|
|
207
|
+
try {
|
|
208
|
+
grant = await this.requestVoiceToken(apiBase, worldId, token, roomId, options.tokenPath);
|
|
209
|
+
}
|
|
210
|
+
catch (e) {
|
|
211
|
+
if (superseded())
|
|
212
|
+
return false;
|
|
213
|
+
throw e;
|
|
214
|
+
}
|
|
215
|
+
if (superseded())
|
|
216
|
+
return false;
|
|
217
|
+
if (!grant) {
|
|
218
|
+
// Voice is not configured for this environment / world — report and stay silent, never throw:
|
|
219
|
+
// a world author's Helix.voice.join() must be safe to call unconditionally.
|
|
220
|
+
this.available = false;
|
|
221
|
+
this.publishState();
|
|
222
|
+
return false;
|
|
223
|
+
}
|
|
224
|
+
this.available = true;
|
|
225
|
+
const lk = (await import('livekit-client'));
|
|
226
|
+
if (superseded())
|
|
227
|
+
return false;
|
|
228
|
+
// Own the mix context so spatial PannerNodes are created on the SAME graph LiveKit plays through
|
|
229
|
+
// (no WebAudio ⇒ LiveKit's internal context; spatial then silently no-ops).
|
|
230
|
+
const mixCtx = typeof AudioContext !== 'undefined' ? new AudioContext() : null;
|
|
231
|
+
this.mixCtx = mixCtx;
|
|
232
|
+
const room = new lk.Room({
|
|
233
|
+
// webAudioMix routes remote playback through a GainNode, so setVolume can boost above 1 (up to
|
|
234
|
+
// MAX_VOLUME) — the default HTMLAudioElement playback path hard-caps volume at 1.0.
|
|
235
|
+
webAudioMix: mixCtx ? { audioContext: mixCtx } : true,
|
|
236
|
+
// One mono Opus track per speaker: 96 kbps (vs the 48 kbps `music` default) is audibly better
|
|
237
|
+
// and costs ≤~1 Mbps downstream even at the 12-player cap.
|
|
238
|
+
publishDefaults: { audioPreset: lk.AudioPresets.musicHighQuality },
|
|
239
|
+
});
|
|
240
|
+
this.events = lk.RoomEvent;
|
|
241
|
+
this.room = room;
|
|
242
|
+
this.wireRoomEvents(room);
|
|
243
|
+
try {
|
|
244
|
+
await room.connect(grant.livekitUrl, grant.token);
|
|
245
|
+
}
|
|
246
|
+
catch (e) {
|
|
247
|
+
// Superseded mid-connect: the teardown that superseded us already released this room and its context.
|
|
248
|
+
if (superseded())
|
|
249
|
+
return false;
|
|
250
|
+
this.room = null;
|
|
251
|
+
this.events = null;
|
|
252
|
+
void mixCtx?.close().catch(() => undefined);
|
|
253
|
+
this.mixCtx = null;
|
|
254
|
+
throw new Error(`Helix.voice: could not connect — ${e instanceof Error ? e.message : String(e)}`);
|
|
255
|
+
}
|
|
256
|
+
if (superseded()) {
|
|
257
|
+
// Connected to a voice room nobody wants any more. this.room is no longer ours to touch (torn down, or a
|
|
258
|
+
// newer join's) — just make sure THIS connection is closed.
|
|
259
|
+
void room.disconnect().catch(() => undefined);
|
|
260
|
+
return false;
|
|
261
|
+
}
|
|
262
|
+
this.connected = true;
|
|
263
|
+
this.pairing = pairing;
|
|
264
|
+
// Reapply the stored channel (set pre-join or surviving a previous session). Fail-soft: an old
|
|
265
|
+
// backend whose grant lacks canUpdateOwnMetadata rejects setAttributes — fall back to ambient
|
|
266
|
+
// rather than failing the whole join.
|
|
267
|
+
if (this.channelId) {
|
|
268
|
+
try {
|
|
269
|
+
await room.localParticipant.setAttributes({ [CHANNEL_ATTR]: this.channelId });
|
|
270
|
+
}
|
|
271
|
+
catch (e) {
|
|
272
|
+
// Superseded first: this is the OLD session's channel write failing (typically because the hop
|
|
273
|
+
// disconnected it). Resetting the channel here would knock the NEW session back to ambient and pin a
|
|
274
|
+
// spurious error on it.
|
|
275
|
+
if (superseded())
|
|
276
|
+
return false;
|
|
277
|
+
this.channelId = null;
|
|
278
|
+
this.error = `voice channel unavailable: ${e instanceof Error ? e.message : String(e)}`;
|
|
279
|
+
}
|
|
280
|
+
if (superseded())
|
|
281
|
+
return false;
|
|
282
|
+
}
|
|
283
|
+
this.ensureAudioRoot();
|
|
284
|
+
this.attachInputListeners();
|
|
285
|
+
await this.applyMicMode();
|
|
286
|
+
if (superseded())
|
|
287
|
+
return false;
|
|
288
|
+
await this.refreshDevices();
|
|
289
|
+
if (superseded())
|
|
290
|
+
return false;
|
|
291
|
+
this.applyAudibility();
|
|
292
|
+
// Consumers such as CharacterVoice outlive a transient room disconnect. Recreate their local
|
|
293
|
+
// analysis graph on the new LiveKit mix context without requiring an unsubscribe/resubscribe.
|
|
294
|
+
this.installRemoteAnalysers();
|
|
295
|
+
this.startRemoteLevelLoop();
|
|
296
|
+
this.publishState();
|
|
297
|
+
return true;
|
|
298
|
+
}
|
|
299
|
+
// Leave the voice room (mic released, audio detached). Safe to call when not joined.
|
|
300
|
+
async leave() {
|
|
301
|
+
this.joinSeq += 1; // supersede a join still in flight — it must not connect (or open the mic) after a leave
|
|
302
|
+
const room = this.room;
|
|
303
|
+
this.teardown();
|
|
304
|
+
if (room)
|
|
305
|
+
await room.disconnect();
|
|
306
|
+
}
|
|
307
|
+
// The voice half of a multiplayer room hop (Helix.multiplayer.switchRoom). A voice room is paired 1:1 to a
|
|
308
|
+
// room INSTANCE (the backend namespaces it v:{worldId}:{roomId}), so when the player changes rooms the old
|
|
309
|
+
// voice room is simply the wrong one: leave it now, and hand back a `resume` that joins the voice room of
|
|
310
|
+
// whichever multiplayer room is current when it is called. The re-join reuses THIS join's per-join options
|
|
311
|
+
// (push-to-talk key, API base, token path); it deliberately does NOT replay the join-time `channel`, because
|
|
312
|
+
// the live channel — along with mic mode, mic tuning and local mutes — already survives teardown and may
|
|
313
|
+
// have been changed since. Resolves null (and touches nothing) when voice is not joined, or was joined with
|
|
314
|
+
// an explicit `roomId`: that voice room was never following the multiplayer room, so a hop leaves it alone.
|
|
315
|
+
//
|
|
316
|
+
// A join STILL IN FLIGHT counts as joined: it resolved its room id before the hop and would finish against
|
|
317
|
+
// the room being left. It is cancelled — leave() supersedes it, so the world's pending join() resolves false
|
|
318
|
+
// and never connects or opens the mic — and the returned `resume` re-targets it at the new room.
|
|
319
|
+
//
|
|
320
|
+
// If the hop then FAILS, resume is simply never called and voice stays left: there is no room to pair with.
|
|
321
|
+
async detachForRoomSwitch() {
|
|
322
|
+
const pairing = this.pairing ?? this.pendingJoin;
|
|
323
|
+
if (!pairing || !pairing.followsRoom)
|
|
324
|
+
return null;
|
|
325
|
+
// leave() tears down BEFORE it disconnects, so by the time a disconnect can fail voice has already left
|
|
326
|
+
// locally. That failure must not cost the resume closure — losing it here would mean voice silently never
|
|
327
|
+
// comes back after the hop — so it is logged and the hand-off continues.
|
|
328
|
+
try {
|
|
329
|
+
await this.leave();
|
|
330
|
+
}
|
|
331
|
+
catch (e) {
|
|
332
|
+
console.warn('[Helix] voice: the old voice room did not disconnect cleanly during a room switch —', e instanceof Error ? e.message : String(e));
|
|
333
|
+
}
|
|
334
|
+
return () => this.join(pairing.options);
|
|
335
|
+
}
|
|
336
|
+
/** The shell-owned settings push (helix:voice-settings) — also callable directly by a world. */
|
|
337
|
+
async applySettings(settings) {
|
|
338
|
+
const next = new Set(settings.mutedUserIds);
|
|
339
|
+
for (const id of [...this.muted])
|
|
340
|
+
if (!next.has(id))
|
|
341
|
+
this.setPlayerMuted(id, false);
|
|
342
|
+
for (const id of next)
|
|
343
|
+
this.setPlayerMuted(id, true);
|
|
344
|
+
if (typeof settings.micVolume === 'number')
|
|
345
|
+
this.setMicVolume(settings.micVolume);
|
|
346
|
+
if (typeof settings.outputVolume === 'number')
|
|
347
|
+
this.setOutputVolume(settings.outputVolume);
|
|
348
|
+
if (typeof settings.micAutoSensitivity === 'boolean')
|
|
349
|
+
this.autoSensitivity = settings.micAutoSensitivity;
|
|
350
|
+
if (typeof settings.micSensitivity === 'number') {
|
|
351
|
+
this.sensitivity = Math.max(0, Math.min(settings.micSensitivity, 1));
|
|
352
|
+
}
|
|
353
|
+
if (settings.micDeviceId && settings.micDeviceId !== this.requestedDeviceId) {
|
|
354
|
+
await this.setDevice(settings.micDeviceId, settings.micDeviceLabel);
|
|
355
|
+
}
|
|
356
|
+
else if (settings.micDeviceId && settings.micDeviceLabel) {
|
|
357
|
+
this.requestedDeviceLabel = settings.micDeviceLabel; // same id re-pushed with a label a prior shell omitted
|
|
358
|
+
}
|
|
359
|
+
await this.setMicMode(settings.micMode);
|
|
360
|
+
}
|
|
361
|
+
/** Outgoing mic gain (0..2, Discord-style 0–200%). Applies live when captured. */
|
|
362
|
+
setMicVolume(volume) {
|
|
363
|
+
this.micVolume = Math.max(0, Math.min(volume, 2));
|
|
364
|
+
if (this.volumeGain)
|
|
365
|
+
this.volumeGain.gain.value = this.micVolume;
|
|
366
|
+
}
|
|
367
|
+
/** Incoming voice master gain (0..2, Discord-style 0–200%). Reapplies to every participant live. */
|
|
368
|
+
setOutputVolume(volume) {
|
|
369
|
+
this.outputVolume = Math.max(0, Math.min(volume, 2));
|
|
370
|
+
this.applyVolumes();
|
|
371
|
+
}
|
|
372
|
+
// Switch the mic mode. All three keep the track published once captured — publish/unpublish is slow,
|
|
373
|
+
// mute is instant: muted = track muted; open = unmuted; ptt = muted at rest, unmuted while held.
|
|
374
|
+
async setMicMode(mode) {
|
|
375
|
+
this.micMode = mode;
|
|
376
|
+
if (this.room && this.connected) {
|
|
377
|
+
if (mode === 'open')
|
|
378
|
+
await this.ensureMic(true);
|
|
379
|
+
else if (mode === 'ptt') {
|
|
380
|
+
// Capture up-front so the first key-press is instant; the capture starts unmuted, so mute
|
|
381
|
+
// immediately unless the key is already held.
|
|
382
|
+
if (!this.captured)
|
|
383
|
+
await this.ensureMic(true);
|
|
384
|
+
if (!this.pttHeld)
|
|
385
|
+
await this.setMicEnabled(false);
|
|
386
|
+
}
|
|
387
|
+
else if (this.captured)
|
|
388
|
+
await this.setMicEnabled(false);
|
|
389
|
+
}
|
|
390
|
+
this.publishState();
|
|
391
|
+
}
|
|
392
|
+
getMicMode() {
|
|
393
|
+
return this.micMode;
|
|
394
|
+
}
|
|
395
|
+
// Push-to-talk gate shared by the programmatic pad/touch leg and the keyboard listener below.
|
|
396
|
+
// Only 'ptt' mode opens the mic, and blur/visibility safeties release either kind of hold.
|
|
397
|
+
setPttDown(down) {
|
|
398
|
+
if (!down) {
|
|
399
|
+
this.releasePtt();
|
|
400
|
+
return;
|
|
401
|
+
}
|
|
402
|
+
if (this.micMode !== 'ptt' || this.pttHeld)
|
|
403
|
+
return;
|
|
404
|
+
// A press inside the world IS a user gesture — the one moment a suspended pipeline can start.
|
|
405
|
+
this.resumeAudio();
|
|
406
|
+
this.pttHeld = true;
|
|
407
|
+
if (this.captured) {
|
|
408
|
+
void this.setMicEnabled(true);
|
|
409
|
+
this.publishState();
|
|
410
|
+
return;
|
|
411
|
+
}
|
|
412
|
+
void this.captureForPtt();
|
|
413
|
+
}
|
|
414
|
+
async captureForPtt() {
|
|
415
|
+
await this.ensureMic(true);
|
|
416
|
+
await this.setMicEnabled(this.shouldTransmit());
|
|
417
|
+
this.publishState();
|
|
418
|
+
}
|
|
419
|
+
// Ask for a specific input device. The published track never changes — we capture the new device
|
|
420
|
+
// and swap the pipeline's SOURCE node. Falls back silently (keeping the current device) when the
|
|
421
|
+
// switch fails — the published activeDeviceId always reports what is REALLY in use. The optional
|
|
422
|
+
// label is the cross-world fallback: deviceIds are minted per origin/session, so a stale saved id
|
|
423
|
+
// re-resolves by label against THIS context's device list (reconcileDeviceLabel).
|
|
424
|
+
async setDevice(deviceId, label) {
|
|
425
|
+
this.requestedDeviceId = deviceId;
|
|
426
|
+
this.requestedDeviceLabel = label ?? null;
|
|
427
|
+
if (this.captured) {
|
|
428
|
+
await this.swapSource(deviceId);
|
|
429
|
+
await this.refreshDevices();
|
|
430
|
+
await this.reconcileDeviceLabel();
|
|
431
|
+
}
|
|
432
|
+
this.publishState();
|
|
433
|
+
}
|
|
434
|
+
// Capture `deviceId` (exact) and swap the pipeline's source node onto it. False when the browser
|
|
435
|
+
// refuses (unplugged / stale origin-scoped id) — callers fall back by label or keep the current device.
|
|
436
|
+
async swapSource(deviceId) {
|
|
437
|
+
if (!this.audioCtx || !this.volumeGain)
|
|
438
|
+
return false;
|
|
439
|
+
try {
|
|
440
|
+
const stream = await navigator.mediaDevices.getUserMedia({
|
|
441
|
+
audio: { deviceId: { exact: deviceId }, echoCancellation: true, noiseSuppression: true, autoGainControl: true },
|
|
442
|
+
});
|
|
443
|
+
this.micSource?.disconnect();
|
|
444
|
+
this.micStream?.getTracks().forEach((t) => t.stop());
|
|
445
|
+
this.micStream = stream;
|
|
446
|
+
this.micSource = this.audioCtx.createMediaStreamSource(stream);
|
|
447
|
+
this.micSource.connect(this.volumeGain);
|
|
448
|
+
this.activeDeviceId = stream.getAudioTracks()[0]?.getSettings?.().deviceId ?? deviceId;
|
|
449
|
+
return true;
|
|
450
|
+
}
|
|
451
|
+
catch {
|
|
452
|
+
return false;
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
// The saved deviceId is origin-scoped (minted where the shell saved it, not necessarily here), so
|
|
456
|
+
// after any capture verify the ACTIVE device against the saved label — the portable identifier —
|
|
457
|
+
// and re-target the source when they disagree. Labels need permission: runs after refreshDevices.
|
|
458
|
+
async reconcileDeviceLabel() {
|
|
459
|
+
const label = this.requestedDeviceLabel;
|
|
460
|
+
if (!label || !this.captured)
|
|
461
|
+
return;
|
|
462
|
+
const active = this.devices.find((d) => d.deviceId === this.activeDeviceId);
|
|
463
|
+
if (!active || active.label === label)
|
|
464
|
+
return;
|
|
465
|
+
const target = this.devices.find((d) => d.deviceId && d.label === label);
|
|
466
|
+
if (target && target.deviceId !== this.activeDeviceId)
|
|
467
|
+
await this.swapSource(target.deviceId);
|
|
468
|
+
}
|
|
469
|
+
// Switch the EXCLUSIVE voice channel: you hear/reach only your own channel; null returns to the
|
|
470
|
+
// ambient space. Ids are free-form (declared channels carry a manifest render policy; dynamic ids
|
|
471
|
+
// like a per-call channel render flat). Safe pre-join — stored and applied when join() connects.
|
|
472
|
+
// Throws on a server rejection (grant without canUpdateOwnMetadata) and reverts, so local and
|
|
473
|
+
// replicated membership can never disagree.
|
|
474
|
+
async setChannel(channelId) {
|
|
475
|
+
const prev = this.channelId;
|
|
476
|
+
this.channelId = channelId;
|
|
477
|
+
if (this.room && this.connected) {
|
|
478
|
+
try {
|
|
479
|
+
await this.room.localParticipant.setAttributes({ [CHANNEL_ATTR]: channelId ?? '' });
|
|
480
|
+
}
|
|
481
|
+
catch (e) {
|
|
482
|
+
this.channelId = prev;
|
|
483
|
+
throw new Error(`Helix.voice: setChannel failed — ${e instanceof Error ? e.message : String(e)}`);
|
|
484
|
+
}
|
|
485
|
+
this.applyAudibility();
|
|
486
|
+
this.emitSpeaking();
|
|
487
|
+
}
|
|
488
|
+
this.publishState();
|
|
489
|
+
}
|
|
490
|
+
/** The local player's current exclusive voice channel (null = the ambient space). */
|
|
491
|
+
channel() {
|
|
492
|
+
return this.channelId;
|
|
493
|
+
}
|
|
494
|
+
/** The channel a remote participant is in (null = ambient; engine render policy reads this). */
|
|
495
|
+
channelOf(userId) {
|
|
496
|
+
const attr = this.findParticipant(userId)?.attributes?.[CHANNEL_ATTR];
|
|
497
|
+
return attr ? attr : null;
|
|
498
|
+
}
|
|
499
|
+
// Locally mute/unmute one player (by platform userId). Pauses SFU delivery of their audio to this
|
|
500
|
+
// client only — bandwidth-free, invisible to them and to everyone else.
|
|
501
|
+
setPlayerMuted(userId, muted) {
|
|
502
|
+
if (muted)
|
|
503
|
+
this.muted.add(userId);
|
|
504
|
+
else
|
|
505
|
+
this.muted.delete(userId);
|
|
506
|
+
if (this.room) {
|
|
507
|
+
const participant = this.findParticipant(userId);
|
|
508
|
+
if (participant) {
|
|
509
|
+
const audible = this.participantAudible(participant);
|
|
510
|
+
participant.audioTrackPublications.forEach((pub) => pub.setEnabled(audible));
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
this.publishState();
|
|
514
|
+
}
|
|
515
|
+
mutedUserIds() {
|
|
516
|
+
return [...this.muted];
|
|
517
|
+
}
|
|
518
|
+
/** Playback gain for one remote player (engine proximity hook; 0 = silent, 1 = as-sent). */
|
|
519
|
+
setParticipantVolume(userId, volume) {
|
|
520
|
+
const base = Math.max(0, Math.min(volume, MAX_VOLUME));
|
|
521
|
+
this.participantBase.set(userId, base);
|
|
522
|
+
const participant = this.room ? this.findParticipant(userId) : null;
|
|
523
|
+
participant?.setVolume(Math.min(base * this.outputVolume, MAX_VOLUME));
|
|
524
|
+
}
|
|
525
|
+
/**
|
|
526
|
+
* Spatialize one remote player's voice: heard from `position` (world coords) relative to the listener
|
|
527
|
+
* pose. Direction ONLY — loudness still flows through setParticipantVolume × outputVolume. null returns
|
|
528
|
+
* them to center-panned render (the engine passes null for channel — radio/phone — voices). Engine
|
|
529
|
+
* proximity hook, called per-frame; silently no-ops where the client audio hook is unavailable.
|
|
530
|
+
*/
|
|
531
|
+
setParticipantPosition(userId, position) {
|
|
532
|
+
if (!position) {
|
|
533
|
+
this.positions.delete(userId);
|
|
534
|
+
if (this.panners.delete(userId))
|
|
535
|
+
this.applyTrackPlugins(userId);
|
|
536
|
+
return;
|
|
537
|
+
}
|
|
538
|
+
this.positions.set(userId, position);
|
|
539
|
+
this.applyParticipantPosition(userId, position);
|
|
540
|
+
}
|
|
541
|
+
/** The listener (camera) pose driving spatial voice. Engine hook, called per-frame. */
|
|
542
|
+
setListenerPose(position, forward, up = { x: 0, y: 1, z: 0 }) {
|
|
543
|
+
this.listenerPose = { position, forward, up };
|
|
544
|
+
this.applyListenerPose();
|
|
545
|
+
}
|
|
546
|
+
/** Connected remote voice participants (platform userIds). */
|
|
547
|
+
participantIds() {
|
|
548
|
+
if (!this.room)
|
|
549
|
+
return [];
|
|
550
|
+
return [...this.room.remoteParticipants.values()].map((p) => p.identity);
|
|
551
|
+
}
|
|
552
|
+
/** Currently speaking userIds (muted-by-me players never appear). Fires on every change. */
|
|
553
|
+
onSpeakingChanged(cb) {
|
|
554
|
+
this.speakingListeners.add(cb);
|
|
555
|
+
return () => this.speakingListeners.delete(cb);
|
|
556
|
+
}
|
|
557
|
+
/** Remote participant set changes (join/leave). Fires with the current userId list. */
|
|
558
|
+
onParticipantsChanged(cb) {
|
|
559
|
+
this.participantListeners.add(cb);
|
|
560
|
+
return () => this.participantListeners.delete(cb);
|
|
561
|
+
}
|
|
562
|
+
/**
|
|
563
|
+
* Receiver-local audio energy for presentation such as avatar jaw motion. The callback receives
|
|
564
|
+
* platform userId plus a clamped 0..1 RMS value at at most about 30 Hz. It is intentionally not exposed
|
|
565
|
+
* to the shell protocol or any network transport. Analysis exists only while at least one listener
|
|
566
|
+
* is registered and degrades to silence when LiveKit's WebAudio plugin hook is unavailable.
|
|
567
|
+
*/
|
|
568
|
+
onParticipantAudioLevel(cb) {
|
|
569
|
+
this.remoteLevelListeners.add(cb);
|
|
570
|
+
this.installRemoteAnalysers();
|
|
571
|
+
this.startRemoteLevelLoop();
|
|
572
|
+
return () => {
|
|
573
|
+
this.remoteLevelListeners.delete(cb);
|
|
574
|
+
if (this.remoteLevelListeners.size === 0 && this.remoteFeatureListeners.size === 0)
|
|
575
|
+
this.stopRemoteLevelAnalysis();
|
|
576
|
+
};
|
|
577
|
+
}
|
|
578
|
+
/**
|
|
579
|
+
* Receiver-local, bounded speech features for replaceable lip-sync solvers. Analysis runs at at
|
|
580
|
+
* most about 30 Hz for at most 32 audible tracks. The five fixed bands and two scalars are ephemeral;
|
|
581
|
+
* raw samples are neither exposed nor retained after each tick.
|
|
582
|
+
*/
|
|
583
|
+
onParticipantAudioFeatures(cb) {
|
|
584
|
+
this.remoteFeatureListeners.add(cb);
|
|
585
|
+
this.installRemoteAnalysers();
|
|
586
|
+
this.startRemoteLevelLoop();
|
|
587
|
+
return () => {
|
|
588
|
+
this.remoteFeatureListeners.delete(cb);
|
|
589
|
+
if (this.remoteLevelListeners.size === 0 && this.remoteFeatureListeners.size === 0)
|
|
590
|
+
this.stopRemoteLevelAnalysis();
|
|
591
|
+
};
|
|
592
|
+
}
|
|
593
|
+
/**
|
|
594
|
+
* The LOCAL speaker's own audio energy, for driving their own avatar's mouth. Nobody plays their
|
|
595
|
+
* own voice back, so `onParticipantAudioLevel` structurally cannot cover this seat.
|
|
596
|
+
*
|
|
597
|
+
* Sampled off the microphone analyser that already drives the meter and the sensitivity gate — no
|
|
598
|
+
* second analyser, no extra timer — and reported only while the room can actually hear you, so a
|
|
599
|
+
* muted or un-held mic leaves the mouth closed. Never networked; nothing is retained.
|
|
600
|
+
*/
|
|
601
|
+
onLocalAudioLevel(cb) {
|
|
602
|
+
this.localLevelListeners.add(cb);
|
|
603
|
+
return () => void this.localLevelListeners.delete(cb);
|
|
604
|
+
}
|
|
605
|
+
/**
|
|
606
|
+
* The local speaker's own speech features, in the same fixed shape and units as the receiver-local
|
|
607
|
+
* ones, so one solver serves both seats. Preferred over `onLocalAudioLevel` when consumed.
|
|
608
|
+
*/
|
|
609
|
+
onLocalAudioFeatures(cb) {
|
|
610
|
+
this.localFeatureListeners.add(cb);
|
|
611
|
+
return () => void this.localFeatureListeners.delete(cb);
|
|
612
|
+
}
|
|
613
|
+
// Browser autoplay unlock: remote audio may be blocked until a user gesture. join() installs a
|
|
614
|
+
// one-time pointerdown/keydown hook that calls this; a world can also call it from its own gesture.
|
|
615
|
+
async startAudio() {
|
|
616
|
+
if (!this.room)
|
|
617
|
+
return;
|
|
618
|
+
try {
|
|
619
|
+
await this.room.startAudio();
|
|
620
|
+
this.audioBlocked = !this.room.canPlaybackAudio;
|
|
621
|
+
}
|
|
622
|
+
catch {
|
|
623
|
+
this.audioBlocked = true;
|
|
624
|
+
}
|
|
625
|
+
this.publishState();
|
|
626
|
+
}
|
|
627
|
+
async requestVoiceToken(apiBase, worldId, token, roomId, tokenPath) {
|
|
628
|
+
const path = tokenPath ??
|
|
629
|
+
`/api/v1/instant-worlds/${encodeURIComponent(worldId)}/voice-token`;
|
|
630
|
+
if (!path.startsWith('/api/v1/') || path.startsWith('//')) {
|
|
631
|
+
throw new Error('Helix.voice: tokenPath must be a same-origin /api/v1/ path');
|
|
632
|
+
}
|
|
633
|
+
const base = new URL(apiBase);
|
|
634
|
+
const url = new URL(path, `${base.origin}/`);
|
|
635
|
+
if (url.origin !== base.origin) {
|
|
636
|
+
throw new Error('Helix.voice: tokenPath must be same-origin with the platform API');
|
|
637
|
+
}
|
|
638
|
+
const res = await fetch(url.toString(), {
|
|
639
|
+
method: 'POST',
|
|
640
|
+
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
|
|
641
|
+
body: JSON.stringify({ roomId }),
|
|
642
|
+
});
|
|
643
|
+
if (res.status === 404)
|
|
644
|
+
return null; // voice off (unconfigured env or unknown world) — fail-soft
|
|
645
|
+
if (!res.ok) {
|
|
646
|
+
const detail = await res.json().catch(() => null);
|
|
647
|
+
const message = (detail && typeof detail.message === 'string' && detail.message) || `HTTP ${res.status}`;
|
|
648
|
+
throw new Error(`Helix.voice: voice-token failed — ${message}`);
|
|
649
|
+
}
|
|
650
|
+
return (await res.json());
|
|
651
|
+
}
|
|
652
|
+
wireRoomEvents(room) {
|
|
653
|
+
const ev = this.events;
|
|
654
|
+
// A voice room's events outlive it: LiveKit reports the close of a room we already left — or of a join a
|
|
655
|
+
// room hop superseded — whenever the transport gets round to it. By then `this.room` may be the NEXT voice
|
|
656
|
+
// room, and an unguarded Disconnected would tear THAT one down. Every handler therefore acts only while its
|
|
657
|
+
// own room is the current one (the same rule the multiplayer module applies to its rooms).
|
|
658
|
+
const on = (event, handler) => {
|
|
659
|
+
room.on(event, (...args) => {
|
|
660
|
+
if (this.room === room)
|
|
661
|
+
handler(...args);
|
|
662
|
+
});
|
|
663
|
+
};
|
|
664
|
+
on(ev.ActiveSpeakersChanged, (speakers) => {
|
|
665
|
+
this.speaking = new Set(speakers.map((s) => s.identity));
|
|
666
|
+
this.emitSpeaking();
|
|
667
|
+
this.publishState();
|
|
668
|
+
});
|
|
669
|
+
on(ev.ParticipantConnected, () => {
|
|
670
|
+
this.applyAudibility();
|
|
671
|
+
this.applyVolumes();
|
|
672
|
+
this.emitParticipants();
|
|
673
|
+
});
|
|
674
|
+
// A participant switched voice channels (their CHANNEL_ATTR changed) — re-gate their delivery.
|
|
675
|
+
// Fires for the local participant too (our own setChannel already applied everything): findParticipant
|
|
676
|
+
// returns null for it, so the guard covers both.
|
|
677
|
+
on(ev.ParticipantAttributesChanged, (_changed, p) => {
|
|
678
|
+
const remote = this.findParticipant(p.identity);
|
|
679
|
+
if (!remote)
|
|
680
|
+
return;
|
|
681
|
+
const audible = this.participantAudible(remote);
|
|
682
|
+
remote.audioTrackPublications.forEach((pub) => pub.setEnabled(audible));
|
|
683
|
+
this.emitSpeaking();
|
|
684
|
+
this.publishState();
|
|
685
|
+
});
|
|
686
|
+
on(ev.ParticipantDisconnected, (p) => {
|
|
687
|
+
this.speaking.delete(p.identity);
|
|
688
|
+
this.participantBase.delete(p.identity);
|
|
689
|
+
this.spatialTracks.delete(p.identity);
|
|
690
|
+
this.panners.delete(p.identity);
|
|
691
|
+
this.remoteAnalysers.delete(p.identity);
|
|
692
|
+
this.emitRemoteLevel(p.identity, 0);
|
|
693
|
+
this.emitRemoteFeatures(p.identity, { rms: 0, zeroCrossingRate: 0, bands: [0, 0, 0, 0, 0] });
|
|
694
|
+
this.installRemoteAnalysers();
|
|
695
|
+
this.positions.delete(p.identity);
|
|
696
|
+
this.emitParticipants();
|
|
697
|
+
this.emitSpeaking();
|
|
698
|
+
this.publishState();
|
|
699
|
+
});
|
|
700
|
+
on(ev.TrackSubscribed, (track, pub, p) => {
|
|
701
|
+
if (pub.kind !== 'audio')
|
|
702
|
+
return;
|
|
703
|
+
// Remote audio only plays once attached to a media element; park them under a hidden host node.
|
|
704
|
+
const el = track.attach();
|
|
705
|
+
this.audioRoot?.appendChild(el);
|
|
706
|
+
this.spatialTracks.set(p.identity, track);
|
|
707
|
+
if (this.remoteLevelListeners.size > 0)
|
|
708
|
+
this.ensureRemoteAnalyser(p.identity);
|
|
709
|
+
// A position pushed before the track arrived (engine loop vs subscribe race) spatializes now.
|
|
710
|
+
const pos = this.positions.get(p.identity);
|
|
711
|
+
if (pos)
|
|
712
|
+
this.applyParticipantPosition(p.identity, pos);
|
|
713
|
+
if (!this.participantAudible(p))
|
|
714
|
+
pub.setEnabled(false);
|
|
715
|
+
});
|
|
716
|
+
on(ev.TrackUnsubscribed, (track, _pub, p) => {
|
|
717
|
+
for (const el of track.detach())
|
|
718
|
+
el.remove();
|
|
719
|
+
if (p) {
|
|
720
|
+
this.spatialTracks.delete(p.identity);
|
|
721
|
+
this.panners.delete(p.identity); // died with the track's chain; the position survives a resubscribe
|
|
722
|
+
this.remoteAnalysers.delete(p.identity);
|
|
723
|
+
this.emitRemoteLevel(p.identity, 0);
|
|
724
|
+
this.emitRemoteFeatures(p.identity, { rms: 0, zeroCrossingRate: 0, bands: [0, 0, 0, 0, 0] });
|
|
725
|
+
this.installRemoteAnalysers();
|
|
726
|
+
}
|
|
727
|
+
});
|
|
728
|
+
on(ev.AudioPlaybackStatusChanged, () => {
|
|
729
|
+
this.audioBlocked = !room.canPlaybackAudio;
|
|
730
|
+
if (this.audioBlocked)
|
|
731
|
+
this.installUnlock();
|
|
732
|
+
this.publishState();
|
|
733
|
+
});
|
|
734
|
+
on(ev.MediaDevicesChanged, () => void this.refreshDevices().then(() => this.publishState()));
|
|
735
|
+
on(ev.Reconnecting, () => {
|
|
736
|
+
this.connected = false;
|
|
737
|
+
this.publishState();
|
|
738
|
+
});
|
|
739
|
+
on(ev.Reconnected, () => {
|
|
740
|
+
this.connected = true;
|
|
741
|
+
this.publishState();
|
|
742
|
+
});
|
|
743
|
+
// Left for good (kicked / same-identity join elsewhere / server close). LiveKit's in-page resume
|
|
744
|
+
// never reaches here; a NEW join (fresh token via join()) is the world's recovery path.
|
|
745
|
+
on(ev.Disconnected, () => this.teardown());
|
|
746
|
+
}
|
|
747
|
+
// Capture (prompting for permission on first use), build the outgoing pipeline, publish its ONE
|
|
748
|
+
// destination track, and set the mute gain. Mute is a gain flip — instant, no track state churn.
|
|
749
|
+
// Permission state is attempt-derived: jsdom/older browsers lack navigator.permissions.
|
|
750
|
+
async ensureMic(unmuted) {
|
|
751
|
+
if (!this.room)
|
|
752
|
+
return;
|
|
753
|
+
if (this.captured) {
|
|
754
|
+
if (this.muteGain)
|
|
755
|
+
this.muteGain.gain.value = unmuted ? 1 : 0;
|
|
756
|
+
return;
|
|
757
|
+
}
|
|
758
|
+
if (!this.capturePromise) {
|
|
759
|
+
const capture = this.captureMic();
|
|
760
|
+
this.capturePromise = capture;
|
|
761
|
+
void capture.finally(() => {
|
|
762
|
+
if (this.capturePromise === capture)
|
|
763
|
+
this.capturePromise = null;
|
|
764
|
+
});
|
|
765
|
+
}
|
|
766
|
+
await this.capturePromise;
|
|
767
|
+
// The hold/mode may have changed while browser permission or LiveKit publishing was pending.
|
|
768
|
+
// Reconcile to current intent instead of the initiating call's stale `unmuted` argument.
|
|
769
|
+
if (this.captured && this.muteGain)
|
|
770
|
+
this.muteGain.gain.value = this.shouldTransmit() ? 1 : 0;
|
|
771
|
+
}
|
|
772
|
+
async captureMic() {
|
|
773
|
+
const room = this.room;
|
|
774
|
+
if (!room)
|
|
775
|
+
return;
|
|
776
|
+
let stream = null;
|
|
777
|
+
let ctx = null;
|
|
778
|
+
try {
|
|
779
|
+
stream = await navigator.mediaDevices.getUserMedia({
|
|
780
|
+
audio: {
|
|
781
|
+
// Saved pref → ideal (quietly falls back if the id is stale here); none → Chromium's virtual
|
|
782
|
+
// 'default' device = the OS default mic (left unconstrained, Chrome picks by its own profile
|
|
783
|
+
// ranking — often the wrong physical device). Firefox/Safari lack 'default'; an unmatched
|
|
784
|
+
// ideal is a fitness hint, never a failure, so they degrade to their own default.
|
|
785
|
+
deviceId: { ideal: this.requestedDeviceId ?? 'default' },
|
|
786
|
+
// NOTE for local testing: two player tabs in ONE browser make this AEC cancel the sender's own
|
|
787
|
+
// voice (the other tab's playout is its reference — headphones don't help). Use two browsers.
|
|
788
|
+
echoCancellation: true,
|
|
789
|
+
noiseSuppression: true,
|
|
790
|
+
autoGainControl: true,
|
|
791
|
+
},
|
|
792
|
+
});
|
|
793
|
+
ctx = new AudioContext();
|
|
794
|
+
// THE OUTGOING TRACK IS THIS CONTEXT'S OUTPUT. A MediaStreamDestination on a SUSPENDED context
|
|
795
|
+
// publishes a live, unmuted track that carries nothing — so every mode reads as transmitting
|
|
796
|
+
// while the room hears silence. Browsers create the context suspended whenever the frame has
|
|
797
|
+
// not had a gesture yet (Safari always without one; Chromium under user-gesture policies, and
|
|
798
|
+
// for any frame whose autoplay is not delegated), and nothing used to resume it. Ask now (it
|
|
799
|
+
// succeeds wherever autoplay is already allowed) and again on the first gesture.
|
|
800
|
+
this.kickContext(ctx);
|
|
801
|
+
const source = ctx.createMediaStreamSource(stream);
|
|
802
|
+
const volume = ctx.createGain();
|
|
803
|
+
volume.gain.value = this.micVolume;
|
|
804
|
+
// Loudness leveling: DynamicsCompressorNode's implicit makeup gain lifts quiet talkers — the
|
|
805
|
+
// "everyone too quiet" fix. Gentle ratio to avoid pumping; sits PRE-analyser so the meter (and
|
|
806
|
+
// the sensitivity threshold sharing its units) reflects what listeners actually receive.
|
|
807
|
+
const compressor = ctx.createDynamicsCompressor();
|
|
808
|
+
compressor.threshold.value = -28;
|
|
809
|
+
compressor.knee.value = 20;
|
|
810
|
+
compressor.ratio.value = 6;
|
|
811
|
+
compressor.attack.value = 0.003;
|
|
812
|
+
compressor.release.value = 0.2;
|
|
813
|
+
const analyser = ctx.createAnalyser();
|
|
814
|
+
analyser.fftSize = 1024;
|
|
815
|
+
const gate = ctx.createGain();
|
|
816
|
+
const mute = ctx.createGain();
|
|
817
|
+
// Never publish a hot track. The current mode/hold gate is applied only after publishing
|
|
818
|
+
// succeeds, which makes a delayed permission prompt or PTT release fail closed.
|
|
819
|
+
mute.gain.value = 0;
|
|
820
|
+
const dest = ctx.createMediaStreamDestination();
|
|
821
|
+
source.connect(volume);
|
|
822
|
+
volume.connect(compressor);
|
|
823
|
+
compressor.connect(analyser);
|
|
824
|
+
analyser.connect(gate);
|
|
825
|
+
gate.connect(mute);
|
|
826
|
+
mute.connect(dest);
|
|
827
|
+
if (this.room !== room) {
|
|
828
|
+
stream.getTracks().forEach((track) => track.stop());
|
|
829
|
+
await ctx.close().catch(() => undefined);
|
|
830
|
+
return;
|
|
831
|
+
}
|
|
832
|
+
await room.localParticipant.publishTrack(dest.stream.getAudioTracks()[0], { source: 'microphone' });
|
|
833
|
+
if (this.room !== room) {
|
|
834
|
+
stream.getTracks().forEach((track) => track.stop());
|
|
835
|
+
await ctx.close().catch(() => undefined);
|
|
836
|
+
return;
|
|
837
|
+
}
|
|
838
|
+
this.audioCtx = ctx;
|
|
839
|
+
this.micStream = stream;
|
|
840
|
+
this.micSource = source;
|
|
841
|
+
this.volumeGain = volume;
|
|
842
|
+
this.compressor = compressor;
|
|
843
|
+
this.analyser = analyser;
|
|
844
|
+
this.gateGain = gate;
|
|
845
|
+
this.muteGain = mute;
|
|
846
|
+
this.captured = true;
|
|
847
|
+
this.permission = 'granted';
|
|
848
|
+
this.error = null;
|
|
849
|
+
this.activeDeviceId =
|
|
850
|
+
stream.getAudioTracks()[0]?.getSettings?.().deviceId ?? this.requestedDeviceId;
|
|
851
|
+
mute.gain.value = this.shouldTransmit() ? 1 : 0;
|
|
852
|
+
if (this.micSuspended())
|
|
853
|
+
this.installUnlock();
|
|
854
|
+
this.startLevelLoop();
|
|
855
|
+
this.publishState();
|
|
856
|
+
await this.refreshDevices();
|
|
857
|
+
await this.reconcileDeviceLabel();
|
|
858
|
+
}
|
|
859
|
+
catch (e) {
|
|
860
|
+
stream?.getTracks().forEach((track) => track.stop());
|
|
861
|
+
await ctx?.close().catch(() => undefined);
|
|
862
|
+
const name = e?.name;
|
|
863
|
+
// A denied permission is a fact about the browser, whichever room asked. The error is not: a capture
|
|
864
|
+
// that failed for a room already left (publishTrack on a disconnected room) says nothing about the
|
|
865
|
+
// session that replaced it.
|
|
866
|
+
if (name === 'NotAllowedError' || name === 'SecurityError')
|
|
867
|
+
this.permission = 'denied';
|
|
868
|
+
if (this.room !== room)
|
|
869
|
+
return;
|
|
870
|
+
this.error = e instanceof Error ? e.message : String(e);
|
|
871
|
+
}
|
|
872
|
+
}
|
|
873
|
+
async setMicEnabled(enabled) {
|
|
874
|
+
if (this.captured)
|
|
875
|
+
await this.ensureMic(enabled);
|
|
876
|
+
}
|
|
877
|
+
shouldTransmit() {
|
|
878
|
+
return this.micMode === 'open' || (this.micMode === 'ptt' && this.pttHeld);
|
|
879
|
+
}
|
|
880
|
+
// Analyser sampling: RMS → meter level (shared units with the sensitivity slider), gate decision,
|
|
881
|
+
// and the throttled shell meter publish. Runs only while captured.
|
|
882
|
+
startLevelLoop() {
|
|
883
|
+
if (this.levelTimer)
|
|
884
|
+
return;
|
|
885
|
+
this.levelTimer = setInterval(() => {
|
|
886
|
+
const analyser = this.analyser;
|
|
887
|
+
if (!analyser)
|
|
888
|
+
return;
|
|
889
|
+
const buf = (this.analyserBuf ??= new Float32Array(analyser.fftSize));
|
|
890
|
+
analyser.getFloatTimeDomainData(buf);
|
|
891
|
+
const { rms, zeroCrossingRate } = timeDomainEnergy(buf);
|
|
892
|
+
this.micLevel = rms;
|
|
893
|
+
this.applyGate();
|
|
894
|
+
if (++this.levelTick % 2 === 0) {
|
|
895
|
+
this.host.publishVoiceLevel?.(Math.round(this.micLevel * 100) / 100);
|
|
896
|
+
}
|
|
897
|
+
this.emitLocalAudio(zeroCrossingRate);
|
|
898
|
+
}, LEVEL_TICK_MS);
|
|
899
|
+
}
|
|
900
|
+
/**
|
|
901
|
+
* The local speaker's own face signal, off the same analyser the meter and the gate already use.
|
|
902
|
+
*
|
|
903
|
+
* Gated on what the room can actually HEAR — the publish mute (mode/PTT) times the sensitivity gate
|
|
904
|
+
* — so a muted or un-held microphone leaves the avatar's mouth closed instead of mouthing silently
|
|
905
|
+
* at everyone. The transition to inaudible emits one explicit silent frame and then stops, so a
|
|
906
|
+
* consumer settles at rest rather than freezing on the last syllable.
|
|
907
|
+
*/
|
|
908
|
+
emitLocalAudio(zeroCrossingRate) {
|
|
909
|
+
if (this.localLevelListeners.size === 0 && this.localFeatureListeners.size === 0)
|
|
910
|
+
return;
|
|
911
|
+
const audible = (this.muteGain?.gain.value ?? 0) > 0 && (this.gateGain?.gain.value ?? 0) > 0;
|
|
912
|
+
if (!audible) {
|
|
913
|
+
if (!this.localWasAudible)
|
|
914
|
+
return;
|
|
915
|
+
this.localWasAudible = false;
|
|
916
|
+
this.emitLocalSilence();
|
|
917
|
+
return;
|
|
918
|
+
}
|
|
919
|
+
this.localWasAudible = true;
|
|
920
|
+
for (const cb of this.localLevelListeners)
|
|
921
|
+
cb(this.micLevel);
|
|
922
|
+
const analyser = this.analyser;
|
|
923
|
+
if (this.localFeatureListeners.size === 0 || !analyser || typeof analyser.getByteFrequencyData !== 'function')
|
|
924
|
+
return;
|
|
925
|
+
const spectrum = (this.analyserSpectrum ??= new Uint8Array(analyser.frequencyBinCount));
|
|
926
|
+
analyser.getByteFrequencyData(spectrum);
|
|
927
|
+
const features = {
|
|
928
|
+
rms: this.micLevel,
|
|
929
|
+
zeroCrossingRate,
|
|
930
|
+
bands: speechBands(spectrum, this.audioCtx?.sampleRate ?? 48_000, analyser.fftSize),
|
|
931
|
+
};
|
|
932
|
+
for (const cb of this.localFeatureListeners)
|
|
933
|
+
cb(features);
|
|
934
|
+
}
|
|
935
|
+
emitLocalSilence() {
|
|
936
|
+
this.localWasAudible = false;
|
|
937
|
+
for (const cb of this.localLevelListeners)
|
|
938
|
+
cb(0);
|
|
939
|
+
for (const cb of this.localFeatureListeners)
|
|
940
|
+
cb(SILENT_FEATURES);
|
|
941
|
+
}
|
|
942
|
+
// Manual sensitivity: cut input below the threshold, with a hold so word gaps don't chop. Auto
|
|
943
|
+
// (default) leaves the gate open — the browser's noise suppression is the only filter.
|
|
944
|
+
applyGate() {
|
|
945
|
+
if (!this.gateGain)
|
|
946
|
+
return;
|
|
947
|
+
let open = true;
|
|
948
|
+
if (!this.autoSensitivity) {
|
|
949
|
+
const now = Date.now();
|
|
950
|
+
if (this.micLevel >= this.sensitivity)
|
|
951
|
+
this.gateOpenUntil = now + GATE_HOLD_MS;
|
|
952
|
+
open = now < this.gateOpenUntil;
|
|
953
|
+
}
|
|
954
|
+
this.gateGain.gain.value = open ? 1 : 0;
|
|
955
|
+
}
|
|
956
|
+
async applyMicMode() {
|
|
957
|
+
await this.setMicMode(this.micMode);
|
|
958
|
+
}
|
|
959
|
+
// Audibility = the ONE gate deciding whose audio the SFU delivers to us: not muted-by-me AND in the
|
|
960
|
+
// same voice space (both compare via the participant's replicated channel attribute; '' = ambient).
|
|
961
|
+
sameChannel(p) {
|
|
962
|
+
return (p.attributes?.[CHANNEL_ATTR] ?? '') === (this.channelId ?? '');
|
|
963
|
+
}
|
|
964
|
+
participantAudible(p) {
|
|
965
|
+
return !this.muted.has(p.identity) && this.sameChannel(p);
|
|
966
|
+
}
|
|
967
|
+
applyAudibility() {
|
|
968
|
+
if (!this.room)
|
|
969
|
+
return;
|
|
970
|
+
this.room.remoteParticipants.forEach((p) => {
|
|
971
|
+
const audible = this.participantAudible(p);
|
|
972
|
+
p.audioTrackPublications.forEach((pub) => pub.setEnabled(audible));
|
|
973
|
+
});
|
|
974
|
+
}
|
|
975
|
+
// Reapply base × outputVolume to every participant — on master-volume changes and joins (a world
|
|
976
|
+
// without a proximity driver never calls setParticipantVolume, so the base defaults to 1).
|
|
977
|
+
applyVolumes() {
|
|
978
|
+
if (!this.room)
|
|
979
|
+
return;
|
|
980
|
+
this.room.remoteParticipants.forEach((p) => {
|
|
981
|
+
const base = this.participantBase.get(p.identity) ?? 1;
|
|
982
|
+
p.setVolume(Math.min(base * this.outputVolume, MAX_VOLUME));
|
|
983
|
+
});
|
|
984
|
+
}
|
|
985
|
+
findParticipant(userId) {
|
|
986
|
+
if (!this.room)
|
|
987
|
+
return null;
|
|
988
|
+
for (const p of this.room.remoteParticipants.values()) {
|
|
989
|
+
if (p.identity === userId)
|
|
990
|
+
return p;
|
|
991
|
+
}
|
|
992
|
+
return null;
|
|
993
|
+
}
|
|
994
|
+
// ── spatial (directional) rendering ───────────────────────────────────────
|
|
995
|
+
// Lazily insert the user's PannerNode into their track's playback chain. Null (= stay non-spatial)
|
|
996
|
+
// when the mix context, the track, or the experimental setWebAudioPlugins hook is missing.
|
|
997
|
+
ensurePanner(userId) {
|
|
998
|
+
const existing = this.panners.get(userId);
|
|
999
|
+
if (existing)
|
|
1000
|
+
return existing;
|
|
1001
|
+
const ctx = this.mixCtx;
|
|
1002
|
+
const track = this.spatialTracks.get(userId);
|
|
1003
|
+
if (!ctx || !track || typeof track.setWebAudioPlugins !== 'function' || typeof ctx.createPanner !== 'function')
|
|
1004
|
+
return null;
|
|
1005
|
+
const panner = ctx.createPanner();
|
|
1006
|
+
// Direction ONLY: HRTF gives real left/right/behind cues; rolloffFactor 0 disables the panner's own
|
|
1007
|
+
// distance attenuation so loudness keeps flowing through the tuned base × outputVolume gain path.
|
|
1008
|
+
panner.panningModel = 'HRTF';
|
|
1009
|
+
panner.rolloffFactor = 0;
|
|
1010
|
+
this.panners.set(userId, panner);
|
|
1011
|
+
this.applyTrackPlugins(userId);
|
|
1012
|
+
return panner;
|
|
1013
|
+
}
|
|
1014
|
+
applyTrackPlugins(userId) {
|
|
1015
|
+
const track = this.spatialTracks.get(userId);
|
|
1016
|
+
if (!track?.setWebAudioPlugins)
|
|
1017
|
+
return;
|
|
1018
|
+
const nodes = [];
|
|
1019
|
+
const panner = this.panners.get(userId);
|
|
1020
|
+
const analyser = this.remoteAnalysers.get(userId)?.node;
|
|
1021
|
+
if (panner)
|
|
1022
|
+
nodes.push(panner);
|
|
1023
|
+
if (analyser)
|
|
1024
|
+
nodes.push(analyser);
|
|
1025
|
+
track.setWebAudioPlugins(nodes);
|
|
1026
|
+
}
|
|
1027
|
+
ensureRemoteAnalyser(userId) {
|
|
1028
|
+
if (this.remoteAnalysers.has(userId) || this.remoteAnalysers.size >= REMOTE_LEVEL_MAX_TRACKS)
|
|
1029
|
+
return;
|
|
1030
|
+
const ctx = this.mixCtx;
|
|
1031
|
+
const track = this.spatialTracks.get(userId);
|
|
1032
|
+
if (!ctx || !track?.setWebAudioPlugins || typeof ctx.createAnalyser !== 'function')
|
|
1033
|
+
return;
|
|
1034
|
+
const node = ctx.createAnalyser();
|
|
1035
|
+
node.fftSize = REMOTE_LEVEL_FFT_SIZE;
|
|
1036
|
+
this.remoteAnalysers.set(userId, {
|
|
1037
|
+
node,
|
|
1038
|
+
samples: new Float32Array(node.fftSize),
|
|
1039
|
+
spectrum: new Uint8Array(node.frequencyBinCount),
|
|
1040
|
+
});
|
|
1041
|
+
this.applyTrackPlugins(userId);
|
|
1042
|
+
}
|
|
1043
|
+
installRemoteAnalysers() {
|
|
1044
|
+
for (const userId of this.spatialTracks.keys())
|
|
1045
|
+
this.ensureRemoteAnalyser(userId);
|
|
1046
|
+
}
|
|
1047
|
+
startRemoteLevelLoop() {
|
|
1048
|
+
if (this.remoteLevelTimer || (this.remoteLevelListeners.size === 0 && this.remoteFeatureListeners.size === 0))
|
|
1049
|
+
return;
|
|
1050
|
+
this.remoteLevelTimer = setInterval(() => {
|
|
1051
|
+
for (const [userId, analyser] of this.remoteAnalysers) {
|
|
1052
|
+
analyser.node.getFloatTimeDomainData(analyser.samples);
|
|
1053
|
+
const { rms, zeroCrossingRate } = timeDomainEnergy(analyser.samples);
|
|
1054
|
+
this.emitRemoteLevel(userId, rms);
|
|
1055
|
+
if (this.remoteFeatureListeners.size > 0) {
|
|
1056
|
+
analyser.node.getByteFrequencyData(analyser.spectrum);
|
|
1057
|
+
this.emitRemoteFeatures(userId, {
|
|
1058
|
+
rms,
|
|
1059
|
+
zeroCrossingRate,
|
|
1060
|
+
bands: speechBands(analyser.spectrum, this.mixCtx?.sampleRate ?? 48_000, analyser.node.fftSize),
|
|
1061
|
+
});
|
|
1062
|
+
}
|
|
1063
|
+
}
|
|
1064
|
+
}, REMOTE_LEVEL_TICK_MS);
|
|
1065
|
+
}
|
|
1066
|
+
emitRemoteLevel(userId, level) {
|
|
1067
|
+
for (const cb of this.remoteLevelListeners)
|
|
1068
|
+
cb(userId, level);
|
|
1069
|
+
}
|
|
1070
|
+
emitRemoteFeatures(userId, features) {
|
|
1071
|
+
for (const cb of this.remoteFeatureListeners)
|
|
1072
|
+
cb(userId, features);
|
|
1073
|
+
}
|
|
1074
|
+
stopRemoteLevelAnalysis(emitSilence = false) {
|
|
1075
|
+
if (this.remoteLevelTimer)
|
|
1076
|
+
clearInterval(this.remoteLevelTimer);
|
|
1077
|
+
this.remoteLevelTimer = null;
|
|
1078
|
+
const users = [...this.remoteAnalysers.keys()];
|
|
1079
|
+
if (emitSilence)
|
|
1080
|
+
for (const userId of users) {
|
|
1081
|
+
this.emitRemoteLevel(userId, 0);
|
|
1082
|
+
this.emitRemoteFeatures(userId, { rms: 0, zeroCrossingRate: 0, bands: [0, 0, 0, 0, 0] });
|
|
1083
|
+
}
|
|
1084
|
+
this.remoteAnalysers.clear();
|
|
1085
|
+
for (const userId of users)
|
|
1086
|
+
this.applyTrackPlugins(userId);
|
|
1087
|
+
}
|
|
1088
|
+
applyParticipantPosition(userId, pos) {
|
|
1089
|
+
const panner = this.ensurePanner(userId);
|
|
1090
|
+
if (!panner)
|
|
1091
|
+
return;
|
|
1092
|
+
if (panner.positionX) {
|
|
1093
|
+
panner.positionX.value = pos.x;
|
|
1094
|
+
panner.positionY.value = pos.y;
|
|
1095
|
+
panner.positionZ.value = pos.z;
|
|
1096
|
+
}
|
|
1097
|
+
else {
|
|
1098
|
+
panner.setPosition(pos.x, pos.y, pos.z);
|
|
1099
|
+
}
|
|
1100
|
+
}
|
|
1101
|
+
applyListenerPose() {
|
|
1102
|
+
const pose = this.listenerPose;
|
|
1103
|
+
const listener = this.mixCtx?.listener;
|
|
1104
|
+
if (!pose || !listener)
|
|
1105
|
+
return;
|
|
1106
|
+
const { position: p, forward: f, up: u } = pose;
|
|
1107
|
+
if (listener.positionX) {
|
|
1108
|
+
listener.positionX.value = p.x;
|
|
1109
|
+
listener.positionY.value = p.y;
|
|
1110
|
+
listener.positionZ.value = p.z;
|
|
1111
|
+
listener.forwardX.value = f.x;
|
|
1112
|
+
listener.forwardY.value = f.y;
|
|
1113
|
+
listener.forwardZ.value = f.z;
|
|
1114
|
+
listener.upX.value = u.x;
|
|
1115
|
+
listener.upY.value = u.y;
|
|
1116
|
+
listener.upZ.value = u.z;
|
|
1117
|
+
}
|
|
1118
|
+
else {
|
|
1119
|
+
// Firefox: AudioListener still lacks the AudioParam form — only the deprecated setters exist.
|
|
1120
|
+
listener.setPosition?.(p.x, p.y, p.z);
|
|
1121
|
+
listener.setOrientation?.(f.x, f.y, f.z, u.x, u.y, u.z);
|
|
1122
|
+
}
|
|
1123
|
+
}
|
|
1124
|
+
async refreshDevices() {
|
|
1125
|
+
const md = typeof navigator !== 'undefined' ? navigator.mediaDevices : undefined;
|
|
1126
|
+
if (!md?.enumerateDevices)
|
|
1127
|
+
return;
|
|
1128
|
+
try {
|
|
1129
|
+
const all = await md.enumerateDevices();
|
|
1130
|
+
this.devices = all
|
|
1131
|
+
.filter((d) => d.kind === 'audioinput')
|
|
1132
|
+
.map((d) => ({ deviceId: d.deviceId, label: d.label }));
|
|
1133
|
+
}
|
|
1134
|
+
catch {
|
|
1135
|
+
// keep the previous list
|
|
1136
|
+
}
|
|
1137
|
+
}
|
|
1138
|
+
// ── push-to-talk key + mic-release safety ─────────────────────────────────
|
|
1139
|
+
// These listeners live INSIDE the world iframe on purpose: during pointer-lock gameplay, keyboard
|
|
1140
|
+
// focus is in this document and never reaches the shell — a shell-side PTT listener would be deaf.
|
|
1141
|
+
attachInputListeners() {
|
|
1142
|
+
window.addEventListener('keydown', this.onKeyDown);
|
|
1143
|
+
window.addEventListener('keyup', this.onKeyUp);
|
|
1144
|
+
window.addEventListener('blur', this.releasePtt);
|
|
1145
|
+
document.addEventListener('visibilitychange', this.onVisibility);
|
|
1146
|
+
}
|
|
1147
|
+
detachInputListeners() {
|
|
1148
|
+
window.removeEventListener('keydown', this.onKeyDown);
|
|
1149
|
+
window.removeEventListener('keyup', this.onKeyUp);
|
|
1150
|
+
window.removeEventListener('blur', this.releasePtt);
|
|
1151
|
+
document.removeEventListener('visibilitychange', this.onVisibility);
|
|
1152
|
+
}
|
|
1153
|
+
onKeyDown = (event) => {
|
|
1154
|
+
// Browser shortcut chords are never PTT. Releasing an existing hold here
|
|
1155
|
+
// also covers browsers that suppress the non-modifier keyup while Meta is
|
|
1156
|
+
// held. A repeat may reacquire after focus moved between shell and iframe;
|
|
1157
|
+
// setPttDown remains idempotent while the same leg is already held.
|
|
1158
|
+
if (event.metaKey || event.ctrlKey || event.altKey) {
|
|
1159
|
+
this.releasePtt();
|
|
1160
|
+
return;
|
|
1161
|
+
}
|
|
1162
|
+
if (this.micMode !== 'ptt' || event.key.toLowerCase() !== this.pttKey)
|
|
1163
|
+
return;
|
|
1164
|
+
const target = event.target;
|
|
1165
|
+
const tag = target?.tagName?.toLowerCase();
|
|
1166
|
+
if (target?.isContentEditable || tag === 'input' || tag === 'textarea' || tag === 'select')
|
|
1167
|
+
return;
|
|
1168
|
+
this.setPttDown(true);
|
|
1169
|
+
};
|
|
1170
|
+
onKeyUp = (event) => {
|
|
1171
|
+
if (event.key.toLowerCase() !== this.pttKey)
|
|
1172
|
+
return;
|
|
1173
|
+
this.releasePtt();
|
|
1174
|
+
};
|
|
1175
|
+
// A missed keyup (tab switch, pointer-lock exit, alt-tab) must never leave a hot mic.
|
|
1176
|
+
releasePtt = () => {
|
|
1177
|
+
if (!this.pttHeld)
|
|
1178
|
+
return;
|
|
1179
|
+
this.pttHeld = false;
|
|
1180
|
+
if (this.micMode === 'ptt')
|
|
1181
|
+
void this.setMicEnabled(false);
|
|
1182
|
+
this.publishState();
|
|
1183
|
+
};
|
|
1184
|
+
onVisibility = () => {
|
|
1185
|
+
if (document.visibilityState === 'hidden')
|
|
1186
|
+
this.releasePtt();
|
|
1187
|
+
};
|
|
1188
|
+
// ── audio unlock + attached-element host ──────────────────────────────────
|
|
1189
|
+
ensureAudioRoot() {
|
|
1190
|
+
if (this.audioRoot || typeof document === 'undefined')
|
|
1191
|
+
return;
|
|
1192
|
+
const root = document.createElement('div');
|
|
1193
|
+
root.style.display = 'none';
|
|
1194
|
+
root.setAttribute('data-helix-voice', '');
|
|
1195
|
+
document.body.appendChild(root);
|
|
1196
|
+
this.audioRoot = root;
|
|
1197
|
+
}
|
|
1198
|
+
// One gesture unlocks BOTH directions: LiveKit's playback (startAudio) and our own contexts — the
|
|
1199
|
+
// mic pipeline's, whose suspension silences what we SEND, and the mix context remote voices play
|
|
1200
|
+
// through. Capture phase, so a world that stops propagation on its own input cannot swallow it.
|
|
1201
|
+
installUnlock() {
|
|
1202
|
+
if (this.unlockInstalled || typeof document === 'undefined')
|
|
1203
|
+
return;
|
|
1204
|
+
this.unlockInstalled = true;
|
|
1205
|
+
const events = ['pointerdown', 'keydown', 'touchend'];
|
|
1206
|
+
const unlock = () => {
|
|
1207
|
+
for (const type of events)
|
|
1208
|
+
document.removeEventListener(type, unlock, true);
|
|
1209
|
+
this.unlockInstalled = false;
|
|
1210
|
+
this.resumeAudio();
|
|
1211
|
+
void this.startAudio();
|
|
1212
|
+
};
|
|
1213
|
+
for (const type of events)
|
|
1214
|
+
document.addEventListener(type, unlock, true);
|
|
1215
|
+
}
|
|
1216
|
+
micSuspended() {
|
|
1217
|
+
return this.captured && this.audioCtx?.state === 'suspended';
|
|
1218
|
+
}
|
|
1219
|
+
// Never awaited: a resume() the browser does not yet allow stays pending rather than rejecting, and
|
|
1220
|
+
// nothing may wait on a gesture that has not happened.
|
|
1221
|
+
resumeAudio() {
|
|
1222
|
+
for (const ctx of [this.audioCtx, this.mixCtx]) {
|
|
1223
|
+
if (ctx?.state === 'suspended' && typeof ctx.resume === 'function') {
|
|
1224
|
+
void ctx.resume().catch(() => undefined);
|
|
1225
|
+
}
|
|
1226
|
+
}
|
|
1227
|
+
}
|
|
1228
|
+
kickContext(ctx) {
|
|
1229
|
+
if (typeof ctx.addEventListener === 'function') {
|
|
1230
|
+
ctx.addEventListener('statechange', () => {
|
|
1231
|
+
if (ctx !== this.audioCtx)
|
|
1232
|
+
return;
|
|
1233
|
+
// Safari can also drop a running context back to suspended/interrupted (calls, Siri).
|
|
1234
|
+
if (ctx.state !== 'running' && ctx.state !== 'closed')
|
|
1235
|
+
this.installUnlock();
|
|
1236
|
+
this.publishState();
|
|
1237
|
+
});
|
|
1238
|
+
}
|
|
1239
|
+
if (ctx.state === 'suspended' && typeof ctx.resume === 'function') {
|
|
1240
|
+
void ctx.resume().catch(() => undefined);
|
|
1241
|
+
}
|
|
1242
|
+
}
|
|
1243
|
+
// ── shell publish + listener fan-out ──────────────────────────────────────
|
|
1244
|
+
// A speaker is visible when you can actually hear them: not muted-by-me AND in your voice space.
|
|
1245
|
+
// An identity with no live participant entry (mid-leave race) counts as ambient — the pre-channel
|
|
1246
|
+
// behavior, so worlds that never call setChannel see exactly the old speaking set.
|
|
1247
|
+
visibleSpeaking() {
|
|
1248
|
+
return [...this.speaking].filter((id) => {
|
|
1249
|
+
if (this.muted.has(id))
|
|
1250
|
+
return false;
|
|
1251
|
+
return (this.findParticipant(id)?.attributes?.[CHANNEL_ATTR] ?? '') === (this.channelId ?? '');
|
|
1252
|
+
});
|
|
1253
|
+
}
|
|
1254
|
+
emitSpeaking() {
|
|
1255
|
+
const ids = this.visibleSpeaking();
|
|
1256
|
+
for (const cb of this.speakingListeners)
|
|
1257
|
+
cb(ids);
|
|
1258
|
+
}
|
|
1259
|
+
emitParticipants() {
|
|
1260
|
+
const ids = this.participantIds();
|
|
1261
|
+
for (const cb of this.participantListeners)
|
|
1262
|
+
cb(ids);
|
|
1263
|
+
}
|
|
1264
|
+
// Signature-guarded like multiplayer's publishRoomState: posts only on a REAL change.
|
|
1265
|
+
publishState() {
|
|
1266
|
+
if (!this.host.publishVoiceState)
|
|
1267
|
+
return;
|
|
1268
|
+
const state = {
|
|
1269
|
+
available: this.available,
|
|
1270
|
+
connected: this.connected,
|
|
1271
|
+
micMode: this.micMode,
|
|
1272
|
+
pttKey: this.pttKey,
|
|
1273
|
+
transmitting: this.connected &&
|
|
1274
|
+
this.captured &&
|
|
1275
|
+
this.permission === 'granted' &&
|
|
1276
|
+
this.shouldTransmit() &&
|
|
1277
|
+
!this.micSuspended(),
|
|
1278
|
+
permission: this.permission,
|
|
1279
|
+
...(this.activeDeviceId ? { activeDeviceId: this.activeDeviceId } : {}),
|
|
1280
|
+
devices: this.devices,
|
|
1281
|
+
speakingUserIds: this.visibleSpeaking(),
|
|
1282
|
+
...(this.channelId ? { channelId: this.channelId } : {}),
|
|
1283
|
+
...(this.audioBlocked || this.micSuspended() ? { audioBlocked: true } : {}),
|
|
1284
|
+
...(this.error ? { error: this.error } : {}),
|
|
1285
|
+
};
|
|
1286
|
+
const sig = JSON.stringify(state);
|
|
1287
|
+
if (sig === this.stateSig)
|
|
1288
|
+
return;
|
|
1289
|
+
this.stateSig = sig;
|
|
1290
|
+
this.host.publishVoiceState(state);
|
|
1291
|
+
}
|
|
1292
|
+
// Reset in-memory state and tell the shell voice is gone. The muted set, mic mode, mic tuning, and
|
|
1293
|
+
// the channel survive (preferences/session intent reapplied on the next join); capture/permission
|
|
1294
|
+
// state does not.
|
|
1295
|
+
teardown() {
|
|
1296
|
+
this.detachInputListeners();
|
|
1297
|
+
this.audioRoot?.remove();
|
|
1298
|
+
this.audioRoot = null;
|
|
1299
|
+
if (this.levelTimer)
|
|
1300
|
+
clearInterval(this.levelTimer);
|
|
1301
|
+
this.levelTimer = null;
|
|
1302
|
+
if (this.captured)
|
|
1303
|
+
this.host.publishVoiceLevel?.(0);
|
|
1304
|
+
// The analyser is going away; leave every local-face consumer at rest, not on the last syllable.
|
|
1305
|
+
this.emitLocalSilence();
|
|
1306
|
+
this.analyserSpectrum = null;
|
|
1307
|
+
this.micStream?.getTracks().forEach((t) => t.stop());
|
|
1308
|
+
this.micStream = null;
|
|
1309
|
+
this.micSource = null;
|
|
1310
|
+
this.volumeGain = null;
|
|
1311
|
+
this.compressor = null;
|
|
1312
|
+
this.analyser = null;
|
|
1313
|
+
this.analyserBuf = null;
|
|
1314
|
+
this.gateGain = null;
|
|
1315
|
+
this.muteGain = null;
|
|
1316
|
+
void this.audioCtx?.close().catch(() => undefined);
|
|
1317
|
+
this.audioCtx = null;
|
|
1318
|
+
// Zero receiver-local face consumers before dropping the old audio graph so a disconnect can
|
|
1319
|
+
// never leave a remote avatar frozen at its last non-zero jaw weight.
|
|
1320
|
+
this.stopRemoteLevelAnalysis(true);
|
|
1321
|
+
this.spatialTracks.clear();
|
|
1322
|
+
this.panners.clear();
|
|
1323
|
+
this.positions.clear();
|
|
1324
|
+
this.listenerPose = null;
|
|
1325
|
+
// Ours, not LiveKit's: an externally provided webAudioMix context is never closed by the room.
|
|
1326
|
+
void this.mixCtx?.close().catch(() => undefined);
|
|
1327
|
+
this.mixCtx = null;
|
|
1328
|
+
this.micLevel = 0;
|
|
1329
|
+
this.gateOpenUntil = 0;
|
|
1330
|
+
if (this.room)
|
|
1331
|
+
this.host.publishVoiceState?.(null);
|
|
1332
|
+
this.room = null;
|
|
1333
|
+
this.pairing = null;
|
|
1334
|
+
this.events = null;
|
|
1335
|
+
this.connected = false;
|
|
1336
|
+
this.captured = false;
|
|
1337
|
+
this.pttHeld = false;
|
|
1338
|
+
this.speaking.clear();
|
|
1339
|
+
this.participantBase.clear();
|
|
1340
|
+
this.devices = [];
|
|
1341
|
+
this.activeDeviceId = null;
|
|
1342
|
+
this.audioBlocked = false;
|
|
1343
|
+
this.error = null;
|
|
1344
|
+
this.stateSig = null;
|
|
1345
|
+
this.unlockInstalled = false;
|
|
1346
|
+
}
|
|
1347
|
+
}
|
|
1348
|
+
//# sourceMappingURL=voice.js.map
|