@needle-tools/engine 5.0.2 → 5.1.0-canary.525aa82

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. package/CHANGELOG.md +24 -0
  2. package/README.md +6 -7
  3. package/SKILL.md +39 -21
  4. package/components.needle.json +1 -1
  5. package/dist/needle-engine.bundle-DPag02s9.min.js +1732 -0
  6. package/dist/needle-engine.bundle-IPMzQpe1.umd.cjs +1732 -0
  7. package/dist/{needle-engine.bundle-BoTyA-Le.js → needle-engine.bundle-qa_NEunk.js} +8881 -8148
  8. package/dist/needle-engine.d.ts +633 -61
  9. package/dist/needle-engine.js +576 -565
  10. package/dist/needle-engine.min.js +1 -1
  11. package/dist/needle-engine.umd.cjs +1 -1
  12. package/dist/{vendor-vHLk8sXu.js → vendor-CAcsI0eU.js} +116 -115
  13. package/dist/{vendor-CntUvmJu.umd.cjs → vendor-CEM38hLE.umd.cjs} +2 -2
  14. package/dist/{vendor-DPbfJJ4d.min.js → vendor-HRlxIBga.min.js} +2 -2
  15. package/lib/engine/api.d.ts +2 -0
  16. package/lib/engine/api.js +2 -0
  17. package/lib/engine/api.js.map +1 -1
  18. package/lib/engine/engine_addressables.js +5 -1
  19. package/lib/engine/engine_addressables.js.map +1 -1
  20. package/lib/engine/engine_animation.d.ts +14 -7
  21. package/lib/engine/engine_animation.js +49 -9
  22. package/lib/engine/engine_animation.js.map +1 -1
  23. package/lib/engine/engine_components.js +33 -4
  24. package/lib/engine/engine_components.js.map +1 -1
  25. package/lib/engine/engine_context.d.ts +7 -2
  26. package/lib/engine/engine_context.js +10 -2
  27. package/lib/engine/engine_context.js.map +1 -1
  28. package/lib/engine/engine_gameobject.d.ts +4 -0
  29. package/lib/engine/engine_gameobject.js.map +1 -1
  30. package/lib/engine/engine_init.js +4 -0
  31. package/lib/engine/engine_init.js.map +1 -1
  32. package/lib/engine/engine_input.js +4 -1
  33. package/lib/engine/engine_input.js.map +1 -1
  34. package/lib/engine/engine_materialpropertyblock.js +1 -20
  35. package/lib/engine/engine_materialpropertyblock.js.map +1 -1
  36. package/lib/engine/engine_networking.d.ts +11 -8
  37. package/lib/engine/engine_networking.js +43 -26
  38. package/lib/engine/engine_networking.js.map +1 -1
  39. package/lib/engine/engine_networking_instantiate.d.ts +100 -5
  40. package/lib/engine/engine_networking_instantiate.js +150 -16
  41. package/lib/engine/engine_networking_instantiate.js.map +1 -1
  42. package/lib/engine/engine_networking_prefabs.d.ts +59 -0
  43. package/lib/engine/engine_networking_prefabs.js +67 -0
  44. package/lib/engine/engine_networking_prefabs.js.map +1 -0
  45. package/lib/engine/engine_physics_rapier.d.ts +3 -0
  46. package/lib/engine/engine_physics_rapier.js +13 -9
  47. package/lib/engine/engine_physics_rapier.js.map +1 -1
  48. package/lib/engine/postprocessing/api.d.ts +2 -0
  49. package/lib/engine/postprocessing/api.js +2 -0
  50. package/lib/engine/postprocessing/api.js.map +1 -0
  51. package/lib/engine/postprocessing/index.d.ts +2 -0
  52. package/lib/engine/postprocessing/index.js +2 -0
  53. package/lib/engine/postprocessing/index.js.map +1 -0
  54. package/lib/engine/postprocessing/postprocessing.d.ts +83 -0
  55. package/lib/engine/postprocessing/postprocessing.js +280 -0
  56. package/lib/engine/postprocessing/postprocessing.js.map +1 -0
  57. package/lib/engine/postprocessing/types.d.ts +39 -0
  58. package/lib/engine/postprocessing/types.js +2 -0
  59. package/lib/engine/postprocessing/types.js.map +1 -0
  60. package/lib/engine/webcomponents/WebXRButtons.js +17 -3
  61. package/lib/engine/webcomponents/WebXRButtons.js.map +1 -1
  62. package/lib/engine/webcomponents/icons.js +3 -1
  63. package/lib/engine/webcomponents/icons.js.map +1 -1
  64. package/lib/engine/xr/NeedleXRSession.d.ts +1 -0
  65. package/lib/engine/xr/NeedleXRSession.js +43 -10
  66. package/lib/engine/xr/NeedleXRSession.js.map +1 -1
  67. package/lib/engine/xr/init.d.ts +4 -0
  68. package/lib/engine/xr/init.js +49 -0
  69. package/lib/engine/xr/init.js.map +1 -0
  70. package/lib/engine-components/AnimationUtils.d.ts +4 -1
  71. package/lib/engine-components/AnimationUtils.js +7 -19
  72. package/lib/engine-components/AnimationUtils.js.map +1 -1
  73. package/lib/engine-components/AnimatorController.d.ts +135 -2
  74. package/lib/engine-components/AnimatorController.js +216 -13
  75. package/lib/engine-components/AnimatorController.js.map +1 -1
  76. package/lib/engine-components/GroundProjection.d.ts +1 -0
  77. package/lib/engine-components/GroundProjection.js +184 -48
  78. package/lib/engine-components/GroundProjection.js.map +1 -1
  79. package/lib/engine-components/OrbitControls.d.ts +4 -0
  80. package/lib/engine-components/OrbitControls.js +5 -1
  81. package/lib/engine-components/OrbitControls.js.map +1 -1
  82. package/lib/engine-components/SeeThrough.d.ts +0 -2
  83. package/lib/engine-components/SeeThrough.js +0 -89
  84. package/lib/engine-components/SeeThrough.js.map +1 -1
  85. package/lib/engine-components/SyncedRoom.d.ts +4 -0
  86. package/lib/engine-components/SyncedRoom.js +23 -8
  87. package/lib/engine-components/SyncedRoom.js.map +1 -1
  88. package/lib/engine-components/SyncedTransform.js +5 -5
  89. package/lib/engine-components/SyncedTransform.js.map +1 -1
  90. package/lib/engine-components/Voip.d.ts +46 -0
  91. package/lib/engine-components/Voip.js +126 -2
  92. package/lib/engine-components/Voip.js.map +1 -1
  93. package/lib/engine-components/api.d.ts +1 -0
  94. package/lib/engine-components/api.js +1 -0
  95. package/lib/engine-components/api.js.map +1 -1
  96. package/lib/engine-components/codegen/components.d.ts +1 -0
  97. package/lib/engine-components/codegen/components.js +1 -0
  98. package/lib/engine-components/codegen/components.js.map +1 -1
  99. package/lib/engine-components/postprocessing/Effects/Tonemapping.d.ts +5 -2
  100. package/lib/engine-components/postprocessing/Effects/Tonemapping.js +11 -18
  101. package/lib/engine-components/postprocessing/Effects/Tonemapping.js.map +1 -1
  102. package/lib/engine-components/postprocessing/PostProcessingEffect.d.ts +3 -4
  103. package/lib/engine-components/postprocessing/PostProcessingEffect.js +6 -15
  104. package/lib/engine-components/postprocessing/PostProcessingEffect.js.map +1 -1
  105. package/lib/engine-components/postprocessing/PostProcessingHandler.d.ts +2 -1
  106. package/lib/engine-components/postprocessing/PostProcessingHandler.js.map +1 -1
  107. package/lib/engine-components/postprocessing/Volume.d.ts +18 -11
  108. package/lib/engine-components/postprocessing/Volume.js +61 -140
  109. package/lib/engine-components/postprocessing/Volume.js.map +1 -1
  110. package/lib/engine-components/postprocessing/index.d.ts +1 -0
  111. package/lib/engine-components/postprocessing/index.js +1 -0
  112. package/lib/engine-components/postprocessing/index.js.map +1 -1
  113. package/lib/engine-components/postprocessing/utils.d.ts +2 -0
  114. package/lib/engine-components/postprocessing/utils.js +2 -0
  115. package/lib/engine-components/postprocessing/utils.js.map +1 -1
  116. package/lib/engine-components/ui/Canvas.js +2 -2
  117. package/lib/engine-components/ui/Canvas.js.map +1 -1
  118. package/lib/engine-components/ui/Graphic.d.ts +3 -3
  119. package/lib/engine-components/ui/Graphic.js +6 -2
  120. package/lib/engine-components/ui/Graphic.js.map +1 -1
  121. package/lib/engine-components/ui/Text.d.ts +64 -11
  122. package/lib/engine-components/ui/Text.js +154 -45
  123. package/lib/engine-components/ui/Text.js.map +1 -1
  124. package/lib/engine-components/ui/index.d.ts +1 -0
  125. package/lib/engine-components/ui/index.js +1 -0
  126. package/lib/engine-components/ui/index.js.map +1 -1
  127. package/lib/engine-components-experimental/networking/PlayerSync.d.ts +25 -3
  128. package/lib/engine-components-experimental/networking/PlayerSync.js +60 -11
  129. package/lib/engine-components-experimental/networking/PlayerSync.js.map +1 -1
  130. package/package.json +6 -5
  131. package/plugins/vite/ai.d.ts +11 -10
  132. package/plugins/vite/ai.js +305 -31
  133. package/src/engine/api.ts +3 -0
  134. package/src/engine/engine_addressables.ts +4 -1
  135. package/src/engine/engine_animation.ts +47 -9
  136. package/src/engine/engine_components.ts +36 -7
  137. package/src/engine/engine_context.ts +11 -2
  138. package/src/engine/engine_gameobject.ts +5 -0
  139. package/src/engine/engine_init.ts +4 -0
  140. package/src/engine/engine_input.ts +2 -1
  141. package/src/engine/engine_materialpropertyblock.ts +1 -20
  142. package/src/engine/engine_networking.ts +46 -23
  143. package/src/engine/engine_networking_instantiate.ts +160 -18
  144. package/src/engine/engine_networking_prefabs.ts +80 -0
  145. package/src/engine/engine_physics_rapier.ts +14 -9
  146. package/src/engine/postprocessing/api.ts +2 -0
  147. package/src/engine/postprocessing/index.ts +2 -0
  148. package/src/engine/postprocessing/postprocessing.ts +322 -0
  149. package/src/engine/postprocessing/types.ts +43 -0
  150. package/src/engine/webcomponents/WebXRButtons.ts +21 -4
  151. package/src/engine/webcomponents/icons.ts +5 -3
  152. package/src/engine/xr/NeedleXRSession.ts +50 -15
  153. package/src/engine/xr/init.ts +56 -0
  154. package/src/engine-components/AnimationUtils.ts +7 -17
  155. package/src/engine-components/AnimatorController.ts +288 -18
  156. package/src/engine-components/GroundProjection.ts +226 -52
  157. package/src/engine-components/OrbitControls.ts +5 -1
  158. package/src/engine-components/SeeThrough.ts +0 -116
  159. package/src/engine-components/SyncedRoom.ts +28 -9
  160. package/src/engine-components/SyncedTransform.ts +5 -5
  161. package/src/engine-components/Voip.ts +129 -2
  162. package/src/engine-components/api.ts +1 -0
  163. package/src/engine-components/codegen/components.ts +1 -0
  164. package/src/engine-components/postprocessing/Effects/Tonemapping.ts +16 -24
  165. package/src/engine-components/postprocessing/PostProcessingEffect.ts +9 -16
  166. package/src/engine-components/postprocessing/PostProcessingHandler.ts +2 -1
  167. package/src/engine-components/postprocessing/Volume.ts +72 -163
  168. package/src/engine-components/postprocessing/index.ts +1 -0
  169. package/src/engine-components/postprocessing/utils.ts +2 -0
  170. package/src/engine-components/ui/Canvas.ts +2 -2
  171. package/src/engine-components/ui/Graphic.ts +7 -3
  172. package/src/engine-components/ui/Text.ts +170 -52
  173. package/src/engine-components/ui/index.ts +2 -1
  174. package/src/engine-components-experimental/networking/PlayerSync.ts +64 -11
  175. package/dist/needle-engine.bundle-B3ywqx5o.min.js +0 -1654
  176. package/dist/needle-engine.bundle-CzOPcOui.umd.cjs +0 -1654
@@ -15,10 +15,10 @@ import { DeviceUtilities, getParam } from "../engine/engine_utils.js";
15
15
  import { NeedleEngineWebComponent } from "../engine/webcomponents/needle-engine.js";
16
16
  import { Camera } from "./Camera.js";
17
17
  import { Behaviour, GameObject } from "./Component.js";
18
+ import { LookAtConstraint } from "./LookAtConstraint.js";
18
19
  import { SyncedTransform } from "./SyncedTransform.js";
19
20
  import { type AfterHandleInputEvent, EventSystem, EventSystemEvents } from "./ui/EventSystem.js";
20
21
  import { tryGetUIComponent } from "./ui/Utils.js";
21
- import { LookAtConstraint } from "./LookAtConstraint.js";
22
22
 
23
23
 
24
24
  const debug = getParam("debugorbit");
@@ -309,6 +309,10 @@ export class OrbitControls extends Behaviour implements ICameraController {
309
309
  set targetLerpDuration(v) { this._lookTargetLerpDuration = v; }
310
310
  private _lookTargetLerpDuration: number = 1;
311
311
 
312
+ /**
313
+ * When set, the camera's look at target will be clamped within the bounds of the specified Object3D. The bounds are defined by the world position and world scale of the assigned Object3D.
314
+ * @default null
315
+ */
312
316
  @serializable(Object3D)
313
317
  targetBounds: Object3D | null = null;
314
318
 
@@ -176,26 +176,6 @@ export class SeeThrough extends Behaviour {
176
176
  onEnable() {
177
177
  this._needsUpdate = true;
178
178
  this._renderer = null;
179
- // SeeThroughUsdzExporterPlugin.components.push(this);
180
- }
181
-
182
- /** @internal */
183
- onDisable() {
184
- // this._renderer?.forEach(r => {
185
- // const original = this.rendererMaterialsOriginal.get(r);
186
- // for (let i = 0; i < r.sharedMaterials.length; i++) {
187
- // const mat = r.sharedMaterials[i];
188
- // if (!mat) continue;
189
- // if (original && original[i]) {
190
- // r.sharedMaterials[i] = original[i];
191
- // }
192
- // }
193
- // this.rendererMaterials.delete(r);
194
- // this.rendererMaterialsOriginal.delete(r);
195
- // });
196
-
197
- // const index = SeeThroughUsdzExporterPlugin.components.indexOf(this);
198
- // if (index !== -1) SeeThroughUsdzExporterPlugin.components.splice(index, 1);
199
179
  }
200
180
 
201
181
  /**
@@ -207,7 +187,6 @@ export class SeeThrough extends Behaviour {
207
187
  if (this._needsUpdate) {
208
188
  this._needsUpdate = false;
209
189
  this._renderer = this.gameObject.getComponentsInChildren(Renderer);
210
-
211
190
  // NOTE: instead of using the object's anchor (gameObject.worldPosition) we could also get the object's bounding box center:
212
191
  // getBoundingBox(this.gameObject); // < import { getBoundingBox } from "@needle-tools/engine";
213
192
  this.updateDirection();
@@ -216,12 +195,9 @@ export class SeeThrough extends Behaviour {
216
195
  this.updateDirection();
217
196
  }
218
197
 
219
-
220
-
221
198
  if (!this.autoUpdate) return;
222
199
  if (!this.referencePoint) return;
223
200
 
224
-
225
201
  const dot = this._referencePointDir.dot(this.context.mainCamera.worldForward);
226
202
  const shouldHide = dot > .2;
227
203
 
@@ -272,38 +248,10 @@ export class SeeThrough extends Behaviour {
272
248
  renderer.gameObject.raycastAllowed = true;
273
249
  }
274
250
 
275
- // if (!this.rendererMaterials.has(renderer)) {
276
- // const originalMaterials = new Array<Material>();
277
- // const clonedMaterials = new Array<MaterialWithState>();
278
-
279
- // // We clone the materials once and store them, so we can modify the opacity without affecting other objects using the same material. This could potentially be optimized further to re-use materials between renderers if multiple renderers use the same material.
280
- // for (let i = 0; i < renderer.sharedMaterials.length; i++) {
281
- // const mat = renderer.sharedMaterials[i];
282
- // if (!mat) continue;
283
- // originalMaterials.push(mat);
284
- // const matClone = mat.clone() as MaterialWithState;
285
- // // @ts-ignore
286
- // matClone.userData = mat.userData || {};
287
- // matClone.userData.seeThrough = {
288
- // initial: {
289
- // opacity: matClone.opacity,
290
- // transparent: matClone.transparent,
291
- // alphaHash: matClone.alphaHash
292
- // }
293
- // }
294
- // clonedMaterials.push(matClone);
295
- // // renderer.sharedMaterials[i] = matClone;
296
- // }
297
-
298
- // this.rendererMaterials.set(renderer, clonedMaterials);
299
- // this.rendererMaterialsOriginal.set(renderer, originalMaterials);
300
- // }
301
-
302
251
  const materials = renderer.sharedMaterials;// : this.rendererMaterials.get(renderer);
303
252
  if (!materials) return;
304
253
 
305
254
  const block = MaterialPropertyBlock.get(renderer.gameObject);
306
-
307
255
  const currentOpacity = (block.getOverride("opacity")?.value ?? materials[0].opacity ?? 1);
308
256
 
309
257
  let newAlpha = Mathf.lerp(currentOpacity, targetAlpha, duration <= 0 ? 1 : this.context.time.deltaTime / duration);;
@@ -315,72 +263,8 @@ export class SeeThrough extends Behaviour {
315
263
  block.setOverride("alphaHash", this.useAlphaHash);
316
264
  block.setOverride("opacity", newAlpha);
317
265
  block.setOverride("transparent", newAlpha >= 0.99999 ? false : !this.useAlphaHash);
318
-
319
-
320
- // for (const mat of materials) {
321
- // if (!mat) continue;
322
-
323
- // let newAlpha = Mathf.lerp(mat.opacity, targetAlpha, duration <= 0 ? 1 : this.context.time.deltaTime / duration);;
324
- // if (newAlpha >= 0.99) newAlpha = 1;
325
- // else if (newAlpha <= 0.01) newAlpha = 0;
326
-
327
-
328
- // const wasTransparent = mat.transparent;
329
- // const wasAlphaHash = mat.alphaHash;
330
- // const previousOpacity = mat.opacity;
331
-
332
- // mat.alphaHash = this.useAlphaHash;
333
-
334
- // if (mat.userData && "seeThrough" in mat.userData) {
335
- // const initial = mat.userData.seeThrough.initial as MaterialState;
336
- // mat.opacity = initial.opacity * newAlpha;
337
- // mat.transparent = mat.opacity >= 1 ? initial.transparent : !this.useAlphaHash;
338
- // }
339
- // else {
340
- // mat.transparent = mat.opacity >= 1 ? false : !this.useAlphaHash;
341
- // }
342
-
343
- // if (wasTransparent !== mat.transparent
344
- // || wasAlphaHash !== mat.alphaHash
345
- // || mat.opacity !== previousOpacity // MeshPhysicsMaterial needs that and maybe other materials too...
346
- // ) {
347
- // mat.needsUpdate = true;
348
- // }
349
- // }
350
266
  });
351
267
  }
352
268
 
353
269
  }
354
270
 
355
-
356
- ;
357
- // class SeeThroughUsdzExporterPlugin implements IUSDExporterExtension {
358
-
359
- // static readonly components: SeeThrough[] = [];
360
-
361
- // get extensionName() {
362
- // return "SeeThrough";
363
- // }
364
-
365
- // // onExportObject(object: Object3D<Object3DEventMap>, model: USDObject, context: USDZExporterContext) {
366
- // // const component = SeeThroughUsdzExporterPlugin.components.find(c => c.gameObject === object);
367
- // // if(!component) return;
368
- // // console.log("OH MY GOD SEE THROUGH USDZ EXPORTER", component, model);
369
-
370
- // // model.materialName = "AlphaHashMaterialInstance"; // we could make this unique per object if needed
371
-
372
- // // model.addEventListener("serialize", (writer, context) => {
373
- // // writer.appendLine(`# SeeThrough component on ${object.name}`);
374
- // // });
375
- // // }
376
-
377
- // }
378
-
379
- // const seeThroughUsdzExporterPlugin = new SeeThroughUsdzExporterPlugin();
380
-
381
- // USDZExporter.beforeExport.addEventListener(args => {
382
- // if (SeeThroughUsdzExporterPlugin.components.length === 0) return;
383
- // if (args.exporter.extensions.includes(seeThroughUsdzExporterPlugin) === false) {
384
- // args.exporter.extensions.push(seeThroughUsdzExporterPlugin);
385
- // }
386
- // });
@@ -121,8 +121,36 @@ export class SyncedRoom extends Behaviour {
121
121
  if (debug) console.log(`SyncedRoom roomName:${this.roomName}, urlParamName:${this.urlParameterName}, joinRandomRoom:${this.joinRandomRoom}`);
122
122
  }
123
123
 
124
+ private _hasConnectedBefore: boolean = false;
125
+
124
126
  /** @internal */
125
127
  onEnable() {
128
+ if (this.createJoinButton) {
129
+ const button = this.createRoomButton();
130
+ this.context.menu.appendChild(button);
131
+ }
132
+ if (this.createViewOnlyButton) {
133
+ this.onEnableViewOnlyButton()
134
+ }
135
+
136
+ // On re-enable (after disable), reconnect immediately
137
+ if (this._hasConnectedBefore) {
138
+ this._connectToRoom();
139
+ }
140
+ }
141
+
142
+ /** @internal */
143
+ start() {
144
+ // Defer initial connection to start() so other components added in the same frame
145
+ // have time to register their listeners in awake/onEnable (e.g. PlayerSync)
146
+ if (!this._hasConnectedBefore) {
147
+ this._connectToRoom();
148
+ }
149
+ }
150
+
151
+ private _connectToRoom() {
152
+ this._hasConnectedBefore = true;
153
+
126
154
  // if the url contains a view parameter override room and join in view mode
127
155
  const viewId = utils.getParam(viewParamName);
128
156
  if (viewId && typeof viewId === "string" && viewId.length > 0) {
@@ -130,16 +158,7 @@ export class SyncedRoom extends Behaviour {
130
158
  this.context.connection.joinRoom(viewId, true);
131
159
  return;
132
160
  }
133
- // If setup to join a random room
134
161
  this.tryJoinRoom();
135
-
136
- if (this.createJoinButton) {
137
- const button = this.createRoomButton();
138
- this.context.menu.appendChild(button);
139
- }
140
- if (this.createViewOnlyButton) {
141
- this.onEnableViewOnlyButton()
142
- }
143
162
  }
144
163
 
145
164
  /** @internal */
@@ -145,7 +145,7 @@ export class SyncedTransform extends Behaviour {
145
145
  */
146
146
  public requestOwnership() {
147
147
  if (debug)
148
- console.log("Request ownership");
148
+ console.log("[SyncedTransform] Request ownership");
149
149
  if (!this._model) {
150
150
  this._shouldRequestOwnership = true;
151
151
  this._needsUpdate = true;
@@ -189,7 +189,7 @@ export class SyncedTransform extends Behaviour {
189
189
  /** @internal */
190
190
  awake() {
191
191
  if (debug)
192
- console.log("new instance", this.guid, this);
192
+ console.log("[SyncedTransform] new instance", this.guid, this);
193
193
  this._receivedDataBefore = false;
194
194
  this._targetPosition = new Vector3();
195
195
  this._targetRotation = new Quaternion();
@@ -245,7 +245,7 @@ export class SyncedTransform extends Behaviour {
245
245
  if (this.destroyed) return;
246
246
  if (typeof data.guid === "function" && data.guid() === this.guid) {
247
247
  if (debug)
248
- console.log("new data", this.context.connection.connectionId, this.context.time.frameCount, this.guid, data);
248
+ console.log("[SyncedTransform] new data", this.context.connection.connectionId, this.context.time.frameCount, this.guid, data);
249
249
  this.receivedUpdate = true;
250
250
  this._receivedFastUpdate = data.fast();
251
251
  const transform = data.transform();
@@ -323,7 +323,7 @@ export class SyncedTransform extends Behaviour {
323
323
 
324
324
  if (!this.context.connection.isInRoom || !this._model) {
325
325
  if (debug)
326
- console.log("no model or room", this.name, this.guid, this.context.connection.isInRoom);
326
+ console.log("[SyncedTransform] no model or room", this.name, this.guid, this.context.connection.isInRoom);
327
327
  return;
328
328
  }
329
329
 
@@ -409,7 +409,7 @@ export class SyncedTransform extends Behaviour {
409
409
  if (this.rb && this.overridePhysics) {
410
410
  if (this._wasKinematic !== undefined) {
411
411
  if (debug)
412
- console.log("reset kinematic", this.rb.name, this._wasKinematic);
412
+ console.log("[SyncedTransform] reset kinematic", this.rb.name, this._wasKinematic);
413
413
  this.rb.isKinematic = this._wasKinematic;
414
414
  }
415
415
 
@@ -3,10 +3,10 @@ import { Application } from "../engine/engine_application.js";
3
3
  import { RoomEvents } from "../engine/engine_networking.js";
4
4
  import { disposeStream, NetworkedStreamEvents, NetworkedStreams, StreamEndedEvent, StreamReceivedEvent } from "../engine/engine_networking_streams.js"
5
5
  import { serializable } from "../engine/engine_serialization_decorator.js";
6
- import { DeviceUtilities, getParam } from "../engine/engine_utils.js";
7
- import { delay } from "../engine/engine_utils.js";
6
+ import { delay, DeviceUtilities, getParam } from "../engine/engine_utils.js";
8
7
  import { getIconElement } from "../engine/webcomponents/icons.js";
9
8
  import { Behaviour } from "./Component.js";
9
+ import { EventList } from "./EventList.js";
10
10
 
11
11
  export const noVoip = "noVoip";
12
12
  const debugParam = getParam("debugvoip");
@@ -75,6 +75,66 @@ export class Voip extends Behaviour {
75
75
  */
76
76
  debug: boolean = false;
77
77
 
78
+ private _volume: number = 1;
79
+
80
+ /**
81
+ * Volume for incoming audio streams (0 = silent, 1 = full volume).
82
+ * Changes apply immediately to all active incoming streams.
83
+ */
84
+ @serializable()
85
+ get volume(): number { return this._volume; }
86
+ set volume(val: number) {
87
+ this._volume = val;
88
+ for (const audio of this._incomingStreams.values()) {
89
+ audio.volume = val;
90
+ }
91
+ }
92
+
93
+ /**
94
+ * Get the incoming audio element for a specific user.
95
+ * Use this to route audio through the Web Audio API for spatial audio, effects, or analysis.
96
+ * @param userId The connection ID of the remote user
97
+ * @returns The HTMLAudioElement for this user's stream, or undefined if not connected
98
+ * @example
99
+ * ```ts
100
+ * const audioEl = voip.getAudioElement(userId);
101
+ * if (audioEl) {
102
+ * const audioCtx = new AudioContext();
103
+ * const source = audioCtx.createMediaElementSource(audioEl);
104
+ * const panner = audioCtx.createPanner();
105
+ * source.connect(panner).connect(audioCtx.destination);
106
+ * }
107
+ * ```
108
+ */
109
+ getAudioElement(userId: string): HTMLAudioElement | undefined {
110
+ return this._incomingStreams.get(userId);
111
+ }
112
+
113
+ /**
114
+ * Get all active incoming audio streams as a Map of userId → HTMLAudioElement.
115
+ * Useful for iterating over all connected voice users.
116
+ */
117
+ get incomingStreams(): ReadonlyMap<string, HTMLAudioElement> {
118
+ return this._incomingStreams;
119
+ }
120
+
121
+ /**
122
+ * Threshold for speaking detection (0–255). When a user's audio amplitude exceeds this,
123
+ * they are considered "speaking". Default is 30.
124
+ */
125
+ @serializable()
126
+ speakingThreshold: number = 30;
127
+
128
+ /**
129
+ * Event fired when a user's speaking state changes.
130
+ * Passes `{ userId: string, isSpeaking: boolean, volume: number }`.
131
+ */
132
+ @serializable(EventList)
133
+ onSpeakingChanged: EventList = new EventList();
134
+
135
+ private _speakingStates = new Map<string, boolean>();
136
+ private _analysers = new Map<string, { analyser: AnalyserNode, data: Uint8Array, context: AudioContext }>();
137
+
78
138
  private _net?: NetworkedStreams;
79
139
  private _menubutton?: HTMLElement;
80
140
 
@@ -141,12 +201,25 @@ export class Voip extends Behaviour {
141
201
  this.onEnabledChanged();
142
202
  this.updateButton();
143
203
  window.removeEventListener("visibilitychange", this.onVisibilityChanged);
204
+ // Clean up analysers
205
+ for (const userId of [...this._analysers.keys()]) {
206
+ this.cleanupAnalyser(userId);
207
+ }
144
208
  }
145
209
 
146
210
  /** @internal */
147
211
  onDestroy(): void {
148
212
  this._menubutton?.remove();
149
213
  this._menubutton = undefined;
214
+ // Clean up all streams and analysers
215
+ for (const userId of [...this._analysers.keys()]) {
216
+ this.cleanupAnalyser(userId);
217
+ }
218
+ for (const incoming of this._incomingStreams.values()) {
219
+ disposeStream(incoming.srcObject as MediaStream);
220
+ }
221
+ this._incomingStreams.clear();
222
+ this._speakingStates.clear();
150
223
  }
151
224
 
152
225
  /** Set via the mic button (e.g. when the websocket connection closes and rejoins but the user was muted before we don't want to enable VOIP again automatically) */
@@ -347,10 +420,61 @@ export class Voip extends Behaviour {
347
420
  disposeStream(incoming.srcObject as MediaStream);
348
421
  }
349
422
  this._incomingStreams.clear();
423
+ for (const userId of this._analysers.keys()) {
424
+ this.cleanupAnalyser(userId);
425
+ }
350
426
  }
351
427
 
352
428
  private _incomingStreams: Map<string, HTMLAudioElement> = new Map();
353
429
 
430
+ /** @internal */
431
+ update() {
432
+ // Only run speaking detection if someone is listening
433
+ if (!this.onSpeakingChanged || this.onSpeakingChanged.listenerCount <= 0) return;
434
+
435
+ for (const [userId, info] of this._analysers) {
436
+ info.analyser.getByteFrequencyData(info.data as Uint8Array<ArrayBuffer>);
437
+ // Average amplitude
438
+ let sum = 0;
439
+ for (let i = 0; i < info.data.length; i++) sum += info.data[i];
440
+ const avg = sum / info.data.length;
441
+
442
+ const wasSpeaking = this._speakingStates.get(userId) ?? false;
443
+ const isSpeaking = avg > this.speakingThreshold;
444
+
445
+ if (isSpeaking !== wasSpeaking) {
446
+ this._speakingStates.set(userId, isSpeaking);
447
+ this.onSpeakingChanged.invoke({ userId, isSpeaking, volume: avg / 255 });
448
+ }
449
+ }
450
+ }
451
+
452
+ private setupAnalyser(userId: string, _audioElement: HTMLAudioElement, stream: MediaStream) {
453
+ // Only set up if someone is listening or might listen
454
+ if (this._analysers.has(userId)) return;
455
+ try {
456
+ const audioCtx = new AudioContext();
457
+ const source = audioCtx.createMediaStreamSource(stream);
458
+ const analyser = audioCtx.createAnalyser();
459
+ analyser.fftSize = 256;
460
+ source.connect(analyser);
461
+ const data = new Uint8Array(analyser.frequencyBinCount);
462
+ this._analysers.set(userId, { analyser, data, context: audioCtx });
463
+ }
464
+ catch (err) {
465
+ if (this.debug) console.warn("VOIP: Failed to create analyser for", userId, err);
466
+ }
467
+ }
468
+
469
+ private cleanupAnalyser(userId: string) {
470
+ const info = this._analysers.get(userId);
471
+ if (info) {
472
+ info.context.close();
473
+ this._analysers.delete(userId);
474
+ }
475
+ this._speakingStates.delete(userId);
476
+ }
477
+
354
478
  private onReceiveStream = (evt: StreamReceivedEvent) => {
355
479
  const userId = evt.target.userId;
356
480
  const stream = evt.stream;
@@ -361,7 +485,9 @@ export class Voip extends Behaviour {
361
485
  this._incomingStreams.set(userId, audioElement);
362
486
  }
363
487
  audioElement.srcObject = stream;
488
+ audioElement.volume = this._volume;
364
489
  audioElement.setAttribute("autoplay", "true");
490
+ this.setupAnalyser(userId, audioElement, stream);
365
491
  // for mobile we need to wait for user interaction to play audio. Auto play doesnt work on android when the page is refreshed
366
492
  Application.registerWaitForInteraction(() => {
367
493
  audioElement?.play().catch((err) => {
@@ -374,6 +500,7 @@ export class Voip extends Behaviour {
374
500
  const existing = this._incomingStreams.get(evt.userId);
375
501
  disposeStream(existing?.srcObject as MediaStream);
376
502
  this._incomingStreams.delete(evt.userId);
503
+ this.cleanupAnalyser(evt.userId);
377
504
  }
378
505
 
379
506
  private onEnabledChanged = () => {
@@ -35,6 +35,7 @@
35
35
  */
36
36
 
37
37
  export * from "./codegen/components.js";
38
+ export { AnimatorControllerBuilder, type ConditionMode, type StateOptions, type TransitionOptions } from "./AnimatorController.js";
38
39
  export { Collider } from "./Collider.js"; // export abstract type
39
40
  export { Behaviour, Component, GameObject } from "./Component.js";
40
41
 
@@ -6,6 +6,7 @@ export { Animation } from "../Animation.js";
6
6
  export { Keyframe } from "../AnimationCurve.js";
7
7
  export { AnimationCurve } from "../AnimationCurve.js";
8
8
  export { Animator } from "../Animator.js";
9
+ export { AnimatorControllerBuilder } from "../AnimatorController.js";
9
10
  export { AnimatorController } from "../AnimatorController.js";
10
11
  export { AudioListener } from "../AudioListener.js";
11
12
  export { AudioSource } from "../AudioSource.js";
@@ -1,11 +1,11 @@
1
1
  import type { ToneMappingEffect as _TonemappingEffect, ToneMappingMode } from "postprocessing";
2
+ import type { ToneMapping } from "three";
2
3
 
3
4
  import { MODULES } from "../../../engine/engine_modules.js";
4
5
  import { serializable } from "../../../engine/engine_serialization.js";
5
6
  import { nameToThreeTonemapping } from "../../../engine/engine_tonemapping.js";
6
7
  import { getParam } from "../../../engine/engine_utils.js";
7
8
  import { EffectProviderResult, PostProcessingEffect } from "../PostProcessingEffect.js";
8
- import { findPostProcessingManager } from "../utils.js";
9
9
  import { VolumeParameter } from "../VolumeParameter.js";
10
10
  import { registerCustomEffectType } from "../VolumeProfile.js";
11
11
  import { NEToneMappingMode, NEToneMappingModeNames, threeToNeedleToneMapping, threeToneMappingToEffectMode, toThreeToneMapping } from "./Tonemapping.utils.js";
@@ -14,8 +14,8 @@ const debug = getParam("debugpost");
14
14
 
15
15
 
16
16
  /**
17
- * [ToneMappingEffect](https://engine.needle.tools/docs/api/ToneMappingEffect) adjusts the brightness and contrast of the rendered scene to map high dynamic range (HDR) colors to a displayable range.
18
- * This effect is essential for achieving realistic lighting and color representation in 3D scenes, as it helps to preserve details in both bright and dark areas.
17
+ * [ToneMappingEffect](https://engine.needle.tools/docs/api/ToneMappingEffect) adjusts the brightness and contrast of the rendered scene to map high dynamic range (HDR) colors to a displayable range.
18
+ * This effect is essential for achieving realistic lighting and color representation in 3D scenes, as it helps to preserve details in both bright and dark areas.
19
19
  * Various tonemapping algorithms can be applied to achieve different visual styles and effects.
20
20
  * @summary Tonemapping Post-Processing Effect
21
21
  * @category Effects
@@ -46,18 +46,23 @@ export class ToneMappingEffect extends PostProcessingEffect {
46
46
 
47
47
  get isToneMapping() { return true; }
48
48
 
49
- onEffectEnabled(): void {
50
- // Tonemapping works with and without a postprocessing manager.
51
- // If there's no manager already in the scene we don't need to create one because tonemapping can also be applied without a postprocessing pass
52
- const ppmanager = findPostProcessingManager(this);
53
- if (!ppmanager) return;
54
- super.onEffectEnabled(ppmanager);
49
+ /** The three.js ToneMapping enum value resolved from the current mode */
50
+ get threeToneMapping(): ToneMapping {
51
+ if (this.mode.isInitialized && this.mode.overrideState)
52
+ return toThreeToneMapping(this.mode.value);
53
+ return this.context.renderer.toneMapping as ToneMapping;
54
+ }
55
+
56
+ /** The exposure value to apply */
57
+ get toneMappingExposure(): number {
58
+ if (this.exposure.overrideState && this.exposure.value !== undefined)
59
+ return Math.max(0.0, this.exposure.value);
60
+ return this.context.renderer.toneMappingExposure;
55
61
  }
56
62
 
57
63
  private _tonemappingEffect: _TonemappingEffect | null = null;
58
64
  onCreateEffect(): EffectProviderResult | undefined {
59
65
 
60
-
61
66
  // ensure the effect tonemapping value is initialized
62
67
  if (this.mode.isInitialized == false) {
63
68
  const mode = threeToNeedleToneMapping(this.context.renderer.toneMapping);
@@ -90,19 +95,6 @@ export class ToneMappingEffect extends PostProcessingEffect {
90
95
  return tonemapping;
91
96
  }
92
97
 
93
-
94
- onBeforeRender(): void {
95
- if (this._tonemappingEffect && this.postprocessingContext?.handler.getEffectIsActive(this._tonemappingEffect)) {
96
- if (this.mode.overrideState)
97
- this.context.renderer.toneMapping = toThreeToneMapping(this.mode.value);
98
- if (this.exposure.overrideState && this.exposure.value !== undefined) {
99
- const newValue = Math.max(0.0, this.exposure.value);
100
- this.context.renderer.toneMappingExposure = newValue;
101
- }
102
- }
103
- }
104
-
105
-
106
98
  }
107
99
 
108
- registerCustomEffectType("Tonemapping", ToneMappingEffect);
100
+ registerCustomEffectType("Tonemapping", ToneMappingEffect);
@@ -3,9 +3,10 @@ import type { Effect, Pass } from "postprocessing";
3
3
  import type { EditorModification, IEditorModification } from "../../engine/engine_editor-sync.js";
4
4
  import { serializable } from "../../engine/engine_serialization.js";
5
5
  import { getParam } from "../../engine/engine_utils.js";
6
+ import type { PostProcessing } from "../../engine/postprocessing/index.js";
7
+ import type { IPostProcessingEffect } from "../../engine/postprocessing/types.js";
6
8
  import { Component } from "../Component.js";
7
9
  import type { PostProcessingHandler } from "./PostProcessingHandler.js";
8
- import { getPostProcessingManager, IPostProcessingManager } from "./utils.js";
9
10
  import { VolumeParameter } from "./VolumeParameter.js";
10
11
 
11
12
  const debug = getParam("debugpost");
@@ -50,8 +51,9 @@ export interface IEffectProvider {
50
51
  *
51
52
  * @category Effects
52
53
  * @group Components
54
+ * @see {@link PostProcessing} for core Needle Engine postprocessing control, also accessible via `context.postprocessing`
53
55
  */
54
- export abstract class PostProcessingEffect extends Component implements IEffectProvider, IEditorModification {
56
+ export abstract class PostProcessingEffect extends Component implements IPostProcessingEffect, IEffectProvider, IEditorModification {
55
57
 
56
58
  get isPostProcessingEffect() { return true; }
57
59
 
@@ -101,31 +103,22 @@ export abstract class PostProcessingEffect extends Component implements IEffectP
101
103
  @serializable()
102
104
  active: boolean = true;
103
105
 
104
- private _manager: IPostProcessingManager | null = null;
105
-
106
106
  onEnable(): void {
107
107
  super.onEnable();
108
108
  if (debug) console.warn("Enable", this.constructor.name + (!this.__internalDidAwakeAndStart ? " (awake)" : ""));
109
- // Dont override the serialized value by enabling (we could also just disable this component / map enabled to active)
110
- if (this.__internalDidAwakeAndStart)
111
- this.active = true;
112
- this.onEffectEnabled();
109
+ // Ensure active is synced with the component's enabled state
110
+ this.active = true;
111
+ // Register directly with the core postprocessing stack
112
+ this.context.postprocessing.addEffect(this);
113
113
  }
114
114
 
115
115
  onDisable(): void {
116
116
  super.onDisable();
117
117
  if (debug) console.warn("Disable", this.constructor.name);
118
- this._manager?.removeEffect(this);
118
+ this.context.postprocessing.removeEffect(this);
119
119
  this.active = false;
120
120
  }
121
121
 
122
- protected onEffectEnabled(manager?: IPostProcessingManager) {
123
- if (manager && manager.isPostProcessingManager === true) this._manager = manager;
124
- else if (!this._manager) this._manager = getPostProcessingManager(this);
125
- this._manager!.addEffect(this);
126
- this._manager!.dirty = true;
127
- }
128
-
129
122
  /** override to initialize bindings on parameters */
130
123
  init() { }
131
124
 
@@ -8,6 +8,7 @@ import { Context } from "../../engine/engine_setup.js";
8
8
  import { Graphics } from "../../engine/engine_three_utils.js";
9
9
  import { Constructor } from "../../engine/engine_types.js";
10
10
  import { getParam } from "../../engine/engine_utils.js";
11
+ import type { IPostProcessingHandler } from "../../engine/postprocessing/types.js";
11
12
  import { Camera } from "../Camera.js";
12
13
  import { threeToneMappingToEffectMode } from "./Effects/Tonemapping.utils.js";
13
14
  import { PostProcessingEffect, PostProcessingEffectContext } from "./PostProcessingEffect.js";
@@ -26,7 +27,7 @@ const previousToneMapping = Symbol("needle:previous-tone-mapping");
26
27
  /**
27
28
  * [PostProcessingHandler](https://engine.needle.tools/docs/api/PostProcessingHandler) Is responsible for applying post processing effects to the scene. It is internally used by the {@link Volume} component
28
29
  */
29
- export class PostProcessingHandler {
30
+ export class PostProcessingHandler implements IPostProcessingHandler {
30
31
 
31
32
  private _composer: EffectComposer | null = null;
32
33
  private _lastVolumeComponents?: PostProcessingEffect[];