@needle-tools/engine 2.28.0-pre → 2.30.0-pre

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/CHANGELOG.md +19 -0
  2. package/dist/needle-engine.d.ts +232 -139
  3. package/dist/needle-engine.js +349 -345
  4. package/dist/needle-engine.js.map +4 -4
  5. package/dist/needle-engine.min.js +24 -20
  6. package/dist/needle-engine.min.js.map +4 -4
  7. package/lib/engine/engine.d.ts +1 -0
  8. package/lib/engine/engine_input.d.ts +13 -1
  9. package/lib/engine/engine_input.js +47 -16
  10. package/lib/engine/engine_input.js.map +1 -1
  11. package/lib/engine/engine_physics.d.ts +1 -0
  12. package/lib/engine/engine_physics.js +2 -1
  13. package/lib/engine/engine_physics.js.map +1 -1
  14. package/lib/engine/engine_playerview.d.ts +26 -0
  15. package/lib/engine/engine_playerview.js +65 -0
  16. package/lib/engine/engine_playerview.js.map +1 -0
  17. package/lib/engine/engine_serialization.d.ts +1 -0
  18. package/lib/engine/engine_serialization.js +1 -0
  19. package/lib/engine/engine_serialization.js.map +1 -1
  20. package/lib/engine/engine_serialization_core.js +5 -0
  21. package/lib/engine/engine_serialization_core.js.map +1 -1
  22. package/lib/engine/engine_setup.d.ts +8 -0
  23. package/lib/engine/engine_setup.js +23 -0
  24. package/lib/engine/engine_setup.js.map +1 -1
  25. package/lib/engine/engine_utils.d.ts +1 -1
  26. package/lib/engine/engine_utils.js +25 -8
  27. package/lib/engine/engine_utils.js.map +1 -1
  28. package/lib/engine/extensions/NEEDLE_deferred_texture.d.ts +1 -1
  29. package/lib/engine/extensions/NEEDLE_deferred_texture.js +26 -14
  30. package/lib/engine/extensions/NEEDLE_deferred_texture.js.map +1 -1
  31. package/lib/engine/extensions/NEEDLE_lighting_settings.js +6 -2
  32. package/lib/engine/extensions/NEEDLE_lighting_settings.js.map +1 -1
  33. package/lib/engine/extensions/extension_utils.js +24 -13
  34. package/lib/engine/extensions/extension_utils.js.map +1 -1
  35. package/lib/engine/extensions/extensions.js +3 -1
  36. package/lib/engine/extensions/extensions.js.map +1 -1
  37. package/lib/engine-components/Camera.js +7 -0
  38. package/lib/engine-components/Camera.js.map +1 -1
  39. package/lib/engine-components/Component.d.ts +1 -1
  40. package/lib/engine-components/Component.js.map +1 -1
  41. package/lib/engine-components/Light.js +1 -0
  42. package/lib/engine-components/Light.js.map +1 -1
  43. package/lib/engine-components/OrbitControls.js +3 -3
  44. package/lib/engine-components/OrbitControls.js.map +1 -1
  45. package/lib/engine-components/ParticleSystem.d.ts +0 -1
  46. package/lib/engine-components/ParticleSystem.js +24 -27
  47. package/lib/engine-components/ParticleSystem.js.map +1 -1
  48. package/lib/engine-components/PlayerColor.js +1 -2
  49. package/lib/engine-components/PlayerColor.js.map +1 -1
  50. package/lib/engine-components/Renderer.d.ts +1 -0
  51. package/lib/engine-components/Renderer.js +10 -3
  52. package/lib/engine-components/Renderer.js.map +1 -1
  53. package/lib/engine-components/ScreenCapture.d.ts +1 -0
  54. package/lib/engine-components/ScreenCapture.js +265 -1
  55. package/lib/engine-components/ScreenCapture.js.map +1 -1
  56. package/lib/engine-components/SpectatorCamera.d.ts +24 -17
  57. package/lib/engine-components/SpectatorCamera.js +435 -182
  58. package/lib/engine-components/SpectatorCamera.js.map +1 -1
  59. package/lib/engine-components/SyncedCamera.d.ts +8 -4
  60. package/lib/engine-components/SyncedCamera.js +15 -18
  61. package/lib/engine-components/SyncedCamera.js.map +1 -1
  62. package/lib/engine-components/SyncedRoom.js +2 -0
  63. package/lib/engine-components/SyncedRoom.js.map +1 -1
  64. package/lib/engine-components/VideoPlayer.d.ts +10 -1
  65. package/lib/engine-components/VideoPlayer.js +64 -15
  66. package/lib/engine-components/VideoPlayer.js.map +1 -1
  67. package/lib/engine-components/Volume.d.ts +4 -0
  68. package/lib/engine-components/Volume.js +44 -3
  69. package/lib/engine-components/Volume.js.map +1 -1
  70. package/lib/engine-components/WebARSessionRoot.d.ts +9 -2
  71. package/lib/engine-components/WebARSessionRoot.js +69 -24
  72. package/lib/engine-components/WebARSessionRoot.js.map +1 -1
  73. package/lib/engine-components/WebXR.d.ts +6 -3
  74. package/lib/engine-components/WebXR.js +43 -7
  75. package/lib/engine-components/WebXR.js.map +1 -1
  76. package/lib/engine-components/WebXRAvatar.d.ts +3 -0
  77. package/lib/engine-components/WebXRAvatar.js +20 -0
  78. package/lib/engine-components/WebXRAvatar.js.map +1 -1
  79. package/lib/engine-components/WebXRController.js +14 -8
  80. package/lib/engine-components/WebXRController.js.map +1 -1
  81. package/lib/engine-components/WebXRSync.js +3 -3
  82. package/lib/engine-components/WebXRSync.js.map +1 -1
  83. package/lib/engine-components/XRFlag.d.ts +2 -1
  84. package/lib/engine-components/XRFlag.js +1 -0
  85. package/lib/engine-components/XRFlag.js.map +1 -1
  86. package/lib/engine-components/ui/CanvasGroup.d.ts +1 -0
  87. package/lib/engine-components/ui/CanvasGroup.js +1 -0
  88. package/lib/engine-components/ui/CanvasGroup.js.map +1 -1
  89. package/lib/engine-components/ui/EventSystem.js +13 -4
  90. package/lib/engine-components/ui/EventSystem.js.map +1 -1
  91. package/lib/engine-components/ui/Graphic.d.ts +1 -0
  92. package/lib/engine-components/ui/Graphic.js +2 -0
  93. package/lib/engine-components/ui/Graphic.js.map +1 -1
  94. package/lib/engine-components/ui/Interfaces.d.ts +2 -0
  95. package/package.json +2 -2
  96. package/src/engine/engine_components.js +16 -0
  97. package/src/engine/engine_input.ts +62 -20
  98. package/src/engine/engine_physics.ts +2 -1
  99. package/src/engine/engine_playerview.ts +80 -0
  100. package/src/engine/engine_serialization.ts +3 -1
  101. package/src/engine/engine_serialization_core.ts +8 -0
  102. package/src/engine/engine_setup.ts +24 -0
  103. package/src/engine/engine_utils.ts +34 -8
  104. package/src/engine/extensions/NEEDLE_deferred_texture.ts +25 -19
  105. package/src/engine/extensions/NEEDLE_lighting_settings.ts +4 -2
  106. package/src/engine/extensions/extension_utils.ts +24 -12
  107. package/src/engine/extensions/extensions.ts +3 -2
  108. package/src/engine-components/Camera.ts +9 -1
  109. package/src/engine-components/Component.ts +1 -1
  110. package/src/engine-components/Light.ts +3 -0
  111. package/src/engine-components/OrbitControls.ts +3 -3
  112. package/src/engine-components/ParticleSystem.ts +25 -26
  113. package/src/engine-components/PlayerColor.ts +1 -1
  114. package/src/engine-components/Renderer.ts +11 -3
  115. package/src/engine-components/ScreenCapture.ts +312 -2
  116. package/src/engine-components/SpectatorCamera.ts +490 -195
  117. package/src/engine-components/SyncedCamera.ts +23 -22
  118. package/src/engine-components/SyncedRoom.ts +1 -0
  119. package/src/engine-components/VideoPlayer.ts +97 -21
  120. package/src/engine-components/Volume.ts +47 -4
  121. package/src/engine-components/WebARSessionRoot.ts +78 -28
  122. package/src/engine-components/WebXR.ts +51 -15
  123. package/src/engine-components/WebXRAvatar.ts +27 -2
  124. package/src/engine-components/WebXRController.ts +21 -15
  125. package/src/engine-components/WebXRSync.ts +3 -3
  126. package/src/engine-components/XRFlag.ts +1 -0
  127. package/src/engine-components/ui/CanvasGroup.ts +2 -0
  128. package/src/engine-components/ui/EventSystem.ts +21 -15
  129. package/src/engine-components/ui/Graphic.ts +3 -0
  130. package/src/engine-components/ui/Interfaces.ts +2 -0
@@ -1,4 +1,4 @@
1
- import { Behaviour, GameObject } from "./Component";
1
+ import { Behaviour, Component, GameObject } from "./Component";
2
2
  import { Camera } from "./Camera";
3
3
  import * as THREE from "three";
4
4
  import { OrbitControls } from "./OrbitControls";
@@ -6,54 +6,102 @@ import { WebXR, WebXREvent } from "./WebXR";
6
6
  import { AvatarMarker } from "./WebXRAvatar";
7
7
  import { XRStateFlag } from "./XRFlag";
8
8
  import { SmoothFollow } from "./SmoothFollow";
9
- import { setWorldPosition, setWorldQuaternion, getWorldPosition, getWorldQuaternion, lookAtInverse } from "../engine/engine_three_utils";
10
- import { ArrayCamera } from "three";
11
- import { KeyCode } from "../engine/engine_input";
9
+ import { Object3D } from "three";
10
+ import { InputEvents, KeyCode } from "../engine/engine_input";
11
+ import { Context } from "../engine/engine_setup";
12
+ import { getParam } from "../engine/engine_utils";
13
+ import { PlayerView, ViewDevice } from "../engine/engine_playerview";
14
+ import { RaycastOptions } from "../engine/engine_physics";
15
+ import { IModel, RoomEvents } from "../engine/engine_networking";
16
+
17
+
18
+ export enum SpectatorMode {
19
+ FirstPerson = 0,
20
+ ThirdPerson = 1,
21
+ }
22
+
23
+ const debug = getParam("debugspectator");
12
24
 
13
25
  export class SpectatorCamera extends Behaviour {
14
26
 
15
27
  cam: Camera | null = null;
16
28
 
17
- private _firstPersonMode: boolean | undefined = true;
18
- get firstPersonMode(): boolean {
19
- return this._firstPersonMode ?? false;
29
+ private _mode: SpectatorMode = SpectatorMode.FirstPerson;
30
+
31
+ get mode() { return this._mode; }
32
+ set mode(val: SpectatorMode) {
33
+ this._mode = val;
34
+ }
35
+
36
+ /** if this user is currently spectating someone else */
37
+ get isSpectating(): boolean {
38
+ return this._handler?.currentTarget !== undefined;
39
+ }
40
+
41
+ isSpectatingUser(userId: string): boolean {
42
+ return this.target?.userId === userId;
43
+ }
44
+
45
+ isFollowedBy(userId: string): boolean {
46
+ return this.followers?.includes(userId);
20
47
  }
21
- set firstPersonMode(val: boolean) {
22
- this._firstPersonMode = val;
23
- // if (this._firstPersonMode) this.enableFirstPersonMode();
24
- // else this.enableThirdPersonMode();
48
+
49
+ /** list of other users that are following me */
50
+ get followers(): string[] {
51
+ return this._networking.followers;
52
+ }
53
+
54
+ stopSpectating() {
55
+ if (this.context.isInXR) {
56
+ this.followSelf();
57
+ return;
58
+ }
59
+ this.target = undefined;
25
60
  }
26
61
 
27
- enableFirstPersonMode() {
28
- this._firstPersonMode = true;
29
- this.updateUI();
30
- if (this.cam) {
31
- if (this.firstPersonFollow) {
32
- this.firstPersonFollow.enabled = true;
33
- this.firstPersonFollow.updateNow(true);
62
+ /** player view to follow */
63
+ set target(target: PlayerView | undefined) {
64
+ if (this._handler) {
65
+
66
+ // if (this.target?.userId) {
67
+ // const isFollowedByThisUser = this.followers.includes(this.target.userId);
68
+ // if (isFollowedByThisUser) {
69
+ // console.warn("Can not follow follower");
70
+ // target = undefined;
71
+ // }
72
+ // }
73
+
74
+ const prev = this._handler.currentTarget?.userId;
75
+ const self = this.context.players.getPlayerView(this.context.connection.connectionId);
76
+
77
+ // if user is in XR and sets target to self disable it
78
+ if (target === undefined || (this.context.isInXR === false && self?.currentObject === target.currentObject)) {
79
+ if (this._handler.currentTarget !== undefined) {
80
+ this._handler.disable();
81
+ GameObject.setActive(this.gameObject, false);
82
+ if (this.orbit) this.orbit.enabled = true;
83
+ this._networking.onSpectatedObjectChanged(target, prev);
84
+ }
34
85
  }
35
- if (this.orbit) {
36
- this.orbit.enabled = false;
86
+ else if (this._handler.currentTarget !== target) {
87
+ this._handler.set(target);
88
+ GameObject.setActive(this.gameObject, true);
89
+ if (this.orbit) this.orbit.enabled = false;
90
+ this._networking.onSpectatedObjectChanged(target, prev);
37
91
  }
38
92
  }
39
93
  }
40
94
 
41
- enableThirdPersonMode() {
42
- this._firstPersonMode = false;
43
- this.updateUI();
44
- if (this.firstPersonFollow) {
45
- this.firstPersonFollow.enabled = false;
46
- }
47
- if (!this.cam) return;
48
- setWorldPosition(this.cam.cam, this._orbitStartPos);
49
- setWorldQuaternion(this.cam.cam, this._orbitStartRot);
50
- setWorldPosition(this.cam.gameObject, this._orbitStartPos2);
51
- setWorldQuaternion(this.cam.gameObject, this._orbitStartRot2);
52
-
53
- if (this.orbit) {
54
- this.orbit.enabled = true;
55
- this.orbit.setFromTargetPosition();
56
- }
95
+ get target(): PlayerView | undefined {
96
+ return this._handler?.currentTarget;
97
+ }
98
+
99
+ requestAllFollowMe() {
100
+ this._networking.onRequestFollowMe();
101
+ }
102
+
103
+ private get isSpectatingSelf() {
104
+ return this.isSpectating && this.target?.currentObject === this.context.players.getPlayerView(this.context.connection.connectionId)?.currentObject;
57
105
  }
58
106
 
59
107
  // private currentViewport : THREE.Vector4 = new THREE.Vector4();
@@ -61,53 +109,31 @@ export class SpectatorCamera extends Behaviour {
61
109
  // private currentScissorTest : boolean = false;
62
110
 
63
111
  private orbit: OrbitControls | null = null;
64
- private firstPersonFollow: SmoothFollow | null = null;
65
- private spectatorUIDomElement: HTMLElement | null = null;
66
-
112
+ private _handler?: ISpectatorHandler;
67
113
  private eventSub_WebXRRequestStartEvent: Function | null = null;
68
114
  private eventSub_WebXRStartEvent: Function | null = null;
69
115
  private eventSub_WebXREndEvent: Function | null = null;
116
+ private _debug?: SpectatorSelectionController;
117
+ private _networking!: SpectatorCamNetworking;
70
118
 
71
119
  awake(): void {
72
120
 
73
- GameObject.setActive(this.gameObject, false);
121
+ this._debug = new SpectatorSelectionController(this.context, this);
122
+ this._networking = new SpectatorCamNetworking(this.context, this);
123
+ this._networking.awake();
74
124
 
75
- if (!this.isSupportedPlatform()) {
76
- console.log("Disable spectator cam", window.navigator.userAgent, this);
77
- return;
78
- }
125
+ GameObject.setActive(this.gameObject, false);
79
126
 
80
127
  this.cam = GameObject.getComponent(this.gameObject, Camera);
81
128
  if (!this.cam) {
82
- // TODO: adding camera component does currently not work properly
83
129
  console.error("Spectator camera needs camera component", this);
84
130
  return;
85
- // this.cam = GameObject.addNewComponent(this.gameObject, Camera) as Camera;
86
131
  }
87
132
 
88
- const uiQuery = "#spectator-camera-ui";
89
- this.spectatorUIDomElement = this.context.domElement.querySelector(uiQuery);
90
- if (!this.spectatorUIDomElement) {
91
- console.warn("Could not find spectator camera UI element", uiQuery);
92
- // this.spectatorUIDomElement = document.createElement("div");
93
- // this.spectatorUIDomElement.id = "spectator-camera-ui";
94
- // this.spectatorUIDomElement.classList.add("desktop");
95
- // this.context.domElement.appendChild(this.spectatorUIDomElement);
96
-
97
- // const toggle = document.createElement("button");
98
- // toggle.id = "toggle-spectator-view";
99
- // this.spectatorUIDomElement.appendChild(toggle);
100
- }
101
- this.spectatorUIDomElement?.classList.add("hidden");
102
133
 
134
+ if (!this._handler && this.cam)
135
+ this._handler = new SpectatorHandler(this.context, this.cam, this);
103
136
 
104
- if (this.cam) {
105
- this.cam.enabled = true;
106
- this._orbitStartPos.copy(getWorldPosition(this.cam.cam));
107
- this._orbitStartRot.copy(getWorldQuaternion(this.cam.cam));
108
- this._orbitStartPos2.copy(getWorldPosition(this.cam.gameObject));
109
- this._orbitStartRot2.copy(getWorldQuaternion(this.cam.gameObject));
110
- }
111
137
 
112
138
  this.eventSub_WebXRRequestStartEvent = this.onXRSessionRequestStart.bind(this);
113
139
  this.eventSub_WebXRStartEvent = this.onXRSessionStart.bind(this);
@@ -117,42 +143,16 @@ export class SpectatorCamera extends Behaviour {
117
143
  WebXR.addEventListener(WebXREvent.XRStarted, this.eventSub_WebXRStartEvent);
118
144
  WebXR.addEventListener(WebXREvent.XRStopped, this.eventSub_WebXREndEvent);
119
145
 
120
- this.context.domElement.querySelector("button#toggle-spectator-view")?.addEventListener("click", this.toggleView.bind(this));
121
- this.updateUI();
146
+ this.orbit = GameObject.getComponent(this.context.mainCamera, OrbitControls);
122
147
  }
123
148
 
124
149
  onDestroy(): void {
150
+ this.stopSpectating();
125
151
  WebXR.removeEventListener(WebXREvent.RequestVRSession, this.eventSub_WebXRStartEvent);
126
152
  WebXR.removeEventListener(WebXREvent.XRStarted, this.eventSub_WebXRStartEvent);
127
153
  WebXR.removeEventListener(WebXREvent.XRStopped, this.eventSub_WebXREndEvent);
128
- }
129
-
130
- private toggleView() {
131
- this.firstPersonMode = !this.firstPersonMode;
132
- }
133
-
134
- private updateUI() {
135
- const el: any = this.context.domElement.querySelector("button#toggle-spectator-view");
136
- if (!el) return;
137
- el.disabled = true;// !this._sessionHasStarted;
138
- if (el.firstChild) {
139
- if (!this._sessionHasStarted) {
140
- el.firstChild.textContent = "Waiting for VR session";
141
- }
142
- else
143
- el.firstChild.textContent = this.firstPersonMode ? "👁️" : "📷";
144
- }
145
-
146
- if (el.children?.length > 1) {
147
- const tooltipElement = el.children[1];
148
- tooltipElement.style.display = !this._sessionHasStarted ? "none" : "block";
149
- if (this.firstPersonMode) {
150
- tooltipElement.textContent = "First-Person View. See what the person in VR sees";
151
- }
152
- else {
153
- tooltipElement.textContent = "Third-Person View. Freely move the camera around"
154
- }
155
- }
154
+ this._handler?.destroy();
155
+ this._networking.destroy();
156
156
  }
157
157
 
158
158
  private isSupportedPlatform() {
@@ -163,60 +163,34 @@ export class SpectatorCamera extends Behaviour {
163
163
  }
164
164
 
165
165
  private onXRSessionRequestStart(_evt) {
166
- this._sessionHasStarted = false;
167
- this.spectatorUIDomElement?.classList.remove("hidden");
168
-
166
+ if (!this.isSupportedPlatform()) return;
169
167
  GameObject.setActive(this.gameObject, true);
170
-
171
- // if (!this.orbit && this.cam) {
172
- // const lookAt = GameObject.addNewComponent(this.gameObject, LookAtConstraint);
173
- // lookAt.constraintActive = true;
174
- // const center = new THREE.Object3D();
175
- // this.context.scene.add(center);
176
- // lookAt.sources = [center];
177
- // this.orbit = GameObject.addNewComponent(this.cam?.gameObject, OrbitControls);
178
- // this.orbit.lookAtConstraint = lookAt;
179
- // this.orbit.debugLog = true;
180
- // this.orbit.enabled = !this.firstPersonMode;
181
- // }
182
-
183
- if (this.cam && this.cam.cam) {
184
- setWorldPosition(this.cam.cam, this._orbitStartPos);
185
- setWorldQuaternion(this.cam.cam, this._orbitStartRot);
186
- setWorldPosition(this.cam.gameObject, this._orbitStartPos2);
187
- setWorldQuaternion(this.cam.gameObject, this._orbitStartRot2);
188
- }
189
-
190
- if (this.firstPersonMode) {
191
- this.enableFirstPersonMode();
192
- }
193
- else this.enableThirdPersonMode();
194
168
  }
195
169
 
196
170
 
197
171
  private onXRSessionStart(_evt) {
198
- this._sessionHasStarted = true;
199
- this.updateUI();
200
-
172
+ if (!this.isSupportedPlatform()) return;
173
+ if (debug) console.log(this.context.mainCamera);
201
174
  if (this.context.mainCamera) {
202
- const cam = this.context.renderer.xr.getCamera(this.context.mainCamera) as ArrayCamera;
203
- this.setupFollowMode(cam);
175
+ this.followSelf();
204
176
  }
205
177
  }
206
178
 
207
179
  private onXRSessionEnded(_evt) {
208
- this._sessionHasStarted = false;
209
- this._firstPersonIsSetup = false;
210
- this.spectatorUIDomElement?.classList.add("hidden");
180
+ this.context.removeCamera(this.cam);
211
181
  GameObject.setActive(this.gameObject, false);
182
+ if (this.orbit) this.orbit.enabled = true;
183
+ this._handler?.set(undefined);
184
+ this._handler?.disable();
185
+ if (this.isSpectatingSelf)
186
+ this.stopSpectating();
212
187
  }
213
188
 
214
- private _sessionHasStarted: boolean = false;
215
- private _firstPersonIsSetup: boolean = false;
216
- private _orbitStartPos: THREE.Vector3 = new THREE.Vector3();
217
- private _orbitStartRot: THREE.Quaternion = new THREE.Quaternion();
218
- private _orbitStartPos2: THREE.Vector3 = new THREE.Vector3();
219
- private _orbitStartRot2: THREE.Quaternion = new THREE.Quaternion();
189
+
190
+ private followSelf() {
191
+ this.target = this.context.players.getPlayerView(this.context.connection.connectionId);
192
+ if (debug) console.log("Follow self", this.target);
193
+ }
220
194
 
221
195
  // TODO: only show Spectator cam for DesktopVR;
222
196
  // don't show for AR, don't show on Quest
@@ -224,19 +198,12 @@ export class SpectatorCamera extends Behaviour {
224
198
  onAfterRender(): void {
225
199
  if (!this.cam) return;
226
200
 
227
- if(this.context.input.isKeyDown(KeyCode.KEY_S))
228
- this.firstPersonMode = !this.firstPersonMode;
229
-
230
- this.updateFollowSettings();
231
-
232
201
  const renderer = this.context.renderer;
233
202
  const xrWasEnabled = renderer.xr.enabled;
234
- // these should not be needed if we don't override viewport/scissor
235
- // renderer.getViewport(this.currentViewport);
236
- // renderer.getScissor(this.currentScissor);
237
- // this.currentScissorTest = renderer.getScissorTest();
238
203
 
239
- if (!renderer.xr.isPresenting) return;
204
+ if (!renderer.xr.isPresenting && !this._handler?.currentTarget) return;
205
+
206
+ this._handler?.update(this._mode);
240
207
 
241
208
  // remember XR render target so we can restore later
242
209
  const previousRenderTarget = renderer.getRenderTarget();
@@ -254,6 +221,12 @@ export class SpectatorCamera extends Behaviour {
254
221
 
255
222
  this.setAvatarFlagsBeforeRender();
256
223
 
224
+ const mainCam = this.context.mainCameraComponent;
225
+
226
+ // these should not be needed if we don't override viewport/scissor
227
+ // renderer.getViewport(this.currentViewport);
228
+ // renderer.getScissor(this.currentScissor);
229
+ // this.currentScissorTest = renderer.getScissorTest();
257
230
  // for scissor rendering (e.g. just a part of the screen / viewport, multiplayer split view)
258
231
  // let left = 0;
259
232
  // let bottom = 100;
@@ -262,7 +235,17 @@ export class SpectatorCamera extends Behaviour {
262
235
  // renderer.setViewport(left, bottom, width, height);
263
236
  // renderer.setScissor(left, bottom, width, height);
264
237
  // renderer.setScissorTest(true);
265
- renderer.setClearColor(new THREE.Color(1, 1, 1));
238
+ if (mainCam) {
239
+ const backgroundColor = mainCam.backgroundColor;
240
+ if (backgroundColor)
241
+ renderer.setClearColor(backgroundColor, backgroundColor.alpha);
242
+ this.cam.backgroundColor = backgroundColor;
243
+ this.cam.clearFlags = mainCam.clearFlags;
244
+ this.cam.nearClipPlane = mainCam.nearClipPlane;
245
+ this.cam.farClipPlane = mainCam.farClipPlane;
246
+ }
247
+ else
248
+ renderer.setClearColor(new THREE.Color(1, 1, 1));
266
249
  renderer.setRenderTarget(null); // null: direct to Canvas
267
250
  renderer.xr.enabled = false;
268
251
  const cam = this.cam?.cam;
@@ -287,55 +270,14 @@ export class SpectatorCamera extends Behaviour {
287
270
  this.resetAvatarFlags();
288
271
  }
289
272
 
290
- private setupFollowMode(object: THREE.Object3D) {
291
- if (!object) return;
292
- if (!this.cam) return;
293
- if (this._firstPersonIsSetup) return;
294
- this._firstPersonIsSetup = true;
295
-
296
-
297
- this.firstPersonFollow = GameObject.addNewComponent(this.cam.gameObject, SmoothFollow);
298
-
299
- const target = new THREE.Object3D();
300
- object.add(target);
301
- target.add(new THREE.AxesHelper(.2))
302
- this.firstPersonFollow.target = target;
303
-
304
- this.updateFollowSettings();
305
-
306
- const perspectiveCamera = this.context.mainCamera as THREE.PerspectiveCamera;
307
- if (perspectiveCamera) {
308
- this.cam.cam.near = perspectiveCamera.near;
309
- this.cam.cam.far = perspectiveCamera.far;
310
- this.cam.cam.updateProjectionMatrix();
311
- }
312
- if (this.orbit) this.orbit.enabled = false;
313
- }
314
-
315
- private updateFollowSettings() {
316
- const target = this.firstPersonFollow?.target;
317
- if (!target || !this.firstPersonFollow) return;
318
- if (this.firstPersonMode === false) {
319
- this.firstPersonFollow.followFactor = 3;
320
- this.firstPersonFollow.rotateFactor = 2;
321
- this.firstPersonFollow.flipForward = false;
322
- target.position.set(0, .5, 1.5);
323
- target.quaternion.identity();
324
- // lookAtInverse(target, new THREE.Vector3(0, 0, 0));
325
- }
326
- else {
327
- target.position.set(0, 0, 0);
328
- target.quaternion.identity();
329
- this.firstPersonFollow.followFactor = 12;
330
- this.firstPersonFollow.rotateFactor = 5;
331
- this.firstPersonFollow.flipForward = false;
332
- }
333
- }
334
-
335
273
  private setAvatarFlagsBeforeRender() {
274
+ const isFirstPersonMode = this._mode === SpectatorMode.FirstPerson;
275
+
336
276
  for (const av of AvatarMarker.instances) {
337
277
  if (av.avatar && "isLocalAvatar" in av.avatar) {
338
- const mask = this.firstPersonMode && this._firstPersonIsSetup && av.avatar.isLocalAvatar ? XRStateFlag.FirstPerson : XRStateFlag.ThirdPerson;
278
+ let mask = XRStateFlag.All;
279
+ if (this.isSpectatingSelf)
280
+ mask = isFirstPersonMode && av.avatar.isLocalAvatar ? XRStateFlag.FirstPerson : XRStateFlag.ThirdPerson;
339
281
  const flags = av.avatar.flags;
340
282
  if (!flags) continue;
341
283
  for (const flag of flags) {
@@ -362,3 +304,356 @@ export class SpectatorCamera extends Behaviour {
362
304
  }
363
305
  }
364
306
  }
307
+
308
+ interface ISpectatorHandler {
309
+ context: Context;
310
+ get currentTarget(): PlayerView | undefined;
311
+ set(target?: PlayerView): void;
312
+ update(mode: SpectatorMode);
313
+ disable();
314
+ destroy();
315
+ }
316
+
317
+ class SpectatorHandler implements ISpectatorHandler {
318
+
319
+ readonly context: Context;
320
+ readonly cam: Camera;
321
+ readonly spectator: SpectatorCamera;
322
+
323
+ private follow?: SmoothFollow;
324
+ private target?: THREE.Object3D;
325
+ private view?: PlayerView;
326
+ private currentObject: Object3D | undefined;
327
+
328
+ get currentTarget(): PlayerView | undefined {
329
+ return this.view;
330
+ }
331
+
332
+ constructor(context: Context, cam: Camera, spectator: SpectatorCamera) {
333
+ this.context = context;
334
+ this.cam = cam;
335
+ this.spectator = spectator;
336
+ }
337
+
338
+ set(view?: PlayerView): void {
339
+ const followObject = view?.currentObject;
340
+ if (!followObject) {
341
+ this.spectator.stopSpectating();
342
+ return;
343
+ }
344
+ if (followObject === this.currentObject) return;
345
+ this.currentObject = followObject;
346
+ this.view = view;
347
+ if (!this.follow)
348
+ this.follow = GameObject.addNewComponent(this.cam.gameObject, SmoothFollow);
349
+ if (!this.target)
350
+ this.target = new THREE.Object3D();
351
+ followObject.add(this.target);
352
+
353
+ this.follow.enabled = true;
354
+ this.follow.target = this.target;
355
+ // this.context.setCurrentCamera(this.cam);
356
+ if (debug) console.log("FOLLOW", followObject);
357
+ if (!this.context.isInXR) {
358
+ this.context.setCurrentCamera(this.cam);
359
+ }
360
+ else this.context.removeCamera(this.cam);
361
+ }
362
+
363
+ disable() {
364
+ if (debug) console.log("STOP FOLLOW", this.currentObject);
365
+ this.view = undefined;
366
+ this.currentObject = undefined;
367
+ this.context.removeCamera(this.cam);
368
+ if (this.follow)
369
+ this.follow.enabled = false;
370
+ }
371
+
372
+ destroy() {
373
+ this.target?.removeFromParent();
374
+ if (this.follow)
375
+ GameObject.destroy(this.follow);
376
+ }
377
+
378
+ update(mode: SpectatorMode) {
379
+ if (this.currentTarget?.isConnected === false || this.currentTarget?.removed === true) {
380
+ if (debug) console.log("Target disconnected or timeout", this.currentTarget);
381
+ this.spectator.stopSpectating();
382
+ return;
383
+ }
384
+ if (this.currentTarget && this.currentTarget?.currentObject !== this.currentObject) {
385
+ if (debug) console.log("Target changed", this.currentObject, "to", this.currentTarget.currentObject);
386
+ this.set(this.currentTarget);
387
+ }
388
+ const perspectiveCamera = this.context.mainCamera as THREE.PerspectiveCamera;
389
+ if (perspectiveCamera) {
390
+ if (this.cam.cam.near !== perspectiveCamera.near || this.cam.cam.far !== perspectiveCamera.far) {
391
+ this.cam.cam.near = perspectiveCamera.near;
392
+ this.cam.cam.far = perspectiveCamera.far;
393
+ this.cam.cam.updateProjectionMatrix();
394
+ }
395
+ }
396
+
397
+ const target = this.follow?.target;
398
+ if (!target || !this.follow) return;
399
+ switch (mode) {
400
+ case SpectatorMode.FirstPerson:
401
+ if (this.view?.viewDevice !== ViewDevice.Browser) {
402
+ // soft follow for AR and VR
403
+ this.follow.followFactor = 5;
404
+ this.follow.rotateFactor = 5;
405
+ }
406
+ else {
407
+ // snappy follow for desktop
408
+ this.follow.followFactor = 50;
409
+ this.follow.rotateFactor = 50;
410
+ }
411
+ target.position.set(0, 0, 0);
412
+ break;
413
+ case SpectatorMode.ThirdPerson:
414
+ this.follow.followFactor = 3;
415
+ this.follow.rotateFactor = 2;
416
+ target.position.set(0, .5, 1.5);
417
+ break;
418
+ }
419
+ this.follow.flipForward = false;
420
+ // console.log(this.view);
421
+ if (this.view?.viewDevice !== ViewDevice.Browser)
422
+ target.quaternion.copy(_inverseYQuat);
423
+ else target.quaternion.identity();
424
+ }
425
+
426
+
427
+ }
428
+
429
+ const _inverseYQuat = new THREE.Quaternion().setFromAxisAngle(new THREE.Vector3(0, 1, 0), Math.PI);
430
+
431
+
432
+ class SpectatorSelectionController {
433
+
434
+ private readonly context: Context;
435
+ private readonly spectator: SpectatorCamera;
436
+
437
+ constructor(context: Context, spectator: SpectatorCamera) {
438
+ this.context = context;
439
+ this.spectator = spectator;
440
+ console.log("Click other avatars or cameras to follow them. Press ESC to exit spectator mode.");
441
+ window.addEventListener("keydown", (evt) => {
442
+ const key = evt.key;
443
+ if (key === "Escape") {
444
+ this.spectator.stopSpectating();
445
+ }
446
+ });
447
+ let downTime: number = 0;
448
+ this.context.input.addEventListener(InputEvents.PointerDown, _ => {
449
+ downTime = this.context.time.time;
450
+ });
451
+ this.context.input.addEventListener(InputEvents.PointerUp, _ => {
452
+ const dt = this.context.time.time - downTime;
453
+ if (dt > 1) {
454
+ this.spectator.stopSpectating();
455
+ }
456
+ else if (this.context.input.getPointerClicked(0) && dt < .3)
457
+ this.trySelectObject();
458
+ });
459
+ }
460
+
461
+ private trySelectObject() {
462
+ const opts = new RaycastOptions();
463
+ opts.setMask(0xffffff);
464
+ // opts.cam = this.spectator.cam?.cam;
465
+ const hits = this.context.physics.raycast(opts);
466
+ if (debug) console.log(...hits);
467
+ if (hits?.length) {
468
+ for (const hit of hits) {
469
+ if (hit.distance < .2) continue;
470
+ const obj = hit.object;
471
+ const avatar = GameObject.getComponentInParent(obj, AvatarMarker);
472
+ const id = avatar?.connectionId;
473
+ if (id) {
474
+ const view = this.context.players.getPlayerView(id);
475
+ this.spectator.target = view;
476
+ if (debug) console.log("spectate", id, avatar);
477
+ break;
478
+ }
479
+ }
480
+ }
481
+ }
482
+ }
483
+
484
+
485
+
486
+
487
+
488
+ class SpectatorFollowerChangedEventModel implements IModel {
489
+ /** the user that is following */
490
+ guid: string;
491
+ readonly dontSave: boolean = true;
492
+
493
+ /** the user being followed */
494
+ targetUserId: string | undefined;
495
+ stoppedFollowing: boolean;
496
+
497
+ constructor(connectionId: string, userId: string | undefined, stoppedFollowing: boolean) {
498
+ this.guid = connectionId;
499
+ this.targetUserId = userId;
500
+ this.stoppedFollowing = stoppedFollowing;
501
+ }
502
+ }
503
+
504
+ class SpectatorFollowEventModel implements IModel {
505
+ guid: string;
506
+ userId: string | undefined;
507
+
508
+ constructor(comp: Component, userId: string | undefined) {
509
+ this.guid = comp.guid;
510
+ this.userId = userId;
511
+ }
512
+ }
513
+
514
+ class SpectatorCamNetworking {
515
+
516
+ readonly followers: string[] = [];
517
+
518
+
519
+ private readonly context: Context;
520
+ private readonly spectator: SpectatorCamera;
521
+ private _followerEventMethod: Function;
522
+ private _requestFollowMethod: Function;
523
+ private _joinedRoomMethod: Function;
524
+
525
+ constructor(context: Context, spectator: SpectatorCamera) {
526
+ this.context = context;
527
+ this.spectator = spectator;
528
+ this._followerEventMethod = this.onFollowerEvent.bind(this);
529
+ this._requestFollowMethod = this.onRequestFollowEvent.bind(this);
530
+ this._joinedRoomMethod = this.onUserJoinedRoom.bind(this);
531
+ }
532
+
533
+ awake() {
534
+ this.context.connection.beginListen("spectator-follower-changed", this._followerEventMethod);
535
+ this.context.connection.beginListen("spectator-request-follow", this._requestFollowMethod);
536
+ this.context.connection.beginListen(RoomEvents.JoinedRoom, this._joinedRoomMethod);
537
+ document.addEventListener("keydown", evt => {
538
+ if (evt.key === "f") {
539
+ this.onRequestFollowMe();
540
+ }
541
+ else if (evt.key === "Escape") {
542
+ this.onRequestFollowMe(true);
543
+ }
544
+ });
545
+ }
546
+
547
+ destroy() {
548
+ this.context.connection.stopListening("spectator-follower-changed", this._followerEventMethod);
549
+ this.context.connection.stopListening("spectator-request-follow", this._requestFollowMethod);
550
+ this.context.connection.stopListening(RoomEvents.JoinedRoom, this._joinedRoomMethod);
551
+ }
552
+
553
+ onSpectatedObjectChanged(target: PlayerView | undefined, _prevId?: string) {
554
+ if (debug)
555
+ console.log(this.context.connection.connectionId, "onSpectatedObjectChanged", target, _prevId);
556
+ if (this.context.connection.connectionId) {
557
+ const stopped = target?.userId === undefined;
558
+ const userId = stopped ? _prevId : target?.userId;
559
+ const evt = new SpectatorFollowerChangedEventModel(this.context.connection.connectionId, userId, stopped);
560
+ this.context.connection.send("spectator-follower-changed", evt)
561
+ }
562
+ }
563
+
564
+ onRequestFollowMe(stop: boolean = false) {
565
+ if (debug)
566
+ console.log("Request follow", this.context.connection.connectionId);
567
+ if (this.context.connection.connectionId) {
568
+ this.spectator.stopSpectating();
569
+ const id = stop ? undefined : this.context.connection.connectionId;
570
+ const model = new SpectatorFollowEventModel(this.spectator, id);
571
+ this.context.connection.send("spectator-request-follow", model);
572
+ }
573
+ }
574
+
575
+ private onUserJoinedRoom() {
576
+ if (getParam("followme")) {
577
+ this.onRequestFollowMe();
578
+ }
579
+ }
580
+
581
+ private onFollowerEvent(evt: SpectatorFollowerChangedEventModel) {
582
+ const userBeingFollowed = evt.targetUserId;
583
+ const userThatIsFollowing = evt.guid;
584
+
585
+ if (debug)
586
+ console.log(evt);
587
+
588
+ if (userBeingFollowed === this.context.connection.connectionId) {
589
+ if (evt.stoppedFollowing) {
590
+ const index = this.followers.indexOf(userThatIsFollowing);
591
+ if (index !== -1) {
592
+ this.followers.splice(index, 1);
593
+ this.removeDisconnectedFollowers();
594
+ console.log(userThatIsFollowing, "unfollows you", this.followers.length);
595
+ }
596
+ }
597
+ else {
598
+ if (!this.followers.includes(userThatIsFollowing)) {
599
+ this.followers.push(userThatIsFollowing);
600
+ this.removeDisconnectedFollowers();
601
+ console.log(userThatIsFollowing, "follows you", this.followers.length);
602
+ }
603
+ }
604
+ }
605
+ }
606
+
607
+ private removeDisconnectedFollowers() {
608
+ for (let i = this.followers.length - 1; i >= 0; i--) {
609
+ const id = this.followers[i];
610
+ if (this.context.connection.userIsInRoom(id) === false) {
611
+ this.followers.splice(i, 1);
612
+ }
613
+ }
614
+ }
615
+
616
+ private _lastRequestFollowUser: SpectatorFollowEventModel | undefined;
617
+
618
+ private onRequestFollowEvent(evt: SpectatorFollowEventModel) {
619
+ this._lastRequestFollowUser = evt;
620
+
621
+ if (evt.userId === this.context.connection.connectionId) {
622
+ this.spectator.stopSpectating();
623
+ }
624
+ else if (evt.userId === undefined) {
625
+ // this will currently also stop spectating if the user is not following you
626
+ this.spectator.stopSpectating();
627
+ }
628
+ else {
629
+ const view = this.context.players.getPlayerView(evt.userId);
630
+ if (view) {
631
+ this.spectator.target = view;
632
+ }
633
+ else {
634
+ if (debug)
635
+ console.warn("Could not find view", evt.userId);
636
+ this.enforceFollow();
637
+ return false;
638
+ }
639
+ }
640
+ return true;
641
+ }
642
+
643
+ private _enforceFollowInterval: any;
644
+ private enforceFollow() {
645
+ if (this._enforceFollowInterval) return;
646
+ this._enforceFollowInterval = setInterval(() => {
647
+ if (this._lastRequestFollowUser === undefined || this._lastRequestFollowUser.userId && this.spectator.isFollowedBy(this._lastRequestFollowUser.userId)) {
648
+ clearInterval(this._enforceFollowInterval);
649
+ this._enforceFollowInterval = undefined;
650
+ }
651
+ else {
652
+ if (debug)
653
+ console.log("REQUEST FOLLOW AGAIN", this._lastRequestFollowUser.userId);
654
+ this.onRequestFollowEvent(this._lastRequestFollowUser);
655
+ }
656
+
657
+ }, 1000);
658
+ }
659
+ }