mudra-skills 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/.claude-plugin/plugin.json +19 -0
  2. package/CLAUDE.md +35 -0
  3. package/bin/install.js +25 -0
  4. package/package.json +28 -0
  5. package/skills/mudra-master/SKILL.md +202 -0
  6. package/skills/mudra-master/mudra-preview/SKILL.md +151 -0
  7. package/skills/mudra-master/mudra-preview/assets/ar-menu.html +382 -0
  8. package/skills/mudra-master/mudra-preview/assets/document-scroller.html +468 -0
  9. package/skills/mudra-master/mudra-preview/assets/drum-machine.html +788 -0
  10. package/skills/mudra-master/mudra-preview/assets/emg-visualizer.html +390 -0
  11. package/skills/mudra-master/mudra-preview/assets/generative-art.html +408 -0
  12. package/skills/mudra-master/mudra-preview/assets/gesture-assistant.html +342 -0
  13. package/skills/mudra-master/mudra-preview/assets/gesture-speech.html +444 -0
  14. package/skills/mudra-master/mudra-preview/assets/hands-free-desktop.html +376 -0
  15. package/skills/mudra-master/mudra-preview/assets/model-rotator.html +365 -0
  16. package/skills/mudra-master/mudra-preview/assets/mudra-duel.html +3200 -0
  17. package/skills/mudra-master/mudra-preview/assets/mudra-monitor.html +1451 -0
  18. package/skills/mudra-master/mudra-preview/assets/mudra-ultimate-template.html +891 -0
  19. package/skills/mudra-master/mudra-preview/assets/music-sequencer.html +455 -0
  20. package/skills/mudra-master/mudra-preview/assets/neural-pong.html +380 -0
  21. package/skills/mudra-master/mudra-preview/assets/neural-snake.html +364 -0
  22. package/skills/mudra-master/mudra-preview/assets/presentation-controller.html +345 -0
  23. package/skills/mudra-master/mudra-preview/assets/pressure-painter.html +285 -0
  24. package/skills/mudra-master/mudra-preview/assets/runner.html +1812 -0
  25. package/skills/mudra-master/mudra-preview/assets/smart-home.html +461 -0
  26. package/skills/mudra-master/mudra-preview/assets/space-invaders.html +1154 -0
  27. package/skills/mudra-master/mudra-preview/assets/waterful-ring-toss.html +1624 -0
  28. package/skills/mudra-master/mudra-preview/references/agent_protocol.json +646 -0
  29. package/skills/mudra-master/mudra-preview/references/promt.md +16829 -0
  30. package/skills/mudra-master/mudra-xr/SKILL.md +245 -0
  31. package/skills/mudra-master/mudra-xr/assets/demos/3dgs-walkthrough.html +242 -0
  32. package/skills/mudra-master/mudra-xr/assets/demos/aisimulator.html +792 -0
  33. package/skills/mudra-master/mudra-xr/assets/demos/balloonpop.html +422 -0
  34. package/skills/mudra-master/mudra-xr/assets/demos/ballpit.html +284 -0
  35. package/skills/mudra-master/mudra-xr/assets/demos/drone.html +41 -0
  36. package/skills/mudra-master/mudra-xr/assets/demos/gemini-icebreakers.html +338 -0
  37. package/skills/mudra-master/mudra-xr/assets/demos/gemini-xrobject.html +302 -0
  38. package/skills/mudra-master/mudra-xr/assets/demos/math3d.html +188 -0
  39. package/skills/mudra-master/mudra-xr/assets/demos/measure.html +247 -0
  40. package/skills/mudra-master/mudra-xr/assets/demos/occlusion.html +251 -0
  41. package/skills/mudra-master/mudra-xr/assets/demos/rain.html +498 -0
  42. package/skills/mudra-master/mudra-xr/assets/demos/screenwiper.html +432 -0
  43. package/skills/mudra-master/mudra-xr/assets/demos/splash.html +519 -0
  44. package/skills/mudra-master/mudra-xr/assets/demos/webcam_gestures.html +253 -0
  45. package/skills/mudra-master/mudra-xr/assets/demos/xremoji.html +833 -0
  46. package/skills/mudra-master/mudra-xr/assets/demos/xrpoet.html +151 -0
  47. package/skills/mudra-master/mudra-xr/assets/samples/depthmap.html +266 -0
  48. package/skills/mudra-master/mudra-xr/assets/samples/depthmesh.html +100 -0
  49. package/skills/mudra-master/mudra-xr/assets/samples/game_rps.html +1099 -0
  50. package/skills/mudra-master/mudra-xr/assets/samples/gestures_custom.html +467 -0
  51. package/skills/mudra-master/mudra-xr/assets/samples/gestures_heuristic.html +256 -0
  52. package/skills/mudra-master/mudra-xr/assets/samples/lighting.html +163 -0
  53. package/skills/mudra-master/mudra-xr/assets/samples/mesh_detection.html +46 -0
  54. package/skills/mudra-master/mudra-xr/assets/samples/modelviewer.html +183 -0
  55. package/skills/mudra-master/mudra-xr/assets/samples/paint.html +133 -0
  56. package/skills/mudra-master/mudra-xr/assets/samples/planar-vst.html +45 -0
  57. package/skills/mudra-master/mudra-xr/assets/samples/reticle.html +95 -0
  58. package/skills/mudra-master/mudra-xr/assets/samples/skybox_agent.html +384 -0
  59. package/skills/mudra-master/mudra-xr/assets/samples/sound.html +390 -0
  60. package/skills/mudra-master/mudra-xr/assets/samples/ui.html +496 -0
  61. package/skills/mudra-master/mudra-xr/assets/samples/virtual-screens.html +808 -0
  62. package/skills/mudra-master/mudra-xr/assets/templates/0_basic.html +64 -0
  63. package/skills/mudra-master/mudra-xr/assets/templates/1_ui.html +107 -0
  64. package/skills/mudra-master/mudra-xr/assets/templates/2_hands.html +183 -0
  65. package/skills/mudra-master/mudra-xr/assets/templates/3_depth.html +89 -0
  66. package/skills/mudra-master/mudra-xr/assets/templates/4_stereo.html +84 -0
  67. package/skills/mudra-master/mudra-xr/assets/templates/5_camera.html +117 -0
  68. package/skills/mudra-master/mudra-xr/assets/templates/6_ai.html +154 -0
  69. package/skills/mudra-master/mudra-xr/assets/templates/7_ai_live.html +253 -0
  70. package/skills/mudra-master/mudra-xr/assets/templates/8_objects.html +95 -0
  71. package/skills/mudra-master/mudra-xr/assets/templates/9_xr-toggle.html +90 -0
  72. package/skills/mudra-master/mudra-xr/assets/templates/heuristic_hand_gestures.html +102 -0
  73. package/skills/mudra-master/mudra-xr/assets/templates/meshes.html +47 -0
  74. package/skills/mudra-master/mudra-xr/assets/templates/planes.html +47 -0
  75. package/skills/mudra-master/mudra-xr/assets/templates/uikit.html +187 -0
  76. package/skills/mudra-master/mudra-xr/references/agent_protocol.json +878 -0
  77. package/skills/mudra-master/mudra-xr/references/promt.md +1516 -0
  78. package/skills/mudra-preview/SKILL.md +151 -0
  79. package/skills/mudra-preview/assets/ar-menu.html +382 -0
  80. package/skills/mudra-preview/assets/document-scroller.html +468 -0
  81. package/skills/mudra-preview/assets/drum-machine.html +788 -0
  82. package/skills/mudra-preview/assets/emg-visualizer.html +390 -0
  83. package/skills/mudra-preview/assets/generative-art.html +408 -0
  84. package/skills/mudra-preview/assets/gesture-assistant.html +342 -0
  85. package/skills/mudra-preview/assets/gesture-speech.html +444 -0
  86. package/skills/mudra-preview/assets/hands-free-desktop.html +376 -0
  87. package/skills/mudra-preview/assets/model-rotator.html +365 -0
  88. package/skills/mudra-preview/assets/mudra-duel.html +3200 -0
  89. package/skills/mudra-preview/assets/mudra-monitor.html +1451 -0
  90. package/skills/mudra-preview/assets/mudra-ultimate-template.html +891 -0
  91. package/skills/mudra-preview/assets/music-sequencer.html +455 -0
  92. package/skills/mudra-preview/assets/neural-pong.html +380 -0
  93. package/skills/mudra-preview/assets/neural-snake.html +364 -0
  94. package/skills/mudra-preview/assets/presentation-controller.html +345 -0
  95. package/skills/mudra-preview/assets/pressure-painter.html +285 -0
  96. package/skills/mudra-preview/assets/runner.html +1812 -0
  97. package/skills/mudra-preview/assets/smart-home.html +461 -0
  98. package/skills/mudra-preview/assets/space-invaders.html +1154 -0
  99. package/skills/mudra-preview/assets/waterful-ring-toss.html +1624 -0
  100. package/skills/mudra-preview/references/agent_protocol.json +646 -0
  101. package/skills/mudra-preview/references/promt.md +16829 -0
  102. package/skills/mudra-xr/SKILL.md +245 -0
  103. package/skills/mudra-xr/assets/demos/3dgs-walkthrough.html +242 -0
  104. package/skills/mudra-xr/assets/demos/aisimulator.html +792 -0
  105. package/skills/mudra-xr/assets/demos/balloonpop.html +422 -0
  106. package/skills/mudra-xr/assets/demos/ballpit.html +284 -0
  107. package/skills/mudra-xr/assets/demos/drone.html +41 -0
  108. package/skills/mudra-xr/assets/demos/gemini-icebreakers.html +338 -0
  109. package/skills/mudra-xr/assets/demos/gemini-xrobject.html +302 -0
  110. package/skills/mudra-xr/assets/demos/math3d.html +188 -0
  111. package/skills/mudra-xr/assets/demos/measure.html +247 -0
  112. package/skills/mudra-xr/assets/demos/occlusion.html +251 -0
  113. package/skills/mudra-xr/assets/demos/rain.html +498 -0
  114. package/skills/mudra-xr/assets/demos/screenwiper.html +432 -0
  115. package/skills/mudra-xr/assets/demos/splash.html +519 -0
  116. package/skills/mudra-xr/assets/demos/webcam_gestures.html +253 -0
  117. package/skills/mudra-xr/assets/demos/xremoji.html +833 -0
  118. package/skills/mudra-xr/assets/demos/xrpoet.html +151 -0
  119. package/skills/mudra-xr/assets/samples/depthmap.html +266 -0
  120. package/skills/mudra-xr/assets/samples/depthmesh.html +100 -0
  121. package/skills/mudra-xr/assets/samples/game_rps.html +1099 -0
  122. package/skills/mudra-xr/assets/samples/gestures_custom.html +467 -0
  123. package/skills/mudra-xr/assets/samples/gestures_heuristic.html +256 -0
  124. package/skills/mudra-xr/assets/samples/lighting.html +163 -0
  125. package/skills/mudra-xr/assets/samples/mesh_detection.html +46 -0
  126. package/skills/mudra-xr/assets/samples/modelviewer.html +183 -0
  127. package/skills/mudra-xr/assets/samples/paint.html +133 -0
  128. package/skills/mudra-xr/assets/samples/planar-vst.html +45 -0
  129. package/skills/mudra-xr/assets/samples/reticle.html +95 -0
  130. package/skills/mudra-xr/assets/samples/skybox_agent.html +384 -0
  131. package/skills/mudra-xr/assets/samples/sound.html +390 -0
  132. package/skills/mudra-xr/assets/samples/ui.html +496 -0
  133. package/skills/mudra-xr/assets/samples/virtual-screens.html +808 -0
  134. package/skills/mudra-xr/assets/templates/0_basic.html +64 -0
  135. package/skills/mudra-xr/assets/templates/1_ui.html +107 -0
  136. package/skills/mudra-xr/assets/templates/2_hands.html +183 -0
  137. package/skills/mudra-xr/assets/templates/3_depth.html +89 -0
  138. package/skills/mudra-xr/assets/templates/4_stereo.html +84 -0
  139. package/skills/mudra-xr/assets/templates/5_camera.html +117 -0
  140. package/skills/mudra-xr/assets/templates/6_ai.html +154 -0
  141. package/skills/mudra-xr/assets/templates/7_ai_live.html +253 -0
  142. package/skills/mudra-xr/assets/templates/8_objects.html +95 -0
  143. package/skills/mudra-xr/assets/templates/9_xr-toggle.html +90 -0
  144. package/skills/mudra-xr/assets/templates/heuristic_hand_gestures.html +102 -0
  145. package/skills/mudra-xr/assets/templates/meshes.html +47 -0
  146. package/skills/mudra-xr/assets/templates/planes.html +47 -0
  147. package/skills/mudra-xr/assets/templates/uikit.html +187 -0
  148. package/skills/mudra-xr/references/agent_protocol.json +878 -0
  149. package/skills/mudra-xr/references/promt.md +1516 -0
package/skills/mudra-xr/assets/samples/virtual-screens.html ADDED
@@ -0,0 +1,808 @@
1
+ <!doctype html>
2
+ <!-- reference: samples/virtual-screens -->
3
+ <html lang="en">
4
+ <head>
5
+ <title>Virtual Screen Viewer | XR Blocks</title>
6
+
7
+ <meta
8
+ name="viewport"
9
+ content="width=device-width, initial-scale=1.0, user-scalable=no"
10
+ />
11
+
12
+ <link type="text/css" rel="stylesheet" href="https://xrblocks.github.io/css/xr.css" />
13
+ <script>
14
+ window.litDisableBundleWarning = true;
15
+ </script>
16
+ <script type="importmap">
17
+ {
18
+ "imports": {
19
+ "lit": "https://cdn.jsdelivr.net/gh/lit/dist@3/core/lit-core.min.js",
20
+ "lit/": "https://esm.run/lit@3/",
21
+ "three": "https://cdn.jsdelivr.net/npm/three@0.182.0/build/three.module.js",
22
+ "three/addons/": "https://cdn.jsdelivr.net/npm/three@0.182.0/examples/jsm/",
23
+ "troika-three-text": "https://cdn.jsdelivr.net/gh/protectwise/troika@028b81cf308f0f22e5aa8e78196be56ec1997af5/packages/troika-three-text/src/index.js",
24
+ "troika-three-utils": "https://cdn.jsdelivr.net/gh/protectwise/troika@v0.52.4/packages/troika-three-utils/src/index.js",
25
+ "troika-worker-utils": "https://cdn.jsdelivr.net/gh/protectwise/troika@v0.52.4/packages/troika-worker-utils/src/index.js",
26
+ "bidi-js": "https://esm.sh/bidi-js@%5E1.0.2?target=es2022",
27
+ "webgl-sdf-generator": "https://esm.sh/webgl-sdf-generator@1.1.1/es2022/webgl-sdf-generator.mjs",
28
+ "xrblocks": "https://cdn.jsdelivr.net/npm/xrblocks@0.11.0/build/xrblocks.js",
29
+ "xrblocks/addons/": "https://cdn.jsdelivr.net/npm/xrblocks@0.11.0/build/addons/"
30
+ }
31
+ }
32
+ </script>
33
+ </head>
34
+
35
+ <body>
36
+ <script type="module">
37
+ import 'xrblocks/addons/simulator/SimulatorAddons.js';
38
+
39
+ import * as THREE from 'three';
40
+ import * as xb from 'xrblocks';
41
+
42
+ // --- Inlined: WebSocketManager.js ---
43
+ class WebSocketManager {
44
+ constructor(port) {
45
+ this.port = port;
46
+ this.ws = null;
47
+ this.requestHandler = null;
48
+ this.binaryHandler = null;
49
+ this.messageQueue = [];
50
+ this.pendingRequests = new Map();
51
+ this.onConnectionError = null;
52
+ this.hasConnectedSuccessfully = false;
53
+ this.shouldReconnect = true;
54
+ this.connect();
55
+ }
56
+
57
+ setRequestHandler(handler) {
58
+ this.requestHandler = handler;
59
+ }
60
+
61
+ setBinaryHandler(handler) {
62
+ this.binaryHandler = handler;
63
+ }
64
+
65
+ connect() {
66
+ this.ws = new WebSocket(`ws://localhost:${this.port}`);
67
+ this.ws.binaryType = 'arraybuffer';
68
+
69
+ this.ws.onopen = () => {
70
+ console.log('WebSocketManager connected.');
71
+ this.hasConnectedSuccessfully = true;
72
+ this._flushMessageQueue();
73
+ };
74
+
75
+ this.ws.onmessage = (event) => {
76
+ if (event.data instanceof ArrayBuffer) {
77
+ if (this.binaryHandler) {
78
+ this.binaryHandler(event.data);
79
+ }
80
+ } else {
81
+ this._handleTextMessage(event.data);
82
+ }
83
+ };
84
+
85
+ this.ws.onclose = () => {
86
+ if (!this.hasConnectedSuccessfully && this.onConnectionError) {
87
+ this.onConnectionError();
88
+ }
89
+ if (this.shouldReconnect) {
90
+ console.log('WebSocketManager disconnected. Reconnecting...');
91
+ setTimeout(() => this.connect(), 2000);
92
+ }
93
+ };
94
+
95
+ this.ws.onerror = (error) => {
96
+ console.error('WebSocketManager error:', error);
97
+ this.ws.close();
98
+ };
99
+ }
100
+
101
+ stopReconnecting() {
102
+ this.shouldReconnect = false;
103
+ if (this.ws) {
104
+ this.ws.close();
105
+ }
106
+ }
107
+
108
+ _handleTextMessage(message) {
109
+ try {
110
+ const data = JSON.parse(message);
111
+
112
+ if (this.pendingRequests.has(data.id)) {
113
+ const promise = this.pendingRequests.get(data.id);
114
+ if (data.error) {
115
+ promise.reject(new Error(data.error));
116
+ } else {
117
+ promise.resolve(data.result);
118
+ }
119
+ this.pendingRequests.delete(data.id);
120
+ return;
121
+ }
122
+
123
+ if (data.params && data.params.target && this.requestHandler) {
124
+ this.requestHandler(data);
125
+ }
126
+ } catch (error) {
127
+ console.error('Error handling server message:', error);
128
+ }
129
+ }
130
+
131
+ send(message) {
132
+ const serializedMessage = JSON.stringify(message);
133
+ if (this.ws && this.ws.readyState === WebSocket.OPEN) {
134
+ this.ws.send(serializedMessage);
135
+ } else {
136
+ this.messageQueue.push(serializedMessage);
137
+ }
138
+ }
139
+
140
+ sendBinary(data) {
141
+ if (this.ws && this.ws.readyState === WebSocket.OPEN) {
142
+ this.ws.send(data);
143
+ } else {
144
+ console.warn('WebSocket not open. Dropping binary data packet.');
145
+ }
146
+ }
147
+
148
+ _flushMessageQueue() {
149
+ while (this.messageQueue.length > 0) {
150
+ this.ws.send(this.messageQueue.shift());
151
+ }
152
+ }
153
+
154
+ request(target, func, args = []) {
155
+ return new Promise((resolve, reject) => {
156
+ const id = `${Date.now()}-${Math.random()}`;
157
+ this.pendingRequests.set(id, {resolve, reject});
158
+ this.send({id, params: {target, func, args}});
159
+ });
160
+ }
161
+ }
162
+
163
+ // --- Inlined: StreamManager.js ---
164
+ class StreamManager {
165
+ constructor(webSocketManager) {
166
+ this.webSocketManager = webSocketManager;
167
+ this.streams = new Map();
168
+ this.onStreamAvailable = null;
169
+ this.onStreamEnded = null;
170
+ this.pollInterval = null;
171
+
172
+ this.webSocketManager.setRequestHandler(
173
+ this.handleServerRequest.bind(this)
174
+ );
175
+ this.webSocketManager.setBinaryHandler(this.onStreamData.bind(this));
176
+ }
177
+
178
+ async handleServerRequest(request) {
179
+ const {target, func, args} = request.params;
180
+ if (target !== 'streamManager') return;
181
+
182
+ if (func === 'onStreamEnded') {
183
+ this._handleStreamEnded(args[0]);
184
+ return;
185
+ }
186
+
187
+ if (typeof this[func] === 'function') {
188
+ this[func].apply(this, args);
189
+ }
190
+ }
191
+
192
+ request(func, args = []) {
193
+ return this.webSocketManager.request('streamManager', func, args);
194
+ }
195
+
196
+ async shareStream(streamId, stream, displaySurface) {
197
+ const videoTrack = stream.getVideoTracks()[0];
198
+ // eslint-disable-next-line no-undef
199
+ const trackProcessor = new MediaStreamTrackProcessor({track: videoTrack});
200
+ const reader = trackProcessor.readable.getReader();
201
+
202
+ const streamState = {
203
+ encoder: null,
204
+ reader,
205
+ isStreaming: true,
206
+ isKeyFrameRequested: true,
207
+ cropRect: null,
208
+ displaySurface: displaySurface,
209
+ };
210
+ this.streams.set(streamId, streamState);
211
+
212
+ videoTrack.onended = () => this.stopStream(streamId);
213
+ this.processFrames(streamId, videoTrack);
214
+ }
215
+
216
+ async stopStream(streamId) {
217
+ const streamState = this.streams.get(streamId);
218
+ if (!streamState) return;
219
+
220
+ streamState.isStreaming = false;
221
+ if (streamState.reader) streamState.reader.cancel();
222
+
223
+ if (streamState.encoder && streamState.encoder.state !== 'closed') {
224
+ await streamState.encoder.flush();
225
+ streamState.encoder.close();
226
+ }
227
+ this.streams.delete(streamId);
228
+ await this.request('stop_stream', [streamId]);
229
+ }
230
+
231
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
232
+ async processFrames(streamId, videoTrack) {
233
+ const streamState = this.streams.get(streamId);
234
+ if (!streamState) return;
235
+
236
+ const {reader} = streamState;
237
+ let {encoder} = streamState;
238
+
239
+ while (streamState.isStreaming) {
240
+ try {
241
+ const {value: frame, done} = await reader.read();
242
+ if (done) break;
243
+
244
+ if (!encoder) {
245
+ let rect = {
246
+ x: 0,
247
+ y: 0,
248
+ width: frame.codedWidth,
249
+ height: frame.codedHeight,
250
+ };
251
+
252
+ if (streamState.displaySurface === 'window' && frame.visibleRect) {
253
+ rect = frame.visibleRect;
254
+ }
255
+
256
+ const codedWidth = rect.width - (rect.width % 2);
257
+ const codedHeight = rect.height - (rect.height % 2);
258
+
259
+ if (codedWidth === 0 || codedHeight === 0) {
260
+ frame.close();
261
+ this.stopStream(streamId);
262
+ return;
263
+ }
264
+
265
+ streamState.cropRect = {
266
+ x: rect.x,
267
+ y: rect.y,
268
+ width: codedWidth,
269
+ height: codedHeight,
270
+ };
271
+
272
+ // eslint-disable-next-line no-undef
273
+ encoder = new VideoEncoder({
274
+ output: (chunk) => this._outputHandler(chunk, streamId),
275
+ error: (e) =>
276
+ console.error(`VideoEncoder error for stream ${streamId}:`, e),
277
+ });
278
+
279
+ await encoder.configure({
280
+ codec: 'vp8',
281
+ width: codedWidth,
282
+ height: codedHeight,
283
+ bitrate: 2_000_000,
284
+ latencyMode: 'realtime',
285
+ });
286
+
287
+ await this.request('start_stream', [
288
+ streamId,
289
+ {width: codedWidth, height: codedHeight},
290
+ ]);
291
+ streamState.encoder = encoder;
292
+ }
293
+
294
+ const frameToEncode = frame.clone({rect: streamState.cropRect});
295
+
296
+ if (encoder.encodeQueueSize < 30) {
297
+ const needsKeyFrame = streamState.isKeyFrameRequested;
298
+ if (needsKeyFrame) streamState.isKeyFrameRequested = false;
299
+ encoder.encode(frameToEncode, {keyFrame: needsKeyFrame});
300
+ }
301
+
302
+ frame.close();
303
+ frameToEncode.close();
304
+ } catch {
305
+ break;
306
+ }
307
+ }
308
+ }
309
+
310
+ _outputHandler(chunk, streamId) {
311
+ const streamIdBytes = new TextEncoder().encode(streamId);
312
+ const headerSize = 1 + streamIdBytes.length + 1;
313
+ const buffer = new ArrayBuffer(chunk.byteLength + headerSize);
314
+ const view = new DataView(buffer);
315
+ let offset = 0;
316
+
317
+ view.setUint8(offset, streamIdBytes.length);
318
+ offset += 1;
319
+ new Uint8Array(buffer, offset, streamIdBytes.length).set(streamIdBytes);
320
+ offset += streamIdBytes.length;
321
+
322
+ view.setUint8(offset, chunk.type === 'key' ? 0 : 1);
323
+ offset += 1;
324
+
325
+ chunk.copyTo(new Uint8Array(buffer, offset));
326
+ this.webSocketManager.sendBinary(buffer);
327
+ }
328
+
329
+ triggerKeyFrame(streamId) {
330
+ const streamState = this.streams.get(streamId);
331
+ if (!streamState) return;
332
+ streamState.isKeyFrameRequested = true;
333
+ }
334
+
335
+ startReceiving(onStreamAvailable, onStreamEnded) {
336
+ this.onStreamAvailable = onStreamAvailable;
337
+ this.onStreamEnded = onStreamEnded;
338
+
339
+ this.pollInterval = setInterval(async () => {
340
+ const activeStreams = await this.request('get_active_streams');
341
+ const activeStreamIds = new Set(Object.keys(activeStreams));
342
+
343
+ for (const streamId of activeStreamIds) {
344
+ if (!this.streams.has(streamId)) {
345
+ this._handleNewStream(streamId, activeStreams[streamId]);
346
+ }
347
+ }
348
+
349
+ for (const streamId of this.streams.keys()) {
350
+ if (!activeStreamIds.has(streamId)) {
351
+ this._handleStreamEnded(streamId);
352
+ }
353
+ }
354
+ }, 1000);
355
+ }
356
+
357
+ _handleStreamEnded(streamId) {
358
+ const streamState = this.streams.get(streamId);
359
+ if (!streamState) return;
360
+
361
+ if (streamState.decoder && streamState.decoder.state !== 'closed') {
362
+ streamState.decoder.close();
363
+ }
364
+ this.streams.delete(streamId);
365
+ if (this.onStreamEnded) this.onStreamEnded(streamId);
366
+ }
367
+
368
+ async _handleNewStream(streamId, streamInfo) {
369
+ this.streams.set(streamId, {
370
+ decoder: null,
371
+ streamInfo: streamInfo,
372
+ isWaitingForKeyFrame: true,
373
+ });
374
+ await this.request('subscribe_to_stream', [streamId]);
375
+ }
376
+
377
+ onStreamData(data) {
378
+ try {
379
+ const view = new DataView(data);
380
+ const streamIdLength = view.getUint8(0);
381
+ let offset = 1;
382
+ const streamId = new TextDecoder().decode(
383
+ new Uint8Array(data, offset, streamIdLength)
384
+ );
385
+ offset += streamIdLength;
386
+ const frameType = view.getUint8(offset);
387
+ offset += 1;
388
+
389
+ const streamState = this.streams.get(streamId);
390
+ if (!streamState) return;
391
+
392
+ // eslint-disable-next-line no-undef
393
+ const chunk = new EncodedVideoChunk({
394
+ type: frameType === 0 ? 'key' : 'delta',
395
+ timestamp: performance.now(),
396
+ data: data.slice(offset),
397
+ });
398
+
399
+ if (streamState.isWaitingForKeyFrame) {
400
+ if (chunk.type === 'key') {
401
+ streamState.isWaitingForKeyFrame = false;
402
+ this.initializeDecoder(streamId);
403
+ } else {
404
+ return;
405
+ }
406
+ }
407
+
408
+ if (streamState.decoder?.state === 'configured') {
409
+ streamState.decoder.decode(chunk);
410
+ }
411
+ } catch (e) {
412
+ console.error('Error processing incoming stream data:', e);
413
+ }
414
+ }
415
+
416
+ initializeDecoder(streamId) {
417
+ const streamState = this.streams.get(streamId);
418
+ if (!streamState) return;
419
+
420
+ const {streamInfo} = streamState;
421
+ const canvas = document.createElement('canvas');
422
+ canvas.width = streamInfo.width;
423
+ canvas.height = streamInfo.height;
424
+ const ctx = canvas.getContext('2d');
425
+
426
+ // eslint-disable-next-line no-undef
427
+ streamState.decoder = new VideoDecoder({
428
+ output: (frame) => {
429
+ ctx.drawImage(frame, 0, 0);
430
+ frame.close();
431
+ },
432
+ error: (e) =>
433
+ console.error(`VideoDecoder error for stream ${streamId}:`, e),
434
+ });
435
+
436
+ streamState.decoder.configure({
437
+ codec: 'vp8',
438
+ width: streamInfo.width,
439
+ height: streamInfo.height,
440
+ });
441
+
442
+ const mediaStream = canvas.captureStream();
443
+ this.onStreamAvailable(streamId, mediaStream, streamInfo);
444
+ }
445
+ }
446
+
447
+ // --- Inlined: WindowStream.js ---
448
+ class WindowStream extends xb.VideoStream {
449
+ constructor() {
450
+ // Disabling 'willCaptureFrequently' uses a hardware-accelerated canvas.
451
+ super({willCaptureFrequently: false});
452
+ this.setState_(xb.StreamState.INITIALIZING);
453
+ }
454
+
455
+ setStream(stream) {
456
+ if (!stream) {
457
+ console.error('WindowStream: Provided stream is null or undefined.');
458
+ this.setState_(xb.StreamState.ERROR, {error: 'Invalid stream provided.'});
459
+ return;
460
+ }
461
+
462
+ this.stream_ = stream;
463
+ this.video_.srcObject = stream;
464
+
465
+ this.video_.onloadedmetadata = () => {
466
+ this.handleVideoStreamLoadedMetadata(
467
+ () => {
468
+ this.setState_(xb.StreamState.STREAMING, {
469
+ aspectRatio: this.aspectRatio,
470
+ });
471
+ },
472
+ (error) => {
473
+ console.error('WindowStream: Failed to load video metadata.', error);
474
+ this.setState_(xb.StreamState.ERROR, {error});
475
+ },
476
+ true
477
+ );
478
+ };
479
+
480
+ this.video_
481
+ .play()
482
+ .catch((e) => console.warn('WindowStream: Autoplay was prevented.', e));
483
+ }
484
+ }
485
+
486
+ // --- Inlined: WindowView.js ---
487
+ class WindowView extends xb.VideoView {
488
+ constructor(options = {}) {
489
+ super(options);
490
+ this.isCurved = this.isCurved ?? false;
491
+ this.curvature = this.curvature ?? 0.5;
492
+ this.sharpness = this.sharpness ?? 0.3;
493
+
494
+ const customMaterial = new THREE.MeshBasicMaterial({
495
+ color: 0xffffff,
496
+ transparent: true,
497
+ });
498
+
499
+ this._curvedGeoAspectRatio = 0;
500
+ customMaterial.onBeforeCompile = (shader) => {
501
+ shader.uniforms.sharpness = {value: this.sharpness};
502
+ shader.fragmentShader = `
503
+ uniform float sharpness;
504
+ ${shader.fragmentShader}
505
+ `;
506
+
507
+ shader.fragmentShader = shader.fragmentShader.replace(
508
+ 'vec4 texelColor = texture2D( map, vMapUv );',
509
+ `
510
+ vec2 ddx = dFdx(vMapUv);
511
+ vec2 ddy = dFdy(vMapUv);
512
+ vec4 texelColor = textureGrad( map, vMapUv, ddx * sharpness, ddy * sharpness );
513
+ `
514
+ );
515
+ };
516
+
517
+ this.material.dispose();
518
+ this.material = customMaterial;
519
+ this.mesh.material = this.material;
520
+ }
521
+
522
+ updateLayout() {
523
+ super.updateLayout();
524
+
525
+ if (isNaN(this.rangeY) || this.rangeY <= 0) {
526
+ this.mesh.visible = false;
527
+ console.error(
528
+ '[WindowView] Invalid parent dimensions detected. Hiding view.'
529
+ );
530
+ return;
531
+ }
532
+ this.mesh.visible = true;
533
+
534
+ if (!this.isCurved || this.curvature <= 0) {
535
+ return;
536
+ }
537
+
538
+ if (this.videoAspectRatio <= 0) {
539
+ return;
540
+ }
541
+
542
+ if (this._curvedGeoAspectRatio !== this.videoAspectRatio) {
543
+ this._curvedGeoAspectRatio = this.videoAspectRatio;
544
+ const aspectRatio = this.videoAspectRatio;
545
+
546
+ const thetaLength = this.curvature * Math.PI;
547
+ const radius = aspectRatio / thetaLength;
548
+
549
+ const cylinderGeometry = new THREE.CylinderGeometry(
550
+ radius,
551
+ radius,
552
+ 1,
553
+ 64,
554
+ 1,
555
+ true,
556
+ -thetaLength / 2,
557
+ thetaLength
558
+ );
559
+
560
+ cylinderGeometry.translate(0, 0, -radius);
561
+
562
+ this.mesh.geometry.dispose();
563
+ this.mesh.geometry = cylinderGeometry;
564
+
565
+ this.mesh.position.z = 0;
566
+ this.mesh.rotation.y = Math.PI;
567
+ this.material.side = THREE.BackSide;
568
+ this.mesh.frustumCulled = false;
569
+ }
570
+
571
+ const scale = this.rangeY;
572
+ this.mesh.scale.set(scale, scale, scale);
573
+ }
574
+
575
+ load(source) {
576
+ super.load(source);
577
+ if (source instanceof xb.VideoStream) {
578
+ this.qualitySetupCallback_ = this.qualitySetupCallback_.bind(this);
579
+ this.stream_.addEventListener('statechange', this.qualitySetupCallback_);
580
+ }
581
+ }
582
+
583
+ qualitySetupCallback_() {
584
+ if (this.stream_?.state === xb.StreamState.STREAMING && this.material.map) {
585
+ const texture = this.material.map;
586
+ texture.generateMipmaps = true;
587
+ texture.minFilter = THREE.LinearMipmapLinearFilter;
588
+ texture.magFilter = THREE.LinearFilter;
589
+ texture.colorSpace = THREE.SRGBColorSpace;
590
+ if (this.isCurved) {
591
+ texture.wrapS = THREE.RepeatWrapping;
592
+ texture.repeat.x = -1;
593
+ }
594
+ const maxAnisotropy = xb.core.renderer.capabilities.getMaxAnisotropy();
595
+ texture.anisotropy = maxAnisotropy;
596
+ texture.needsUpdate = true;
597
+ this.material.needsUpdate = true;
598
+ this.stream_.removeEventListener(
599
+ 'statechange',
600
+ this.qualitySetupCallback_
601
+ );
602
+ }
603
+ }
604
+
605
+ dispose() {
606
+ if (this.stream_ && this.qualitySetupCallback_) {
607
+ this.stream_.removeEventListener(
608
+ 'statechange',
609
+ this.qualitySetupCallback_
610
+ );
611
+ }
612
+ super.dispose();
613
+ }
614
+ }
615
+
616
+ // --- Inlined: main_receive.js (WindowReceiver) ---
617
+ class WindowReceiver extends xb.Script {
618
+ constructor() {
619
+ super();
620
+ this.sharedWindows = new Map();
621
+ this.panels = [];
622
+ this.screenDistance = -0.4;
623
+ this.screenHeight = 0.4;
624
+ this.screenCurvature = 0.2;
625
+ this.curveScreens = true;
626
+ this.layerOffset = new THREE.Vector3(0.1, 0.1, 0.1);
627
+ this.frameBufferScaleFactor = 1.0;
628
+
629
+ this.simulatorRunning = false;
630
+ this.localPreviewStarted = false;
631
+ }
632
+
633
+ init() {
634
+ const isLocalhost =
635
+ window.location.hostname === 'localhost' ||
636
+ window.location.hostname === '127.0.0.1';
637
+
638
+ if (isLocalhost) {
639
+ this.webSocketManager = new WebSocketManager(8765);
640
+ this.streamManager = new StreamManager(this.webSocketManager);
641
+
642
+ this.streamManager.startReceiving(
643
+ (streamId, mediaStream, streamInfo) =>
644
+ this.onNewStream(streamId, mediaStream, streamInfo),
645
+ (streamId) => this.onStreamEnded(streamId)
646
+ );
647
+ } else {
648
+ console.log('Not running on localhost, WebSocket connection disabled.');
649
+ this.webSocketManager = {
650
+ hasConnectedSuccessfully: false,
651
+ stopReconnecting: () => {},
652
+ };
653
+ }
654
+
655
+ xb.core.renderer.setPixelRatio(window.devicePixelRatio);
656
+ xb.core.renderer.xr.setFramebufferScaleFactor(this.frameBufferScaleFactor);
657
+ }
658
+
659
+ onNewStream(streamId, mediaStream, streamInfo) {
660
+ if (this.sharedWindows.has(streamId)) {
661
+ console.warn(`Stream with ID ${streamId} already exists. Ignoring.`);
662
+ return;
663
+ }
664
+
665
+ const {width, height} = streamInfo;
666
+ if (!width || !height) {
667
+ console.error(`Stream ${streamId} has invalid video dimensions.`);
668
+ return;
669
+ }
670
+
671
+ console.log(
672
+ `New stream received: ${streamId} with resolution ${width}x${height}.`
673
+ );
674
+
675
+ const aspectRatio = width / height;
676
+ const panelWidth = this.screenHeight * aspectRatio;
677
+
678
+ const windowStream = new WindowStream();
679
+ windowStream.setStream(mediaStream);
680
+
681
+ let position;
682
+ let rotation = null;
683
+
684
+ if (this.panels.length > 0) {
685
+ const lastPanel = this.panels[this.panels.length - 1];
686
+ const rotatedOffset = this.layerOffset
687
+ .clone()
688
+ .applyQuaternion(lastPanel.quaternion);
689
+ position = lastPanel.position.clone().add(rotatedOffset);
690
+ rotation = lastPanel.quaternion.clone();
691
+ } else {
692
+ position = new THREE.Vector3(0, xb.core.user.height, this.screenDistance);
693
+ }
694
+
695
+ const panel = new xb.SpatialPanel({
696
+ width: panelWidth,
697
+ height: this.screenHeight,
698
+ isDraggable: true,
699
+ isClosable: true,
700
+ showEdge: true,
701
+ backgroundColor: '#ffffff00',
702
+ onClose: () => this.onStreamEnded(streamId),
703
+ });
704
+ panel.position.set(position.x, position.y, position.z);
705
+
706
+ if (rotation) {
707
+ panel.quaternion.copy(rotation);
708
+ }
709
+
710
+ const videoView = new WindowView({
711
+ isCurved: this.curveScreens,
712
+ curvature: this.screenCurvature,
713
+ });
714
+ panel.add(videoView);
715
+ videoView.load(windowStream);
716
+
717
+ this.add(panel);
718
+ this.sharedWindows.set(streamId, {panel, stream: windowStream});
719
+ this.panels.push(panel);
720
+ panel.fadeOut(0);
721
+ panel.show();
722
+ panel.fadeIn();
723
+ }
724
+
725
+ async _manageLocalStreamCreation() {
726
+ let addAnother = window.confirm(
727
+ 'This sample requires a server to run on localhost, which cannot be found. Would you like to share a local screen for simulation?'
728
+ );
729
+
730
+ while (addAnother) {
731
+ const success = await this._addLocalStream();
732
+ if (success) {
733
+ addAnother = window.confirm('Would you like to share another screen?');
734
+ } else {
735
+ addAnother = false;
736
+ }
737
+ }
738
+ }
739
+
740
+ async _addLocalStream() {
741
+ try {
742
+ const mediaStream = await navigator.mediaDevices.getDisplayMedia({
743
+ audio: false,
744
+ video: {width: {max: 2560}, height: {max: 1440}},
745
+ });
746
+
747
+ const videoTrack = mediaStream.getVideoTracks()[0];
748
+ if (!videoTrack) {
749
+ console.error('No video track found in the selected stream.');
750
+ return false;
751
+ }
752
+
753
+ const {width, height} = videoTrack.getSettings();
754
+ const streamInfo = {width, height};
755
+ const streamId = `local-preview-${crypto.randomUUID()}`;
756
+ this.onNewStream(streamId, mediaStream, streamInfo);
757
+ videoTrack.onended = () => {
758
+ this.onStreamEnded(streamId);
759
+ };
760
+ return true;
761
+ } catch (err) {
762
+ console.log('Could not start local screen preview:', err.name);
763
+ return false;
764
+ }
765
+ }
766
+
767
+ onStreamEnded(streamId) {
768
+ if (!this.sharedWindows.has(streamId)) {
769
+ return;
770
+ }
771
+ console.log(`Cleaning up stream: ${streamId}`);
772
+ const {panel, stream} = this.sharedWindows.get(streamId);
773
+ this.sharedWindows.delete(streamId);
774
+ this.panels = this.panels.filter((p) => p !== panel);
775
+
776
+ panel.fadeOut(1, () => {
777
+ this.remove(panel);
778
+ panel.dispose();
779
+ stream.dispose();
780
+ console.log(`Cleanup complete for stream: ${streamId}`);
781
+ });
782
+ }
783
+
784
+ onSimulatorStarted() {
785
+ this.simulatorRunning = true;
786
+ if (!this.webSocketManager.hasConnectedSuccessfully) {
787
+ this.webSocketManager.stopReconnecting();
788
+ if (!this.localPreviewStarted) {
789
+ this.localPreviewStarted = true;
790
+ this._manageLocalStreamCreation();
791
+ }
792
+ }
793
+ }
794
+ }
795
+
796
+ // --- Main entry point ---
797
+ document.addEventListener('DOMContentLoaded', function () {
798
+ const options = new xb.Options();
799
+ options.simulator.instructions.enabled = false;
800
+ options.enableUI();
801
+
802
+ xb.add(new WindowReceiver());
803
+ xb.init(options);
804
+ });
805
+
806
+ </script>
807
+ </body>
808
+ </html>