mudra-skills 3.0.2 → 3.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. package/package.json +1 -1
  2. package/skills/mudra-master/SKILL.md +0 -196
  3. package/skills/mudra-master/mudra-preview/SKILL.md +0 -150
  4. package/skills/mudra-master/mudra-preview/assets/ar-menu.html +0 -382
  5. package/skills/mudra-master/mudra-preview/assets/document-scroller.html +0 -468
  6. package/skills/mudra-master/mudra-preview/assets/drum-machine.html +0 -788
  7. package/skills/mudra-master/mudra-preview/assets/emg-visualizer.html +0 -390
  8. package/skills/mudra-master/mudra-preview/assets/generative-art.html +0 -408
  9. package/skills/mudra-master/mudra-preview/assets/gesture-assistant.html +0 -342
  10. package/skills/mudra-master/mudra-preview/assets/gesture-speech.html +0 -444
  11. package/skills/mudra-master/mudra-preview/assets/hands-free-desktop.html +0 -376
  12. package/skills/mudra-master/mudra-preview/assets/model-rotator.html +0 -365
  13. package/skills/mudra-master/mudra-preview/assets/mudra-duel.html +0 -3200
  14. package/skills/mudra-master/mudra-preview/assets/mudra-monitor.html +0 -1451
  15. package/skills/mudra-master/mudra-preview/assets/mudra-ultimate-template.html +0 -891
  16. package/skills/mudra-master/mudra-preview/assets/music-sequencer.html +0 -455
  17. package/skills/mudra-master/mudra-preview/assets/neural-pong.html +0 -380
  18. package/skills/mudra-master/mudra-preview/assets/neural-snake.html +0 -364
  19. package/skills/mudra-master/mudra-preview/assets/presentation-controller.html +0 -345
  20. package/skills/mudra-master/mudra-preview/assets/pressure-painter.html +0 -285
  21. package/skills/mudra-master/mudra-preview/assets/runner.html +0 -1812
  22. package/skills/mudra-master/mudra-preview/assets/smart-home.html +0 -461
  23. package/skills/mudra-master/mudra-preview/assets/space-invaders.html +0 -1154
  24. package/skills/mudra-master/mudra-preview/assets/waterful-ring-toss.html +0 -1624
  25. package/skills/mudra-master/mudra-preview/references/agent_protocol.json +0 -634
  26. package/skills/mudra-master/mudra-preview/references/prompt.md +0 -16746
  27. package/skills/mudra-master/mudra-xr/SKILL.md +0 -245
  28. package/skills/mudra-master/mudra-xr/assets/demos/3dgs-walkthrough.html +0 -242
  29. package/skills/mudra-master/mudra-xr/assets/demos/aisimulator.html +0 -792
  30. package/skills/mudra-master/mudra-xr/assets/demos/balloonpop.html +0 -422
  31. package/skills/mudra-master/mudra-xr/assets/demos/ballpit.html +0 -284
  32. package/skills/mudra-master/mudra-xr/assets/demos/drone.html +0 -41
  33. package/skills/mudra-master/mudra-xr/assets/demos/gemini-icebreakers.html +0 -338
  34. package/skills/mudra-master/mudra-xr/assets/demos/gemini-xrobject.html +0 -302
  35. package/skills/mudra-master/mudra-xr/assets/demos/math3d.html +0 -188
  36. package/skills/mudra-master/mudra-xr/assets/demos/measure.html +0 -247
  37. package/skills/mudra-master/mudra-xr/assets/demos/occlusion.html +0 -251
  38. package/skills/mudra-master/mudra-xr/assets/demos/rain.html +0 -498
  39. package/skills/mudra-master/mudra-xr/assets/demos/screenwiper.html +0 -432
  40. package/skills/mudra-master/mudra-xr/assets/demos/splash.html +0 -519
  41. package/skills/mudra-master/mudra-xr/assets/demos/webcam_gestures.html +0 -253
  42. package/skills/mudra-master/mudra-xr/assets/demos/xremoji.html +0 -833
  43. package/skills/mudra-master/mudra-xr/assets/demos/xrpoet.html +0 -151
  44. package/skills/mudra-master/mudra-xr/assets/samples/depthmap.html +0 -266
  45. package/skills/mudra-master/mudra-xr/assets/samples/depthmesh.html +0 -100
  46. package/skills/mudra-master/mudra-xr/assets/samples/game_rps.html +0 -1099
  47. package/skills/mudra-master/mudra-xr/assets/samples/gestures_custom.html +0 -467
  48. package/skills/mudra-master/mudra-xr/assets/samples/gestures_heuristic.html +0 -256
  49. package/skills/mudra-master/mudra-xr/assets/samples/lighting.html +0 -163
  50. package/skills/mudra-master/mudra-xr/assets/samples/mesh_detection.html +0 -46
  51. package/skills/mudra-master/mudra-xr/assets/samples/modelviewer.html +0 -183
  52. package/skills/mudra-master/mudra-xr/assets/samples/paint.html +0 -133
  53. package/skills/mudra-master/mudra-xr/assets/samples/planar-vst.html +0 -45
  54. package/skills/mudra-master/mudra-xr/assets/samples/reticle.html +0 -95
  55. package/skills/mudra-master/mudra-xr/assets/samples/skybox_agent.html +0 -384
  56. package/skills/mudra-master/mudra-xr/assets/samples/sound.html +0 -390
  57. package/skills/mudra-master/mudra-xr/assets/samples/ui.html +0 -496
  58. package/skills/mudra-master/mudra-xr/assets/samples/virtual-screens.html +0 -808
  59. package/skills/mudra-master/mudra-xr/assets/templates/0_basic.html +0 -64
  60. package/skills/mudra-master/mudra-xr/assets/templates/1_ui.html +0 -107
  61. package/skills/mudra-master/mudra-xr/assets/templates/2_hands.html +0 -183
  62. package/skills/mudra-master/mudra-xr/assets/templates/3_depth.html +0 -89
  63. package/skills/mudra-master/mudra-xr/assets/templates/4_stereo.html +0 -84
  64. package/skills/mudra-master/mudra-xr/assets/templates/5_camera.html +0 -117
  65. package/skills/mudra-master/mudra-xr/assets/templates/6_ai.html +0 -154
  66. package/skills/mudra-master/mudra-xr/assets/templates/7_ai_live.html +0 -253
  67. package/skills/mudra-master/mudra-xr/assets/templates/8_objects.html +0 -95
  68. package/skills/mudra-master/mudra-xr/assets/templates/9_xr-toggle.html +0 -90
  69. package/skills/mudra-master/mudra-xr/assets/templates/heuristic_hand_gestures.html +0 -102
  70. package/skills/mudra-master/mudra-xr/assets/templates/meshes.html +0 -47
  71. package/skills/mudra-master/mudra-xr/assets/templates/planes.html +0 -47
  72. package/skills/mudra-master/mudra-xr/assets/templates/uikit.html +0 -187
  73. package/skills/mudra-master/mudra-xr/references/agent_protocol.json +0 -865
  74. package/skills/mudra-master/mudra-xr/references/prompt.md +0 -1605
  75. package/skills/mudra-preview/SKILL.md +0 -159
  76. package/skills/mudra-preview/assets/ar-menu.html +0 -382
  77. package/skills/mudra-preview/assets/document-scroller.html +0 -468
  78. package/skills/mudra-preview/assets/drum-machine.html +0 -788
  79. package/skills/mudra-preview/assets/emg-visualizer.html +0 -390
  80. package/skills/mudra-preview/assets/generative-art.html +0 -408
  81. package/skills/mudra-preview/assets/gesture-assistant.html +0 -342
  82. package/skills/mudra-preview/assets/gesture-speech.html +0 -444
  83. package/skills/mudra-preview/assets/hands-free-desktop.html +0 -376
  84. package/skills/mudra-preview/assets/model-rotator.html +0 -365
  85. package/skills/mudra-preview/assets/mudra-duel.html +0 -3200
  86. package/skills/mudra-preview/assets/mudra-monitor.html +0 -1451
  87. package/skills/mudra-preview/assets/mudra-ultimate-template.html +0 -997
  88. package/skills/mudra-preview/assets/music-sequencer.html +0 -455
  89. package/skills/mudra-preview/assets/neural-pong.html +0 -380
  90. package/skills/mudra-preview/assets/neural-snake.html +0 -364
  91. package/skills/mudra-preview/assets/presentation-controller.html +0 -345
  92. package/skills/mudra-preview/assets/pressure-painter.html +0 -285
  93. package/skills/mudra-preview/assets/runner.html +0 -1812
  94. package/skills/mudra-preview/assets/smart-home.html +0 -461
  95. package/skills/mudra-preview/assets/space-invaders.html +0 -1154
  96. package/skills/mudra-preview/assets/waterful-ring-toss.html +0 -1624
  97. package/skills/mudra-preview/references/agent_protocol.json +0 -701
  98. package/skills/mudra-preview/references/prompt.md +0 -16731
  99. package/skills/mudra-xr/SKILL.md +0 -245
  100. package/skills/mudra-xr/assets/demos/3dgs-walkthrough.html +0 -242
  101. package/skills/mudra-xr/assets/demos/aisimulator.html +0 -792
  102. package/skills/mudra-xr/assets/demos/balloonpop.html +0 -422
  103. package/skills/mudra-xr/assets/demos/ballpit.html +0 -284
  104. package/skills/mudra-xr/assets/demos/drone.html +0 -41
  105. package/skills/mudra-xr/assets/demos/gemini-icebreakers.html +0 -338
  106. package/skills/mudra-xr/assets/demos/gemini-xrobject.html +0 -302
  107. package/skills/mudra-xr/assets/demos/math3d.html +0 -188
  108. package/skills/mudra-xr/assets/demos/measure.html +0 -247
  109. package/skills/mudra-xr/assets/demos/occlusion.html +0 -251
  110. package/skills/mudra-xr/assets/demos/rain.html +0 -498
  111. package/skills/mudra-xr/assets/demos/screenwiper.html +0 -432
  112. package/skills/mudra-xr/assets/demos/splash.html +0 -519
  113. package/skills/mudra-xr/assets/demos/webcam_gestures.html +0 -253
  114. package/skills/mudra-xr/assets/demos/xremoji.html +0 -833
  115. package/skills/mudra-xr/assets/demos/xrpoet.html +0 -151
  116. package/skills/mudra-xr/assets/samples/depthmap.html +0 -266
  117. package/skills/mudra-xr/assets/samples/depthmesh.html +0 -100
  118. package/skills/mudra-xr/assets/samples/game_rps.html +0 -1099
  119. package/skills/mudra-xr/assets/samples/gestures_custom.html +0 -467
  120. package/skills/mudra-xr/assets/samples/gestures_heuristic.html +0 -256
  121. package/skills/mudra-xr/assets/samples/lighting.html +0 -163
  122. package/skills/mudra-xr/assets/samples/mesh_detection.html +0 -46
  123. package/skills/mudra-xr/assets/samples/modelviewer.html +0 -183
  124. package/skills/mudra-xr/assets/samples/paint.html +0 -133
  125. package/skills/mudra-xr/assets/samples/planar-vst.html +0 -45
  126. package/skills/mudra-xr/assets/samples/reticle.html +0 -95
  127. package/skills/mudra-xr/assets/samples/skybox_agent.html +0 -384
  128. package/skills/mudra-xr/assets/samples/sound.html +0 -390
  129. package/skills/mudra-xr/assets/samples/ui.html +0 -496
  130. package/skills/mudra-xr/assets/samples/virtual-screens.html +0 -808
  131. package/skills/mudra-xr/assets/templates/0_basic.html +0 -64
  132. package/skills/mudra-xr/assets/templates/1_ui.html +0 -107
  133. package/skills/mudra-xr/assets/templates/2_hands.html +0 -183
  134. package/skills/mudra-xr/assets/templates/3_depth.html +0 -89
  135. package/skills/mudra-xr/assets/templates/4_stereo.html +0 -84
  136. package/skills/mudra-xr/assets/templates/5_camera.html +0 -117
  137. package/skills/mudra-xr/assets/templates/6_ai.html +0 -154
  138. package/skills/mudra-xr/assets/templates/7_ai_live.html +0 -253
  139. package/skills/mudra-xr/assets/templates/8_objects.html +0 -95
  140. package/skills/mudra-xr/assets/templates/9_xr-toggle.html +0 -90
  141. package/skills/mudra-xr/assets/templates/heuristic_hand_gestures.html +0 -102
  142. package/skills/mudra-xr/assets/templates/meshes.html +0 -47
  143. package/skills/mudra-xr/assets/templates/planes.html +0 -47
  144. package/skills/mudra-xr/assets/templates/uikit.html +0 -187
  145. package/skills/mudra-xr/references/agent_protocol.json +0 -865
  146. package/skills/mudra-xr/references/prompt.md +0 -1605
package/skills/mudra-xr/assets/demos/gemini-xrobject.html DELETED
@@ -1,302 +0,0 @@
1
- <!doctype html>
2
- <!-- reference: demos/gemini-xrobject -->
3
- <html lang="en">
4
- <head>
5
- <title>Gemini XR-Objects</title>
6
-
7
- <meta
8
- name="viewport"
9
- content="width=device-width, initial-scale=1.0, user-scalable=no"
10
- />
11
-
12
- <link type="text/css" rel="stylesheet" href="https://xrblocks.github.io/css/xr.css" />
13
- <script>
14
- window.litDisableBundleWarning = true;
15
- </script>
16
- <script type="importmap">
17
- {
18
- "imports": {
19
- "lit": "https://cdn.jsdelivr.net/gh/lit/dist@3/core/lit-core.min.js",
20
- "lit/": "https://esm.run/lit@3/",
21
- "three": "https://cdn.jsdelivr.net/npm/three@0.182.0/build/three.module.js",
22
- "three/addons/": "https://cdn.jsdelivr.net/npm/three@0.182.0/examples/jsm/",
23
- "troika-three-text": "https://cdn.jsdelivr.net/gh/protectwise/troika@028b81cf308f0f22e5aa8e78196be56ec1997af5/packages/troika-three-text/src/index.js",
24
- "troika-three-utils": "https://cdn.jsdelivr.net/gh/protectwise/troika@v0.52.4/packages/troika-three-utils/src/index.js",
25
- "troika-worker-utils": "https://cdn.jsdelivr.net/gh/protectwise/troika@v0.52.4/packages/troika-worker-utils/src/index.js",
26
- "bidi-js": "https://esm.sh/bidi-js@%5E1.0.2?target=es2022",
27
- "webgl-sdf-generator": "https://esm.sh/webgl-sdf-generator@1.1.1/es2022/webgl-sdf-generator.mjs",
28
- "xrblocks": "https://cdn.jsdelivr.net/npm/xrblocks@0.11.0/build/xrblocks.js",
29
- "xrblocks/addons/": "https://cdn.jsdelivr.net/npm/xrblocks@0.11.0/build/addons/",
30
- "@google/genai": "https://cdn.jsdelivr.net/npm/@google/genai/+esm"
31
- }
32
- }
33
- </script>
34
- </head>
35
-
36
- <body>
37
- <script type="module">
38
- import 'xrblocks/addons/simulator/SimulatorAddons.js';
39
- import {LongSelectHandler} from 'xrblocks/addons/ui/LongSelectHandler.js';
40
-
41
- import * as THREE from 'three';
42
- import {Text} from 'troika-three-text';
43
- import * as xb from 'xrblocks';
44
-
45
- const HANDEDNESS = xb.Handedness;
46
-
47
- // --- TouchableSphere inlined ---
48
- class TouchableSphere extends xb.MeshScript {
49
- static ICON_SIZE_MULTIPLIER = 1.2;
50
-
51
- constructor(detectedObject, radius = 0.2, iconName = 'help') {
52
- const inactiveColor = new THREE.Color(0xd1e2ff);
53
- const geometry = new THREE.SphereGeometry(radius, 32, 16);
54
- const material = new THREE.MeshBasicMaterial({color: inactiveColor, transparent: true, opacity: 0.9});
55
- super(geometry, material);
56
- this.inactiveColor = inactiveColor;
57
- this.activeColor = new THREE.Color(0x4970ff);
58
- this.textColor = new THREE.Color(0xffffff);
59
- this.textFontSize = 0.05;
60
- this.textAnchorX = 'center';
61
- this.textAnchorY = 'bottom';
62
- this.textOffsetY = 0.01;
63
- this.touchDistanceThreshold = radius * 2;
64
- this.sphereRadius = radius;
65
- this.labelText = detectedObject.label;
66
- this.object = detectedObject;
67
- this.wasTouchedLastFrame = false;
68
- this.position.copy(detectedObject.position);
69
- this.iconFont = 'https://fonts.gstatic.com/s/materialicons/v143/flUhRq6tzZclQEJ-Vdg-IuiaDsNa.woff';
70
- this.iconName = iconName;
71
- this.iconFontSize = this.sphereRadius * TouchableSphere.ICON_SIZE_MULTIPLIER;
72
- this.iconColor = new THREE.Color(0xffffff);
73
- this.iconMesh = null;
74
- this.raycaster = null;
75
- this.textLabel = null;
76
- }
77
-
78
- init(xrCoreInstance) {
79
- super.init(xrCoreInstance);
80
- if (this.scene && !this.parent) this.scene.add(this);
81
-
82
- this.textLabel = new Text();
83
- this.textLabel.text = this.labelText;
84
- this.textLabel.fontSize = this.textFontSize;
85
- this.textLabel.color = this.textColor;
86
- this.textLabel.anchorX = this.textAnchorX;
87
- this.textLabel.anchorY = this.textAnchorY;
88
- this.textLabel.position.set(0, this.sphereRadius + this.textOffsetY, 0);
89
- this.add(this.textLabel);
90
- this.textLabel.sync();
91
-
92
- this.iconMesh = new Text();
93
- this.iconMesh.text = this.iconName;
94
- this.iconMesh.font = this.iconFont;
95
- this.iconMesh.fontSize = this.iconFontSize;
96
- this.iconMesh.color = this.iconColor;
97
- this.iconMesh.anchorX = 'center';
98
- this.iconMesh.anchorY = 'middle';
99
- this.iconMesh.material.depthTest = false;
100
- this.iconMesh.renderOrder = this.renderOrder + 1;
101
- this.iconMesh.position.set(0, 0, 0);
102
- this.add(this.iconMesh);
103
- this.iconMesh.sync();
104
-
105
- this.raycaster = new THREE.Raycaster();
106
- }
107
-
108
- update() {
109
- if (!this.material || !xb.core.user || !this.textLabel || !xb.core?.camera) return;
110
- if ((xb.core.user.controllers && !this.raycaster) || !this.iconMesh) return;
111
-
112
- let isTouchedThisFrame = false;
113
- let touchInitiator = null;
114
-
115
- for (const controller of xb.core.user.controllers) {
116
- if (controller && controller.visible) {
117
- this.raycaster.setFromXRController(controller);
118
- const intersections = this.raycaster.intersectObject(this, false);
119
- if (intersections.length > 0) { isTouchedThisFrame = true; touchInitiator = controller; break; }
120
- }
121
- }
122
-
123
- if (xb.core.user.hands && !isTouchedThisFrame) {
124
- const sphereWorldPosition = new THREE.Vector3();
125
- this.getWorldPosition(sphereWorldPosition);
126
- const handednessToCheck = [HANDEDNESS.LEFT, HANDEDNESS.RIGHT];
127
- for (const handSide of handednessToCheck) {
128
- const hand = xb.core.user.hands.hands[handSide];
129
- if (Object.keys(hand.joints).length === 0) break;
130
- const indexTip = xb.core.user.hands.getIndexTip(handSide);
131
- if (indexTip) {
132
- const jointWorldPosition = new THREE.Vector3();
133
- indexTip.getWorldPosition(jointWorldPosition);
134
- const distanceToJoint = sphereWorldPosition.distanceTo(jointWorldPosition);
135
- if (distanceToJoint <= this.sphereRadius + this.touchDistanceThreshold) {
136
- isTouchedThisFrame = true;
137
- touchInitiator = {type: 'hand', side: handSide, joint: indexTip};
138
- break;
139
- }
140
- }
141
- }
142
- }
143
-
144
- const selectEvent = {target: this, initiator: touchInitiator};
145
- if (isTouchedThisFrame && !this.wasTouchedLastFrame) {
146
- this.material.color.set(this.activeColor);
147
- this.onSelectStart(selectEvent);
148
- this.onSelect(selectEvent);
149
- } else if (!isTouchedThisFrame && this.wasTouchedLastFrame) {
150
- this.material.color.set(this.inactiveColor);
151
- this.onSelectEnd(selectEvent);
152
- } else if (isTouchedThisFrame) {
153
- this.onSelect(selectEvent);
154
- }
155
-
156
- this.wasTouchedLastFrame = isTouchedThisFrame;
157
- const cameraPosition = xb.core.camera.position;
158
- this.iconMesh.lookAt(cameraPosition);
159
- this.textLabel.lookAt(cameraPosition);
160
- }
161
-
162
- setActive(isActive) {
163
- if (!this.material) return;
164
- this.material.color.set(isActive ? this.activeColor : this.inactiveColor);
165
- }
166
-
167
- dispose() {
168
- if (this.textLabel) { this.remove(this.textLabel); this.textLabel.dispose(); this.textLabel = null; }
169
- if (this.iconMesh) { this.remove(this.iconMesh); this.iconMesh.dispose(); this.iconMesh = null; }
170
- super.dispose();
171
- }
172
- }
173
-
174
- // --- XRObjectManager inlined ---
175
- class XRObjectManager extends xb.Script {
176
- constructor() {
177
- super();
178
- this.objectSphereRadius = 0.03;
179
- this.activeSphere = null;
180
-
181
- this.geminiConfig = {
182
- userQuery: {
183
- thinkingConfig: {thinkingBudget: 0},
184
- systemInstruction: [{text: `You're an informative and helpful AI assistant specializing in identifying and describing objects within images. Your primary goal is to provide detailed yet concise answers to user questions, making a best effort to respond even if you're not entirely sure or the image quality is poor. When describing objects, strive for maximum detail without being verbose, focusing on key characteristics. Please ignore any hands or other human body parts present in the image. User queries will always be structured like this: {object: '...', question: '...'}`}],
185
- responseMimeType: 'application/json',
186
- responseSchema: {type: 'OBJECT', required: ['answer'], properties: {answer: {type: 'STRING'}}},
187
- },
188
- };
189
- }
190
-
191
- init() {
192
- if (xb.core.sound.speechRecognizer) {
193
- xb.core.sound.speechRecognizer.addEventListener('result', this.handleSpeechResult.bind(this));
194
- xb.core.sound.speechRecognizer.addEventListener('end', () => { this.activeSphere?.setActive(false); this.activeSphere = null; });
195
- } else {
196
- console.error('Speech recognizer not available at init.');
197
- }
198
- }
199
-
200
- handleSpeechResult(event) {
201
- const {transcript, isFinal} = event;
202
- if (!isFinal) return;
203
- if (!this.activeSphere) { console.warn('Speech result received, but no active sphere is set.'); return; }
204
- if (!this.activeSphere.object.image) {
205
- const warningMsg = "I don't have a specific image for that object, so I can't answer questions about it.";
206
- console.warn(warningMsg);
207
- xb.core.sound.speechSynthesizer.speak(warningMsg);
208
- return;
209
- }
210
- const prompt = {question: transcript, object: this.activeSphere.object.label};
211
- this.queryObjectInformation(JSON.stringify(prompt), this.activeSphere.object.image)
212
- .then((response) => {
213
- try {
214
- const parsedResponse = JSON.parse(response);
215
- xb.core.sound.speechSynthesizer.speak(parsedResponse.answer);
216
- } catch (e) { console.error('Error parsing AI response JSON:', e); }
217
- })
218
- .catch((error) => {
219
- const errorMsg = "I'm sorry, I had trouble processing that request. Please try again.";
220
- console.error('Failed to get information about the object:', error);
221
- xb.core.sound.speechSynthesizer.speak(errorMsg);
222
- });
223
- }
224
-
225
- async queryObjectionDetection() {
226
- if (!xb.core.world?.objects) { console.error('ObjectDetector is not available. Ensure it is enabled in the options.'); return; }
227
- const detectedObjects = await xb.core.world.objects.runDetection();
228
- for (const detectedObject of detectedObjects) this.createSphereWithLabel(detectedObject);
229
- }
230
-
231
- queryObjectInformation(textPrompt, objectImageBase64) {
232
- if (!xb.core.ai.isAvailable()) { const errorMsg = 'Gemini is unavailable for object query.'; console.error(errorMsg); return Promise.reject(new Error(errorMsg)); }
233
- xb.core.options.ai.gemini.config = this.geminiConfig.userQuery;
234
- let {mimeType, strippedBase64} = xb.parseBase64DataURL(objectImageBase64);
235
- return xb.core.ai.query({type: 'multiPart', parts: [{inlineData: {mimeType: mimeType, data: strippedBase64}}, {text: textPrompt}]})
236
- .catch((error) => { console.error('AI query for object information failed:', error); throw error; });
237
- }
238
-
239
- onSphereTouchStart(event) {
240
- if (typeof event.target !== 'object' || !(event.target instanceof TouchableSphere)) return;
241
- if (xb.core.sound.speechSynthesizer.isSpeaking) { event.target.setActive(false); return; }
242
- this.activeSphere = event.target;
243
- this.activeSphere.setActive(true);
244
- xb.core.sound.speechRecognizer.start();
245
- }
246
-
247
- onSphereTouchEnd(event) {
248
- if (typeof event.target !== 'object' || !(event.target instanceof TouchableSphere)) return;
249
- xb.core.sound.speechRecognizer.stop();
250
- }
251
-
252
- createSphereWithLabel(detectedObject) {
253
- const touchableSphereInstance = new TouchableSphere(detectedObject, this.objectSphereRadius, 'live_help');
254
- touchableSphereInstance.onSelectStart = (event) => this.onSphereTouchStart(event);
255
- touchableSphereInstance.onSelectEnd = (event) => this.onSphereTouchEnd(event);
256
- xb.add(touchableSphereInstance);
257
- }
258
- }
259
-
260
- // --- main ---
261
- const options = new xb.Options();
262
- options.simulator.instructions.enabled = false;
263
- options.deviceCamera.enabled = true;
264
- options.deviceCamera.videoConstraints = {width: {ideal: 1280}, height: {ideal: 720}, facingMode: 'environment'};
265
- options.permissions.camera = true;
266
- options.reticles.enabled = false;
267
- options.controllers.visualizeRays = false;
268
- options.world.enableObjectDetection();
269
- options.depth.enabled = true;
270
- options.depth.depthMesh.enabled = true;
271
- options.depth.depthMesh.updateFullResolutionGeometry = true;
272
- options.depth.depthMesh.renderShadow = true;
273
- options.depth.depthTexture.enabled = false;
274
- options.depth.matchDepthView = false;
275
- options.hands.enabled = true;
276
- options.hands.visualization = false;
277
- options.hands.visualizeMeshes = false;
278
- options.sound.speechSynthesizer.enabled = true;
279
- options.sound.speechRecognizer.enabled = true;
280
- options.sound.speechRecognizer.playSimulatorActivationSounds = true;
281
- options.ai.enabled = true;
282
- options.ai.gemini.enabled = true;
283
- options.ai.gemini.model = 'gemini-2.5-flash';
284
- options.setAppTitle('Gemini XR-Objects');
285
- options.setAppDescription('Recognize objects with Gemini and ask questions about them. Perform a long pinch / press to start!');
286
- options.xrButton.showEnterSimulatorButton = true;
287
-
288
- function start() {
289
- const xrObjectManager = new XRObjectManager();
290
- const longSelectHandler = new LongSelectHandler(xrObjectManager.queryObjectionDetection.bind(xrObjectManager));
291
- xb.add(xrObjectManager);
292
- xb.add(longSelectHandler);
293
- xb.init(options);
294
- }
295
-
296
- document.addEventListener('DOMContentLoaded', function () {
297
- start();
298
- });
299
-
300
- </script>
301
- </body>
302
- </html>
package/skills/mudra-xr/assets/demos/math3d.html DELETED
@@ -1,188 +0,0 @@
1
- <!doctype html>
2
- <!-- reference: demos/math3d -->
3
- <html lang="en">
4
- <head>
5
- <title>Math 3D</title>
6
-
7
- <meta
8
- name="viewport"
9
- content="width=device-width, initial-scale=1.0, user-scalable=no"
10
- />
11
-
12
- <link type="text/css" rel="stylesheet" href="https://xrblocks.github.io/css/xr.css" />
13
- <script>
14
- window.litDisableBundleWarning = true;
15
- </script>
16
- <script type="importmap">
17
- {
18
- "imports": {
19
- "three": "https://cdn.jsdelivr.net/npm/three@0.182.0/build/three.module.js",
20
- "three/addons/": "https://cdn.jsdelivr.net/npm/three@0.182.0/examples/jsm/",
21
- "troika-three-text": "https://cdn.jsdelivr.net/gh/protectwise/troika@028b81cf308f0f22e5aa8e78196be56ec1997af5/packages/troika-three-text/src/index.js",
22
- "troika-three-utils": "https://cdn.jsdelivr.net/gh/protectwise/troika@v0.52.4/packages/troika-three-utils/src/index.js",
23
- "troika-worker-utils": "https://cdn.jsdelivr.net/gh/protectwise/troika@v0.52.4/packages/troika-worker-utils/src/index.js",
24
- "bidi-js": "https://esm.sh/bidi-js@%5E1.0.2?target=es2022",
25
- "webgl-sdf-generator": "https://esm.sh/webgl-sdf-generator@1.1.1/es2022/webgl-sdf-generator.mjs",
26
- "expr-eval": "https://esm.sh/expr-eval",
27
- "xrblocks": "https://cdn.jsdelivr.net/npm/xrblocks@0.11.0/build/xrblocks.js",
28
- "xrblocks/addons/": "https://cdn.jsdelivr.net/npm/xrblocks@0.11.0/build/addons/"
29
- }
30
- }
31
- </script>
32
- </head>
33
-
34
- <body>
35
- <script type="module">
36
- import {Parser} from 'expr-eval';
37
- import * as THREE from 'three';
38
- import {ParametricGeometry} from 'three/addons/geometries/ParametricGeometry.js';
39
- import * as xb from 'xrblocks';
40
- import {Keyboard} from 'xrblocks/addons/virtualkeyboard/Keyboard.js';
41
-
42
- // --- Math3D inlined ---
43
- const MATH_OBJECTS = [
44
- {functionText: 'x^2 - y^2'},
45
- ];
46
-
47
- class Math3D extends xb.Script {
48
- constructor() {
49
- super();
50
- this.mathObjects = MATH_OBJECTS;
51
- this.graph = null;
52
- this.functionDisplay = null;
53
- this.keyboard = null;
54
-
55
- const panel = new xb.SpatialPanel({backgroundColor: '#00000000', useDefaultPosition: false, showEdge: false});
56
- panel.isRoot = true;
57
- this.add(panel);
58
-
59
- this.descriptionPagerState = new xb.PagerState({pages: this.mathObjects.length});
60
- const grid = panel.addGrid();
61
-
62
- const imageRow = grid.addRow({weight: 1.0});
63
- this.imagePager = new xb.HorizontalPager({state: this.descriptionPagerState});
64
- imageRow.addCol({weight: 1.0}).add(this.imagePager);
65
- const meshAxes = this.createCoordinateGridAxes();
66
- for (let i = 0; i < this.mathObjects.length; i++) {
67
- this.imagePager.children[i].add(meshAxes[0]);
68
- this.imagePager.children[i].add(meshAxes[1]);
69
- this.imagePager.children[i].add(meshAxes[2]);
70
- }
71
- grid.addRow({weight: 0.1});
72
- const controlRow = grid.addRow({weight: 0.4});
73
-
74
- const ctrlPanel = controlRow.addPanel({backgroundColor: '#000000D9'});
75
- const ctrlGrid = ctrlPanel.addGrid();
76
- {
77
- const leftColumn = ctrlGrid.addCol({weight: 0.1});
78
- const midColumn = ctrlGrid.addCol({weight: 0.8});
79
- const descRow = midColumn.addRow({weight: 0.8});
80
-
81
- this.add(this.descriptionPagerState);
82
- this.descriptionPager = new xb.HorizontalPager({state: this.descriptionPagerState, enableRaycastOnChildren: false});
83
- descRow.add(this.descriptionPager);
84
-
85
- for (let i = 0; i < this.mathObjects.length; i++) {
86
- const initialFunction = this.mathObjects[i].functionText;
87
- this.functionDisplay = this.descriptionPager.children[i].add(
88
- new xb.TextButton({text: initialFunction, fontColor: '#ffffff', fontSize: 0.2, backgroundColor: '#00000000'})
89
- );
90
- }
91
- }
92
-
93
- const orbiter = ctrlGrid.addOrbiter();
94
- orbiter.addExitButton();
95
- panel.updateLayouts();
96
- this.panel = panel;
97
- }
98
-
99
- init() {
100
- xb.core.renderer.localClippingEnabled = true;
101
- this.add(new THREE.HemisphereLight(0x888877, 0x777788, 3));
102
- const light = new THREE.DirectionalLight(0xffffff, 5.0);
103
- light.position.set(-0.5, 4, 1.0);
104
- this.add(light);
105
- this.panel.position.set(0, 1.9, -1.0);
106
-
107
- this.keyboard = new Keyboard();
108
- this.add(this.keyboard);
109
- this.keyboard.position.set(0, -0.3, 0);
110
-
111
- this.keyboard.onEnterPressed = (newFunctionText) => {
112
- this.mathObjects[this.descriptionPagerState.currentPage].functionText = newFunctionText;
113
- this.updateGraph(newFunctionText);
114
- };
115
-
116
- this.keyboard.onTextChanged = (currentText) => {
117
- if (this.functionDisplay) this.functionDisplay.text = currentText;
118
- };
119
-
120
- this.updateGraph('x^2 - y^2');
121
- console.log('Math3D UIs: ', xb.core.ui.views);
122
- }
123
-
124
- updateGraph(functionText) {
125
- if (!functionText || functionText.trim() === '') return;
126
- try {
127
- if (this.graph && this.graph.parent) { this.graph.parent.remove(this.graph); this.graph.geometry.dispose(); this.graph.material.dispose(); }
128
- this.graph = this.createParametricMesh(functionText);
129
- const currentPage = this.descriptionPagerState.currentPage;
130
- this.imagePager.children[currentPage].add(this.graph);
131
- this.functionDisplay.text = functionText;
132
- } catch (e) { console.error('Error creating graph:', e); this.functionDisplay.text = 'Error: Invalid function'; }
133
- }
134
-
135
- createCoordinateGridAxes(gridLength = 0.25) {
136
- const gridVectors = [
137
- {startVector: new THREE.Vector3(gridLength, 0, 0), endVector: new THREE.Vector3(-gridLength, 0, 0), color: 0xff0000},
138
- {startVector: new THREE.Vector3(0, gridLength, 0), endVector: new THREE.Vector3(0, -gridLength, 0), color: 0x0000ff},
139
- {startVector: new THREE.Vector3(0, 0, gridLength), endVector: new THREE.Vector3(0, 0, -gridLength), color: 0x00ff00},
140
- ];
141
- var axes = [];
142
- for (let i = 0; i < gridVectors.length; i++) {
143
- var axisGeometry = new THREE.BufferGeometry().setFromPoints([gridVectors[i].startVector, gridVectors[i].endVector]);
144
- var axisMaterial = new THREE.LineBasicMaterial({color: gridVectors[i].color});
145
- axes.push(new THREE.Line(axisGeometry, axisMaterial));
146
- }
147
- return axes;
148
- }
149
-
150
- createParametricMesh(zFunctionText) {
151
- var xMin = -5, xMax = 5, xRange = xMax - xMin;
152
- var yMin = -5, yMax = 5, yRange = yMax - yMin;
153
- var zFunction = Parser.parse(zFunctionText).toJSFunction(['x', 'y']);
154
- var parametricFunction = function (x, y, target) {
155
- var x = xRange * x + xMin;
156
- var y = yRange * y + yMin;
157
- var z = zFunction(x, y);
158
- target.set(x, y, z);
159
- };
160
- const slices = 25, stacks = 25;
161
- var graphGeometry = new ParametricGeometry(parametricFunction, slices, stacks, true);
162
- graphGeometry.rotateX(-Math.PI / 2);
163
- graphGeometry.scale(0.015, 0.015, 0.015);
164
- const textureLoader = new THREE.TextureLoader();
165
- const texture = textureLoader.load('images/gradient.png');
166
- const material = new THREE.MeshBasicMaterial({map: texture, transparent: true, side: THREE.DoubleSide});
167
- material.opacity = 0.75;
168
- return new THREE.Mesh(graphGeometry, material);
169
- }
170
- }
171
-
172
- const options = new xb.Options();
173
- options.simulator.instructions.enabled = false;
174
- options.reticles.enabled = true;
175
- options.visualizeRays = true;
176
-
177
- function start() {
178
- xb.add(new Math3D());
179
- xb.init(options);
180
- }
181
-
182
- document.addEventListener('DOMContentLoaded', function () {
183
- start();
184
- });
185
-
186
- </script>
187
- </body>
188
- </html>