mudra-skills 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/.claude-plugin/plugin.json +19 -0
  2. package/CLAUDE.md +35 -0
  3. package/bin/install.js +25 -0
  4. package/package.json +28 -0
  5. package/skills/mudra-master/SKILL.md +202 -0
  6. package/skills/mudra-master/mudra-preview/SKILL.md +151 -0
  7. package/skills/mudra-master/mudra-preview/assets/ar-menu.html +382 -0
  8. package/skills/mudra-master/mudra-preview/assets/document-scroller.html +468 -0
  9. package/skills/mudra-master/mudra-preview/assets/drum-machine.html +788 -0
  10. package/skills/mudra-master/mudra-preview/assets/emg-visualizer.html +390 -0
  11. package/skills/mudra-master/mudra-preview/assets/generative-art.html +408 -0
  12. package/skills/mudra-master/mudra-preview/assets/gesture-assistant.html +342 -0
  13. package/skills/mudra-master/mudra-preview/assets/gesture-speech.html +444 -0
  14. package/skills/mudra-master/mudra-preview/assets/hands-free-desktop.html +376 -0
  15. package/skills/mudra-master/mudra-preview/assets/model-rotator.html +365 -0
  16. package/skills/mudra-master/mudra-preview/assets/mudra-duel.html +3200 -0
  17. package/skills/mudra-master/mudra-preview/assets/mudra-monitor.html +1451 -0
  18. package/skills/mudra-master/mudra-preview/assets/mudra-ultimate-template.html +891 -0
  19. package/skills/mudra-master/mudra-preview/assets/music-sequencer.html +455 -0
  20. package/skills/mudra-master/mudra-preview/assets/neural-pong.html +380 -0
  21. package/skills/mudra-master/mudra-preview/assets/neural-snake.html +364 -0
  22. package/skills/mudra-master/mudra-preview/assets/presentation-controller.html +345 -0
  23. package/skills/mudra-master/mudra-preview/assets/pressure-painter.html +285 -0
  24. package/skills/mudra-master/mudra-preview/assets/runner.html +1812 -0
  25. package/skills/mudra-master/mudra-preview/assets/smart-home.html +461 -0
  26. package/skills/mudra-master/mudra-preview/assets/space-invaders.html +1154 -0
  27. package/skills/mudra-master/mudra-preview/assets/waterful-ring-toss.html +1624 -0
  28. package/skills/mudra-master/mudra-preview/references/agent_protocol.json +646 -0
  29. package/skills/mudra-master/mudra-preview/references/promt.md +16829 -0
  30. package/skills/mudra-master/mudra-xr/SKILL.md +245 -0
  31. package/skills/mudra-master/mudra-xr/assets/demos/3dgs-walkthrough.html +242 -0
  32. package/skills/mudra-master/mudra-xr/assets/demos/aisimulator.html +792 -0
  33. package/skills/mudra-master/mudra-xr/assets/demos/balloonpop.html +422 -0
  34. package/skills/mudra-master/mudra-xr/assets/demos/ballpit.html +284 -0
  35. package/skills/mudra-master/mudra-xr/assets/demos/drone.html +41 -0
  36. package/skills/mudra-master/mudra-xr/assets/demos/gemini-icebreakers.html +338 -0
  37. package/skills/mudra-master/mudra-xr/assets/demos/gemini-xrobject.html +302 -0
  38. package/skills/mudra-master/mudra-xr/assets/demos/math3d.html +188 -0
  39. package/skills/mudra-master/mudra-xr/assets/demos/measure.html +247 -0
  40. package/skills/mudra-master/mudra-xr/assets/demos/occlusion.html +251 -0
  41. package/skills/mudra-master/mudra-xr/assets/demos/rain.html +498 -0
  42. package/skills/mudra-master/mudra-xr/assets/demos/screenwiper.html +432 -0
  43. package/skills/mudra-master/mudra-xr/assets/demos/splash.html +519 -0
  44. package/skills/mudra-master/mudra-xr/assets/demos/webcam_gestures.html +253 -0
  45. package/skills/mudra-master/mudra-xr/assets/demos/xremoji.html +833 -0
  46. package/skills/mudra-master/mudra-xr/assets/demos/xrpoet.html +151 -0
  47. package/skills/mudra-master/mudra-xr/assets/samples/depthmap.html +266 -0
  48. package/skills/mudra-master/mudra-xr/assets/samples/depthmesh.html +100 -0
  49. package/skills/mudra-master/mudra-xr/assets/samples/game_rps.html +1099 -0
  50. package/skills/mudra-master/mudra-xr/assets/samples/gestures_custom.html +467 -0
  51. package/skills/mudra-master/mudra-xr/assets/samples/gestures_heuristic.html +256 -0
  52. package/skills/mudra-master/mudra-xr/assets/samples/lighting.html +163 -0
  53. package/skills/mudra-master/mudra-xr/assets/samples/mesh_detection.html +46 -0
  54. package/skills/mudra-master/mudra-xr/assets/samples/modelviewer.html +183 -0
  55. package/skills/mudra-master/mudra-xr/assets/samples/paint.html +133 -0
  56. package/skills/mudra-master/mudra-xr/assets/samples/planar-vst.html +45 -0
  57. package/skills/mudra-master/mudra-xr/assets/samples/reticle.html +95 -0
  58. package/skills/mudra-master/mudra-xr/assets/samples/skybox_agent.html +384 -0
  59. package/skills/mudra-master/mudra-xr/assets/samples/sound.html +390 -0
  60. package/skills/mudra-master/mudra-xr/assets/samples/ui.html +496 -0
  61. package/skills/mudra-master/mudra-xr/assets/samples/virtual-screens.html +808 -0
  62. package/skills/mudra-master/mudra-xr/assets/templates/0_basic.html +64 -0
  63. package/skills/mudra-master/mudra-xr/assets/templates/1_ui.html +107 -0
  64. package/skills/mudra-master/mudra-xr/assets/templates/2_hands.html +183 -0
  65. package/skills/mudra-master/mudra-xr/assets/templates/3_depth.html +89 -0
  66. package/skills/mudra-master/mudra-xr/assets/templates/4_stereo.html +84 -0
  67. package/skills/mudra-master/mudra-xr/assets/templates/5_camera.html +117 -0
  68. package/skills/mudra-master/mudra-xr/assets/templates/6_ai.html +154 -0
  69. package/skills/mudra-master/mudra-xr/assets/templates/7_ai_live.html +253 -0
  70. package/skills/mudra-master/mudra-xr/assets/templates/8_objects.html +95 -0
  71. package/skills/mudra-master/mudra-xr/assets/templates/9_xr-toggle.html +90 -0
  72. package/skills/mudra-master/mudra-xr/assets/templates/heuristic_hand_gestures.html +102 -0
  73. package/skills/mudra-master/mudra-xr/assets/templates/meshes.html +47 -0
  74. package/skills/mudra-master/mudra-xr/assets/templates/planes.html +47 -0
  75. package/skills/mudra-master/mudra-xr/assets/templates/uikit.html +187 -0
  76. package/skills/mudra-master/mudra-xr/references/agent_protocol.json +878 -0
  77. package/skills/mudra-master/mudra-xr/references/promt.md +1516 -0
  78. package/skills/mudra-preview/SKILL.md +151 -0
  79. package/skills/mudra-preview/assets/ar-menu.html +382 -0
  80. package/skills/mudra-preview/assets/document-scroller.html +468 -0
  81. package/skills/mudra-preview/assets/drum-machine.html +788 -0
  82. package/skills/mudra-preview/assets/emg-visualizer.html +390 -0
  83. package/skills/mudra-preview/assets/generative-art.html +408 -0
  84. package/skills/mudra-preview/assets/gesture-assistant.html +342 -0
  85. package/skills/mudra-preview/assets/gesture-speech.html +444 -0
  86. package/skills/mudra-preview/assets/hands-free-desktop.html +376 -0
  87. package/skills/mudra-preview/assets/model-rotator.html +365 -0
  88. package/skills/mudra-preview/assets/mudra-duel.html +3200 -0
  89. package/skills/mudra-preview/assets/mudra-monitor.html +1451 -0
  90. package/skills/mudra-preview/assets/mudra-ultimate-template.html +891 -0
  91. package/skills/mudra-preview/assets/music-sequencer.html +455 -0
  92. package/skills/mudra-preview/assets/neural-pong.html +380 -0
  93. package/skills/mudra-preview/assets/neural-snake.html +364 -0
  94. package/skills/mudra-preview/assets/presentation-controller.html +345 -0
  95. package/skills/mudra-preview/assets/pressure-painter.html +285 -0
  96. package/skills/mudra-preview/assets/runner.html +1812 -0
  97. package/skills/mudra-preview/assets/smart-home.html +461 -0
  98. package/skills/mudra-preview/assets/space-invaders.html +1154 -0
  99. package/skills/mudra-preview/assets/waterful-ring-toss.html +1624 -0
  100. package/skills/mudra-preview/references/agent_protocol.json +646 -0
  101. package/skills/mudra-preview/references/promt.md +16829 -0
  102. package/skills/mudra-xr/SKILL.md +245 -0
  103. package/skills/mudra-xr/assets/demos/3dgs-walkthrough.html +242 -0
  104. package/skills/mudra-xr/assets/demos/aisimulator.html +792 -0
  105. package/skills/mudra-xr/assets/demos/balloonpop.html +422 -0
  106. package/skills/mudra-xr/assets/demos/ballpit.html +284 -0
  107. package/skills/mudra-xr/assets/demos/drone.html +41 -0
  108. package/skills/mudra-xr/assets/demos/gemini-icebreakers.html +338 -0
  109. package/skills/mudra-xr/assets/demos/gemini-xrobject.html +302 -0
  110. package/skills/mudra-xr/assets/demos/math3d.html +188 -0
  111. package/skills/mudra-xr/assets/demos/measure.html +247 -0
  112. package/skills/mudra-xr/assets/demos/occlusion.html +251 -0
  113. package/skills/mudra-xr/assets/demos/rain.html +498 -0
  114. package/skills/mudra-xr/assets/demos/screenwiper.html +432 -0
  115. package/skills/mudra-xr/assets/demos/splash.html +519 -0
  116. package/skills/mudra-xr/assets/demos/webcam_gestures.html +253 -0
  117. package/skills/mudra-xr/assets/demos/xremoji.html +833 -0
  118. package/skills/mudra-xr/assets/demos/xrpoet.html +151 -0
  119. package/skills/mudra-xr/assets/samples/depthmap.html +266 -0
  120. package/skills/mudra-xr/assets/samples/depthmesh.html +100 -0
  121. package/skills/mudra-xr/assets/samples/game_rps.html +1099 -0
  122. package/skills/mudra-xr/assets/samples/gestures_custom.html +467 -0
  123. package/skills/mudra-xr/assets/samples/gestures_heuristic.html +256 -0
  124. package/skills/mudra-xr/assets/samples/lighting.html +163 -0
  125. package/skills/mudra-xr/assets/samples/mesh_detection.html +46 -0
  126. package/skills/mudra-xr/assets/samples/modelviewer.html +183 -0
  127. package/skills/mudra-xr/assets/samples/paint.html +133 -0
  128. package/skills/mudra-xr/assets/samples/planar-vst.html +45 -0
  129. package/skills/mudra-xr/assets/samples/reticle.html +95 -0
  130. package/skills/mudra-xr/assets/samples/skybox_agent.html +384 -0
  131. package/skills/mudra-xr/assets/samples/sound.html +390 -0
  132. package/skills/mudra-xr/assets/samples/ui.html +496 -0
  133. package/skills/mudra-xr/assets/samples/virtual-screens.html +808 -0
  134. package/skills/mudra-xr/assets/templates/0_basic.html +64 -0
  135. package/skills/mudra-xr/assets/templates/1_ui.html +107 -0
  136. package/skills/mudra-xr/assets/templates/2_hands.html +183 -0
  137. package/skills/mudra-xr/assets/templates/3_depth.html +89 -0
  138. package/skills/mudra-xr/assets/templates/4_stereo.html +84 -0
  139. package/skills/mudra-xr/assets/templates/5_camera.html +117 -0
  140. package/skills/mudra-xr/assets/templates/6_ai.html +154 -0
  141. package/skills/mudra-xr/assets/templates/7_ai_live.html +253 -0
  142. package/skills/mudra-xr/assets/templates/8_objects.html +95 -0
  143. package/skills/mudra-xr/assets/templates/9_xr-toggle.html +90 -0
  144. package/skills/mudra-xr/assets/templates/heuristic_hand_gestures.html +102 -0
  145. package/skills/mudra-xr/assets/templates/meshes.html +47 -0
  146. package/skills/mudra-xr/assets/templates/planes.html +47 -0
  147. package/skills/mudra-xr/assets/templates/uikit.html +187 -0
  148. package/skills/mudra-xr/references/agent_protocol.json +878 -0
  149. package/skills/mudra-xr/references/promt.md +1516 -0
@@ -0,0 +1,154 @@
1
+ <!doctype html>
2
+ <!-- reference: templates/6_ai -->
3
+ <html lang="en">
4
+ <head>
5
+
6
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
7
+
8
+ <title>XR Blocks - AI Query Demo</title>
9
+ <link type="text/css" rel="stylesheet" href="https://xrblocks.github.io/css/xr.css" />
10
+ <script>
11
+ window.litDisableBundleWarning = true;
12
+ </script>
13
+ <script type="importmap">
14
+ {
15
+ "imports": {
16
+ "three": "https://cdn.jsdelivr.net/npm/three@0.182.0/build/three.module.js",
17
+ "three/addons/": "https://cdn.jsdelivr.net/npm/three@0.182.0/examples/jsm/",
18
+ "troika-three-text": "https://cdn.jsdelivr.net/gh/protectwise/troika@028b81cf308f0f22e5aa8e78196be56ec1997af5/packages/troika-three-text/src/index.js",
19
+ "troika-three-utils": "https://cdn.jsdelivr.net/gh/protectwise/troika@v0.52.4/packages/troika-three-utils/src/index.js",
20
+ "troika-worker-utils": "https://cdn.jsdelivr.net/gh/protectwise/troika@v0.52.4/packages/troika-worker-utils/src/index.js",
21
+ "bidi-js": "https://esm.sh/bidi-js@%5E1.0.2?target=es2022",
22
+ "webgl-sdf-generator": "https://esm.sh/webgl-sdf-generator@1.1.1/es2022/webgl-sdf-generator.mjs",
23
+ "lit": "https://cdn.jsdelivr.net/gh/lit/dist@3/core/lit-core.min.js",
24
+ "lit/": "https://esm.run/lit@3/",
25
+ "xrblocks": "https://cdn.jsdelivr.net/npm/xrblocks@0.11.0/build/xrblocks.js",
26
+ "xrblocks/addons/": "https://cdn.jsdelivr.net/npm/xrblocks@0.11.0/build/addons/",
27
+ "@google/genai": "https://cdn.jsdelivr.net/npm/@google/genai/+esm"
28
+ }
29
+ }
30
+ </script>
31
+ </head>
32
+
33
+ <body>
34
+ <script type="module">
35
+ import 'xrblocks/addons/simulator/SimulatorAddons.js';
36
+
37
+ import * as xb from 'xrblocks';
38
+
39
+ // Inlined from GeminiQueryManager.js
40
+ class GeminiQueryManager extends xb.Script {
41
+ constructor() {
42
+ super();
43
+ this.panel = null;
44
+ this.isProcessing = false;
45
+ this.responseDisplay = null;
46
+ }
47
+
48
+ init() {
49
+ this.ai = xb.core.ai;
50
+
51
+ this.createPanel();
52
+ }
53
+
54
+ createPanel() {
55
+ this.panel = new xb.SpatialPanel({
56
+ width: 2.5,
57
+ height: 1.5,
58
+ backgroundColor: '#1a1a1abb',
59
+ });
60
+ this.panel.position.set(0, 1.6, -2);
61
+ this.add(this.panel);
62
+
63
+ const grid = this.panel.addGrid();
64
+
65
+ // Response area
66
+ const responseRow = grid.addRow({weight: 0.8});
67
+ this.responseDisplay = new xb.ScrollingTroikaTextView({
68
+ text: '',
69
+ fontSize: 0.04,
70
+ });
71
+ responseRow.add(this.responseDisplay);
72
+
73
+ const buttonRow = grid.addRow({weight: 0.2});
74
+ const textCol = buttonRow.addCol({weight: 0.5});
75
+ const textButton = textCol.addTextButton({
76
+ text: 'Ask about WebXR',
77
+ fontColor: '#ffffff',
78
+ backgroundColor: '#4285f4',
79
+ fontSize: 0.24,
80
+ });
81
+
82
+ const imageCol = buttonRow.addCol({weight: 0.5});
83
+ const imageButton = imageCol.addTextButton({
84
+ text: 'Send Sample Image',
85
+ fontColor: '#ffffff',
86
+ backgroundColor: '#34a853',
87
+ fontSize: 0.24,
88
+ });
89
+
90
+ textButton.onTriggered = () => this.askText();
91
+ imageButton.onTriggered = () => this.askImage();
92
+ }
93
+
94
+ async ask(parts, displayText) {
95
+ if (this.isProcessing || !this.ai?.isAvailable()) return;
96
+
97
+ this.isProcessing = true;
98
+ this.responseDisplay.addText(displayText);
99
+
100
+ try {
101
+ const response = await this.ai.query({
102
+ type: 'multiPart',
103
+ parts: parts,
104
+ });
105
+ this.responseDisplay.addText(`🤖 AI: ${response.text}\n\n`);
106
+ } catch (error) {
107
+ this.responseDisplay.addText(`❌ Error: ${error.message}\n\n`);
108
+ }
109
+
110
+ this.isProcessing = false;
111
+ }
112
+
113
+ askText() {
114
+ const question = 'Hello! What is WebXR?';
115
+ const parts = [{text: question + ' reply succinctly.'}];
116
+ const displayText = `💬 You: ${question}\n\n`;
117
+ this.ask(parts, displayText);
118
+ }
119
+
120
+ askImage() {
121
+ const question = 'What do you see in this image?';
122
+ const image = {
123
+ inlineData: {
124
+ data: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==',
125
+ mimeType: 'image/png',
126
+ },
127
+ };
128
+ const parts = [image, {text: question}];
129
+ const displayText = `💬 You: ${question}\n📸 [Sample image sent]\n\n`;
130
+ this.ask(parts, displayText);
131
+ }
132
+ }
133
+
134
+ const options = new xb.Options();
135
+ options.simulator.instructions.enabled = false;
136
+ options.enableUI();
137
+ options.enableAI();
138
+
139
+ function start() {
140
+ try {
141
+ xb.init(options);
142
+ xb.add(new GeminiQueryManager());
143
+ } catch (error) {
144
+ console.error('Failed to initialize XR app:', error);
145
+ }
146
+ }
147
+
148
+ document.addEventListener('DOMContentLoaded', function () {
149
+ start();
150
+ });
151
+
152
+ </script>
153
+ </body>
154
+ </html>
@@ -0,0 +1,253 @@
1
+ <!doctype html>
2
+ <!-- reference: templates/7_ai_live -->
3
+ <html lang="en">
4
+ <head>
5
+
6
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
7
+
8
+ <title>XR Blocks - Gemini Live AI Demo</title>
9
+ <link type="text/css" rel="stylesheet" href="https://xrblocks.github.io/css/xr.css" />
10
+ <script>
11
+ window.litDisableBundleWarning = true;
12
+ </script>
13
+ <script type="importmap">
14
+ {
15
+ "imports": {
16
+ "three": "https://cdn.jsdelivr.net/npm/three@0.182.0/build/three.module.js",
17
+ "three/addons/": "https://cdn.jsdelivr.net/npm/three@0.182.0/examples/jsm/",
18
+ "troika-three-text": "https://cdn.jsdelivr.net/gh/protectwise/troika@028b81cf308f0f22e5aa8e78196be56ec1997af5/packages/troika-three-text/src/index.js",
19
+ "troika-three-utils": "https://cdn.jsdelivr.net/gh/protectwise/troika@v0.52.4/packages/troika-three-utils/src/index.js",
20
+ "troika-worker-utils": "https://cdn.jsdelivr.net/gh/protectwise/troika@v0.52.4/packages/troika-worker-utils/src/index.js",
21
+ "bidi-js": "https://esm.sh/bidi-js@%5E1.0.2?target=es2022",
22
+ "webgl-sdf-generator": "https://esm.sh/webgl-sdf-generator@1.1.1/es2022/webgl-sdf-generator.mjs",
23
+ "lit": "https://cdn.jsdelivr.net/gh/lit/dist@3/core/lit-core.min.js",
24
+ "lit/": "https://esm.run/lit@3/",
25
+ "xrblocks": "https://cdn.jsdelivr.net/npm/xrblocks@0.11.0/build/xrblocks.js",
26
+ "xrblocks/addons/": "https://cdn.jsdelivr.net/npm/xrblocks@0.11.0/build/addons/",
27
+ "@google/genai": "https://cdn.jsdelivr.net/npm/@google/genai/+esm"
28
+ }
29
+ }
30
+ </script>
31
+ </head>
32
+
33
+ <body>
34
+ <script type="module">
35
+ import 'xrblocks/addons/simulator/SimulatorAddons.js';
36
+
37
+ import * as xb from 'xrblocks';
38
+ import {GeminiManager as CoreGeminiManager} from 'xrblocks/addons/ai/GeminiManager.js';
39
+
40
+ // Inlined from TranscriptionManager.js
41
+ class TranscriptionManager {
42
+ constructor(responseDisplay) {
43
+ this.responseDisplay = responseDisplay;
44
+ this.currentInputText = '';
45
+ this.currentOutputText = '';
46
+ this.conversationHistory = [];
47
+ }
48
+
49
+ handleInputTranscription(text) {
50
+ if (!text) return;
51
+ this.currentInputText += text;
52
+ this.updateLiveDisplay();
53
+ }
54
+
55
+ handleOutputTranscription(text) {
56
+ if (!text) return;
57
+ this.currentOutputText += text;
58
+ this.updateLiveDisplay();
59
+ }
60
+
61
+ finalizeTurn() {
62
+ if (this.currentInputText.trim()) {
63
+ this.conversationHistory.push({
64
+ speaker: 'You',
65
+ text: this.currentInputText.trim(),
66
+ });
67
+ }
68
+ if (this.currentOutputText.trim()) {
69
+ this.conversationHistory.push({
70
+ speaker: 'AI',
71
+ text: this.currentOutputText.trim(),
72
+ });
73
+ }
74
+ this.currentInputText = '';
75
+ this.currentOutputText = '';
76
+ this.updateFinalDisplay();
77
+ }
78
+
79
+ updateLiveDisplay() {
80
+ let displayText = '';
81
+ for (const entry of this.conversationHistory.slice(-2)) {
82
+ displayText += `${entry.speaker}: ${entry.text}\n\n`;
83
+ }
84
+ if (this.currentInputText.trim()) {
85
+ displayText += `You: ${this.currentInputText}`;
86
+ }
87
+ if (this.currentOutputText.trim()) {
88
+ if (this.currentInputText.trim()) displayText += '\n\n';
89
+ displayText += `AI: ${this.currentOutputText}`;
90
+ }
91
+ this.responseDisplay?.setText(displayText);
92
+ }
93
+
94
+ updateFinalDisplay() {
95
+ let displayText = '';
96
+ for (const entry of this.conversationHistory) {
97
+ displayText += `${entry.speaker}: ${entry.text}\n\n`;
98
+ }
99
+ this.responseDisplay?.setText(displayText);
100
+ }
101
+
102
+ clear() {
103
+ this.currentInputText = '';
104
+ this.currentOutputText = '';
105
+ this.conversationHistory = [];
106
+ }
107
+
108
+ addText(text) {
109
+ this.responseDisplay?.addText(text + '\n\n');
110
+ }
111
+
112
+ setText(text) {
113
+ this.responseDisplay?.setText(text);
114
+ }
115
+ }
116
+
117
+ // Inlined from GeminiManager.js
118
+ class GeminiManager extends CoreGeminiManager {
119
+ constructor() {
120
+ super();
121
+ this.defaultText = 'Say "Start" to begin...';
122
+ }
123
+
124
+ init() {
125
+ super.init();
126
+ this.createTextDisplay();
127
+
128
+ // Hook into events from the base class
129
+ this.addEventListener('inputTranscription', (event) => {
130
+ this.transcription?.handleInputTranscription(event.message);
131
+ });
132
+ this.addEventListener('outputTranscription', (event) => {
133
+ this.transcription?.handleOutputTranscription(event.message);
134
+ });
135
+ this.addEventListener('turnComplete', () => {
136
+ this.transcription?.finalizeTurn();
137
+ });
138
+ this.addEventListener('interrupted', () => {
139
+ // Optional: handle interruption visual cues if needed
140
+ });
141
+ }
142
+
143
+ async toggleGeminiLive() {
144
+ return this.isAIRunning ? this.stopGeminiLive() : this.startGeminiLive();
145
+ }
146
+
147
+ async startGeminiLive() {
148
+ try {
149
+ await super.startGeminiLive();
150
+ this.updateButtonState();
151
+ } catch (error) {
152
+ console.error('Failed to start AI session:', error);
153
+ this.transcription?.addText(
154
+ 'Error: Failed to start AI session - ' + error.message
155
+ );
156
+ this.cleanup(); // Clean up on failure
157
+ this.updateButtonState();
158
+ }
159
+ }
160
+
161
+ async stopGeminiLive() {
162
+ await super.stopGeminiLive();
163
+ this.updateButtonState();
164
+ this.transcription?.clear();
165
+ this.transcription?.setText(this.defaultText);
166
+ }
167
+
168
+ createTextDisplay() {
169
+ this.textPanel = new xb.SpatialPanel({
170
+ width: 3,
171
+ height: 1.5,
172
+ backgroundColor: '#1a1a1abb',
173
+ });
174
+ const grid = this.textPanel.addGrid();
175
+
176
+ const responseDisplay = new xb.ScrollingTroikaTextView({
177
+ text: this.defaultText,
178
+ fontSize: 0.03,
179
+ textAlign: 'left',
180
+ });
181
+ grid.addRow({weight: 0.7}).add(responseDisplay);
182
+ this.transcription = new TranscriptionManager(responseDisplay);
183
+
184
+ this.toggleButton = grid.addRow({weight: 0.3}).addTextButton({
185
+ text: '▶ Start',
186
+ fontColor: '#ffffff',
187
+ backgroundColor: '#006644',
188
+ fontSize: 0.2,
189
+ });
190
+ this.toggleButton.onTriggered = () => this.toggleGeminiLive();
191
+
192
+ this.textPanel.position.set(0, 1.2, -2);
193
+ this.add(this.textPanel);
194
+ }
195
+
196
+ updateButtonState() {
197
+ this.toggleButton?.setText(this.isAIRunning ? '⏹ Stop' : '▶ Start');
198
+ }
199
+ }
200
+
201
+ const options = new xb.Options();
202
+ options.simulator.instructions.enabled = false;
203
+ options.enableUI();
204
+ options.enableHands();
205
+ options.enableAI();
206
+ options.enableCamera();
207
+ options.deviceCamera = new xb.DeviceCameraOptions({
208
+ enabled: true,
209
+ videoConstraints: {
210
+ width: {ideal: 1280},
211
+ height: {ideal: 720},
212
+ facingMode: 'environment',
213
+ },
214
+ });
215
+
216
+ async function requestAudioPermission() {
217
+ try {
218
+ const stream = await navigator.mediaDevices.getUserMedia({
219
+ audio: {
220
+ sampleRate: 16000,
221
+ channelCount: 1,
222
+ echoCancellation: true,
223
+ noiseSuppression: true,
224
+ },
225
+ });
226
+ stream.getTracks().forEach((track) => track.stop());
227
+ return stream;
228
+ } catch (error) {
229
+ console.error('Audio permission denied or not available:', error);
230
+ alert(
231
+ 'Audio permission is required for Gemini Live AI features. Please enable microphone access and refresh the page.'
232
+ );
233
+ return null;
234
+ }
235
+ }
236
+
237
+ async function start() {
238
+ try {
239
+ await requestAudioPermission();
240
+ xb.init(options);
241
+ xb.add(new GeminiManager());
242
+ } catch (error) {
243
+ console.error('Failed to initialize XR app:', error);
244
+ }
245
+ }
246
+
247
+ document.addEventListener('DOMContentLoaded', function () {
248
+ start();
249
+ });
250
+
251
+ </script>
252
+ </body>
253
+ </html>
@@ -0,0 +1,95 @@
1
+ <!doctype html>
2
+ <!-- reference: templates/8_objects -->
3
+ <html lang="en">
4
+ <head>
5
+ <title>World Component: Object Detection | XR Blocks Template</title>
6
+
7
+ <meta
8
+ name="viewport"
9
+ content="width=device-width, initial-scale=1.0, user-scalable=no"
10
+ />
11
+ <link type="text/css" rel="stylesheet" href="https://xrblocks.github.io/css/xr.css" />
12
+ <script>
13
+ window.litDisableBundleWarning = true;
14
+ </script>
15
+ <script type="importmap">
16
+ {
17
+ "imports": {
18
+ "three": "https://cdn.jsdelivr.net/npm/three@0.182.0/build/three.module.js",
19
+ "three/addons/": "https://cdn.jsdelivr.net/npm/three@0.182.0/examples/jsm/",
20
+ "lit": "https://cdn.jsdelivr.net/gh/lit/dist@3/core/lit-core.min.js",
21
+ "lit/": "https://esm.run/lit@3/",
22
+ "troika-three-text": "https://cdn.jsdelivr.net/gh/protectwise/troika@028b81cf308f0f22e5aa8e78196be56ec1997af5/packages/troika-three-text/src/index.js",
23
+ "troika-three-utils": "https://cdn.jsdelivr.net/gh/protectwise/troika@v0.52.4/packages/troika-three-utils/src/index.js",
24
+ "troika-worker-utils": "https://cdn.jsdelivr.net/gh/protectwise/troika@v0.52.4/packages/troika-worker-utils/src/index.js",
25
+ "bidi-js": "https://esm.sh/bidi-js@%5E1.0.2?target=es2022",
26
+ "webgl-sdf-generator": "https://esm.sh/webgl-sdf-generator@1.1.1/es2022/webgl-sdf-generator.mjs",
27
+ "xrblocks": "https://cdn.jsdelivr.net/npm/xrblocks@0.11.0/build/xrblocks.js",
28
+ "xrblocks/addons/": "https://cdn.jsdelivr.net/npm/xrblocks@0.11.0/build/addons/",
29
+ "@google/genai": "https://cdn.jsdelivr.net/npm/@google/genai/+esm"
30
+ }
31
+ }
32
+ </script>
33
+ </head>
34
+
35
+ <body>
36
+ <script type="module">
37
+ import * as THREE from 'three';
38
+ import * as xb from 'xrblocks';
39
+
40
+ /**
41
+ * A basic example of using the XR Blocks SDK's world component to detect
42
+ * objects in the real world using Gemini.
43
+ */
44
+ class MainScript extends xb.Script {
45
+ init() {
46
+ this.add(new THREE.HemisphereLight(0xffffff, 0x666666, /*intensity=*/ 3));
47
+ }
48
+
49
+ /**
50
+ * Runs object detection on select (click in simulator, pinch in XR device).
51
+ * The results of the detection are automatically handled by the
52
+ * ObjectDetector, which will create debug visuals for each detected object.
53
+ * This behavior is enabled when `showDebugVisualizations` is set to true.
54
+ * @param {XRInputSourceEvent} event event.target holds controller or hand
55
+ * data.
56
+ */
57
+ async onSelectEnd() {
58
+ console.log('Running object detection...');
59
+ const detectedObjects = await xb.world.objects.runDetection();
60
+
61
+ // `detectedObjects` is an array of THREE.Object3D instances, each
62
+ // representing a detected object. These objects contain the 3D world
63
+ // position and other metadata returned by the detection model.
64
+ if (detectedObjects.length > 0) {
65
+ console.log('Detected objects:', detectedObjects);
66
+ } else {
67
+ console.log('No objects detected.');
68
+ }
69
+ }
70
+ }
71
+
72
+ document.addEventListener('DOMContentLoaded', function () {
73
+ const options = new xb.Options();
74
+
75
+ options.simulator.instructions.enabled = false;
76
+ // AI is required for the object detection backend. Please set your
77
+ options.enableAI();
78
+
79
+ // Enable the environment camera to provide the video feed to the AI module.
80
+ options.enableCamera('environment');
81
+
82
+ // Depth is required to project the 2D detections from the AI module into 3D.
83
+ options.enableDepth();
84
+
85
+ // Enable the object detection feature and its debug visualizations.
86
+ options.world.enableObjectDetection();
87
+ options.world.objects.showDebugVisualizations = true;
88
+
89
+ xb.add(new MainScript());
90
+ xb.init(options);
91
+ });
92
+
93
+ </script>
94
+ </body>
95
+ </html>
@@ -0,0 +1,90 @@
1
+ <!doctype html>
2
+ <!-- reference: templates/9_xr-toggle -->
3
+ <html lang="en">
4
+ <head>
5
+ <title>Transition between AR and VR | XR Blocks Template</title>
6
+
7
+ <meta
8
+ name="viewport"
9
+ content="width=device-width, initial-scale=1.0, user-scalable=no"
10
+ />
11
+ <link type="text/css" rel="stylesheet" href="https://xrblocks.github.io/css/xr.css" />
12
+ <script>
13
+ window.litDisableBundleWarning = true;
14
+ </script>
15
+ <script type="importmap">
16
+ {
17
+ "imports": {
18
+ "three": "https://cdn.jsdelivr.net/npm/three@0.182.0/build/three.module.js",
19
+ "three/addons/": "https://cdn.jsdelivr.net/npm/three@0.182.0/examples/jsm/",
20
+ "lit": "https://cdn.jsdelivr.net/gh/lit/dist@3/core/lit-core.min.js",
21
+ "lit/": "https://esm.run/lit@3/",
22
+ "xrblocks": "https://cdn.jsdelivr.net/npm/xrblocks@0.11.0/build/xrblocks.js",
23
+ "xrblocks/addons/": "https://cdn.jsdelivr.net/npm/xrblocks@0.11.0/build/addons/"
24
+ }
25
+ }
26
+ </script>
27
+ </head>
28
+
29
+ <body>
30
+ <script type="module">
31
+ import * as THREE from 'three';
32
+ import * as xb from 'xrblocks';
33
+
34
+ /**
35
+ * Demonstrates how to use the XRTransition component to smoothly switch
36
+ * between AR and VR backgrounds.
37
+ */
38
+ class MainScript extends xb.Script {
39
+ init() {
40
+ this.add(new THREE.HemisphereLight(0xffffff, 0x666666, /*intensity=*/ 3));
41
+
42
+ const geometry = new THREE.CylinderGeometry(
43
+ 0.2,
44
+ 0.2,
45
+ 0.4,
46
+ /*segments=*/ 32
47
+ );
48
+ const material = new THREE.MeshPhongMaterial({
49
+ color: 0xffffff,
50
+ transparent: true,
51
+ opacity: 0.8,
52
+ });
53
+ this.cylinder = new THREE.Mesh(geometry, material);
54
+ this.cylinder.position.set(
55
+ 0,
56
+ xb.core.user.height - 0.5,
57
+ -xb.core.user.objectDistance
58
+ );
59
+ this.add(this.cylinder);
60
+ }
61
+
62
+ /**
63
+ * On pinch, toggle between AR and VR modes and update cylinder color.
64
+ */
65
+ onSelectEnd() {
66
+ if (!xb.core.transition) {
67
+ console.warn('XRTransition not enabled.');
68
+ return;
69
+ }
70
+ this.cylinder.material.color.set(Math.random() * 0xffffff);
71
+
72
+ // Toggle between AR and VR based on the current mode.
73
+ if (xb.core.transition.currentMode === 'AR') {
74
+ xb.core.transition.toVR({color: Math.random() * 0xffffff});
75
+ } else {
76
+ xb.core.transition.toAR();
77
+ }
78
+ }
79
+ }
80
+
81
+ document.addEventListener('DOMContentLoaded', function () {
82
+ const options = new xb.Options().enableXRTransitions();
83
+ options.simulator.instructions.enabled = false;
84
+ xb.add(new MainScript());
85
+ xb.init(options);
86
+ });
87
+
88
+ </script>
89
+ </body>
90
+ </html>
@@ -0,0 +1,102 @@
1
+ <!doctype html>
2
+ <!-- reference: templates/heuristic_hand_gestures -->
3
+ <html lang="en">
4
+ <head>
5
+ <title>Gestures: Heuristic Logging | XR Blocks Template</title>
6
+
7
+ <meta
8
+ name="viewport"
9
+ content="width=device-width, initial-scale=1.0, user-scalable=no"
10
+ />
11
+ <link type="text/css" rel="stylesheet" href="https://xrblocks.github.io/css/xr.css" />
12
+ <script>
13
+ window.litDisableBundleWarning = true;
14
+ </script>
15
+ <script type="importmap">
16
+ {
17
+ "imports": {
18
+ "three": "https://cdn.jsdelivr.net/npm/three@0.182.0/build/three.module.js",
19
+ "three/addons/": "https://cdn.jsdelivr.net/npm/three@0.182.0/examples/jsm/",
20
+ "troika-three-text": "https://cdn.jsdelivr.net/gh/protectwise/troika@028b81cf308f0f22e5aa8e78196be56ec1997af5/packages/troika-three-text/src/index.js",
21
+ "troika-three-utils": "https://cdn.jsdelivr.net/gh/protectwise/troika@v0.52.4/packages/troika-three-utils/src/index.js",
22
+ "troika-worker-utils": "https://cdn.jsdelivr.net/gh/protectwise/troika@v0.52.4/packages/troika-worker-utils/src/index.js",
23
+ "bidi-js": "https://esm.sh/bidi-js@%5E1.0.2?target=es2022",
24
+ "webgl-sdf-generator": "https://esm.sh/webgl-sdf-generator@1.1.1/es2022/webgl-sdf-generator.mjs",
25
+ "lit": "https://cdn.jsdelivr.net/gh/lit/dist@3/core/lit-core.min.js",
26
+ "lit/": "https://esm.run/lit@3/",
27
+ "xrblocks": "https://cdn.jsdelivr.net/npm/xrblocks@0.11.0/build/xrblocks.js",
28
+ "xrblocks/addons/": "https://cdn.jsdelivr.net/npm/xrblocks@0.11.0/build/addons/"
29
+ }
30
+ }
31
+ </script>
32
+ </head>
33
+
34
+ <body>
35
+ <script type="module">
36
+ import 'xrblocks/addons/simulator/SimulatorAddons.js';
37
+
38
+ import * as xb from 'xrblocks';
39
+
40
+ const options = new xb.Options();
41
+ options.simulator.instructions.enabled = false;
42
+ options.enableReticles();
43
+ options.enableGestures();
44
+
45
+ options.gestures.setGestureEnabled('point', true);
46
+ options.gestures.setGestureEnabled('spread', true);
47
+
48
+ options.hands.enabled = true;
49
+ options.hands.visualization = true;
50
+ options.hands.visualizeJoints = true;
51
+ options.hands.visualizeMeshes = true;
52
+
53
+ options.simulator.defaultMode = xb.SimulatorMode.POSE;
54
+
55
+ class GestureLogger extends xb.Script {
56
+ init() {
57
+ const gestures = xb.core.gestureRecognition;
58
+ if (!gestures) {
59
+ console.warn(
60
+ '[GestureLogger] GestureRecognition is unavailable. ' +
61
+ 'Make sure options.enableGestures() is called before xb.init().'
62
+ );
63
+ return;
64
+ }
65
+ this._onGestureStart = (event) => {
66
+ const {hand, name, confidence = 0} = event.detail;
67
+ console.log(
68
+ `[gesture] ${hand} hand started ${name} (${confidence.toFixed(2)})`
69
+ );
70
+ };
71
+ this._onGestureEnd = (event) => {
72
+ const {hand, name} = event.detail;
73
+ console.log(`[gesture] ${hand} hand ended ${name}`);
74
+ };
75
+ gestures.addEventListener('gesturestart', this._onGestureStart);
76
+ gestures.addEventListener('gestureend', this._onGestureEnd);
77
+ }
78
+
79
+ dispose() {
80
+ const gestures = xb.core.gestureRecognition;
81
+ if (!gestures) return;
82
+ if (this._onGestureStart) {
83
+ gestures.removeEventListener('gesturestart', this._onGestureStart);
84
+ }
85
+ if (this._onGestureEnd) {
86
+ gestures.removeEventListener('gestureend', this._onGestureEnd);
87
+ }
88
+ }
89
+ }
90
+
91
+ function start() {
92
+ xb.add(new GestureLogger());
93
+ xb.init(options);
94
+ }
95
+
96
+ document.addEventListener('DOMContentLoaded', () => {
97
+ start();
98
+ });
99
+
100
+ </script>
101
+ </body>
102
+ </html>