mudra-skills 3.0.2 → 3.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/skills/mudra-master/SKILL.md +0 -196
- package/skills/mudra-master/mudra-preview/SKILL.md +0 -150
- package/skills/mudra-master/mudra-preview/assets/ar-menu.html +0 -382
- package/skills/mudra-master/mudra-preview/assets/document-scroller.html +0 -468
- package/skills/mudra-master/mudra-preview/assets/drum-machine.html +0 -788
- package/skills/mudra-master/mudra-preview/assets/emg-visualizer.html +0 -390
- package/skills/mudra-master/mudra-preview/assets/generative-art.html +0 -408
- package/skills/mudra-master/mudra-preview/assets/gesture-assistant.html +0 -342
- package/skills/mudra-master/mudra-preview/assets/gesture-speech.html +0 -444
- package/skills/mudra-master/mudra-preview/assets/hands-free-desktop.html +0 -376
- package/skills/mudra-master/mudra-preview/assets/model-rotator.html +0 -365
- package/skills/mudra-master/mudra-preview/assets/mudra-duel.html +0 -3200
- package/skills/mudra-master/mudra-preview/assets/mudra-monitor.html +0 -1451
- package/skills/mudra-master/mudra-preview/assets/mudra-ultimate-template.html +0 -891
- package/skills/mudra-master/mudra-preview/assets/music-sequencer.html +0 -455
- package/skills/mudra-master/mudra-preview/assets/neural-pong.html +0 -380
- package/skills/mudra-master/mudra-preview/assets/neural-snake.html +0 -364
- package/skills/mudra-master/mudra-preview/assets/presentation-controller.html +0 -345
- package/skills/mudra-master/mudra-preview/assets/pressure-painter.html +0 -285
- package/skills/mudra-master/mudra-preview/assets/runner.html +0 -1812
- package/skills/mudra-master/mudra-preview/assets/smart-home.html +0 -461
- package/skills/mudra-master/mudra-preview/assets/space-invaders.html +0 -1154
- package/skills/mudra-master/mudra-preview/assets/waterful-ring-toss.html +0 -1624
- package/skills/mudra-master/mudra-preview/references/agent_protocol.json +0 -634
- package/skills/mudra-master/mudra-preview/references/prompt.md +0 -16746
- package/skills/mudra-master/mudra-xr/SKILL.md +0 -245
- package/skills/mudra-master/mudra-xr/assets/demos/3dgs-walkthrough.html +0 -242
- package/skills/mudra-master/mudra-xr/assets/demos/aisimulator.html +0 -792
- package/skills/mudra-master/mudra-xr/assets/demos/balloonpop.html +0 -422
- package/skills/mudra-master/mudra-xr/assets/demos/ballpit.html +0 -284
- package/skills/mudra-master/mudra-xr/assets/demos/drone.html +0 -41
- package/skills/mudra-master/mudra-xr/assets/demos/gemini-icebreakers.html +0 -338
- package/skills/mudra-master/mudra-xr/assets/demos/gemini-xrobject.html +0 -302
- package/skills/mudra-master/mudra-xr/assets/demos/math3d.html +0 -188
- package/skills/mudra-master/mudra-xr/assets/demos/measure.html +0 -247
- package/skills/mudra-master/mudra-xr/assets/demos/occlusion.html +0 -251
- package/skills/mudra-master/mudra-xr/assets/demos/rain.html +0 -498
- package/skills/mudra-master/mudra-xr/assets/demos/screenwiper.html +0 -432
- package/skills/mudra-master/mudra-xr/assets/demos/splash.html +0 -519
- package/skills/mudra-master/mudra-xr/assets/demos/webcam_gestures.html +0 -253
- package/skills/mudra-master/mudra-xr/assets/demos/xremoji.html +0 -833
- package/skills/mudra-master/mudra-xr/assets/demos/xrpoet.html +0 -151
- package/skills/mudra-master/mudra-xr/assets/samples/depthmap.html +0 -266
- package/skills/mudra-master/mudra-xr/assets/samples/depthmesh.html +0 -100
- package/skills/mudra-master/mudra-xr/assets/samples/game_rps.html +0 -1099
- package/skills/mudra-master/mudra-xr/assets/samples/gestures_custom.html +0 -467
- package/skills/mudra-master/mudra-xr/assets/samples/gestures_heuristic.html +0 -256
- package/skills/mudra-master/mudra-xr/assets/samples/lighting.html +0 -163
- package/skills/mudra-master/mudra-xr/assets/samples/mesh_detection.html +0 -46
- package/skills/mudra-master/mudra-xr/assets/samples/modelviewer.html +0 -183
- package/skills/mudra-master/mudra-xr/assets/samples/paint.html +0 -133
- package/skills/mudra-master/mudra-xr/assets/samples/planar-vst.html +0 -45
- package/skills/mudra-master/mudra-xr/assets/samples/reticle.html +0 -95
- package/skills/mudra-master/mudra-xr/assets/samples/skybox_agent.html +0 -384
- package/skills/mudra-master/mudra-xr/assets/samples/sound.html +0 -390
- package/skills/mudra-master/mudra-xr/assets/samples/ui.html +0 -496
- package/skills/mudra-master/mudra-xr/assets/samples/virtual-screens.html +0 -808
- package/skills/mudra-master/mudra-xr/assets/templates/0_basic.html +0 -64
- package/skills/mudra-master/mudra-xr/assets/templates/1_ui.html +0 -107
- package/skills/mudra-master/mudra-xr/assets/templates/2_hands.html +0 -183
- package/skills/mudra-master/mudra-xr/assets/templates/3_depth.html +0 -89
- package/skills/mudra-master/mudra-xr/assets/templates/4_stereo.html +0 -84
- package/skills/mudra-master/mudra-xr/assets/templates/5_camera.html +0 -117
- package/skills/mudra-master/mudra-xr/assets/templates/6_ai.html +0 -154
- package/skills/mudra-master/mudra-xr/assets/templates/7_ai_live.html +0 -253
- package/skills/mudra-master/mudra-xr/assets/templates/8_objects.html +0 -95
- package/skills/mudra-master/mudra-xr/assets/templates/9_xr-toggle.html +0 -90
- package/skills/mudra-master/mudra-xr/assets/templates/heuristic_hand_gestures.html +0 -102
- package/skills/mudra-master/mudra-xr/assets/templates/meshes.html +0 -47
- package/skills/mudra-master/mudra-xr/assets/templates/planes.html +0 -47
- package/skills/mudra-master/mudra-xr/assets/templates/uikit.html +0 -187
- package/skills/mudra-master/mudra-xr/references/agent_protocol.json +0 -865
- package/skills/mudra-master/mudra-xr/references/prompt.md +0 -1605
- package/skills/mudra-preview/SKILL.md +0 -159
- package/skills/mudra-preview/assets/ar-menu.html +0 -382
- package/skills/mudra-preview/assets/document-scroller.html +0 -468
- package/skills/mudra-preview/assets/drum-machine.html +0 -788
- package/skills/mudra-preview/assets/emg-visualizer.html +0 -390
- package/skills/mudra-preview/assets/generative-art.html +0 -408
- package/skills/mudra-preview/assets/gesture-assistant.html +0 -342
- package/skills/mudra-preview/assets/gesture-speech.html +0 -444
- package/skills/mudra-preview/assets/hands-free-desktop.html +0 -376
- package/skills/mudra-preview/assets/model-rotator.html +0 -365
- package/skills/mudra-preview/assets/mudra-duel.html +0 -3200
- package/skills/mudra-preview/assets/mudra-monitor.html +0 -1451
- package/skills/mudra-preview/assets/mudra-ultimate-template.html +0 -997
- package/skills/mudra-preview/assets/music-sequencer.html +0 -455
- package/skills/mudra-preview/assets/neural-pong.html +0 -380
- package/skills/mudra-preview/assets/neural-snake.html +0 -364
- package/skills/mudra-preview/assets/presentation-controller.html +0 -345
- package/skills/mudra-preview/assets/pressure-painter.html +0 -285
- package/skills/mudra-preview/assets/runner.html +0 -1812
- package/skills/mudra-preview/assets/smart-home.html +0 -461
- package/skills/mudra-preview/assets/space-invaders.html +0 -1154
- package/skills/mudra-preview/assets/waterful-ring-toss.html +0 -1624
- package/skills/mudra-preview/references/agent_protocol.json +0 -701
- package/skills/mudra-preview/references/prompt.md +0 -16731
- package/skills/mudra-xr/SKILL.md +0 -245
- package/skills/mudra-xr/assets/demos/3dgs-walkthrough.html +0 -242
- package/skills/mudra-xr/assets/demos/aisimulator.html +0 -792
- package/skills/mudra-xr/assets/demos/balloonpop.html +0 -422
- package/skills/mudra-xr/assets/demos/ballpit.html +0 -284
- package/skills/mudra-xr/assets/demos/drone.html +0 -41
- package/skills/mudra-xr/assets/demos/gemini-icebreakers.html +0 -338
- package/skills/mudra-xr/assets/demos/gemini-xrobject.html +0 -302
- package/skills/mudra-xr/assets/demos/math3d.html +0 -188
- package/skills/mudra-xr/assets/demos/measure.html +0 -247
- package/skills/mudra-xr/assets/demos/occlusion.html +0 -251
- package/skills/mudra-xr/assets/demos/rain.html +0 -498
- package/skills/mudra-xr/assets/demos/screenwiper.html +0 -432
- package/skills/mudra-xr/assets/demos/splash.html +0 -519
- package/skills/mudra-xr/assets/demos/webcam_gestures.html +0 -253
- package/skills/mudra-xr/assets/demos/xremoji.html +0 -833
- package/skills/mudra-xr/assets/demos/xrpoet.html +0 -151
- package/skills/mudra-xr/assets/samples/depthmap.html +0 -266
- package/skills/mudra-xr/assets/samples/depthmesh.html +0 -100
- package/skills/mudra-xr/assets/samples/game_rps.html +0 -1099
- package/skills/mudra-xr/assets/samples/gestures_custom.html +0 -467
- package/skills/mudra-xr/assets/samples/gestures_heuristic.html +0 -256
- package/skills/mudra-xr/assets/samples/lighting.html +0 -163
- package/skills/mudra-xr/assets/samples/mesh_detection.html +0 -46
- package/skills/mudra-xr/assets/samples/modelviewer.html +0 -183
- package/skills/mudra-xr/assets/samples/paint.html +0 -133
- package/skills/mudra-xr/assets/samples/planar-vst.html +0 -45
- package/skills/mudra-xr/assets/samples/reticle.html +0 -95
- package/skills/mudra-xr/assets/samples/skybox_agent.html +0 -384
- package/skills/mudra-xr/assets/samples/sound.html +0 -390
- package/skills/mudra-xr/assets/samples/ui.html +0 -496
- package/skills/mudra-xr/assets/samples/virtual-screens.html +0 -808
- package/skills/mudra-xr/assets/templates/0_basic.html +0 -64
- package/skills/mudra-xr/assets/templates/1_ui.html +0 -107
- package/skills/mudra-xr/assets/templates/2_hands.html +0 -183
- package/skills/mudra-xr/assets/templates/3_depth.html +0 -89
- package/skills/mudra-xr/assets/templates/4_stereo.html +0 -84
- package/skills/mudra-xr/assets/templates/5_camera.html +0 -117
- package/skills/mudra-xr/assets/templates/6_ai.html +0 -154
- package/skills/mudra-xr/assets/templates/7_ai_live.html +0 -253
- package/skills/mudra-xr/assets/templates/8_objects.html +0 -95
- package/skills/mudra-xr/assets/templates/9_xr-toggle.html +0 -90
- package/skills/mudra-xr/assets/templates/heuristic_hand_gestures.html +0 -102
- package/skills/mudra-xr/assets/templates/meshes.html +0 -47
- package/skills/mudra-xr/assets/templates/planes.html +0 -47
- package/skills/mudra-xr/assets/templates/uikit.html +0 -187
- package/skills/mudra-xr/references/agent_protocol.json +0 -865
- package/skills/mudra-xr/references/prompt.md +0 -1605
|
@@ -1,792 +0,0 @@
|
|
|
1
|
-
<!doctype html>
|
|
2
|
-
<!-- reference: demos/aisimulator -->
|
|
3
|
-
<html lang="en">
|
|
4
|
-
<head>
|
|
5
|
-
<title>XR Blocks AI Simulator</title>
|
|
6
|
-
|
|
7
|
-
<meta
|
|
8
|
-
name="viewport"
|
|
9
|
-
content="width=device-width, initial-scale=1.0, user-scalable=no"
|
|
10
|
-
/>
|
|
11
|
-
|
|
12
|
-
<link type="text/css" rel="stylesheet" href="https://xrblocks.github.io/css/xr.css" />
|
|
13
|
-
<link
|
|
14
|
-
rel="stylesheet"
|
|
15
|
-
href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:opsz,wght,FILL,GRAD@24,400,0,0&icon_names=mic"
|
|
16
|
-
/>
|
|
17
|
-
<script>
|
|
18
|
-
window.litDisableBundleWarning = true;
|
|
19
|
-
</script>
|
|
20
|
-
<script type="importmap">
|
|
21
|
-
{
|
|
22
|
-
"imports": {
|
|
23
|
-
"three": "https://cdn.jsdelivr.net/npm/three@0.182.0/build/three.module.js",
|
|
24
|
-
"three/addons/": "https://cdn.jsdelivr.net/npm/three@0.182.0/examples/jsm/",
|
|
25
|
-
"troika-three-text": "https://cdn.jsdelivr.net/gh/protectwise/troika@028b81cf308f0f22e5aa8e78196be56ec1997af5/packages/troika-three-text/src/index.js",
|
|
26
|
-
"troika-three-utils": "https://cdn.jsdelivr.net/gh/protectwise/troika@v0.52.4/packages/troika-three-utils/src/index.js",
|
|
27
|
-
"troika-worker-utils": "https://cdn.jsdelivr.net/gh/protectwise/troika@v0.52.4/packages/troika-worker-utils/src/index.js",
|
|
28
|
-
"bidi-js": "https://esm.sh/bidi-js@%5E1.0.2?target=es2022",
|
|
29
|
-
"webgl-sdf-generator": "https://esm.sh/webgl-sdf-generator@1.1.1/es2022/webgl-sdf-generator.mjs",
|
|
30
|
-
"lit": "https://cdn.jsdelivr.net/gh/lit/dist@3/core/lit-core.min.js",
|
|
31
|
-
"lit/": "https://esm.run/lit@3/",
|
|
32
|
-
"xrblocks": "https://cdn.jsdelivr.net/npm/xrblocks@0.11.0/build/xrblocks.js",
|
|
33
|
-
"xrblocks/addons/": "https://cdn.jsdelivr.net/npm/xrblocks@0.11.0/build/addons/",
|
|
34
|
-
"@google/genai": "https://cdn.jsdelivr.net/npm/@google/genai/+esm"
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
</script>
|
|
38
|
-
</head>
|
|
39
|
-
|
|
40
|
-
<body>
|
|
41
|
-
<div
|
|
42
|
-
id="background-image"
|
|
43
|
-
class="background-image"
|
|
44
|
-
style="background-image: url('textures/background.webp'); z-index: 0"
|
|
45
|
-
></div>
|
|
46
|
-
<script type="module">
|
|
47
|
-
import 'xrblocks/addons/simulator/SimulatorAddons.js';
|
|
48
|
-
import 'xrblocks/addons/simulator/ui/MicButton.js';
|
|
49
|
-
import 'xrblocks/addons/simulator/ui/GeminiLiveApiKeyInput.js';
|
|
50
|
-
|
|
51
|
-
import {css, html, LitElement} from 'lit';
|
|
52
|
-
import {createRef, ref} from 'lit/directives/ref.js';
|
|
53
|
-
import {
|
|
54
|
-
ApiKeyEnteredEvent,
|
|
55
|
-
MicButtonPressedEvent,
|
|
56
|
-
} from 'xrblocks/addons/simulator/ui/GeminiLiveEvents.js';
|
|
57
|
-
import {GoogleGenAI, Modality} from '@google/genai';
|
|
58
|
-
|
|
59
|
-
import * as THREE from 'three';
|
|
60
|
-
import * as xb from 'xrblocks';
|
|
61
|
-
|
|
62
|
-
// --- model_data.js inlined ---
|
|
63
|
-
const ASSETS_BASE_URL = 'https://cdn.jsdelivr.net/gh/xrblocks/assets@main/';
|
|
64
|
-
const PROPRIETARY_ASSETS_BASE_URL =
|
|
65
|
-
'https://cdn.jsdelivr.net/gh/xrblocks/proprietary-assets@main/';
|
|
66
|
-
|
|
67
|
-
const DATA = [
|
|
68
|
-
{
|
|
69
|
-
model: {
|
|
70
|
-
scale: {x: 4.0, y: 4.0, z: 4.0},
|
|
71
|
-
path: PROPRIETARY_ASSETS_BASE_URL + 'monalisa/',
|
|
72
|
-
model: 'mona_lisa_picture_frame_compressed.glb',
|
|
73
|
-
verticallyAlignObject: false,
|
|
74
|
-
},
|
|
75
|
-
position: {x: -2.1, y: 1.75, z: -1.92},
|
|
76
|
-
rotation: {x: 0, y: Math.PI / 2, z: 0},
|
|
77
|
-
prompt: '"What is she smiling about?"',
|
|
78
|
-
},
|
|
79
|
-
{
|
|
80
|
-
model: {
|
|
81
|
-
scale: {x: 0.01, y: 0.01, z: 0.01},
|
|
82
|
-
rotation: {x: 0, y: 0, z: 0},
|
|
83
|
-
position: {x: 0, y: -0.2, z: -3.0},
|
|
84
|
-
path: PROPRIETARY_ASSETS_BASE_URL + 'chess/',
|
|
85
|
-
model: 'chess_compressed.glb',
|
|
86
|
-
verticallyAlignObject: false,
|
|
87
|
-
},
|
|
88
|
-
position: {x: 0, y: 1.0, z: -1.2},
|
|
89
|
-
rotation: {x: 0, y: 0, z: 0},
|
|
90
|
-
prompt: '"What\'s a good strategy for this game?"',
|
|
91
|
-
},
|
|
92
|
-
{
|
|
93
|
-
model: {
|
|
94
|
-
path: ASSETS_BASE_URL + 'models/',
|
|
95
|
-
model: 'Parasaurolophus.glb',
|
|
96
|
-
scale: {x: 0.3, y: 0.3, z: 0.3},
|
|
97
|
-
position: {x: 0, y: -0.6, z: 0},
|
|
98
|
-
verticallyAlignObject: false,
|
|
99
|
-
horizontallyAlignObject: false,
|
|
100
|
-
},
|
|
101
|
-
position: {x: 2.0, y: 1.0, z: -3.0},
|
|
102
|
-
rotation: {x: 0, y: 0, z: 0},
|
|
103
|
-
prompt: '"If this dinosaur could talk, what would it say?"',
|
|
104
|
-
},
|
|
105
|
-
];
|
|
106
|
-
|
|
107
|
-
// --- GeminiLiveWebInterface inlined ---
|
|
108
|
-
class GeminiLiveWebInterface {
|
|
109
|
-
constructor(apiKey) {
|
|
110
|
-
this.ai = new GoogleGenAI({apiKey: apiKey});
|
|
111
|
-
this.model = 'gemini-2.0-flash-live-001';
|
|
112
|
-
this.config = {
|
|
113
|
-
responseModalities: [Modality.AUDIO],
|
|
114
|
-
speechConfig: {voiceConfig: {prebuiltVoiceConfig: {voiceName: 'Aoede'}}},
|
|
115
|
-
outputAudioTranscription: {},
|
|
116
|
-
inputAudioTranscription: {},
|
|
117
|
-
};
|
|
118
|
-
|
|
119
|
-
this.session = null;
|
|
120
|
-
this.isRecording = false;
|
|
121
|
-
this.isCapturingScreen = false;
|
|
122
|
-
this.isGeminiSpeaking = false;
|
|
123
|
-
this.lastAudioChunkReceived = 0;
|
|
124
|
-
this.audioFinalizationTimeout = null;
|
|
125
|
-
|
|
126
|
-
this.audioContext = null;
|
|
127
|
-
this.audioWorkletNode = null;
|
|
128
|
-
this.mediaStream = null;
|
|
129
|
-
this.mediaStreamSource = null;
|
|
130
|
-
|
|
131
|
-
this.screenshotInterval = null;
|
|
132
|
-
this.screenshotIntervalMs = 3000;
|
|
133
|
-
this.displayStream = null;
|
|
134
|
-
this.displayVideo = null;
|
|
135
|
-
|
|
136
|
-
this.audioQueue = [];
|
|
137
|
-
this.isPlayingAudio = false;
|
|
138
|
-
|
|
139
|
-
this.currentInputText = '';
|
|
140
|
-
this.currentOutputText = '';
|
|
141
|
-
this.currentInputId = null;
|
|
142
|
-
this.currentOutputId = null;
|
|
143
|
-
this.conversationHistory = [];
|
|
144
|
-
this.isSynthesizing = false;
|
|
145
|
-
|
|
146
|
-
this.onTranscription = null;
|
|
147
|
-
this.onError = null;
|
|
148
|
-
this.onSessionStateChange = null;
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
async initialize() {
|
|
152
|
-
try {
|
|
153
|
-
this.audioContext = new AudioContext({sampleRate: 16000});
|
|
154
|
-
if (this.audioContext.state === 'suspended') {
|
|
155
|
-
await this.audioContext.resume();
|
|
156
|
-
}
|
|
157
|
-
const micPermission = await this.requestMicrophonePermission();
|
|
158
|
-
if (!micPermission) return false;
|
|
159
|
-
return true;
|
|
160
|
-
} catch (error) {
|
|
161
|
-
console.error('Failed to initialize audio context:', error);
|
|
162
|
-
if (this.onError) this.onError(error);
|
|
163
|
-
return false;
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
async requestMicrophonePermission() {
|
|
168
|
-
try {
|
|
169
|
-
this.mediaStream = await navigator.mediaDevices.getUserMedia({
|
|
170
|
-
audio: {sampleRate: 16000, channelCount: 1, echoCancellation: true, noiseSuppression: true, autoGainControl: true},
|
|
171
|
-
});
|
|
172
|
-
return true;
|
|
173
|
-
} catch (error) {
|
|
174
|
-
console.error('Failed to get microphone access:', error);
|
|
175
|
-
if (this.onError) this.onError(error);
|
|
176
|
-
return false;
|
|
177
|
-
}
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
async startSession() {
|
|
181
|
-
if (this.session) return;
|
|
182
|
-
try {
|
|
183
|
-
this.session = await this.ai.live.connect({
|
|
184
|
-
model: this.model,
|
|
185
|
-
callbacks: {
|
|
186
|
-
onopen: () => {
|
|
187
|
-
if (this.onSessionStateChange) this.onSessionStateChange('connected');
|
|
188
|
-
this.startAudioRecording();
|
|
189
|
-
this.startScreenCapture();
|
|
190
|
-
},
|
|
191
|
-
onmessage: (message) => { this.processMessageStream(message); },
|
|
192
|
-
onerror: (e) => { console.error('Session error:', e.message); if (this.onError) this.onError(e); },
|
|
193
|
-
onclose: (e) => { if (this.onSessionStateChange) this.onSessionStateChange('disconnected'); this.cleanup(); },
|
|
194
|
-
},
|
|
195
|
-
config: this.config,
|
|
196
|
-
});
|
|
197
|
-
} catch (error) {
|
|
198
|
-
console.error('Failed to connect to Gemini Live:', error);
|
|
199
|
-
if (this.onError) this.onError(error);
|
|
200
|
-
throw error;
|
|
201
|
-
}
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
processMessageStream(message) {
|
|
205
|
-
try {
|
|
206
|
-
if (message.data) {
|
|
207
|
-
this.isGeminiSpeaking = true;
|
|
208
|
-
this.lastAudioChunkReceived = Date.now();
|
|
209
|
-
if (this.audioFinalizationTimeout) clearTimeout(this.audioFinalizationTimeout);
|
|
210
|
-
this.playAudioChunk(message.data);
|
|
211
|
-
this.audioFinalizationTimeout = setTimeout(() => { this.isGeminiSpeaking = false; }, 1000);
|
|
212
|
-
}
|
|
213
|
-
if (message.serverContent) {
|
|
214
|
-
if (message.serverContent.inputTranscription) this.handleInputTranscription(message.serverContent.inputTranscription);
|
|
215
|
-
if (message.serverContent.outputTranscription) this.handleOutputTranscription(message.serverContent.outputTranscription);
|
|
216
|
-
if (message.serverContent.turnComplete) this.handleTurnComplete();
|
|
217
|
-
}
|
|
218
|
-
} catch (error) {
|
|
219
|
-
console.error('Error processing message:', error);
|
|
220
|
-
if (this.onError) this.onError(error);
|
|
221
|
-
}
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
handleInputTranscription(transcription) {
|
|
225
|
-
const text = transcription.text;
|
|
226
|
-
if (!text) return;
|
|
227
|
-
this.currentInputText += text;
|
|
228
|
-
if (this.onTranscription) {
|
|
229
|
-
if (this.currentInputId === null) {
|
|
230
|
-
this.currentInputId = this.generateId();
|
|
231
|
-
this.onTranscription({type: 'input', text: this.currentInputText, id: this.currentInputId, isPartial: true, action: 'create'});
|
|
232
|
-
} else {
|
|
233
|
-
this.onTranscription({type: 'input', text: this.currentInputText, id: this.currentInputId, isPartial: true, action: 'update'});
|
|
234
|
-
}
|
|
235
|
-
}
|
|
236
|
-
}
|
|
237
|
-
|
|
238
|
-
handleOutputTranscription(transcription) {
|
|
239
|
-
const text = transcription.text;
|
|
240
|
-
if (!text) return;
|
|
241
|
-
this.currentOutputText += text;
|
|
242
|
-
if (this.onTranscription) {
|
|
243
|
-
if (this.currentOutputId === null) {
|
|
244
|
-
this.currentOutputId = this.generateId();
|
|
245
|
-
this.onTranscription({type: 'output', text: this.currentOutputText, id: this.currentOutputId, isPartial: true, action: 'create'});
|
|
246
|
-
} else {
|
|
247
|
-
this.onTranscription({type: 'output', text: this.currentOutputText, id: this.currentOutputId, isPartial: true, action: 'update'});
|
|
248
|
-
}
|
|
249
|
-
}
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
handleTurnComplete() {
|
|
253
|
-
if (this.currentInputId !== null && this.currentInputText.trim()) {
|
|
254
|
-
if (this.onTranscription) this.onTranscription({type: 'input', text: this.currentInputText.trim(), id: this.currentInputId, isPartial: false, action: 'finalize'});
|
|
255
|
-
this.conversationHistory.push({type: 'input', text: this.currentInputText.trim(), timestamp: new Date(), id: this.currentInputId});
|
|
256
|
-
}
|
|
257
|
-
if (this.currentOutputId !== null && this.currentOutputText.trim()) {
|
|
258
|
-
if (this.onTranscription) this.onTranscription({type: 'output', text: this.currentOutputText.trim(), id: this.currentOutputId, isPartial: false, action: 'finalize'});
|
|
259
|
-
this.conversationHistory.push({type: 'output', text: this.currentOutputText.trim(), timestamp: new Date(), id: this.currentOutputId});
|
|
260
|
-
}
|
|
261
|
-
if (this.onTranscription) this.onTranscription({type: 'turnComplete'});
|
|
262
|
-
this.currentInputText = '';
|
|
263
|
-
this.currentOutputText = '';
|
|
264
|
-
this.currentInputId = null;
|
|
265
|
-
this.currentOutputId = null;
|
|
266
|
-
}
|
|
267
|
-
|
|
268
|
-
generateId() { return 'msg_' + Date.now() + '_' + Math.random().toString(36).substr(2, 9); }
|
|
269
|
-
|
|
270
|
-
async startAudioRecording() {
|
|
271
|
-
if (this.isRecording) return;
|
|
272
|
-
try {
|
|
273
|
-
if (!this.mediaStream) {
|
|
274
|
-
const micPermission = await this.requestMicrophonePermission();
|
|
275
|
-
if (!micPermission) return;
|
|
276
|
-
}
|
|
277
|
-
this.mediaStreamSource = this.audioContext.createMediaStreamSource(this.mediaStream);
|
|
278
|
-
await this.createAudioWorklet();
|
|
279
|
-
this.isRecording = true;
|
|
280
|
-
} catch (error) {
|
|
281
|
-
console.error('Failed to start audio recording:', error);
|
|
282
|
-
if (this.onError) this.onError(error);
|
|
283
|
-
}
|
|
284
|
-
}
|
|
285
|
-
|
|
286
|
-
async createAudioWorklet() {
|
|
287
|
-
const bufferSize = 4096;
|
|
288
|
-
const processor = this.audioContext.createScriptProcessor(bufferSize, 1, 1);
|
|
289
|
-
processor.onaudioprocess = (event) => {
|
|
290
|
-
if (!this.isRecording || !this.session || this.isSynthesizing) return;
|
|
291
|
-
const inputData = event.inputBuffer.getChannelData(0);
|
|
292
|
-
const int16Array = new Int16Array(inputData.length);
|
|
293
|
-
for (let i = 0; i < inputData.length; i++) {
|
|
294
|
-
const sample = Math.max(-1, Math.min(1, inputData[i]));
|
|
295
|
-
int16Array[i] = sample < 0 ? sample * 0x8000 : sample * 0x7fff;
|
|
296
|
-
}
|
|
297
|
-
const base64Audio = this.arrayBufferToBase64(int16Array.buffer);
|
|
298
|
-
try { this.session.sendRealtimeInput({audio: {data: base64Audio, mimeType: 'audio/pcm;rate=16000'}}); } catch (error) { console.error('Error sending audio:', error); }
|
|
299
|
-
};
|
|
300
|
-
this.mediaStreamSource.connect(processor);
|
|
301
|
-
processor.connect(this.audioContext.destination);
|
|
302
|
-
this.audioWorkletNode = processor;
|
|
303
|
-
}
|
|
304
|
-
|
|
305
|
-
arrayBufferToBase64(buffer) {
|
|
306
|
-
const bytes = new Uint8Array(buffer);
|
|
307
|
-
let binary = '';
|
|
308
|
-
for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]);
|
|
309
|
-
return btoa(binary);
|
|
310
|
-
}
|
|
311
|
-
|
|
312
|
-
base64ToArrayBuffer(base64) {
|
|
313
|
-
const binaryString = atob(base64);
|
|
314
|
-
const bytes = new Uint8Array(binaryString.length);
|
|
315
|
-
for (let i = 0; i < binaryString.length; i++) bytes[i] = binaryString.charCodeAt(i);
|
|
316
|
-
return bytes.buffer;
|
|
317
|
-
}
|
|
318
|
-
|
|
319
|
-
async playAudioChunk(audioData) {
|
|
320
|
-
try {
|
|
321
|
-
const arrayBuffer = this.base64ToArrayBuffer(audioData);
|
|
322
|
-
const audioBuffer = this.audioContext.createBuffer(1, arrayBuffer.byteLength / 2, 24000);
|
|
323
|
-
const channelData = audioBuffer.getChannelData(0);
|
|
324
|
-
const int16View = new Int16Array(arrayBuffer);
|
|
325
|
-
for (let i = 0; i < int16View.length; i++) channelData[i] = int16View[i] / 32768.0;
|
|
326
|
-
this.audioQueue.push(audioBuffer);
|
|
327
|
-
if (!this.isPlayingAudio) this.playNextAudioBuffer();
|
|
328
|
-
} catch (error) { console.error('Error playing audio chunk:', error); }
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
playNextAudioBuffer() {
|
|
332
|
-
if (this.audioQueue.length === 0) { this.isPlayingAudio = false; return; }
|
|
333
|
-
this.isPlayingAudio = true;
|
|
334
|
-
const audioBuffer = this.audioQueue.shift();
|
|
335
|
-
const source = this.audioContext.createBufferSource();
|
|
336
|
-
source.buffer = audioBuffer;
|
|
337
|
-
source.connect(this.audioContext.destination);
|
|
338
|
-
source.onended = () => { this.playNextAudioBuffer(); };
|
|
339
|
-
source.start();
|
|
340
|
-
}
|
|
341
|
-
|
|
342
|
-
async startScreenCapture() {
|
|
343
|
-
if (this.isCapturingScreen) return;
|
|
344
|
-
try {
|
|
345
|
-
await this.setupPersistentScreenCapture();
|
|
346
|
-
this.isCapturingScreen = true;
|
|
347
|
-
const captureScreenshot = async () => {
|
|
348
|
-
try {
|
|
349
|
-
if (this.isGeminiSpeaking) return;
|
|
350
|
-
if (this.displayVideo && this.displayVideo.readyState === this.displayVideo.HAVE_ENOUGH_DATA) {
|
|
351
|
-
await this.captureFromPersistentStream();
|
|
352
|
-
} else {
|
|
353
|
-
await this.captureViaCanvas();
|
|
354
|
-
}
|
|
355
|
-
} catch (error) { console.error('Error capturing screenshot:', error); }
|
|
356
|
-
};
|
|
357
|
-
await captureScreenshot();
|
|
358
|
-
this.screenshotInterval = setInterval(captureScreenshot, this.screenshotIntervalMs);
|
|
359
|
-
} catch (error) {
|
|
360
|
-
console.error('Failed to start screen capture:', error);
|
|
361
|
-
if (this.onError) this.onError(error);
|
|
362
|
-
}
|
|
363
|
-
}
|
|
364
|
-
|
|
365
|
-
async setupPersistentScreenCapture() {
|
|
366
|
-
try {
|
|
367
|
-
if (navigator.mediaDevices && navigator.mediaDevices.getDisplayMedia) {
|
|
368
|
-
this.displayStream = await navigator.mediaDevices.getDisplayMedia({
|
|
369
|
-
video: {width: {ideal: 1920, max: 1920}, height: {ideal: 1080, max: 1080}, frameRate: {ideal: 5, max: 10}},
|
|
370
|
-
audio: false,
|
|
371
|
-
displaySurface: 'monitor',
|
|
372
|
-
});
|
|
373
|
-
this.displayVideo = document.createElement('video');
|
|
374
|
-
this.displayVideo.srcObject = this.displayStream;
|
|
375
|
-
this.displayVideo.autoplay = true;
|
|
376
|
-
this.displayVideo.muted = true;
|
|
377
|
-
this.displayVideo.playsInline = true;
|
|
378
|
-
this.displayVideo.style.cssText = 'position:fixed;top:10px;right:10px;width:200px;height:112px;border:2px solid red;z-index:9999;background-color:black;';
|
|
379
|
-
document.body.appendChild(this.displayVideo);
|
|
380
|
-
this.displayStream.getVideoTracks()[0].addEventListener('ended', () => { this.stopScreenCapture(); });
|
|
381
|
-
await new Promise((resolve) => { this.displayVideo.onloadedmetadata = resolve; });
|
|
382
|
-
await new Promise((resolve) => { if (this.displayVideo.readyState >= 2) resolve(); else this.displayVideo.oncanplay = resolve; });
|
|
383
|
-
} else {
|
|
384
|
-
throw new Error('Screen Capture API not available');
|
|
385
|
-
}
|
|
386
|
-
} catch (error) {
|
|
387
|
-
console.error('Failed to setup persistent screen capture:', error);
|
|
388
|
-
throw error;
|
|
389
|
-
}
|
|
390
|
-
}
|
|
391
|
-
|
|
392
|
-
async captureFromPersistentStream() {
|
|
393
|
-
try {
|
|
394
|
-
if (!this.displayVideo || this.displayVideo.readyState < 2) return;
|
|
395
|
-
if (this.displayVideo.videoWidth === 0 || this.displayVideo.videoHeight === 0) return;
|
|
396
|
-
const canvas = document.createElement('canvas');
|
|
397
|
-
const ctx = canvas.getContext('2d');
|
|
398
|
-
canvas.width = this.displayVideo.videoWidth;
|
|
399
|
-
canvas.height = this.displayVideo.videoHeight;
|
|
400
|
-
ctx.fillStyle = 'black';
|
|
401
|
-
ctx.fillRect(0, 0, canvas.width, canvas.height);
|
|
402
|
-
ctx.drawImage(this.displayVideo, 0, 0, canvas.width, canvas.height);
|
|
403
|
-
canvas.toBlob((blob) => {
|
|
404
|
-
if (!blob) return;
|
|
405
|
-
const reader = new FileReader();
|
|
406
|
-
reader.onload = () => {
|
|
407
|
-
const base64 = reader.result.split(',')[1];
|
|
408
|
-
if (this.session && base64) this.session.sendRealtimeInput({video: {data: base64, mimeType: 'image/jpeg'}});
|
|
409
|
-
};
|
|
410
|
-
reader.readAsDataURL(blob);
|
|
411
|
-
}, 'image/jpeg', 0.8);
|
|
412
|
-
} catch (error) { console.error('Error capturing from persistent stream:', error); throw error; }
|
|
413
|
-
}
|
|
414
|
-
|
|
415
|
-
async captureViaCanvas() {
|
|
416
|
-
try {
|
|
417
|
-
const canvas = document.createElement('canvas');
|
|
418
|
-
const ctx = canvas.getContext('2d');
|
|
419
|
-
canvas.width = window.innerWidth;
|
|
420
|
-
canvas.height = window.innerHeight;
|
|
421
|
-
ctx.fillStyle = '#ffffff';
|
|
422
|
-
ctx.fillRect(0, 0, canvas.width, canvas.height);
|
|
423
|
-
ctx.fillStyle = '#000000';
|
|
424
|
-
ctx.font = '16px Arial';
|
|
425
|
-
ctx.fillText('Page content capture (fallback method)', 20, 50);
|
|
426
|
-
ctx.fillText(`Timestamp: ${new Date().toISOString()}`, 20, 80);
|
|
427
|
-
canvas.toBlob((blob) => {
|
|
428
|
-
const reader = new FileReader();
|
|
429
|
-
reader.onload = () => {
|
|
430
|
-
const base64 = reader.result.split(',')[1];
|
|
431
|
-
if (this.session) this.session.sendRealtimeInput({video: {data: base64, mimeType: 'image/jpeg'}});
|
|
432
|
-
};
|
|
433
|
-
reader.readAsDataURL(blob);
|
|
434
|
-
}, 'image/jpeg', 0.8);
|
|
435
|
-
} catch (error) { console.error('Canvas capture failed:', error); throw error; }
|
|
436
|
-
}
|
|
437
|
-
|
|
438
|
-
stopScreenCapture() {
|
|
439
|
-
if (!this.isCapturingScreen) return;
|
|
440
|
-
this.isCapturingScreen = false;
|
|
441
|
-
if (this.screenshotInterval) { clearInterval(this.screenshotInterval); this.screenshotInterval = null; }
|
|
442
|
-
if (this.displayStream) { this.displayStream.getTracks().forEach((track) => track.stop()); this.displayStream = null; }
|
|
443
|
-
if (this.displayVideo) { if (this.displayVideo.parentNode) this.displayVideo.parentNode.removeChild(this.displayVideo); this.displayVideo.srcObject = null; this.displayVideo = null; }
|
|
444
|
-
}
|
|
445
|
-
|
|
446
|
-
stopAudioRecording() {
|
|
447
|
-
if (!this.isRecording) return;
|
|
448
|
-
this.isRecording = false;
|
|
449
|
-
if (this.mediaStream) { this.mediaStream.getTracks().forEach((track) => track.stop()); this.mediaStream = null; }
|
|
450
|
-
if (this.audioWorkletNode) { this.audioWorkletNode.disconnect(); this.audioWorkletNode = null; }
|
|
451
|
-
if (this.mediaStreamSource) { this.mediaStreamSource.disconnect(); this.mediaStreamSource = null; }
|
|
452
|
-
}
|
|
453
|
-
|
|
454
|
-
async stopSession() {
|
|
455
|
-
if (!this.session) return;
|
|
456
|
-
this.stopAudioRecording();
|
|
457
|
-
this.stopScreenCapture();
|
|
458
|
-
if (this.audioFinalizationTimeout) { clearTimeout(this.audioFinalizationTimeout); this.audioFinalizationTimeout = null; }
|
|
459
|
-
this.session.close();
|
|
460
|
-
this.session = null;
|
|
461
|
-
}
|
|
462
|
-
|
|
463
|
-
cleanup() {
|
|
464
|
-
this.stopAudioRecording();
|
|
465
|
-
this.stopScreenCapture();
|
|
466
|
-
if (this.audioContext && this.audioContext.state !== 'closed') { this.audioContext.close(); this.audioContext = null; }
|
|
467
|
-
this.audioQueue = [];
|
|
468
|
-
this.isPlayingAudio = false;
|
|
469
|
-
this.session = null;
|
|
470
|
-
this.displayStream = null;
|
|
471
|
-
this.displayVideo = null;
|
|
472
|
-
}
|
|
473
|
-
|
|
474
|
-
setScreenCaptureInterval(intervalMs) { this.screenshotIntervalMs = intervalMs; }
|
|
475
|
-
setCallbacks(callbacks) { this.onTranscription = callbacks.onTranscription || null; this.onError = callbacks.onError || null; this.onSessionStateChange = callbacks.onSessionStateChange || null; }
|
|
476
|
-
isConnected() { return this.session !== null; }
|
|
477
|
-
|
|
478
|
-
async generateSpeech(text) {
|
|
479
|
-
const response = await this.ai.models.generateContent({
|
|
480
|
-
model: 'gemini-2.5-flash-preview-tts',
|
|
481
|
-
contents: [{parts: [{text: `Say cheerfully: ${text}`}]}],
|
|
482
|
-
config: {responseModalities: ['AUDIO'], speechConfig: {voiceConfig: {prebuiltVoiceConfig: {voiceName: 'Kore'}}}},
|
|
483
|
-
});
|
|
484
|
-
const data = response.candidates?.[0]?.content?.parts?.[0]?.inlineData;
|
|
485
|
-
return this.resampleL16(data);
|
|
486
|
-
}
|
|
487
|
-
|
|
488
|
-
resampleL16(inlineData) {
|
|
489
|
-
const base64String = inlineData.data;
|
|
490
|
-
const binaryString = atob(base64String);
|
|
491
|
-
const originalLength = binaryString.length / 2;
|
|
492
|
-
const originalSamples = new Int16Array(originalLength);
|
|
493
|
-
for (let i = 0; i < originalLength; i++) {
|
|
494
|
-
originalSamples[i] = (binaryString.charCodeAt(i * 2 + 1) << 8) | binaryString.charCodeAt(i * 2);
|
|
495
|
-
}
|
|
496
|
-
const originalRate = 24000, targetRate = 16000;
|
|
497
|
-
const newLength = Math.round(originalLength * (targetRate / originalRate));
|
|
498
|
-
const resampledSamples = new Int16Array(newLength);
|
|
499
|
-
const ratio = (originalLength - 1) / (newLength - 1);
|
|
500
|
-
for (let i = 0; i < newLength; i++) {
|
|
501
|
-
let index = i * ratio;
|
|
502
|
-
let lowerIndex = Math.floor(index), upperIndex = Math.ceil(index);
|
|
503
|
-
let fraction = index - lowerIndex;
|
|
504
|
-
resampledSamples[i] = originalSamples[lowerIndex] + (originalSamples[upperIndex] - originalSamples[lowerIndex]) * fraction;
|
|
505
|
-
}
|
|
506
|
-
const resampledBytes = new Uint8Array(resampledSamples.buffer);
|
|
507
|
-
let newBinaryString = '';
|
|
508
|
-
for (let i = 0; i < resampledBytes.length; i++) newBinaryString += String.fromCharCode(resampledBytes[i]);
|
|
509
|
-
return {inlineData: {data: btoa(newBinaryString), mimeType: 'audio/L16;codec=pcm;rate=16000'}};
|
|
510
|
-
}
|
|
511
|
-
|
|
512
|
-
sendAudio(audioData) {
|
|
513
|
-
if (!this.session) return;
|
|
514
|
-
if (audioData) this.session.sendRealtimeInput({audio: {data: audioData.inlineData.data, mimeType: 'audio/pcm;rate=16000'}});
|
|
515
|
-
}
|
|
516
|
-
}
|
|
517
|
-
|
|
518
|
-
// --- GeminiLivePanel inlined ---
|
|
519
|
-
class GeminiLivePanel extends LitElement {
|
|
520
|
-
static properties = {
|
|
521
|
-
micRecording: {type: Boolean},
|
|
522
|
-
responseText: {type: String},
|
|
523
|
-
};
|
|
524
|
-
static styles = css`
|
|
525
|
-
:host {
|
|
526
|
-
position: absolute;
|
|
527
|
-
bottom: 0;
|
|
528
|
-
left: 50%;
|
|
529
|
-
width: 60rem;
|
|
530
|
-
-webkit-transform: translateX(-50%);
|
|
531
|
-
transform: translateX(-50%);
|
|
532
|
-
}
|
|
533
|
-
.control-panel {
|
|
534
|
-
width: 30rem;
|
|
535
|
-
height: 3rem;
|
|
536
|
-
display: flex;
|
|
537
|
-
margin: 1rem auto;
|
|
538
|
-
column-gap: 1rem;
|
|
539
|
-
}
|
|
540
|
-
.text-input {
|
|
541
|
-
flex-grow: 1;
|
|
542
|
-
border-radius: 3rem;
|
|
543
|
-
height: 100%;
|
|
544
|
-
background: #00000088;
|
|
545
|
-
border: none;
|
|
546
|
-
color: white;
|
|
547
|
-
padding: 0rem 1rem;
|
|
548
|
-
}
|
|
549
|
-
.material-symbols-outlined {
|
|
550
|
-
font-variation-settings: 'FILL' 0, 'wght' 400, 'GRAD' 0, 'opsz' 24;
|
|
551
|
-
}
|
|
552
|
-
.response-panel-wrapper {
|
|
553
|
-
display: flex;
|
|
554
|
-
flex-direction: row;
|
|
555
|
-
background: #00000088;
|
|
556
|
-
border-radius: 1rem;
|
|
557
|
-
margin: 1rem auto;
|
|
558
|
-
padding: 1rem;
|
|
559
|
-
width: 50rem;
|
|
560
|
-
border: 1px solid #333;
|
|
561
|
-
box-sizing: border-box;
|
|
562
|
-
}
|
|
563
|
-
.response-panel {
|
|
564
|
-
flex-grow: 1;
|
|
565
|
-
padding: 1rem;
|
|
566
|
-
height: 6rem;
|
|
567
|
-
color: white;
|
|
568
|
-
font-family: monospace;
|
|
569
|
-
font-size: 0.9rem;
|
|
570
|
-
line-height: 1.4;
|
|
571
|
-
overflow-y: auto;
|
|
572
|
-
white-space: pre-wrap;
|
|
573
|
-
word-wrap: break-word;
|
|
574
|
-
}
|
|
575
|
-
.response-panel::-webkit-scrollbar { display: none; }
|
|
576
|
-
`;
|
|
577
|
-
|
|
578
|
-
textInputRef = createRef();
|
|
579
|
-
responsePanelRef = createRef();
|
|
580
|
-
|
|
581
|
-
constructor() {
|
|
582
|
-
super();
|
|
583
|
-
this.micRecording = false;
|
|
584
|
-
this.responseText = '';
|
|
585
|
-
this.apiKey = '';
|
|
586
|
-
this.apiKeyInputElement = null;
|
|
587
|
-
this.addEventListener(MicButtonPressedEvent.type, this.onMicButtonClicked.bind(this));
|
|
588
|
-
this.geminiLive = null;
|
|
589
|
-
this.currentTranscriptionId = null;
|
|
590
|
-
this.currentTranscriptionText = null;
|
|
591
|
-
if (!this.apiKey) {
|
|
592
|
-
this.showApiKeyPrompt();
|
|
593
|
-
}
|
|
594
|
-
}
|
|
595
|
-
|
|
596
|
-
showApiKeyPrompt() {
|
|
597
|
-
if (this.apiKeyInputElement) return;
|
|
598
|
-
this.apiKeyInputElement = document.createElement('xrblocks-simulator-geminilive-apikeyinput');
|
|
599
|
-
this.apiKeyInputElement.addEventListener(ApiKeyEnteredEvent.type, (e) => {
|
|
600
|
-
this.apiKey = e.apiKey;
|
|
601
|
-
this.apiKeyInputElement.remove();
|
|
602
|
-
this.apiKeyInputElement = null;
|
|
603
|
-
this.onMicButtonClicked();
|
|
604
|
-
});
|
|
605
|
-
document.body.appendChild(this.apiKeyInputElement);
|
|
606
|
-
}
|
|
607
|
-
|
|
608
|
-
async connectGeminiLive() {
|
|
609
|
-
const apiKey = this.apiKey;
|
|
610
|
-
if (!apiKey && !this.apiKeyInputElement) { this.showApiKeyPrompt(); return; }
|
|
611
|
-
if (!apiKey) return;
|
|
612
|
-
try {
|
|
613
|
-
this.geminiLive = new GeminiLiveWebInterface(apiKey);
|
|
614
|
-
this.geminiLive.setCallbacks({onTranscription: (data) => this.handleTranscription(data)});
|
|
615
|
-
this.geminiLive.setScreenCaptureInterval(3000);
|
|
616
|
-
const initialized = await this.geminiLive.initialize();
|
|
617
|
-
if (!initialized) return;
|
|
618
|
-
await this.geminiLive.startSession();
|
|
619
|
-
} catch (error) { console.error('Connection failed:', error); }
|
|
620
|
-
}
|
|
621
|
-
|
|
622
|
-
async disconnectGeminiLive() {
|
|
623
|
-
if (this.geminiLive) { await this.geminiLive.stopSession(); this.geminiLive.cleanup(); this.geminiLive = null; }
|
|
624
|
-
}
|
|
625
|
-
|
|
626
|
-
handleTranscription(data) {
|
|
627
|
-
const timestamp = new Date().toLocaleTimeString();
|
|
628
|
-
if (data.type === 'input' || data.type === 'output') {
|
|
629
|
-
if (data.action === 'create') {
|
|
630
|
-
const typeLabel = data.type === 'input' ? 'You' : 'Gemini';
|
|
631
|
-
const newEntry = `[${timestamp}] ${typeLabel}: ${data.text}`;
|
|
632
|
-
if (data.isPartial) {
|
|
633
|
-
if (this.currentTranscriptionId !== data.id) {
|
|
634
|
-
this.currentTranscriptionId = data.id;
|
|
635
|
-
this.responseText += newEntry + '\n';
|
|
636
|
-
} else {
|
|
637
|
-
const lines = this.responseText.split('\n');
|
|
638
|
-
lines[lines.length - 2] = newEntry;
|
|
639
|
-
this.responseText = lines.join('\n');
|
|
640
|
-
}
|
|
641
|
-
} else {
|
|
642
|
-
this.responseText += newEntry + '\n';
|
|
643
|
-
}
|
|
644
|
-
} else if (data.action === 'update') {
|
|
645
|
-
if (this.currentTranscriptionId === data.id) {
|
|
646
|
-
const typeLabel = data.type === 'input' ? 'You' : 'Gemini';
|
|
647
|
-
const updatedEntry = `[${timestamp}] ${typeLabel}: ${data.text}`;
|
|
648
|
-
const lines = this.responseText.split('\n');
|
|
649
|
-
lines[lines.length - 2] = updatedEntry;
|
|
650
|
-
this.responseText = lines.join('\n');
|
|
651
|
-
}
|
|
652
|
-
} else if (data.action === 'finalize') {
|
|
653
|
-
if (this.currentTranscriptionId === data.id) {
|
|
654
|
-
this.currentTranscriptionId = null;
|
|
655
|
-
this.currentTranscriptionText = null;
|
|
656
|
-
}
|
|
657
|
-
}
|
|
658
|
-
}
|
|
659
|
-
this.requestUpdate();
|
|
660
|
-
this.updateComplete.then(() => {
|
|
661
|
-
if (this.responsePanelRef.value) this.responsePanelRef.value.scrollTop = this.responsePanelRef.value.scrollHeight;
|
|
662
|
-
});
|
|
663
|
-
}
|
|
664
|
-
|
|
665
|
-
firstUpdated() {
|
|
666
|
-
this.textInputRef.value.addEventListener('keydown', this.textInputKeyDown.bind(this));
|
|
667
|
-
if (this.apiKey) this.onMicButtonClicked();
|
|
668
|
-
}
|
|
669
|
-
|
|
670
|
-
async onMicButtonClicked() {
|
|
671
|
-
if (this.micRecording) { await this.disconnectGeminiLive(); } else { await this.connectGeminiLive(); }
|
|
672
|
-
if (!this.apiKey) return;
|
|
673
|
-
this.micRecording = !this.micRecording;
|
|
674
|
-
}
|
|
675
|
-
|
|
676
|
-
textInputKeyDown(event) {
|
|
677
|
-
if (event.key === 'Enter') { const text = event.target.value; this.responseText += '\n' + text; event.target.value = ''; }
|
|
678
|
-
event.stopPropagation();
|
|
679
|
-
}
|
|
680
|
-
|
|
681
|
-
render() {
|
|
682
|
-
return html`
|
|
683
|
-
<div class="response-panel-wrapper">
|
|
684
|
-
<p class="response-panel" ${ref(this.responsePanelRef)}>${this.responseText}</p>
|
|
685
|
-
</div>
|
|
686
|
-
<div class="control-panel">
|
|
687
|
-
<xrblocks-simulator-geminilive-micbutton ?micRecording=${this.micRecording}></xrblocks-simulator-geminilive-micbutton>
|
|
688
|
-
<input type="text" class="text-input" placeholder="Ask Gemini" ${ref(this.textInputRef)} />
|
|
689
|
-
</div>
|
|
690
|
-
`;
|
|
691
|
-
}
|
|
692
|
-
}
|
|
693
|
-
customElements.define('xrblocks-simulator-geminilive', GeminiLivePanel);
|
|
694
|
-
|
|
695
|
-
// --- AISimulator inlined ---
|
|
696
|
-
class AISimulator extends xb.Script {
|
|
697
|
-
constructor() {
|
|
698
|
-
super();
|
|
699
|
-
this.data = DATA;
|
|
700
|
-
this.models = [];
|
|
701
|
-
}
|
|
702
|
-
|
|
703
|
-
init(...args) {
|
|
704
|
-
super.init(...args);
|
|
705
|
-
xb.core.renderer.localClippingEnabled = true;
|
|
706
|
-
|
|
707
|
-
this.add(new THREE.HemisphereLight(0x888877, 0x777788, 3));
|
|
708
|
-
const light = new THREE.DirectionalLight(0xffffff, 5.0);
|
|
709
|
-
light.position.set(-0.5, 4, 1.0);
|
|
710
|
-
this.add(light);
|
|
711
|
-
|
|
712
|
-
console.log('Gemini Quest UIs: ', xb.core.ui.views);
|
|
713
|
-
this.loadModels();
|
|
714
|
-
document.addEventListener('keydown', (event) => {
|
|
715
|
-
if (event.code === 'KeyI') {
|
|
716
|
-
this.onTriggerRandomQuestion();
|
|
717
|
-
}
|
|
718
|
-
});
|
|
719
|
-
}
|
|
720
|
-
|
|
721
|
-
onSimulatorStarted() {
|
|
722
|
-
const backgroundImageElement = document.getElementById('background-image');
|
|
723
|
-
if (backgroundImageElement) backgroundImageElement.style.zIndex = -1;
|
|
724
|
-
}
|
|
725
|
-
|
|
726
|
-
queryGemini() {
|
|
727
|
-
const response = xb.core.ai.query('what is it?', this.image.toBase64());
|
|
728
|
-
}
|
|
729
|
-
|
|
730
|
-
changeMeshColor() {}
|
|
731
|
-
onSelectStart(event) {}
|
|
732
|
-
onSelecting(id) {}
|
|
733
|
-
|
|
734
|
-
update() {
|
|
735
|
-
const deltaTime = xb.getDeltaTime();
|
|
736
|
-
for (const model of this.data) {
|
|
737
|
-
model.modelAnimation?.update(deltaTime);
|
|
738
|
-
}
|
|
739
|
-
}
|
|
740
|
-
|
|
741
|
-
loadModels() {
|
|
742
|
-
for (let i = 0; i < this.data.length; i++) {
|
|
743
|
-
if (this.data[i].model) {
|
|
744
|
-
const data = this.data[i];
|
|
745
|
-
const model = new xb.ModelViewer({});
|
|
746
|
-
model.loadGLTFModel({
|
|
747
|
-
data: this.data[i].model,
|
|
748
|
-
setupPlatform: false,
|
|
749
|
-
setupRaycastCylinder: false,
|
|
750
|
-
setupRaycastBox: true,
|
|
751
|
-
renderer: xb.core.renderer,
|
|
752
|
-
onSceneLoaded: () => {
|
|
753
|
-
console.log('scene loaded');
|
|
754
|
-
model.position.copy(data.position);
|
|
755
|
-
model.rotation.set(data.rotation.x, data.rotation.y, data.rotation.z, 'YXZ');
|
|
756
|
-
this.add(model);
|
|
757
|
-
},
|
|
758
|
-
addOcclusionToShader: true,
|
|
759
|
-
});
|
|
760
|
-
this.models[i] = model;
|
|
761
|
-
}
|
|
762
|
-
}
|
|
763
|
-
}
|
|
764
|
-
|
|
765
|
-
onTriggerRandomQuestion() {
|
|
766
|
-
const geminiLivePanel = document.querySelector('xrblocks-simulator-geminilive');
|
|
767
|
-
if (geminiLivePanel && geminiLivePanel.geminiLive && geminiLivePanel.geminiLive.isConnected()) {
|
|
768
|
-
const randomIndex = Math.floor(Math.random() * this.data.length);
|
|
769
|
-
const randomPrompt = this.data[randomIndex].prompt;
|
|
770
|
-
const geminiLive = geminiLivePanel.geminiLive;
|
|
771
|
-
geminiLivePanel.responseText += `\n> Synthesizing speech for: ${randomPrompt}...\n`;
|
|
772
|
-
geminiLive.generateSpeech(randomPrompt).then((resampledData) => { geminiLive.sendAudio(resampledData); });
|
|
773
|
-
}
|
|
774
|
-
}
|
|
775
|
-
}
|
|
776
|
-
|
|
777
|
-
// --- main ---
|
|
778
|
-
const options = new xb.Options();
|
|
779
|
-
options.simulator.instructions.enabled = false;
|
|
780
|
-
options.simulator.geminiLivePanel.enabled = true;
|
|
781
|
-
options.depth.enabled = true;
|
|
782
|
-
options.depth.depthTexture.enabled = true;
|
|
783
|
-
options.depth.occlusion.enabled = true;
|
|
784
|
-
|
|
785
|
-
document.addEventListener('DOMContentLoaded', function () {
|
|
786
|
-
xb.add(new AISimulator());
|
|
787
|
-
xb.init(options);
|
|
788
|
-
});
|
|
789
|
-
|
|
790
|
-
</script>
|
|
791
|
-
</body>
|
|
792
|
-
</html>
|