npm - mudra-skills - Versions diffs - 1.0.0 - Mend

mudra-skills 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (149) hide show

package/skills/mudra-xr/assets/demos/aisimulator.html ADDED Viewed

@@ -0,0 +1,792 @@
+<!doctype html>
+<!-- reference: demos/aisimulator -->
+<html lang="en">
+  <head>
+    <title>XR Blocks AI Simulator</title>
+    <meta
+      name="viewport"
+      content="width=device-width, initial-scale=1.0, user-scalable=no"
+    />
+<link type="text/css" rel="stylesheet" href="https://xrblocks.github.io/css/xr.css" />
+    <link
+      rel="stylesheet"
+      href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:opsz,wght,FILL,GRAD@24,400,0,0&icon_names=mic"
+    />
+    <script>
+      window.litDisableBundleWarning = true;
+    </script>
+    <script type="importmap">
+      {
+        "imports": {
+          "three": "https://cdn.jsdelivr.net/npm/three@0.182.0/build/three.module.js",
+          "three/addons/": "https://cdn.jsdelivr.net/npm/three@0.182.0/examples/jsm/",
+          "troika-three-text": "https://cdn.jsdelivr.net/gh/protectwise/troika@028b81cf308f0f22e5aa8e78196be56ec1997af5/packages/troika-three-text/src/index.js",
+          "troika-three-utils": "https://cdn.jsdelivr.net/gh/protectwise/troika@v0.52.4/packages/troika-three-utils/src/index.js",
+          "troika-worker-utils": "https://cdn.jsdelivr.net/gh/protectwise/troika@v0.52.4/packages/troika-worker-utils/src/index.js",
+          "bidi-js": "https://esm.sh/bidi-js@%5E1.0.2?target=es2022",
+          "webgl-sdf-generator": "https://esm.sh/webgl-sdf-generator@1.1.1/es2022/webgl-sdf-generator.mjs",
+          "lit": "https://cdn.jsdelivr.net/gh/lit/dist@3/core/lit-core.min.js",
+          "lit/": "https://esm.run/lit@3/",
+          "xrblocks": "https://cdn.jsdelivr.net/npm/xrblocks@0.11.0/build/xrblocks.js",
+          "xrblocks/addons/": "https://cdn.jsdelivr.net/npm/xrblocks@0.11.0/build/addons/",
+          "@google/genai": "https://cdn.jsdelivr.net/npm/@google/genai/+esm"
+        }
+      }
+    </script>
+  </head>
+  <body>
+    <div
+      id="background-image"
+      class="background-image"
+      style="background-image: url('textures/background.webp'); z-index: 0"
+    ></div>
+    <script type="module">
+import 'xrblocks/addons/simulator/SimulatorAddons.js';
+import 'xrblocks/addons/simulator/ui/MicButton.js';
+import 'xrblocks/addons/simulator/ui/GeminiLiveApiKeyInput.js';
+import {css, html, LitElement} from 'lit';
+import {createRef, ref} from 'lit/directives/ref.js';
+import {
+  ApiKeyEnteredEvent,
+  MicButtonPressedEvent,
+} from 'xrblocks/addons/simulator/ui/GeminiLiveEvents.js';
+import {GoogleGenAI, Modality} from '@google/genai';
+import * as THREE from 'three';
+import * as xb from 'xrblocks';
+// --- model_data.js inlined ---
+const ASSETS_BASE_URL = 'https://cdn.jsdelivr.net/gh/xrblocks/assets@main/';
+const PROPRIETARY_ASSETS_BASE_URL =
+  'https://cdn.jsdelivr.net/gh/xrblocks/proprietary-assets@main/';
+const DATA = [
+  {
+    model: {
+      scale: {x: 4.0, y: 4.0, z: 4.0},
+      path: PROPRIETARY_ASSETS_BASE_URL + 'monalisa/',
+      model: 'mona_lisa_picture_frame_compressed.glb',
+      verticallyAlignObject: false,
+    },
+    position: {x: -2.1, y: 1.75, z: -1.92},
+    rotation: {x: 0, y: Math.PI / 2, z: 0},
+    prompt: '"What is she smiling about?"',
+  },
+  {
+    model: {
+      scale: {x: 0.01, y: 0.01, z: 0.01},
+      rotation: {x: 0, y: 0, z: 0},
+      position: {x: 0, y: -0.2, z: -3.0},
+      path: PROPRIETARY_ASSETS_BASE_URL + 'chess/',
+      model: 'chess_compressed.glb',
+      verticallyAlignObject: false,
+    },
+    position: {x: 0, y: 1.0, z: -1.2},
+    rotation: {x: 0, y: 0, z: 0},
+    prompt: '"What\'s a good strategy for this game?"',
+  },
+  {
+    model: {
+      path: ASSETS_BASE_URL + 'models/',
+      model: 'Parasaurolophus.glb',
+      scale: {x: 0.3, y: 0.3, z: 0.3},
+      position: {x: 0, y: -0.6, z: 0},
+      verticallyAlignObject: false,
+      horizontallyAlignObject: false,
+    },
+    position: {x: 2.0, y: 1.0, z: -3.0},
+    rotation: {x: 0, y: 0, z: 0},
+    prompt: '"If this dinosaur could talk, what would it say?"',
+  },
+];
+// --- GeminiLiveWebInterface inlined ---
+class GeminiLiveWebInterface {
+  constructor(apiKey) {
+    this.ai = new GoogleGenAI({apiKey: apiKey});
+    this.model = 'gemini-2.0-flash-live-001';
+    this.config = {
+      responseModalities: [Modality.AUDIO],
+      speechConfig: {voiceConfig: {prebuiltVoiceConfig: {voiceName: 'Aoede'}}},
+      outputAudioTranscription: {},
+      inputAudioTranscription: {},
+    };
+    this.session = null;
+    this.isRecording = false;
+    this.isCapturingScreen = false;
+    this.isGeminiSpeaking = false;
+    this.lastAudioChunkReceived = 0;
+    this.audioFinalizationTimeout = null;
+    this.audioContext = null;
+    this.audioWorkletNode = null;
+    this.mediaStream = null;
+    this.mediaStreamSource = null;
+    this.screenshotInterval = null;
+    this.screenshotIntervalMs = 3000;
+    this.displayStream = null;
+    this.displayVideo = null;
+    this.audioQueue = [];
+    this.isPlayingAudio = false;
+    this.currentInputText = '';
+    this.currentOutputText = '';
+    this.currentInputId = null;
+    this.currentOutputId = null;
+    this.conversationHistory = [];
+    this.isSynthesizing = false;
+    this.onTranscription = null;
+    this.onError = null;
+    this.onSessionStateChange = null;
+  }
+  async initialize() {
+    try {
+      this.audioContext = new AudioContext({sampleRate: 16000});
+      if (this.audioContext.state === 'suspended') {
+        await this.audioContext.resume();
+      }
+      const micPermission = await this.requestMicrophonePermission();
+      if (!micPermission) return false;
+      return true;
+    } catch (error) {
+      console.error('Failed to initialize audio context:', error);
+      if (this.onError) this.onError(error);
+      return false;
+    }
+  }
+  async requestMicrophonePermission() {
+    try {
+      this.mediaStream = await navigator.mediaDevices.getUserMedia({
+        audio: {sampleRate: 16000, channelCount: 1, echoCancellation: true, noiseSuppression: true, autoGainControl: true},
+      });
+      return true;
+    } catch (error) {
+      console.error('Failed to get microphone access:', error);
+      if (this.onError) this.onError(error);
+      return false;
+    }
+  }
+  async startSession() {
+    if (this.session) return;
+    try {
+      this.session = await this.ai.live.connect({
+        model: this.model,
+        callbacks: {
+          onopen: () => {
+            if (this.onSessionStateChange) this.onSessionStateChange('connected');
+            this.startAudioRecording();
+            this.startScreenCapture();
+          },
+          onmessage: (message) => { this.processMessageStream(message); },
+          onerror: (e) => { console.error('Session error:', e.message); if (this.onError) this.onError(e); },
+          onclose: (e) => { if (this.onSessionStateChange) this.onSessionStateChange('disconnected'); this.cleanup(); },
+        },
+        config: this.config,
+      });
+    } catch (error) {
+      console.error('Failed to connect to Gemini Live:', error);
+      if (this.onError) this.onError(error);
+      throw error;
+    }
+  }
+  processMessageStream(message) {
+    try {
+      if (message.data) {
+        this.isGeminiSpeaking = true;
+        this.lastAudioChunkReceived = Date.now();
+        if (this.audioFinalizationTimeout) clearTimeout(this.audioFinalizationTimeout);
+        this.playAudioChunk(message.data);
+        this.audioFinalizationTimeout = setTimeout(() => { this.isGeminiSpeaking = false; }, 1000);
+      }
+      if (message.serverContent) {
+        if (message.serverContent.inputTranscription) this.handleInputTranscription(message.serverContent.inputTranscription);
+        if (message.serverContent.outputTranscription) this.handleOutputTranscription(message.serverContent.outputTranscription);
+        if (message.serverContent.turnComplete) this.handleTurnComplete();
+      }
+    } catch (error) {
+      console.error('Error processing message:', error);
+      if (this.onError) this.onError(error);
+    }
+  }
+  handleInputTranscription(transcription) {
+    const text = transcription.text;
+    if (!text) return;
+    this.currentInputText += text;
+    if (this.onTranscription) {
+      if (this.currentInputId === null) {
+        this.currentInputId = this.generateId();
+        this.onTranscription({type: 'input', text: this.currentInputText, id: this.currentInputId, isPartial: true, action: 'create'});
+      } else {
+        this.onTranscription({type: 'input', text: this.currentInputText, id: this.currentInputId, isPartial: true, action: 'update'});
+      }
+    }
+  }
+  handleOutputTranscription(transcription) {
+    const text = transcription.text;
+    if (!text) return;
+    this.currentOutputText += text;
+    if (this.onTranscription) {
+      if (this.currentOutputId === null) {
+        this.currentOutputId = this.generateId();
+        this.onTranscription({type: 'output', text: this.currentOutputText, id: this.currentOutputId, isPartial: true, action: 'create'});
+      } else {
+        this.onTranscription({type: 'output', text: this.currentOutputText, id: this.currentOutputId, isPartial: true, action: 'update'});
+      }
+    }
+  }
+  handleTurnComplete() {
+    if (this.currentInputId !== null && this.currentInputText.trim()) {
+      if (this.onTranscription) this.onTranscription({type: 'input', text: this.currentInputText.trim(), id: this.currentInputId, isPartial: false, action: 'finalize'});
+      this.conversationHistory.push({type: 'input', text: this.currentInputText.trim(), timestamp: new Date(), id: this.currentInputId});
+    }
+    if (this.currentOutputId !== null && this.currentOutputText.trim()) {
+      if (this.onTranscription) this.onTranscription({type: 'output', text: this.currentOutputText.trim(), id: this.currentOutputId, isPartial: false, action: 'finalize'});
+      this.conversationHistory.push({type: 'output', text: this.currentOutputText.trim(), timestamp: new Date(), id: this.currentOutputId});
+    }
+    if (this.onTranscription) this.onTranscription({type: 'turnComplete'});
+    this.currentInputText = '';
+    this.currentOutputText = '';
+    this.currentInputId = null;
+    this.currentOutputId = null;
+  }
+  generateId() { return 'msg_' + Date.now() + '_' + Math.random().toString(36).substr(2, 9); }
+  async startAudioRecording() {
+    if (this.isRecording) return;
+    try {
+      if (!this.mediaStream) {
+        const micPermission = await this.requestMicrophonePermission();
+        if (!micPermission) return;
+      }
+      this.mediaStreamSource = this.audioContext.createMediaStreamSource(this.mediaStream);
+      await this.createAudioWorklet();
+      this.isRecording = true;
+    } catch (error) {
+      console.error('Failed to start audio recording:', error);
+      if (this.onError) this.onError(error);
+    }
+  }
+  async createAudioWorklet() {
+    const bufferSize = 4096;
+    const processor = this.audioContext.createScriptProcessor(bufferSize, 1, 1);
+    processor.onaudioprocess = (event) => {
+      if (!this.isRecording || !this.session || this.isSynthesizing) return;
+      const inputData = event.inputBuffer.getChannelData(0);
+      const int16Array = new Int16Array(inputData.length);
+      for (let i = 0; i < inputData.length; i++) {
+        const sample = Math.max(-1, Math.min(1, inputData[i]));
+        int16Array[i] = sample < 0 ? sample * 0x8000 : sample * 0x7fff;
+      }
+      const base64Audio = this.arrayBufferToBase64(int16Array.buffer);
+      try { this.session.sendRealtimeInput({audio: {data: base64Audio, mimeType: 'audio/pcm;rate=16000'}}); } catch (error) { console.error('Error sending audio:', error); }
+    };
+    this.mediaStreamSource.connect(processor);
+    processor.connect(this.audioContext.destination);
+    this.audioWorkletNode = processor;
+  }
+  arrayBufferToBase64(buffer) {
+    const bytes = new Uint8Array(buffer);
+    let binary = '';
+    for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]);
+    return btoa(binary);
+  }
+  base64ToArrayBuffer(base64) {
+    const binaryString = atob(base64);
+    const bytes = new Uint8Array(binaryString.length);
+    for (let i = 0; i < binaryString.length; i++) bytes[i] = binaryString.charCodeAt(i);
+    return bytes.buffer;
+  }
+  async playAudioChunk(audioData) {
+    try {
+      const arrayBuffer = this.base64ToArrayBuffer(audioData);
+      const audioBuffer = this.audioContext.createBuffer(1, arrayBuffer.byteLength / 2, 24000);
+      const channelData = audioBuffer.getChannelData(0);
+      const int16View = new Int16Array(arrayBuffer);
+      for (let i = 0; i < int16View.length; i++) channelData[i] = int16View[i] / 32768.0;
+      this.audioQueue.push(audioBuffer);
+      if (!this.isPlayingAudio) this.playNextAudioBuffer();
+    } catch (error) { console.error('Error playing audio chunk:', error); }
+  }
+  playNextAudioBuffer() {
+    if (this.audioQueue.length === 0) { this.isPlayingAudio = false; return; }
+    this.isPlayingAudio = true;
+    const audioBuffer = this.audioQueue.shift();
+    const source = this.audioContext.createBufferSource();
+    source.buffer = audioBuffer;
+    source.connect(this.audioContext.destination);
+    source.onended = () => { this.playNextAudioBuffer(); };
+    source.start();
+  }
+  async startScreenCapture() {
+    if (this.isCapturingScreen) return;
+    try {
+      await this.setupPersistentScreenCapture();
+      this.isCapturingScreen = true;
+      const captureScreenshot = async () => {
+        try {
+          if (this.isGeminiSpeaking) return;
+          if (this.displayVideo && this.displayVideo.readyState === this.displayVideo.HAVE_ENOUGH_DATA) {
+            await this.captureFromPersistentStream();
+          } else {
+            await this.captureViaCanvas();
+          }
+        } catch (error) { console.error('Error capturing screenshot:', error); }
+      };
+      await captureScreenshot();
+      this.screenshotInterval = setInterval(captureScreenshot, this.screenshotIntervalMs);
+    } catch (error) {
+      console.error('Failed to start screen capture:', error);
+      if (this.onError) this.onError(error);
+    }
+  }
+  async setupPersistentScreenCapture() {
+    try {
+      if (navigator.mediaDevices && navigator.mediaDevices.getDisplayMedia) {
+        this.displayStream = await navigator.mediaDevices.getDisplayMedia({
+          video: {width: {ideal: 1920, max: 1920}, height: {ideal: 1080, max: 1080}, frameRate: {ideal: 5, max: 10}},
+          audio: false,
+          displaySurface: 'monitor',
+        });
+        this.displayVideo = document.createElement('video');
+        this.displayVideo.srcObject = this.displayStream;
+        this.displayVideo.autoplay = true;
+        this.displayVideo.muted = true;
+        this.displayVideo.playsInline = true;
+        this.displayVideo.style.cssText = 'position:fixed;top:10px;right:10px;width:200px;height:112px;border:2px solid red;z-index:9999;background-color:black;';
+        document.body.appendChild(this.displayVideo);
+        this.displayStream.getVideoTracks()[0].addEventListener('ended', () => { this.stopScreenCapture(); });
+        await new Promise((resolve) => { this.displayVideo.onloadedmetadata = resolve; });
+        await new Promise((resolve) => { if (this.displayVideo.readyState >= 2) resolve(); else this.displayVideo.oncanplay = resolve; });
+      } else {
+        throw new Error('Screen Capture API not available');
+      }
+    } catch (error) {
+      console.error('Failed to setup persistent screen capture:', error);
+      throw error;
+    }
+  }
+  async captureFromPersistentStream() {
+    try {
+      if (!this.displayVideo || this.displayVideo.readyState < 2) return;
+      if (this.displayVideo.videoWidth === 0 || this.displayVideo.videoHeight === 0) return;
+      const canvas = document.createElement('canvas');
+      const ctx = canvas.getContext('2d');
+      canvas.width = this.displayVideo.videoWidth;
+      canvas.height = this.displayVideo.videoHeight;
+      ctx.fillStyle = 'black';
+      ctx.fillRect(0, 0, canvas.width, canvas.height);
+      ctx.drawImage(this.displayVideo, 0, 0, canvas.width, canvas.height);
+      canvas.toBlob((blob) => {
+        if (!blob) return;
+        const reader = new FileReader();
+        reader.onload = () => {
+          const base64 = reader.result.split(',')[1];
+          if (this.session && base64) this.session.sendRealtimeInput({video: {data: base64, mimeType: 'image/jpeg'}});
+        };
+        reader.readAsDataURL(blob);
+      }, 'image/jpeg', 0.8);
+    } catch (error) { console.error('Error capturing from persistent stream:', error); throw error; }
+  }
+  async captureViaCanvas() {
+    try {
+      const canvas = document.createElement('canvas');
+      const ctx = canvas.getContext('2d');
+      canvas.width = window.innerWidth;
+      canvas.height = window.innerHeight;
+      ctx.fillStyle = '#ffffff';
+      ctx.fillRect(0, 0, canvas.width, canvas.height);
+      ctx.fillStyle = '#000000';
+      ctx.font = '16px Arial';
+      ctx.fillText('Page content capture (fallback method)', 20, 50);
+      ctx.fillText(`Timestamp: ${new Date().toISOString()}`, 20, 80);
+      canvas.toBlob((blob) => {
+        const reader = new FileReader();
+        reader.onload = () => {
+          const base64 = reader.result.split(',')[1];
+          if (this.session) this.session.sendRealtimeInput({video: {data: base64, mimeType: 'image/jpeg'}});
+        };
+        reader.readAsDataURL(blob);
+      }, 'image/jpeg', 0.8);
+    } catch (error) { console.error('Canvas capture failed:', error); throw error; }
+  }
+  stopScreenCapture() {
+    if (!this.isCapturingScreen) return;
+    this.isCapturingScreen = false;
+    if (this.screenshotInterval) { clearInterval(this.screenshotInterval); this.screenshotInterval = null; }
+    if (this.displayStream) { this.displayStream.getTracks().forEach((track) => track.stop()); this.displayStream = null; }
+    if (this.displayVideo) { if (this.displayVideo.parentNode) this.displayVideo.parentNode.removeChild(this.displayVideo); this.displayVideo.srcObject = null; this.displayVideo = null; }
+  }
+  stopAudioRecording() {
+    if (!this.isRecording) return;
+    this.isRecording = false;
+    if (this.mediaStream) { this.mediaStream.getTracks().forEach((track) => track.stop()); this.mediaStream = null; }
+    if (this.audioWorkletNode) { this.audioWorkletNode.disconnect(); this.audioWorkletNode = null; }
+    if (this.mediaStreamSource) { this.mediaStreamSource.disconnect(); this.mediaStreamSource = null; }
+  }
+  async stopSession() {
+    if (!this.session) return;
+    this.stopAudioRecording();
+    this.stopScreenCapture();
+    if (this.audioFinalizationTimeout) { clearTimeout(this.audioFinalizationTimeout); this.audioFinalizationTimeout = null; }
+    this.session.close();
+    this.session = null;
+  }
+  cleanup() {
+    this.stopAudioRecording();
+    this.stopScreenCapture();
+    if (this.audioContext && this.audioContext.state !== 'closed') { this.audioContext.close(); this.audioContext = null; }
+    this.audioQueue = [];
+    this.isPlayingAudio = false;
+    this.session = null;
+    this.displayStream = null;
+    this.displayVideo = null;
+  }
+  setScreenCaptureInterval(intervalMs) { this.screenshotIntervalMs = intervalMs; }
+  setCallbacks(callbacks) { this.onTranscription = callbacks.onTranscription || null; this.onError = callbacks.onError || null; this.onSessionStateChange = callbacks.onSessionStateChange || null; }
+  isConnected() { return this.session !== null; }
+  async generateSpeech(text) {
+    const response = await this.ai.models.generateContent({
+      model: 'gemini-2.5-flash-preview-tts',
+      contents: [{parts: [{text: `Say cheerfully: ${text}`}]}],
+      config: {responseModalities: ['AUDIO'], speechConfig: {voiceConfig: {prebuiltVoiceConfig: {voiceName: 'Kore'}}}},
+    });
+    const data = response.candidates?.[0]?.content?.parts?.[0]?.inlineData;
+    return this.resampleL16(data);
+  }
+  resampleL16(inlineData) {
+    const base64String = inlineData.data;
+    const binaryString = atob(base64String);
+    const originalLength = binaryString.length / 2;
+    const originalSamples = new Int16Array(originalLength);
+    for (let i = 0; i < originalLength; i++) {
+      originalSamples[i] = (binaryString.charCodeAt(i * 2 + 1) << 8) | binaryString.charCodeAt(i * 2);
+    }
+    const originalRate = 24000, targetRate = 16000;
+    const newLength = Math.round(originalLength * (targetRate / originalRate));
+    const resampledSamples = new Int16Array(newLength);
+    const ratio = (originalLength - 1) / (newLength - 1);
+    for (let i = 0; i < newLength; i++) {
+      let index = i * ratio;
+      let lowerIndex = Math.floor(index), upperIndex = Math.ceil(index);
+      let fraction = index - lowerIndex;
+      resampledSamples[i] = originalSamples[lowerIndex] + (originalSamples[upperIndex] - originalSamples[lowerIndex]) * fraction;
+    }
+    const resampledBytes = new Uint8Array(resampledSamples.buffer);
+    let newBinaryString = '';
+    for (let i = 0; i < resampledBytes.length; i++) newBinaryString += String.fromCharCode(resampledBytes[i]);
+    return {inlineData: {data: btoa(newBinaryString), mimeType: 'audio/L16;codec=pcm;rate=16000'}};
+  }
+  sendAudio(audioData) {
+    if (!this.session) return;
+    if (audioData) this.session.sendRealtimeInput({audio: {data: audioData.inlineData.data, mimeType: 'audio/pcm;rate=16000'}});
+  }
+}
+// --- GeminiLivePanel inlined ---
+class GeminiLivePanel extends LitElement {
+  static properties = {
+    micRecording: {type: Boolean},
+    responseText: {type: String},
+  };
+  static styles = css`
+    :host {
+      position: absolute;
+      bottom: 0;
+      left: 50%;
+      width: 60rem;
+      -webkit-transform: translateX(-50%);
+      transform: translateX(-50%);
+    }
+    .control-panel {
+      width: 30rem;
+      height: 3rem;
+      display: flex;
+      margin: 1rem auto;
+      column-gap: 1rem;
+    }
+    .text-input {
+      flex-grow: 1;
+      border-radius: 3rem;
+      height: 100%;
+      background: #00000088;
+      border: none;
+      color: white;
+      padding: 0rem 1rem;
+    }
+    .material-symbols-outlined {
+      font-variation-settings: 'FILL' 0, 'wght' 400, 'GRAD' 0, 'opsz' 24;
+    }
+    .response-panel-wrapper {
+      display: flex;
+      flex-direction: row;
+      background: #00000088;
+      border-radius: 1rem;
+      margin: 1rem auto;
+      padding: 1rem;
+      width: 50rem;
+      border: 1px solid #333;
+      box-sizing: border-box;
+    }
+    .response-panel {
+      flex-grow: 1;
+      padding: 1rem;
+      height: 6rem;
+      color: white;
+      font-family: monospace;
+      font-size: 0.9rem;
+      line-height: 1.4;
+      overflow-y: auto;
+      white-space: pre-wrap;
+      word-wrap: break-word;
+    }
+    .response-panel::-webkit-scrollbar { display: none; }
+  `;
+  textInputRef = createRef();
+  responsePanelRef = createRef();
+  constructor() {
+    super();
+    this.micRecording = false;
+    this.responseText = '';
+    this.apiKey = '';
+    this.apiKeyInputElement = null;
+    this.addEventListener(MicButtonPressedEvent.type, this.onMicButtonClicked.bind(this));
+    this.geminiLive = null;
+    this.currentTranscriptionId = null;
+    this.currentTranscriptionText = null;
+    if (!this.apiKey) {
+      this.showApiKeyPrompt();
+    }
+  }
+  showApiKeyPrompt() {
+    if (this.apiKeyInputElement) return;
+    this.apiKeyInputElement = document.createElement('xrblocks-simulator-geminilive-apikeyinput');
+    this.apiKeyInputElement.addEventListener(ApiKeyEnteredEvent.type, (e) => {
+      this.apiKey = e.apiKey;
+      this.apiKeyInputElement.remove();
+      this.apiKeyInputElement = null;
+      this.onMicButtonClicked();
+    });
+    document.body.appendChild(this.apiKeyInputElement);
+  }
+  async connectGeminiLive() {
+    const apiKey = this.apiKey;
+    if (!apiKey && !this.apiKeyInputElement) { this.showApiKeyPrompt(); return; }
+    if (!apiKey) return;
+    try {
+      this.geminiLive = new GeminiLiveWebInterface(apiKey);
+      this.geminiLive.setCallbacks({onTranscription: (data) => this.handleTranscription(data)});
+      this.geminiLive.setScreenCaptureInterval(3000);
+      const initialized = await this.geminiLive.initialize();
+      if (!initialized) return;
+      await this.geminiLive.startSession();
+    } catch (error) { console.error('Connection failed:', error); }
+  }
+  async disconnectGeminiLive() {
+    if (this.geminiLive) { await this.geminiLive.stopSession(); this.geminiLive.cleanup(); this.geminiLive = null; }
+  }
+  handleTranscription(data) {
+    const timestamp = new Date().toLocaleTimeString();
+    if (data.type === 'input' || data.type === 'output') {
+      if (data.action === 'create') {
+        const typeLabel = data.type === 'input' ? 'You' : 'Gemini';
+        const newEntry = `[${timestamp}] ${typeLabel}: ${data.text}`;
+        if (data.isPartial) {
+          if (this.currentTranscriptionId !== data.id) {
+            this.currentTranscriptionId = data.id;
+            this.responseText += newEntry + '\n';
+          } else {
+            const lines = this.responseText.split('\n');
+            lines[lines.length - 2] = newEntry;
+            this.responseText = lines.join('\n');
+          }
+        } else {
+          this.responseText += newEntry + '\n';
+        }
+      } else if (data.action === 'update') {
+        if (this.currentTranscriptionId === data.id) {
+          const typeLabel = data.type === 'input' ? 'You' : 'Gemini';
+          const updatedEntry = `[${timestamp}] ${typeLabel}: ${data.text}`;
+          const lines = this.responseText.split('\n');
+          lines[lines.length - 2] = updatedEntry;
+          this.responseText = lines.join('\n');
+        }
+      } else if (data.action === 'finalize') {
+        if (this.currentTranscriptionId === data.id) {
+          this.currentTranscriptionId = null;
+          this.currentTranscriptionText = null;
+        }
+      }
+    }
+    this.requestUpdate();
+    this.updateComplete.then(() => {
+      if (this.responsePanelRef.value) this.responsePanelRef.value.scrollTop = this.responsePanelRef.value.scrollHeight;
+    });
+  }
+  firstUpdated() {
+    this.textInputRef.value.addEventListener('keydown', this.textInputKeyDown.bind(this));
+    if (this.apiKey) this.onMicButtonClicked();
+  }
+  async onMicButtonClicked() {
+    if (this.micRecording) { await this.disconnectGeminiLive(); } else { await this.connectGeminiLive(); }
+    if (!this.apiKey) return;
+    this.micRecording = !this.micRecording;
+  }
+  textInputKeyDown(event) {
+    if (event.key === 'Enter') { const text = event.target.value; this.responseText += '\n' + text; event.target.value = ''; }
+    event.stopPropagation();
+  }
+  render() {
+    return html`
+      <div class="response-panel-wrapper">
+        <p class="response-panel" ${ref(this.responsePanelRef)}>${this.responseText}</p>
+      </div>
+      <div class="control-panel">
+        <xrblocks-simulator-geminilive-micbutton ?micRecording=${this.micRecording}></xrblocks-simulator-geminilive-micbutton>
+        <input type="text" class="text-input" placeholder="Ask Gemini" ${ref(this.textInputRef)} />
+      </div>
+    `;
+  }
+}
+customElements.define('xrblocks-simulator-geminilive', GeminiLivePanel);
+// --- AISimulator inlined ---
+class AISimulator extends xb.Script {
+  constructor() {
+    super();
+    this.data = DATA;
+    this.models = [];
+  }
+  init(...args) {
+    super.init(...args);
+    xb.core.renderer.localClippingEnabled = true;
+    this.add(new THREE.HemisphereLight(0x888877, 0x777788, 3));
+    const light = new THREE.DirectionalLight(0xffffff, 5.0);
+    light.position.set(-0.5, 4, 1.0);
+    this.add(light);
+    console.log('Gemini Quest UIs: ', xb.core.ui.views);
+    this.loadModels();
+    document.addEventListener('keydown', (event) => {
+      if (event.code === 'KeyI') {
+        this.onTriggerRandomQuestion();
+      }
+    });
+  }
+  onSimulatorStarted() {
+    const backgroundImageElement = document.getElementById('background-image');
+    if (backgroundImageElement) backgroundImageElement.style.zIndex = -1;
+  }
+  queryGemini() {
+    const response = xb.core.ai.query('what is it?', this.image.toBase64());
+  }
+  changeMeshColor() {}
+  onSelectStart(event) {}
+  onSelecting(id) {}
+  update() {
+    const deltaTime = xb.getDeltaTime();
+    for (const model of this.data) {
+      model.modelAnimation?.update(deltaTime);
+    }
+  }
+  loadModels() {
+    for (let i = 0; i < this.data.length; i++) {
+      if (this.data[i].model) {
+        const data = this.data[i];
+        const model = new xb.ModelViewer({});
+        model.loadGLTFModel({
+          data: this.data[i].model,
+          setupPlatform: false,
+          setupRaycastCylinder: false,
+          setupRaycastBox: true,
+          renderer: xb.core.renderer,
+          onSceneLoaded: () => {
+            console.log('scene loaded');
+            model.position.copy(data.position);
+            model.rotation.set(data.rotation.x, data.rotation.y, data.rotation.z, 'YXZ');
+            this.add(model);
+          },
+          addOcclusionToShader: true,
+        });
+        this.models[i] = model;
+      }
+    }
+  }
+  onTriggerRandomQuestion() {
+    const geminiLivePanel = document.querySelector('xrblocks-simulator-geminilive');
+    if (geminiLivePanel && geminiLivePanel.geminiLive && geminiLivePanel.geminiLive.isConnected()) {
+      const randomIndex = Math.floor(Math.random() * this.data.length);
+      const randomPrompt = this.data[randomIndex].prompt;
+      const geminiLive = geminiLivePanel.geminiLive;
+      geminiLivePanel.responseText += `\n> Synthesizing speech for: ${randomPrompt}...\n`;
+      geminiLive.generateSpeech(randomPrompt).then((resampledData) => { geminiLive.sendAudio(resampledData); });
+    }
+  }
+}
+// --- main ---
+const options = new xb.Options();
+options.simulator.instructions.enabled = false;
+options.simulator.geminiLivePanel.enabled = true;
+options.depth.enabled = true;
+options.depth.depthTexture.enabled = true;
+options.depth.occlusion.enabled = true;
+document.addEventListener('DOMContentLoaded', function () {
+  xb.add(new AISimulator());
+  xb.init(options);
+});
+</script>
+  </body>
+</html>