@jlongo78/agent-spaces 0.7.7 → 0.8.0

package/.next/standalone/src/app/vr/vr-pane.tsx

@@ -0,0 +1,195 @@
+'use client';
+
+import { useMemo, useEffect } from 'react';
+import { Text } from '@react-three/drei';
+import * as THREE from 'three';
+import { useVRTerminal } from './vr-terminal';
+import { useVR } from './vr-app';
+import { useSpeechRecognition } from '@/hooks/use-speech-recognition';
+import type { PaneData } from '@/lib/db/queries';
+
+interface VRPaneProps {
+  pane: PaneData;
+  position: [number, number, number];
+  workspaceColor: string;
+  isFocused: boolean;
+  onFocus: () => void;
+}
+
+const PANE_WIDTH = 4;
+const PANE_HEIGHT = 2.5;
+
+const SOFT_KEYS = [
+  { label: 'Esc', data: '\x1b' },
+  { label: 'Tab', data: '\t' },
+  { label: '↑', data: '\x1b[A' },
+  { label: '↓', data: '\x1b[B' },
+  { label: '←', data: '\x1b[D' },
+  { label: '→', data: '\x1b[C' },
+  { label: '⏎', data: '\r' },
+  { label: 'Ctrl-C', data: '\x03' },
+  { label: '|', data: '|' },
+  { label: '~', data: '~' },
+];
+
+export function VRPane({ pane, position, workspaceColor, isFocused, onFocus }: VRPaneProps) {
+  const { terminalToken } = useVR();
+  const { texture, textureReady, focus, scroll, send } = useVRTerminal({
+    paneId: pane.id,
+    cwd: pane.cwd || '~',
+    agentType: pane.agentType || 'shell',
+    terminalToken,
+    isFocused,
+  });
+
+
+  const {
+    isListening,
+    isSupported: micSupported,
+    transcript,
+    startListening,
+    stopListening,
+  } = useSpeechRecognition();
+
+  // Auto-start dictation when pane is focused
+  useEffect(() => {
+    if (isFocused && micSupported) {
+      try { startListening(); } catch {}
+    }
+    return () => {
+      if (isListening) {
+        try { stopListening(); } catch {}
+      }
+    };
+  }, [isFocused, micSupported]);
+
+  // Auto-restart after each utterance
+  useEffect(() => {
+    if (isFocused && micSupported && !isListening) {
+      const timer = setTimeout(() => {
+        try { startListening(); } catch {}
+      }, 500);
+      return () => clearTimeout(timer);
+    }
+  }, [isListening, isFocused, micSupported]);
+
+  // Send transcript to terminal
+  useEffect(() => {
+    if (transcript && isFocused) {
+      send(transcript);
+    }
+  }, [transcript, isFocused, send]);
+
+  const paneColor = useMemo(() => new THREE.Color(pane.color || workspaceColor || '#6366f1'), [pane.color, workspaceColor]);
+  const darkPaneColor = useMemo(() => paneColor.clone().multiplyScalar(0.3), [paneColor]);
+
+  const rotation = useMemo(() => {
+    const lookTarget = new THREE.Vector3(0, position[1], 0);
+    const panePos = new THREE.Vector3(...position);
+    const direction = new THREE.Vector3().subVectors(lookTarget, panePos).normalize();
+    return new THREE.Euler().setFromQuaternion(
+      new THREE.Quaternion().setFromUnitVectors(new THREE.Vector3(0, 0, 1), direction)
+    );
+  }, [position]);
+
+  const handleClick = () => {
+    onFocus();
+  };
+
+  // Toolbar: soft keys + keyboard fallback
+  const allButtons = [
+    ...SOFT_KEYS.map(k => ({ label: k.label, action: () => send(k.data), color: '#1a1a2e', emissive: '#6366f1' })),
+    { label: '⌨', action: () => focus(), color: '#1a1a2e', emissive: '#6366f1' },
+  ];
+
+  return (
+    <group position={position} rotation={rotation}>
+      {/* Background quad */}
+      <mesh onClick={handleClick}>
+        <planeGeometry args={[PANE_WIDTH, PANE_HEIGHT]} />
+        <meshStandardMaterial color="#0b0b0f" roughness={0.7} />
+      </mesh>
+
+      {/* Terminal content texture */}
+      {textureReady && texture.current ? (
+        <mesh position={[0, -0.15, 0.005]}>
+          <planeGeometry args={[PANE_WIDTH - 0.1, PANE_HEIGHT - 0.35]} />
+          <meshBasicMaterial map={texture.current} toneMapped={false} />
+        </mesh>
+      ) : (
+        <Text position={[0, 0, 0.005]} fontSize={0.12} color="#555" anchorX="center" anchorY="middle">
+          Connecting to terminal...
+        </Text>
+      )}
+
+      {/* Title bar */}
+      <mesh position={[0, PANE_HEIGHT / 2 - 0.06, 0.005]}>
+        <planeGeometry args={[PANE_WIDTH, 0.12]} />
+        <meshStandardMaterial color={darkPaneColor} emissive={paneColor} emissiveIntensity={0.08} />
+      </mesh>
+
+      {/* Accent line */}
+      <mesh position={[0, PANE_HEIGHT / 2 - 0.13, 0.005]}>
+        <planeGeometry args={[PANE_WIDTH, 0.015]} />
+        <meshStandardMaterial color={paneColor} emissive={paneColor} emissiveIntensity={0.6} toneMapped={false} />
+      </mesh>
+
+      {/* Focus highlight border */}
+      {isFocused && (
+        <mesh position={[0, 0, -0.005]}>
+          <planeGeometry args={[PANE_WIDTH + 0.06, PANE_HEIGHT + 0.06]} />
+          <meshStandardMaterial color={paneColor} emissive={paneColor} emissiveIntensity={0.3} toneMapped={false} />
+        </mesh>
+      )}
+
+      {/* Title text */}
+      <Text
+        position={[-PANE_WIDTH / 2 + 0.15, PANE_HEIGHT / 2 - 0.06, 0.01]}
+        fontSize={0.08}
+        color="white"
+        anchorX="left"
+        anchorY="middle"
+      >
+        {pane.title || pane.agentType} — {pane.agentType}
+      </Text>
+
+      {/* Dictation status — shown on title bar when focused */}
+      {isFocused && micSupported && (
+        <Text
+          position={[PANE_WIDTH / 2 - 0.3, PANE_HEIGHT / 2 - 0.06, 0.01]}
+          fontSize={0.06}
+          color={isListening ? '#22c55e' : '#666'}
+          anchorX="right"
+          anchorY="middle"
+        >
+          {isListening ? '🎤 Listening...' : '🎤 Off'}
+        </Text>
+      )}
+
+      {/* Toolbar — shown when focused */}
+      {isFocused && (
+        <group position={[0, -PANE_HEIGHT / 2 - 0.15, 0.01]}>
+          {allButtons.map((btn, i) => {
+            const totalWidth = allButtons.length * 0.4;
+            const x = -totalWidth / 2 + i * 0.4 + 0.2;
+            return (
+              <group key={btn.label} position={[x, 0, 0]}>
+                <mesh onClick={btn.action}>
+                  <planeGeometry args={[0.35, 0.18]} />
+                  <meshStandardMaterial
+                    color={btn.color}
+                    emissive={btn.emissive}
+                    emissiveIntensity={0.08}
+                  />
+                </mesh>
+                <Text position={[0, 0, 0.005]} fontSize={0.05} color="#ccc" anchorX="center" anchorY="middle">
+                  {btn.label}
+                </Text>
+              </group>
+            );
+          })}
+        </group>
+      )}
+    </group>
+  );
+}

package/.next/standalone/src/app/vr/vr-room.tsx

@@ -0,0 +1,79 @@
+'use client';
+
+import { useState, useEffect } from 'react';
+import { Text } from '@react-three/drei';
+import { useVR } from './vr-app';
+import { VRPane } from './vr-pane';
+import { VREnvironment } from './vr-environment';
+import { computePanePositions } from './vr-layout';
+import type { PaneData } from '@/lib/db/queries';
+
+export function VRRoom() {
+  const { workspace, returnToLobby } = useVR();
+  const [panes, setPanes] = useState<PaneData[]>([]);
+  const [loading, setLoading] = useState(true);
+  const [focusedPane, setFocusedPane] = useState<string | null>(null);
+
+  useEffect(() => {
+    if (!workspace) return;
+    fetch(`/api/panes?workspace_id=${workspace.id}`)
+      .then(r => r.json())
+      .then(data => {
+        if (Array.isArray(data)) setPanes(data);
+        setLoading(false);
+      })
+      .catch(() => setLoading(false));
+
+    return () => {
+      setPanes([]);
+    };
+  }, [workspace]);
+
+  const positions = computePanePositions(panes.length);
+  const wsColor = workspace?.color || '#6366f1';
+
+  return (
+    <group>
+      <VREnvironment variant="room" accentColor={wsColor} />
+
+      {/* Back button — low and close, in front of user */}
+      <group position={[0, 0.8, -1.5]}>
+        <mesh onClick={returnToLobby}>
+          <planeGeometry args={[1.2, 0.3]} />
+          <meshStandardMaterial color="#1a1a2e" emissive="#6366f1" emissiveIntensity={0.1} />
+        </mesh>
+        <Text position={[0, 0, 0.01]} fontSize={0.1} color="white" anchorX="center">
+          ← Back to Lobby
+        </Text>
+      </group>
+
+      {/* Loading / empty states */}
+      {loading && (
+        <Text position={[0, 1.6, 5]} fontSize={0.15} color="#888" anchorX="center">
+          Loading panes...
+        </Text>
+      )}
+      {!loading && panes.length === 0 && (
+        <Text position={[0, 1.6, 5]} fontSize={0.15} color="#888" anchorX="center">
+          No panes in this workspace.
+        </Text>
+      )}
+
+      {/* Terminal panes */}
+      {panes.map((pane, i) => {
+        const pos = positions[i];
+        if (!pos) return null;
+        return (
+          <VRPane
+            key={pane.id}
+            pane={pane}
+            position={[pos.x, pos.y, pos.z]}
+            workspaceColor={wsColor}
+            isFocused={focusedPane === pane.id}
+            onFocus={() => setFocusedPane(pane.id)}
+          />
+        );
+      })}
+    </group>
+  );
+}

package/.next/standalone/src/app/vr/vr-terminal.tsx

@@ -0,0 +1,303 @@
+'use client';
+
+import { useRef, useEffect, useState } from 'react';
+import { useFrame } from '@react-three/fiber';
+import * as THREE from 'three';
+import { Terminal } from 'xterm';
+
+const WS_PATH = process.env.NEXT_PUBLIC_WS_PATH;
+
+// ANSI color palette for rendering
+const ANSI_COLORS: Record<number, string> = {
+  0: '#27272a', 1: '#ef4444', 2: '#22c55e', 3: '#eab308',
+  4: '#6366f1', 5: '#a855f7', 6: '#06b6d4', 7: '#e4e4e7',
+  8: '#52525b', 9: '#f87171', 10: '#4ade80', 11: '#facc15',
+  12: '#818cf8', 13: '#c084fc', 14: '#22d3ee', 15: '#fafafa',
+};
+
+interface VRTerminalProps {
+  paneId: string;
+  cwd: string;
+  agentType: string;
+  terminalToken: string;
+  cols?: number;
+  rows?: number;
+  isFocused?: boolean;
+}
+
+/**
+ * Renders xterm buffer content directly to a canvas using 2D context.
+ * Bypasses CanvasAddon entirely — works on Quest browser.
+ */
+export function useVRTerminal({
+  paneId,
+  cwd,
+  agentType,
+  terminalToken,
+  cols = 100,
+  rows = 30,
+  isFocused = false,
+}: VRTerminalProps) {
+  const textureRef = useRef<THREE.CanvasTexture | null>(null);
+  const termRef = useRef<Terminal | null>(null);
+  const canvasRef = useRef<HTMLCanvasElement | null>(null);
+  const ctxRef = useRef<CanvasRenderingContext2D | null>(null);
+  const containerRef = useRef<HTMLDivElement | null>(null);
+  const dirtyRef = useRef(true);
+  const elapsedRef = useRef(0);
+  const wsRef = useRef<WebSocket | null>(null);
+  const [textureReady, setTextureReady] = useState(false);
+
+  useEffect(() => {
+    // Create our own canvas for rendering terminal text
+    // Higher res canvas for readable text on Quest
+    const fontSize = 12;
+    const charWidth = 7.2;
+    const lineHeight = 15;
+    const canvasWidth = 2048;
+    const canvasHeight = 1024;
+
+    const canvas = document.createElement('canvas');
+    canvas.width = canvasWidth;
+    canvas.height = canvasHeight;
+    canvasRef.current = canvas;
+
+    const ctx = canvas.getContext('2d');
+    if (!ctx) return;
+    ctxRef.current = ctx;
+
+    // Fill with terminal background
+    ctx.fillStyle = '#0a0a0f';
+    ctx.fillRect(0, 0, canvasWidth, canvasHeight);
+
+    // Create texture from our canvas
+    const texture = new THREE.CanvasTexture(canvas);
+    texture.minFilter = THREE.LinearFilter;
+    texture.magFilter = THREE.LinearFilter;
+    textureRef.current = texture;
+    setTextureReady(true);
+
+    // Create hidden container for xterm (it needs a DOM element)
+    const container = document.createElement('div');
+    container.style.position = 'absolute';
+    container.style.left = '-9999px';
+    container.style.top = '-9999px';
+    container.style.width = `${cols * 10}px`;
+    container.style.height = `${rows * 20}px`;
+    document.body.appendChild(container);
+    containerRef.current = container;
+
+    // Create terminal (DOM renderer only — no CanvasAddon needed)
+    const term = new Terminal({
+      cols,
+      rows,
+      cursorBlink: false,
+      scrollback: 1000,
+    });
+    term.open(container);
+    termRef.current = term;
+
+    // When terminal renders, mark dirty so we redraw our canvas
+    term.onRender(() => { dirtyRef.current = true; });
+
+    // Connect WebSocket
+    const proto = location.protocol === 'https:' ? 'wss:' : 'ws:';
+    const basePath = process.env.NEXT_PUBLIC_SPACES_BASE_PATH || '';
+    const wsPath = WS_PATH || `${basePath}/ws`;
+    const params = new URLSearchParams({
+      paneId,
+      cwd: cwd || '~',
+      agentType: agentType || 'shell',
+      cols: String(cols),
+      rows: String(rows),
+    });
+    if (terminalToken) params.set('terminalToken', terminalToken);
+
+    const wsUrl = `${proto}//${location.host}${wsPath}?${params}`;
+
+    const ws = new WebSocket(wsUrl);
+    wsRef.current = ws;
+
+    ws.onmessage = (event) => {
+      let data = '';
+      try {
+        const msg = JSON.parse(event.data);
+        if (msg.type === 'data' && msg.data) {
+          data = msg.data;
+        }
+      } catch {
+        data = event.data;
+      }
+
+      if (data) {
+        term.write(data);
+        dirtyRef.current = true;
+      }
+    };
+
+    ws.onerror = () => {
+      term.write('\r\n\x1b[31m[Connection error]\x1b[0m\r\n');
+      dirtyRef.current = true;
+    };
+    ws.onclose = () => {
+      term.write('\r\n\x1b[33m[Disconnected]\x1b[0m\r\n');
+      dirtyRef.current = true;
+    };
+
+    // Prevent xterm from handling keydown for printable chars —
+    // the textarea 'input' event handles those. This avoids double input
+    // on Quest's virtual keyboard which fires both keydown and input.
+    // Special keys (Ctrl, arrows, Esc) still go through keydown.
+    term.attachCustomKeyEventHandler((ev: KeyboardEvent) => {
+      if (ev.type === 'keydown' && ev.key.length === 1 && !ev.ctrlKey && !ev.altKey && !ev.metaKey) {
+        return false; // let input event handle it
+      }
+      return true;
+    });
+
+    term.onData((data) => {
+      if (ws.readyState === WebSocket.OPEN) {
+        ws.send(JSON.stringify({ type: 'data', data }));
+      }
+    });
+
+    return () => {
+      ws.close();
+      term.dispose();
+      container.remove();
+      textureRef.current?.dispose();
+    };
+  }, [paneId, cwd, agentType, terminalToken, cols, rows]);
+
+  // Throttled: read xterm buffer → draw to canvas → update texture
+  useFrame((_, delta) => {
+    if (delta > 0.016 && !isFocused) return;
+
+    elapsedRef.current += delta;
+    const interval = isFocused ? 0.1 : 0.333;
+
+    if (elapsedRef.current >= interval && dirtyRef.current && textureRef.current) {
+      renderTerminalToCanvas();
+      textureRef.current.needsUpdate = true;
+      dirtyRef.current = false;
+      elapsedRef.current = 0;
+    }
+  });
+
+  function renderTerminalToCanvas() {
+    const term = termRef.current;
+    const ctx = ctxRef.current;
+    if (!term || !ctx) return;
+
+    const charW = 7.2;
+    const lineH = 15;
+    const buffer = term.buffer.active;
+    const defaultFg = '#e4e4e7';
+
+    // ANSI 256-color palette (first 16)
+    const palette = [
+      '#27272a', '#ef4444', '#22c55e', '#eab308', '#6366f1', '#a855f7', '#06b6d4', '#e4e4e7',
+      '#52525b', '#f87171', '#4ade80', '#facc15', '#818cf8', '#c084fc', '#22d3ee', '#fafafa',
+    ];
+
+    // Clear
+    ctx.fillStyle = '#0a0a0f';
+    ctx.fillRect(0, 0, ctx.canvas.width, ctx.canvas.height);
+
+    ctx.font = '12px monospace';
+    ctx.textBaseline = 'top';
+
+    // Two passes: backgrounds first, then foreground text
+    for (let row = 0; row < rows; row++) {
+      const line = buffer.getLine(row + buffer.baseY);
+      if (!line) continue;
+
+      // Pass 1: Draw background colors
+      for (let col = 0; col < cols; col++) {
+        const cell = line.getCell(col);
+        if (!cell) continue;
+
+        const bgColor = cell.getBgColor();
+        const bgModeRaw = cell.getBgColorMode();
+        const bgMode = bgModeRaw > 0 ? (bgModeRaw >> 24) || bgModeRaw : 0;
+
+        let bg: string | null = null;
+        if (bgMode === 1) {
+          bg = palette[bgColor] || null;
+        } else if (bgMode === 2) {
+          bg = get256Color(bgColor);
+        } else if (bgMode === 3) {
+          const r = (bgColor >> 16) & 0xff;
+          const g = (bgColor >> 8) & 0xff;
+          const b = bgColor & 0xff;
+          bg = `rgb(${r},${g},${b})`;
+        }
+
+        if (bg) {
+          ctx.fillStyle = bg;
+          ctx.fillRect(col * charW + 4, row * lineH, charW * (cell.getWidth() || 1), lineH);
+        }
+      }
+
+      // Pass 2: Draw foreground text
+      let x = 4;
+      for (let col = 0; col < cols; col++) {
+        const cell = line.getCell(col);
+        if (!cell) { x += charW; continue; }
+
+        const char = cell.getChars();
+        if (!char || char === ' ') { x += charW; continue; }
+
+        const fgColor = cell.getFgColor();
+        const fgModeRaw = cell.getFgColorMode();
+        const fgMode = fgModeRaw > 0 ? (fgModeRaw >> 24) || fgModeRaw : 0;
+
+        if (fgMode === 1) {
+          ctx.fillStyle = palette[fgColor] || defaultFg;
+        } else if (fgMode === 2) {
+          ctx.fillStyle = get256Color(fgColor);
+        } else if (fgMode === 3) {
+          const r = (fgColor >> 16) & 0xff;
+          const g = (fgColor >> 8) & 0xff;
+          const b = fgColor & 0xff;
+          ctx.fillStyle = `rgb(${r},${g},${b})`;
+        } else {
+          ctx.fillStyle = defaultFg;
+        }
+
+        ctx.fillText(char, x, row * lineH + 2);
+        x += charW * (cell.getWidth() || 1);
+      }
+    }
+
+  }
+
+  function get256Color(idx: number): string {
+    if (idx < 16) return '#e4e4e7';
+    if (idx < 232) {
+      const v = idx - 16;
+      const b = v % 6; const rest = (v - b) / 6;
+      const g = rest % 6; const r = (rest - g) / 6;
+      return `rgb(${r * 51},${g * 51},${b * 51})`;
+    }
+    const gray = 8 + (idx - 232) * 10;
+    return `rgb(${gray},${gray},${gray})`;
+  }
+
+  const focus = () => {
+    const textarea = containerRef.current?.querySelector('.xterm-helper-textarea') as HTMLElement;
+    textarea?.focus();
+  };
+
+  const scroll = (lines: number) => {
+    termRef.current?.scrollLines(lines);
+  };
+
+  const send = (data: string) => {
+    if (wsRef.current?.readyState === WebSocket.OPEN) {
+      wsRef.current.send(JSON.stringify({ type: 'data', data }));
+    }
+  };
+
+  return { texture: textureRef, textureReady, term: termRef, focus, scroll, send };
+}