openvoiceui 1.0.0

package/routes/elevenlabs_hybrid.py

@@ -0,0 +1,306 @@
+"""
+routes/elevenlabs_hybrid.py — ElevenLabs + OpenClaw Hybrid Blueprint (P7-T5)
+
+Provides two endpoints that form the bridge between ElevenLabs Conversational
+AI (voice layer) and OpenClaw / Clawdbot Gateway (brain layer):
+
+    POST /api/elevenlabs-llm
+        Custom LLM endpoint configured in the ElevenLabs hybrid agent.
+        Receives the conversation context, extracts the latest user message,
+        forwards it to the Clawdbot Gateway using the persistent WebSocket
+        connection, strips canvas/HTML markers from the response, then
+        returns clean text to ElevenLabs in OpenAI-compatible SSE format
+        so ElevenLabs TTS can begin speaking as the first tokens arrive.
+
+    GET  /api/canvas-pending
+        Side-channel for canvas commands extracted from OpenClaw responses.
+        Returns and clears the pending canvas command queue so the frontend
+        adapter (ElevenLabsHybridAdapter._startCanvasPolling) can load the
+        correct iframe without the agent reading HTML aloud.
+
+Architecture:
+    Browser (ElevenLabs SDK)
+        → POST /api/elevenlabs-llm  (this module)
+            → gateway_connection.stream_to_queue(session='voice-elevenlabs-hybrid')
+            ← streaming text chunks
+        → SSE to ElevenLabs TTS
+
+    OpenClaw response:  "Dashboard ready! {canvas:present,url:/pages/stats.html} Check it out."
+    Spoken text:        "Dashboard ready!  Check it out."
+    Canvas queue:       [{"action": "present", "url": "/pages/stats.html"}]
+    GET /api/canvas-pending returns → {"commands": [{"action": "present", "url": "..."}]}
+
+Ref: future-dev-plans/16-ELEVENLABS-OPENCLAW-HYBRID.md
+Ref: ADR-008 (Fallback chains — graceful degradation on Gateway unavailability)
+"""
+
+import json
+import logging
+import os
+import queue
+import re
+import threading
+from collections import deque
+
+from flask import Blueprint, Response, jsonify, request
+
+from services.gateway import gateway_connection
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Blueprint
+# ---------------------------------------------------------------------------
+
+elevenlabs_hybrid_bp = Blueprint('elevenlabs_hybrid', __name__)
+
+# ---------------------------------------------------------------------------
+# Session key
+# Separate from ClawdBot's voice-main-N key so histories don't collide.
+# ---------------------------------------------------------------------------
+
+HYBRID_SESSION_KEY = os.getenv('ELEVENLABS_HYBRID_SESSION_KEY', 'voice-elevenlabs-hybrid')
+
+# Optional shared secret for validating requests from ElevenLabs
+HYBRID_LLM_SECRET = os.getenv('ELEVENLABS_HYBRID_LLM_SECRET', '')
+
+# ---------------------------------------------------------------------------
+# Canvas command side-channel
+# Thread-safe deque; items are dicts: {"action": "present"|"close", "url": str}
+# ---------------------------------------------------------------------------
+
+_canvas_pending: deque = deque()
+_canvas_lock = threading.Lock()
+
+# ---------------------------------------------------------------------------
+# Canvas extraction helpers
+# ---------------------------------------------------------------------------
+
+_CANVAS_PATTERN = re.compile(r'\{canvas:(\w+),url:([^}]+)\}')
+_HTML_BLOCK_PATTERN = re.compile(r'```html[\s\S]*?```', re.IGNORECASE)
+
+
+def _extract_canvas_commands(text: str) -> list:
+    """
+    Parse {canvas:action,url:path} markers from OpenClaw response text.
+
+    Returns a list of {"action": str, "url": str} dicts.
+    """
+    commands = []
+    for match in _CANVAS_PATTERN.finditer(text):
+        commands.append({
+            'action': match.group(1),   # 'present', 'close', etc.
+            'url':    match.group(2).strip(),
+        })
+    return commands
+
+
+def _strip_canvas_markers(text: str) -> str:
+    """
+    Remove {canvas:...} markers and raw ```html``` blocks from spoken text
+    so ElevenLabs TTS doesn't read them aloud.
+    """
+    text = _CANVAS_PATTERN.sub('', text)
+    text = _HTML_BLOCK_PATTERN.sub('', text)
+    return text.strip()
+
+
+def _queue_canvas_commands(commands: list) -> None:
+    """Append extracted canvas commands to the pending queue (thread-safe)."""
+    if not commands:
+        return
+    with _canvas_lock:
+        for cmd in commands:
+            _canvas_pending.append(cmd)
+
+
+# ---------------------------------------------------------------------------
+# POST /api/elevenlabs-llm — custom LLM endpoint
+# ---------------------------------------------------------------------------
+
+@elevenlabs_hybrid_bp.route('/api/elevenlabs-llm', methods=['POST'])
+def elevenlabs_custom_llm():
+    """
+    Bridge ElevenLabs voice to OpenClaw brain.
+
+    ElevenLabs sends the full conversation context in OpenAI chat format:
+        {"messages": [{"role": "system"/"user"/"assistant", "content": "..."}]}
+
+    We extract the latest user message, forward to Gateway, stream the
+    response back as OpenAI-compatible SSE so ElevenLabs TTS can start
+    speaking on the first sentence rather than waiting for the full reply.
+
+    Fallback (ADR-008): if Gateway is unreachable, returns a graceful
+    error response so ElevenLabs speaks an apology rather than hanging.
+    """
+
+    # ── Optional shared-secret auth ──────────────────────────────────────────
+    if HYBRID_LLM_SECRET:
+        auth_header = request.headers.get('Authorization', '')
+        if auth_header != f'Bearer {HYBRID_LLM_SECRET}':
+            return jsonify({'error': 'Unauthorized'}), 401
+
+    # ── Parse request ─────────────────────────────────────────────────────────
+    data     = request.get_json(silent=True) or {}
+    messages = data.get('messages', [])
+
+    if not messages:
+        return _openai_error_response("No messages provided"), 400
+
+    # Extract the latest user turn (last message with role 'user')
+    user_message = ''
+    for msg in reversed(messages):
+        if msg.get('role') == 'user':
+            user_message = msg.get('content', '')
+            break
+
+    if not user_message:
+        return _openai_error_response("No user message found in context"), 400
+
+    logger.info(f"[ElevenLabsHybrid] Custom LLM request: {user_message[:80]!r}")
+
+    # ── Check Gateway availability ────────────────────────────────────────────
+    if not gateway_connection.is_configured():
+        logger.warning('[ElevenLabsHybrid] Gateway not configured — returning fallback response')
+        return _openai_sync_response(
+            "Sorry, my connection to the server brain is not configured right now. "
+            "Please check the CLAWDBOT_AUTH_TOKEN environment variable."
+        )
+
+    # ── Stream from Gateway → SSE to ElevenLabs ──────────────────────────────
+
+    def generate():
+        """Generator: reads Gateway events, yields OpenAI SSE chunks."""
+        event_queue: queue.Queue = queue.Queue()
+        captured_actions = []
+        full_response_parts = []
+
+        # Run stream_to_queue in a background thread so we can yield from
+        # the main thread (Flask's response generator must be synchronous).
+        stream_thread = threading.Thread(
+            target=gateway_connection.stream_to_queue,
+            args=(event_queue, user_message, HYBRID_SESSION_KEY, captured_actions),
+            daemon=True,
+        )
+        stream_thread.start()
+
+        try:
+            while True:
+                try:
+                    event = event_queue.get(timeout=60)
+                except queue.Empty:
+                    logger.warning('[ElevenLabsHybrid] Gateway stream timeout')
+                    break
+
+                etype = event.get('type')
+
+                if etype == 'delta':
+                    chunk_text = event.get('text', '')
+                    if chunk_text:
+                        full_response_parts.append(chunk_text)
+                        # Strip canvas markers from streaming chunks
+                        clean_chunk = _strip_canvas_markers(chunk_text)
+                        if clean_chunk:
+                            yield _sse_delta(clean_chunk)
+
+                elif etype == 'text_done':
+                    full_response = event.get('response') or ''.join(full_response_parts)
+                    # Extract canvas commands from the full response
+                    canvas_cmds = _extract_canvas_commands(full_response)
+                    _queue_canvas_commands(canvas_cmds)
+                    if canvas_cmds:
+                        logger.info(f'[ElevenLabsHybrid] Queued {len(canvas_cmds)} canvas command(s)')
+                    break
+
+                elif etype == 'error':
+                    error_msg = event.get('error', 'Unknown Gateway error')
+                    logger.error(f'[ElevenLabsHybrid] Gateway error: {error_msg}')
+                    yield _sse_delta("I'm having trouble connecting right now. Please try again.")
+                    break
+
+                elif etype == 'handshake':
+                    # Connection established — nothing to yield to ElevenLabs
+                    continue
+
+                # 'action' events: tool use / lifecycle events — log only
+                elif etype == 'action':
+                    action_name = event.get('action', {}).get('type', 'unknown')
+                    logger.debug(f'[ElevenLabsHybrid] Gateway action: {action_name}')
+
+        finally:
+            yield 'data: [DONE]\n\n'
+
+        stream_thread.join(timeout=5)
+
+    return Response(
+        generate(),
+        mimetype='text/event-stream',
+        headers={
+            'Cache-Control':   'no-cache',
+            'X-Accel-Buffering': 'no',   # disable nginx buffering for SSE
+        },
+    )
+
+
+# ---------------------------------------------------------------------------
+# GET /api/canvas-pending — canvas command side-channel
+# ---------------------------------------------------------------------------
+
+@elevenlabs_hybrid_bp.route('/api/canvas-pending', methods=['GET'])
+def canvas_pending():
+    """
+    Return and clear the pending canvas command queue.
+
+    The ElevenLabsHybridAdapter frontend polls this endpoint every second
+    during a hybrid conversation.  When OpenClaw creates a canvas page, the
+    command appears here; the frontend then loads the iframe.
+
+    Response:
+        {"commands": [{"action": "present", "url": "/pages/stats.html"}, ...]}
+
+    Commands are consumed (cleared) on each call.
+    """
+    with _canvas_lock:
+        commands = list(_canvas_pending)
+        _canvas_pending.clear()
+
+    return jsonify({'commands': commands})
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _sse_delta(text: str) -> str:
+    """Format a text chunk as an OpenAI-compatible SSE delta event."""
+    payload = json.dumps({
+        'choices': [{
+            'delta': {
+                'role':    'assistant',
+                'content': text,
+            },
+            'finish_reason': None,
+        }]
+    })
+    return f'data: {payload}\n\n'
+
+
+def _openai_sync_response(text: str):
+    """
+    Return a non-streaming OpenAI-compatible response.
+    Used for fallback / error paths where we have the full text immediately.
+    """
+    return jsonify({
+        'choices': [{
+            'message': {
+                'role':    'assistant',
+                'content': text,
+            },
+            'finish_reason': 'stop',
+        }]
+    })
+
+
+def _openai_error_response(message: str):
+    """Return an OpenAI-compatible error body."""
+    return jsonify({'error': {'message': message, 'type': 'invalid_request_error'}})

package/routes/greetings.py

@@ -0,0 +1,98 @@
+"""
+routes/greetings.py — Greetings API
+
+GET  /api/greetings          — return full greetings.json
+GET  /api/greetings/random   — return a single random greeting (optional ?user=mike)
+POST /api/greetings/add      — append a contextual greeting (agent use)
+"""
+
+import json
+import logging
+import random
+from pathlib import Path
+
+from flask import Blueprint, jsonify, request
+
+logger = logging.getLogger(__name__)
+
+greetings_bp = Blueprint('greetings', __name__)
+
+GREETINGS_PATH = Path(__file__).parent.parent / 'greetings.json'
+
+
+def _load() -> dict:
+    try:
+        with open(GREETINGS_PATH) as f:
+            return json.load(f)
+    except Exception as e:
+        logger.error(f'Failed to load greetings.json: {e}')
+        return {'greetings': {'generic': {'classic_annoyed': ['What do you want?']}, 'mike': {}, 'contextual': []}}
+
+
+def _save(data: dict) -> None:
+    """Persist greetings (atomic write)."""
+    tmp = GREETINGS_PATH.with_suffix('.tmp')
+    tmp.write_text(json.dumps(data, indent=2))
+    tmp.replace(GREETINGS_PATH)
+
+
+@greetings_bp.route('/api/greetings', methods=['GET'])
+def get_greetings():
+    return jsonify(_load())
+
+
+@greetings_bp.route('/api/greetings/random', methods=['GET'])
+def random_greeting():
+    """Return a random greeting. Pass ?user=mike for Mike-specific categories."""
+    user = request.args.get('user', '').lower().strip()
+    data = _load()
+    greetings = data.get('greetings', {})
+
+    pool = []
+
+    # Check for a queued next_greeting first
+    if data.get('next_greeting'):
+        next_g = data['next_greeting']
+        data['next_greeting'] = None
+        _save(data)
+        return jsonify({'greeting': next_g, 'category': 'queued', 'user': user})
+
+    # Add contextual greetings (highest priority, 3x weight)
+    contextual = greetings.get('contextual', [])
+    pool.extend(contextual * 3)
+
+    # Add user-specific greetings if recognized
+    if user == 'mike':
+        mike_cats = greetings.get('mike', {})
+        for cat_greetings in mike_cats.values():
+            pool.extend(cat_greetings)
+
+    # Always add generic greetings
+    generic_cats = greetings.get('generic', {})
+    for cat_greetings in generic_cats.values():
+        pool.extend(cat_greetings)
+
+    if not pool:
+        return jsonify({'greeting': 'What do you want?', 'category': 'fallback', 'user': user})
+
+    greeting = random.choice(pool)
+    return jsonify({'greeting': greeting, 'category': 'random', 'user': user})
+
+
+@greetings_bp.route('/api/greetings/add', methods=['POST'])
+def add_greeting():
+    """Agent can queue a contextual greeting for the next session start."""
+    body = request.get_json(silent=True) or {}
+    greeting = (body.get('greeting') or '').strip()
+    if not greeting:
+        return jsonify({'ok': False, 'error': 'Missing greeting'}), 400
+    if len(greeting) > 300:
+        return jsonify({'ok': False, 'error': 'Greeting too long (max 300 chars)'}), 400
+
+    data = _load()
+    contextual = data['greetings'].get('contextual', [])
+    contextual.append(greeting)
+    data['greetings']['contextual'] = contextual[-20:]  # keep last 20
+    _save(data)
+    logger.info(f'Contextual greeting added: {greeting[:80]}')
+    return jsonify({'ok': True, 'total_contextual': len(data['greetings']['contextual'])})

package/routes/icons.py

@@ -0,0 +1,279 @@
+"""
+Icon library & AI icon generation.
+
+Static icons:
+  GET /api/icons/library                        → list all icon names
+  GET /api/icons/library/search?q=<term>        → search icons by name
+  GET /api/icons/library/<name>.svg             → serve a Lucide SVG icon
+
+Generated icons:
+  POST /api/icons/generate                      → generate icon via Gemini
+  GET  /api/icons/generated                     → list user's generated icons
+  GET  /api/icons/generated/<filename>          → serve a generated icon
+"""
+
+import os
+import re
+import json
+import base64
+import hashlib
+import time
+from pathlib import Path
+
+import requests
+from flask import Blueprint, jsonify, request, send_file, Response
+
+from services.paths import RUNTIME_DIR
+
+icons_bp = Blueprint('icons', __name__)
+
+# ── Static icon library (Lucide SVGs, shared across all clients) ──
+LUCIDE_DIR = Path('/mnt/system/base/icons/lucide')
+
+# ── Per-user generated icons ──
+GENERATED_DIR = RUNTIME_DIR / 'icons' / 'generated'
+
+# ── Gemini config ──
+GEMINI_API_KEY = os.getenv('GEMINI_API_KEY', '')
+GEMINI_MODEL = 'gemini-2.5-flash-image'
+GEMINI_URL = f'https://generativelanguage.googleapis.com/v1beta/models/{GEMINI_MODEL}:generateContent'
+
+# Cache icon list (rebuilt on first request)
+_icon_list_cache = None
+
+
+def _get_icon_list():
+    """Get sorted list of all Lucide icon names."""
+    global _icon_list_cache
+    if _icon_list_cache is None:
+        if LUCIDE_DIR.exists():
+            _icon_list_cache = sorted(
+                p.stem for p in LUCIDE_DIR.glob('*.svg')
+            )
+        else:
+            _icon_list_cache = []
+    return _icon_list_cache
+
+
+def _ensure_generated_dir():
+    """Create per-user generated icons directory."""
+    GENERATED_DIR.mkdir(parents=True, exist_ok=True)
+    return GENERATED_DIR
+
+
+# ══════════════════════════════════════════════════════════════
+#  STATIC ICON LIBRARY
+# ══════════════════════════════════════════════════════════════
+
+@icons_bp.route('/api/icons/library')
+def list_icons():
+    """List all available icon names."""
+    icons = _get_icon_list()
+    return jsonify({
+        'count': len(icons),
+        'icons': icons,
+    })
+
+
+@icons_bp.route('/api/icons/library/search')
+def search_icons():
+    """Search icons by name. ?q=folder&limit=20"""
+    q = request.args.get('q', '').lower().strip()
+    limit = min(int(request.args.get('limit', 50)), 200)
+
+    if not q:
+        return jsonify({'error': 'Missing ?q= parameter'}), 400
+
+    icons = _get_icon_list()
+    # Exact prefix matches first, then contains
+    prefix = [n for n in icons if n.startswith(q)]
+    contains = [n for n in icons if q in n and n not in prefix]
+    results = (prefix + contains)[:limit]
+
+    return jsonify({
+        'query': q,
+        'count': len(results),
+        'icons': results,
+    })
+
+
+@icons_bp.route('/api/icons/library/<name>.svg')
+def serve_icon(name):
+    """Serve a Lucide SVG icon by name."""
+    # Sanitize name
+    safe = re.sub(r'[^a-z0-9\-]', '', name.lower())
+    path = LUCIDE_DIR / f'{safe}.svg'
+
+    if not path.exists():
+        return Response('<!-- icon not found -->', status=404, mimetype='image/svg+xml')
+
+    return send_file(str(path), mimetype='image/svg+xml',
+                     max_age=86400)  # cache 1 day
+
+
+# ══════════════════════════════════════════════════════════════
+#  AI ICON GENERATION (Gemini)
+# ══════════════════════════════════════════════════════════════
+
+@icons_bp.route('/api/icons/generate', methods=['POST'])
+def generate_icon():
+    """
+    Generate a custom icon via Gemini image generation.
+
+    POST body:
+      { "prompt": "description of icon",
+        "name": "optional-filename-slug",
+        "style": "optional style override" }
+
+    Returns:
+      { "url": "/api/icons/generated/my-icon.png",
+        "name": "my-icon",
+        "prompt": "..." }
+    """
+    if not GEMINI_API_KEY:
+        return jsonify({'error': 'GEMINI_API_KEY not configured'}), 500
+
+    data = request.get_json(silent=True) or {}
+    user_prompt = data.get('prompt', '').strip()
+    if not user_prompt:
+        return jsonify({'error': 'Missing "prompt" field'}), 400
+
+    name_slug = data.get('name', '').strip()
+    style = data.get('style', '').strip()
+
+    # Build the generation prompt
+    style_instruction = style or (
+        'Windows XP style icon, clean vector art, vibrant colors, '
+        'slight 3D shading, white or transparent background'
+    )
+    full_prompt = (
+        f'Generate a single app icon: {user_prompt}. '
+        f'Style: {style_instruction}. '
+        f'The icon should be simple, recognizable at 48x48 pixels, centered on the canvas, '
+        f'with no text or labels. Square aspect ratio. Professional quality.'
+    )
+
+    # Generate filename
+    if not name_slug:
+        # Derive from prompt
+        name_slug = re.sub(r'[^a-z0-9]+', '-', user_prompt.lower())[:40].strip('-')
+    safe_name = re.sub(r'[^a-z0-9\-]', '', name_slug)
+    if not safe_name:
+        safe_name = 'icon-' + hashlib.md5(user_prompt.encode()).hexdigest()[:8]
+
+    # Call Gemini API
+    try:
+        resp = requests.post(
+            f'{GEMINI_URL}?key={GEMINI_API_KEY}',
+            json={
+                'contents': [{'parts': [{'text': full_prompt}]}],
+                'generationConfig': {
+                    'responseModalities': ['IMAGE', 'TEXT'],
+                },
+            },
+            timeout=60,
+        )
+        resp.raise_for_status()
+        result = resp.json()
+    except requests.RequestException as e:
+        return jsonify({'error': f'Gemini API error: {str(e)}'}), 502
+
+    # Extract image from response
+    image_data = None
+    mime_type = 'image/png'
+    try:
+        for candidate in result.get('candidates', []):
+            for part in candidate.get('content', {}).get('parts', []):
+                if 'inlineData' in part:
+                    image_data = base64.b64decode(part['inlineData']['data'])
+                    mime_type = part['inlineData'].get('mimeType', 'image/png')
+                    break
+            if image_data:
+                break
+    except (KeyError, TypeError):
+        pass
+
+    if not image_data:
+        return jsonify({
+            'error': 'Gemini did not return an image',
+            'raw': result.get('candidates', [{}])[0].get('content', {}).get('parts', []),
+        }), 502
+
+    # Determine extension
+    ext = '.png'
+    if 'jpeg' in mime_type:
+        ext = '.jpg'
+    elif 'webp' in mime_type:
+        ext = '.webp'
+
+    # Save to server immediately (NEVER lose generated content)
+    out_dir = _ensure_generated_dir()
+    filename = f'{safe_name}{ext}'
+    out_path = out_dir / filename
+
+    # Don't overwrite — add timestamp suffix
+    if out_path.exists():
+        filename = f'{safe_name}-{int(time.time())}{ext}'
+        out_path = out_dir / filename
+
+    out_path.write_bytes(image_data)
+
+    # Save metadata alongside
+    meta_path = out_dir / f'{filename}.meta.json'
+    meta_path.write_text(json.dumps({
+        'prompt': user_prompt,
+        'full_prompt': full_prompt,
+        'style': style_instruction,
+        'generated_at': time.strftime('%Y-%m-%d %H:%M:%S'),
+        'size': len(image_data),
+        'mime': mime_type,
+    }, indent=2))
+
+    url = f'/api/icons/generated/{filename}'
+
+    return jsonify({
+        'url': url,
+        'name': safe_name,
+        'filename': filename,
+        'prompt': user_prompt,
+        'size': len(image_data),
+    })
+
+
+# ══════════════════════════════════════════════════════════════
+#  GENERATED ICONS — LIST & SERVE
+# ══════════════════════════════════════════════════════════════
+
+@icons_bp.route('/api/icons/generated')
+def list_generated():
+    """List user's generated icons."""
+    out_dir = _ensure_generated_dir()
+    icons = []
+    for p in sorted(out_dir.iterdir(), key=lambda x: x.stat().st_mtime, reverse=True):
+        if p.suffix in ('.png', '.jpg', '.jpeg', '.webp') and not p.name.endswith('.meta.json'):
+            meta = {}
+            meta_path = out_dir / f'{p.name}.meta.json'
+            if meta_path.exists():
+                try:
+                    meta = json.loads(meta_path.read_text())
+                except Exception:
+                    pass
+            icons.append({
+                'name': p.stem,
+                'filename': p.name,
+                'url': f'/api/icons/generated/{p.name}',
+                'size': p.stat().st_size,
+                'prompt': meta.get('prompt', ''),
+                'generated_at': meta.get('generated_at', ''),
+            })
+    return jsonify({'count': len(icons), 'icons': icons})
+
+
+@icons_bp.route('/api/icons/generated/<filename>')
+def serve_generated(filename):
+    """Serve a generated icon."""
+    safe = re.sub(r'[^\w.\-]', '', filename)
+    path = _ensure_generated_dir() / safe
+    if not path.exists():
+        return jsonify({'error': 'Not found'}), 404
+    return send_file(str(path), max_age=3600)