openvoiceui 1.0.0

package/services/__init__.py

@@ -0,0 +1 @@
+# services package

package/services/auth.py

@@ -0,0 +1,143 @@
+"""
+Clerk JWT authentication middleware for OpenVoiceUI.
+
+Verifies Clerk session tokens from:
+  1. Authorization: Bearer <token> header
+  2. __session cookie (set automatically by Clerk for browser requests)
+
+Usage:
+    from services.auth import verify_clerk_token, get_token_from_request
+
+    token = get_token_from_request()
+    user_id = verify_clerk_token(token)   # returns str or None
+"""
+import logging
+import os
+import time
+from functools import lru_cache
+from typing import Optional
+
+import jwt
+import requests
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Configuration
+# ---------------------------------------------------------------------------
+
+def _derive_clerk_domain(key: str) -> str:
+    """Derive the Clerk frontend domain from a publishable key (pk_live_/pk_test_)."""
+    import base64
+    try:
+        suffix = key.split('_', 2)[-1]
+        padding = (4 - len(suffix) % 4) % 4
+        decoded = base64.b64decode(suffix + '=' * padding).decode('utf-8').rstrip('$')
+        return decoded
+    except Exception:
+        return ''
+
+_raw_clerk_key = (os.getenv('CLERK_PUBLISHABLE_KEY') or os.getenv('VITE_CLERK_PUBLISHABLE_KEY', '')).strip()
+_CLERK_FRONTEND_DOMAIN = os.getenv('CLERK_FRONTEND_API') or (_derive_clerk_domain(_raw_clerk_key) if _raw_clerk_key else '')
+_JWKS_URL = f'https://{_CLERK_FRONTEND_DOMAIN}/.well-known/jwks.json'
+_JWKS_CACHE_TTL = 3600  # refresh keys every 60 minutes
+
+# Allowlist of Clerk user IDs permitted to access this deployment.
+# Set ALLOWED_USER_IDS=user_abc123,user_xyz789 in .env
+# If the env var is empty or unset, the check is SKIPPED (open to any valid Clerk user).
+# Always set this in production — agents have full access.
+_raw_allowed = os.getenv('ALLOWED_USER_IDS', '')
+_ALLOWED_USER_IDS: set[str] = {uid.strip() for uid in _raw_allowed.split(',') if uid.strip()}
+
+# ---------------------------------------------------------------------------
+# JWKS cache
+# ---------------------------------------------------------------------------
+
+_jwks_cache: dict = {'keys': None, 'fetched_at': 0}
+
+
+def _get_jwks() -> list:
+    """Return cached JWKS key list, refreshing if stale."""
+    now = time.time()
+    if _jwks_cache['keys'] is None or (now - _jwks_cache['fetched_at']) > _JWKS_CACHE_TTL:
+        try:
+            resp = requests.get(_JWKS_URL, timeout=10)
+            resp.raise_for_status()
+            _jwks_cache['keys'] = resp.json().get('keys', [])
+            _jwks_cache['fetched_at'] = now
+            logger.debug('JWKS refreshed (%d keys)', len(_jwks_cache['keys']))
+        except Exception as exc:
+            logger.error('Failed to fetch JWKS from %s: %s', _JWKS_URL, exc)
+            # Return stale keys if available
+            if _jwks_cache['keys']:
+                return _jwks_cache['keys']
+            return []
+    return _jwks_cache['keys']
+
+
+# ---------------------------------------------------------------------------
+# Token verification
+# ---------------------------------------------------------------------------
+
+def verify_clerk_token(token: str) -> Optional[str]:
+    """
+    Verify a Clerk JWT and return the user_id (sub claim) if valid.
+
+    Returns None if the token is missing, malformed, or invalid.
+    """
+    if not token:
+        return None
+
+    keys = _get_jwks()
+    if not keys:
+        logger.warning('No JWKS keys available — cannot verify token')
+        return None
+
+    for key_data in keys:
+        try:
+            public_key = jwt.algorithms.RSAAlgorithm.from_jwk(key_data)
+            payload = jwt.decode(
+                token,
+                public_key,
+                algorithms=['RS256'],
+                options={'verify_aud': False},  # Clerk tokens don't use aud in all configs
+            )
+            user_id = payload.get('sub')
+            if not user_id:
+                return None
+            # Log user_id on every successful auth so it can be captured for ALLOWED_USER_IDS
+            logger.info('Clerk auth: user_id=%s', user_id)
+            # Enforce allowlist if configured
+            if _ALLOWED_USER_IDS and user_id not in _ALLOWED_USER_IDS:
+                logger.warning('Clerk auth: user_id=%s not in ALLOWED_USER_IDS — access denied', user_id)
+                return None
+            return user_id
+        except jwt.ExpiredSignatureError:
+            logger.debug('Token expired')
+            return None
+        except jwt.InvalidTokenError:
+            continue  # try next key
+
+    logger.debug('Token did not validate against any JWKS key')
+    return None
+
+
+def get_token_from_request() -> Optional[str]:
+    """
+    Extract Clerk session token from the current Flask request.
+
+    Checks in order:
+      1. Authorization: Bearer <token>
+      2. __session cookie
+    """
+    from flask import request
+
+    auth_header = request.headers.get('Authorization', '')
+    if auth_header.startswith('Bearer '):
+        return auth_header[7:].strip()
+
+    cookie_token = request.cookies.get('__session')
+    if cookie_token:
+        return cookie_token
+
+    return None

package/services/canvas_versioning.py

@@ -0,0 +1,239 @@
+"""
+Canvas Page Versioning — automatic version history for canvas pages.
+
+Watches the canvas-pages directory for file changes and saves previous
+versions to a .versions/ subdirectory. This catches ALL writes regardless
+of source (agent tool, API, manual edit).
+
+Usage:
+    from services.canvas_versioning import start_version_watcher, list_versions, restore_version
+
+Version files are stored as:
+    canvas-pages/.versions/<page-stem>.<unix-timestamp>.html
+
+Auto-cleanup keeps the last MAX_VERSIONS_PER_PAGE versions per page.
+"""
+
+import hashlib
+import logging
+import os
+import shutil
+import threading
+import time
+from pathlib import Path
+
+from services.paths import CANVAS_PAGES_DIR
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Configuration
+# ---------------------------------------------------------------------------
+
+MAX_VERSIONS_PER_PAGE = 20          # Keep last N versions per page
+CHECK_INTERVAL_SECONDS = 15         # How often to scan for changes
+VERSIONS_DIRNAME = '.versions'      # Subdirectory name inside canvas-pages
+
+# ---------------------------------------------------------------------------
+# Internal state
+# ---------------------------------------------------------------------------
+
+_file_hashes: dict[str, str] = {}    # filename -> content hash
+_file_contents: dict[str, bytes] = {}  # filename -> last-known content (for saving)
+_watcher_thread: threading.Thread | None = None
+_stop_event = threading.Event()
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _versions_dir() -> Path:
+    """Return the .versions/ directory path, creating it if needed."""
+    vdir = CANVAS_PAGES_DIR / VERSIONS_DIRNAME
+    vdir.mkdir(parents=True, exist_ok=True)
+    return vdir
+
+
+def _content_hash(data: bytes) -> str:
+    """SHA-256 hash of file content."""
+    return hashlib.sha256(data).hexdigest()
+
+
+def _save_version(filename: str, old_content: bytes) -> Path | None:
+    """Save old_content as a timestamped version file."""
+    try:
+        stem = Path(filename).stem
+        timestamp = int(time.time())
+        version_name = f'{stem}.{timestamp}.html'
+        version_path = _versions_dir() / version_name
+        version_path.write_bytes(old_content)
+        logger.info(f'Canvas version saved: {version_name} ({len(old_content)} bytes)')
+        _cleanup_versions(stem)
+        return version_path
+    except Exception as exc:
+        logger.error(f'Failed to save canvas version for {filename}: {exc}')
+        return None
+
+
+def _cleanup_versions(page_stem: str) -> None:
+    """Keep only the latest MAX_VERSIONS_PER_PAGE versions for a page."""
+    vdir = _versions_dir()
+    versions = sorted(
+        vdir.glob(f'{page_stem}.*.html'),
+        key=lambda p: p.stat().st_mtime,
+        reverse=True,
+    )
+    for old_version in versions[MAX_VERSIONS_PER_PAGE:]:
+        try:
+            old_version.unlink()
+            logger.debug(f'Pruned old version: {old_version.name}')
+        except Exception:
+            pass
+
+
+def _initial_scan() -> None:
+    """Scan all existing pages and record their hashes (no versioning on startup)."""
+    if not CANVAS_PAGES_DIR.exists():
+        return
+    for page_path in CANVAS_PAGES_DIR.glob('*.html'):
+        try:
+            content = page_path.read_bytes()
+            _file_hashes[page_path.name] = _content_hash(content)
+            _file_contents[page_path.name] = content
+        except Exception as exc:
+            logger.debug(f'Initial scan: could not read {page_path.name}: {exc}')
+
+
+def _check_for_changes() -> None:
+    """Scan canvas-pages for modified files and save versions."""
+    if not CANVAS_PAGES_DIR.exists():
+        return
+
+    current_files = set()
+    for page_path in CANVAS_PAGES_DIR.glob('*.html'):
+        filename = page_path.name
+        current_files.add(filename)
+        try:
+            content = page_path.read_bytes()
+            new_hash = _content_hash(content)
+
+            if filename in _file_hashes:
+                if new_hash != _file_hashes[filename]:
+                    # File changed — save the OLD content as a version
+                    old_content = _file_contents.get(filename)
+                    if old_content:
+                        _save_version(filename, old_content)
+                    _file_hashes[filename] = new_hash
+                    _file_contents[filename] = content
+            else:
+                # New file — just record it (no previous version to save)
+                _file_hashes[filename] = new_hash
+                _file_contents[filename] = content
+
+        except Exception as exc:
+            logger.debug(f'Version check: could not read {filename}: {exc}')
+
+    # Note: we don't remove deleted files from tracking — they might come back
+
+
+# ---------------------------------------------------------------------------
+# Background watcher
+# ---------------------------------------------------------------------------
+
+def _watcher_loop() -> None:
+    """Background thread that periodically checks for file changes."""
+    logger.info(f'Canvas version watcher started (interval={CHECK_INTERVAL_SECONDS}s, max_versions={MAX_VERSIONS_PER_PAGE})')
+    _initial_scan()
+
+    while not _stop_event.is_set():
+        try:
+            _check_for_changes()
+        except Exception as exc:
+            logger.error(f'Canvas version watcher error: {exc}')
+        _stop_event.wait(CHECK_INTERVAL_SECONDS)
+
+    logger.info('Canvas version watcher stopped')
+
+
+def start_version_watcher() -> None:
+    """Start the background version watcher thread."""
+    global _watcher_thread
+    if _watcher_thread and _watcher_thread.is_alive():
+        logger.debug('Canvas version watcher already running')
+        return
+    _stop_event.clear()
+    _watcher_thread = threading.Thread(
+        target=_watcher_loop,
+        name='canvas-version-watcher',
+        daemon=True,
+    )
+    _watcher_thread.start()
+
+
+def stop_version_watcher() -> None:
+    """Stop the background version watcher thread."""
+    _stop_event.set()
+    if _watcher_thread:
+        _watcher_thread.join(timeout=5)
+
+
+# ---------------------------------------------------------------------------
+# Public API (used by canvas.py routes)
+# ---------------------------------------------------------------------------
+
+def list_versions(page_id: str) -> list[dict]:
+    """List all saved versions for a page, newest first."""
+    vdir = _versions_dir()
+    versions = []
+    for vpath in sorted(vdir.glob(f'{page_id}.*.html'), key=lambda p: p.stat().st_mtime, reverse=True):
+        # Parse timestamp from filename: page-id.1709510400.html
+        parts = vpath.stem.rsplit('.', 1)
+        if len(parts) == 2:
+            try:
+                ts = int(parts[1])
+            except ValueError:
+                ts = int(vpath.stat().st_mtime)
+        else:
+            ts = int(vpath.stat().st_mtime)
+
+        versions.append({
+            'filename': vpath.name,
+            'timestamp': ts,
+            'iso': time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(ts)),
+            'size': vpath.stat().st_size,
+        })
+    return versions
+
+
+def restore_version(page_id: str, timestamp: int) -> bool:
+    """Restore a specific version, saving the current as a new version first."""
+    version_file = _versions_dir() / f'{page_id}.{timestamp}.html'
+    current_file = CANVAS_PAGES_DIR / f'{page_id}.html'
+
+    if not version_file.exists():
+        logger.error(f'Version not found: {version_file}')
+        return False
+
+    # Save current as a version before restoring
+    if current_file.exists():
+        current_content = current_file.read_bytes()
+        _save_version(current_file.name, current_content)
+
+    # Restore the old version
+    restored_content = version_file.read_bytes()
+    current_file.write_bytes(restored_content)
+
+    # Update tracking
+    _file_hashes[current_file.name] = _content_hash(restored_content)
+    _file_contents[current_file.name] = restored_content
+
+    logger.info(f'Restored canvas page {page_id} to version {timestamp}')
+    return True
+
+
+def get_version_content(page_id: str, timestamp: int) -> bytes | None:
+    """Get the content of a specific version."""
+    version_file = _versions_dir() / f'{page_id}.{timestamp}.html'
+    if version_file.exists():
+        return version_file.read_bytes()
+    return None

package/services/db_pool.py

@@ -0,0 +1,107 @@
+# services/db_pool.py
+"""SQLite connection pool with WAL mode enabled.
+
+Recipe R2 — SQLite WAL + Connection Pool
+Fixes SQLite write contention under concurrent Flask request handling.
+
+Usage:
+    from services.db_pool import SQLitePool
+    db = SQLitePool("usage.db")
+
+    # Write (auto-commit, retries on busy)
+    db.execute("INSERT INTO conversation_log VALUES (?)", (message,))
+
+    # Read (concurrent reads via WAL)
+    results = db.query("SELECT * FROM conversation_log LIMIT 10")
+
+    # Manual connection (for row_factory etc.)
+    with db.get_connection() as conn:
+        conn.row_factory = sqlite3.Row
+        rows = conn.execute("SELECT * FROM foo").fetchall()
+"""
+import sqlite3
+import threading
+from contextlib import contextmanager
+from queue import Queue, Empty
+import time
+
+
+class SQLitePool:
+    """Thread-safe SQLite connection pool with WAL mode."""
+
+    def __init__(self, db_path: str, pool_size: int = 5):
+        self.db_path = str(db_path)
+        self.pool_size = pool_size
+        self._pool = Queue(maxsize=pool_size)
+        self._lock = threading.Lock()
+
+        # Initialize pool with WAL-enabled connections
+        for _ in range(pool_size):
+            self._pool.put(self._create_connection())
+
+    def _create_connection(self) -> sqlite3.Connection:
+        """Create a new connection with WAL mode and optimized settings."""
+        conn = sqlite3.connect(
+            self.db_path,
+            timeout=30.0,
+            check_same_thread=False,
+        )
+
+        # Enable WAL mode for concurrent reads without blocking writers
+        conn.execute("PRAGMA journal_mode=WAL")
+        # NORMAL is safe with WAL and much faster than FULL
+        conn.execute("PRAGMA synchronous=NORMAL")
+        # 64 MB page cache per connection
+        conn.execute("PRAGMA cache_size=-64000")
+        # 30 s busy timeout (belt-and-suspenders alongside pool timeout)
+        conn.execute("PRAGMA busy_timeout=30000")
+
+        return conn
+
+    @contextmanager
+    def get_connection(self):
+        """Get a connection from the pool (blocks up to 35 s before raising)."""
+        try:
+            conn = self._pool.get(timeout=35)
+        except Empty:
+            raise RuntimeError(
+                "SQLitePool: timed out waiting for a free connection. "
+                f"Pool size is {self.pool_size}."
+            )
+        try:
+            yield conn
+        finally:
+            self._pool.put(conn)
+
+    def execute(self, query: str, params: tuple = ()):
+        """Execute a write query with automatic retry on busy."""
+        max_retries = 3
+        for attempt in range(max_retries):
+            try:
+                with self.get_connection() as conn:
+                    cursor = conn.execute(query, params)
+                    conn.commit()
+                    return cursor
+            except sqlite3.OperationalError as e:
+                if "locked" in str(e) and attempt < max_retries - 1:
+                    time.sleep(0.1 * (attempt + 1))
+                    continue
+                raise
+
+    def query(self, query: str, params: tuple = ()) -> list:
+        """Execute a read query and return all rows."""
+        with self.get_connection() as conn:
+            cursor = conn.execute(query, params)
+            return cursor.fetchall()
+
+    def close(self):
+        """Drain the pool and close all connections."""
+        closed = 0
+        while not self._pool.empty():
+            try:
+                conn = self._pool.get_nowait()
+                conn.close()
+                closed += 1
+            except Empty:
+                break
+        return closed

package/services/gateway.py

@@ -0,0 +1,16 @@
+"""
+Backwards-compatibility shim.
+
+The gateway implementation has moved to:
+  services/gateways/openclaw.py   — OpenClaw persistent WS gateway
+  services/gateway_manager.py     — registry, plugin loader, router
+
+New code should import from gateway_manager:
+    from services.gateway_manager import gateway_manager
+
+Existing code that imports gateway_connection continues to work unchanged:
+    from services.gateway import gateway_connection
+"""
+from services.gateway_manager import gateway_manager as gateway_connection
+
+__all__ = ['gateway_connection']