openclaw-agent-dashboard 1.0.39 → 1.0.41

package/dashboard/core/fallback_manager.py

@@ -0,0 +1,81 @@
+"""
+REQ_003-SPEC-04：按错误类型注册集中降级策略；供状态计算、列表聚合等路径调用。
+
+REQ_003-AC-003：网络/IO 等错误在重试仍失败或未覆盖时，可读 StatusCache 中的最近状态（见 status_cache.get_stale_fallback）。
+"""
+from __future__ import annotations
+
+import threading
+from typing import Any, Callable, Dict, Optional
+
+_Handler = Callable[..., Any]
+
+_lock = threading.Lock()
+_handlers: Dict[str, _Handler] = {}
+_defaults_registered = False
+
+
+def register_fallback(error_category: str, handler: _Handler) -> None:
+    """注册某 classify_exception 类别对应的降级函数；handler 签名为 (agent_id=None, **kwargs) -> Any。"""
+    with _lock:
+        _handlers[error_category] = handler
+
+
+def run_fallback(error_category: str, *, agent_id: Optional[str] = None, **kwargs: Any) -> Any:
+    """按类别执行已注册降级；无匹配则返回 None。"""
+    _ensure_default_fallbacks()
+    with _lock:
+        h = _handlers.get(error_category)
+    if h is None:
+        return None
+    # NFR-R-005: Record fallback attempt (success if returns non-None)
+    try:
+        result = h(agent_id=agent_id, **kwargs)
+        from core.error_handler import record_fallback_attempt
+
+        record_fallback_attempt(success=result is not None)
+        return result
+    except Exception:
+        from core.error_handler import record_fallback_attempt
+
+        record_fallback_attempt(success=False)
+        raise
+
+
+def _stale_agent_status_handler(agent_id: Optional[str] = None, **_: Any) -> Optional[str]:
+    if not agent_id:
+        return None
+    from core.config_fortify import get_fortify_config
+
+    if not get_fortify_config().fallback_cache_on_io:
+        return None
+    from status.status_cache import get_cache
+
+    row = get_cache().get_stale_fallback(agent_id)
+    if not row:
+        return None
+    s = row.get("status")
+    if s in ("idle", "working", "down"):
+        return str(s)
+    return None
+
+
+def _ensure_default_fallbacks() -> None:
+    global _defaults_registered
+    if _defaults_registered:
+        return
+    with _lock:
+        if _defaults_registered:
+            return
+        for cat in ("network", "io-error", "timeout", "permission-error"):
+            if cat not in _handlers:
+                _handlers[cat] = _stale_agent_status_handler
+        _defaults_registered = True
+
+
+def reset_fallback_handlers_for_tests() -> None:
+    """单测隔离：清空注册表并允许重新挂载默认处理器。"""
+    global _handlers, _defaults_registered
+    with _lock:
+        _handlers.clear()
+        _defaults_registered = False

package/dashboard/core/logging_config.py

@@ -0,0 +1,217 @@
+"""
+NFR-S-003: Logging storage security configuration.
+
+Provides secure logging setup with:
+- File rotation (size-based)
+- Compression of rotated files
+- Automatic cleanup of old logs based on retention policy
+- File permission hardening
+
+Usage:
+    from core.logging_config import setup_secure_logging
+    setup_secure_logging()
+
+Configuration via environment variables:
+    OPENCLAW_LOG_RETENTION_DAYS: Days to retain log files (default: 30)
+    OPENCLAW_LOG_MAX_SIZE_MB: Max size per log file in MB (default: 100)
+    OPENCLAW_LOG_BACKUP_COUNT: Number of backup files to keep (default: 5)
+    OPENCLAW_LOG_FILE_PATH: Custom log file path (optional)
+    OPENCLAW_LOG_COMPRESSION: Compress rotated logs (default: true)
+"""
+from __future__ import annotations
+
+import logging
+import logging.handlers
+import os
+import sys
+from pathlib import Path
+from typing import Optional
+
+from core.config_fortify import get_fortify_config
+
+
+def get_log_file_path() -> Optional[Path]:
+    """Determine the log file path based on configuration."""
+    cfg = get_fortify_config()
+    if cfg.log_file_path:
+        return Path(cfg.log_file_path)
+
+    # Default path: logs/openclaw.log in project root
+    project_root = Path(__file__).parent.parent.parent
+    log_dir = project_root / "logs"
+    return log_dir / "openclaw.log"
+
+
+def ensure_log_directory(log_path: Path) -> None:
+    """Ensure log directory exists with proper permissions."""
+    log_dir = log_path.parent
+    log_dir.mkdir(parents=True, exist_ok=True)
+
+    # Set directory permissions to 0o750 (owner rwx, group r-x, others none)
+    # Note: This may fail on Windows or if running as non-owner
+    try:
+        os.chmod(log_dir, 0o750)
+    except (OSError, PermissionError):
+        pass  # Skip on platforms that don't support chmod
+
+
+def setup_secure_logging() -> None:
+    """
+    Configure secure logging with rotation, compression, and retention.
+
+    This sets up handlers for all openclaw.* loggers:
+    - Console handler for development
+    - Rotating file handler with compression for production
+    """
+    cfg = get_fortify_config()
+    log_path = get_log_file_path()
+
+    if log_path is None:
+        # No file logging, just console
+        return
+
+    ensure_log_directory(log_path)
+
+    # Determine which loggers to configure
+    logger_names = ["openclaw", "openclaw.fortify", "openclaw.fortify.watcher",
+                    "openclaw.fortify.audit", "openclaw.fortify.cache_probe"]
+
+    # Create rotating file handler
+    max_bytes = cfg.log_max_size_mb * 1024 * 1024
+    backup_count = cfg.log_backup_count
+
+    # Base rotating handler
+    if cfg.log_compression:
+        # Use custom rotating handler with gzip compression
+        handler: logging.Handler = _CompressedRotatingFileHandler(
+            filename=str(log_path),
+            maxBytes=max_bytes,
+            backupCount=backup_count,
+            encoding="utf-8",
+        )
+    else:
+        handler = logging.handlers.RotatingFileHandler(
+            filename=str(log_path),
+            maxBytes=max_bytes,
+            backupCount=backup_count,
+            encoding="utf-8",
+        )
+
+    # Set file permissions (owner read/write only)
+    try:
+        os.chmod(log_path, 0o600)
+    except (OSError, PermissionError):
+        pass
+
+    formatter = logging.Formatter(
+        fmt="%(asctime)s | %(levelname)-8s | %(name)s | %(message)s",
+        datefmt="%Y-%m-%dT%H:%M:%S%z",
+    )
+    handler.setFormatter(formatter)
+
+    # Apply to all relevant loggers
+    for logger_name in logger_names:
+        logger = logging.getLogger(logger_name)
+        # Avoid duplicate handlers
+        if not any(isinstance(h, (logging.handlers.RotatingFileHandler, _CompressedRotatingFileHandler))
+                   for h in logger.handlers):
+            logger.addHandler(handler)
+
+    # Set levels based on config
+    level = getattr(logging, cfg.error_log_level, logging.INFO)
+    for logger_name in logger_names:
+        logging.getLogger(logger_name).setLevel(level)
+
+    # Schedule cleanup of old logs (best-effort)
+    _schedule_log_cleanup(log_path, cfg.log_retention_days)
+
+
+class _CompressedRotatingFileHandler(logging.handlers.RotatingFileHandler):
+    """
+    Rotating file handler that compresses old log files using gzip.
+
+    Rotated files are renamed to <filename>.1.gz, <filename>.2.gz, etc.
+    """
+
+    def __init__(self, filename: str, maxBytes: int = 0, backupCount: int = 0,
+                 encoding: str = "utf-8", compress: bool = True):
+        super().__init__(filename, maxBytes=maxBytes, backupCount=backupCount, encoding=encoding)
+        self._compress = compress
+
+    def rotate(self, source: str, dest: str) -> None:
+        """Compress the rotated file."""
+        super().rotate(source, dest)
+
+        if self._compress and os.path.exists(dest):
+            try:
+                import gzip
+                with open(dest, "rb") as f_in:
+                    with gzip.open(dest + ".gz", "wb", compresslevel=6) as f_out:
+                        f_out.writelines(f_in)
+                os.remove(dest)
+            except Exception:
+                # Compression failed, keep uncompressed file
+                pass
+
+    def shouldRollover(self, record: logging.LogRecord) -> int:
+        """Check if rollover should occur."""
+        if self.stream is None:
+            self.stream = self._open()
+
+        if self.maxBytes > 0:
+            msg = "%s\n" % self.format(record)
+            if self.stream.tell() + len(msg) >= self.maxBytes:
+                return 1
+        return 0
+
+
+def _schedule_log_cleanup(log_path: Path, retention_days: int) -> None:
+    """
+    Schedule cleanup of log files older than retention period.
+
+    This is a best-effort cleanup that runs on startup.
+    For production, use an external cron job or logrotate.
+    """
+    import time
+
+    def _cleanup():
+        try:
+            cutoff = time.time() - (retention_days * 86400)
+            log_dir = log_path.parent
+
+            for pattern in ["*.log*", "*.gz"]:
+                for file_path in log_dir.glob(pattern):
+                    if file_path.is_file() and file_path.stat().st_mtime < cutoff:
+                        try:
+                            file_path.unlink()
+                        except OSError:
+                            pass
+        except Exception:
+            pass  # Best-effort cleanup
+
+    # Run cleanup in background thread
+    import threading
+    t = threading.Thread(target=_cleanup, daemon=True)
+    t.start()
+
+
+def get_logging_config_summary() -> dict:
+    """Get a summary of the logging configuration for diagnostics."""
+    cfg = get_fortify_config()
+    log_path = get_log_file_path()
+
+    summary = {
+        "log_retention_days": cfg.log_retention_days,
+        "log_max_size_mb": cfg.log_max_size_mb,
+        "log_backup_count": cfg.log_backup_count,
+        "log_file_path": str(log_path) if log_path else None,
+        "log_compression": cfg.log_compression,
+        "log_directory_exists": log_path.parent.exists() if log_path else False,
+    }
+
+    if log_path and log_path.exists():
+        stat = log_path.stat()
+        summary["current_log_size_bytes"] = stat.st_size
+        summary["current_log_size_mb"] = round(stat.st_size / (1024 * 1024), 2)
+
+    return summary

package/dashboard/core/safe_api_error.py

@@ -0,0 +1,76 @@
+"""
+面向浏览器/API 客户端的错误文案脱敏（NFR-S-001）。
+服务端日志仍由 record_error 记录完整信息（默认不截断）。
+"""
+from __future__ import annotations
+
+import re
+from typing import Any, Dict
+
+
+def sanitize_client_error_text(raw: str, max_len: int = 1200) -> str:
+    """去除常见密钥/路径/邮箱形态，压缩长度；含 Traceback 时整段替换。"""
+    if not raw:
+        return "internal error"
+    s = raw.replace("\r", " ").replace("\n", " ")
+    if len(s) > max_len * 2:
+        s = s[: max_len * 2]
+    if "Traceback (most recent call last)" in raw or '\n  File "' in raw:
+        return "Internal error (details redacted; see server logs)"
+
+    s = re.sub(r"\bsk-[a-zA-Z0-9]{12,}\b", "sk-[REDACTED]", s, flags=re.I)
+    s = re.sub(r"\bxox[baprs]-[a-zA-Z0-9-]{10,}\b", "[slack-token]", s)
+    s = re.sub(r"Bearer\s+[a-zA-Z0-9._=+\/-]{12,}", "Bearer [REDACTED]", s, flags=re.I)
+    s = re.sub(
+        r"\bAKIA[0-9A-Z]{16}\b",
+        "AKIA[REDACTED]",
+        s,
+    )
+    s = re.sub(r"(?i)password\s*[=:]\s*[^\s,}\"]{2,}", "password=[REDACTED]", s)
+    s = re.sub(
+        r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b",
+        "[email]",
+        s,
+    )
+    s = re.sub(r"(?:/home/|/Users/)[^\s:]{1,80}/[^\s:]{0,200}", "[path]", s)
+    s = re.sub(r"[A-Za-z]:\\(?:[^\\\s]+\\){0,8}[^\s\\]{0,120}", "[path]", s)
+    s = re.sub(r"/[^\s:]{0,16}\.openclaw(?:/[^\s:]{0,160})?", "[openclaw-path]", s)
+
+    if len(s) > max_len:
+        s = s[:max_len] + "…"
+    return s
+
+
+def safe_client_string(message: str) -> str:
+    """JSON 响应体中的 error 等字符串字段脱敏。"""
+    from core.config_fortify import get_fortify_config
+
+    raw = message or ""
+    if not get_fortify_config().sanitize_api_errors:
+        return raw[:4000]
+    return sanitize_client_error_text(raw)
+
+
+def safe_api_error_detail(exc: BaseException) -> str:
+    """HTTP 500 等返回给客户端的 detail 字符串。"""
+    from core.config_fortify import get_fortify_config
+
+    raw = str(exc).strip() or type(exc).__name__
+    if not get_fortify_config().sanitize_api_errors:
+        return raw[:4000]
+    return sanitize_client_error_text(raw)
+
+
+def redact_framework_stats_for_client(data: Dict[str, Any]) -> Dict[str, Any]:
+    """为 /api/errors/stats 等接口脱敏 last_error.detail。"""
+    from core.config_fortify import get_fortify_config
+
+    if not get_fortify_config().sanitize_api_errors:
+        return data
+    out = dict(data)
+    le = out.get("last_error")
+    if isinstance(le, dict) and le.get("detail"):
+        le = dict(le)
+        le["detail"] = sanitize_client_error_text(str(le["detail"]))
+        out["last_error"] = le
+    return out

package/dashboard/core/schemas/__init__.py

@@ -0,0 +1,16 @@
+from core.schemas.base import SchemaValidator, ValidationResult
+from core.schemas.session_schema import (
+    session_envelope_schema,
+    session_message_schema,
+    sessions_index_schema,
+)
+from core.schemas.subagent_schema import subagent_runs_root_schema
+
+__all__ = [
+    "SchemaValidator",
+    "ValidationResult",
+    "session_envelope_schema",
+    "session_message_schema",
+    "sessions_index_schema",
+    "subagent_runs_root_schema",
+]

package/dashboard/core/schemas/base.py

@@ -0,0 +1,43 @@
+"""JSON Schema validation wrapper (jsonschema)."""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+import jsonschema
+from jsonschema import Draft202012Validator
+
+
+@dataclass
+class ValidationResult:
+    is_valid: bool
+    errors: List[str] = field(default_factory=list)
+
+    @property
+    def error_message(self) -> str:
+        return "; ".join(self.errors) if self.errors else ""
+
+
+class SchemaValidator:
+    """Validate dict data against a JSON Schema dict."""
+
+    def __init__(self, schema: Dict[str, Any], strict: bool = True):
+        self.schema = schema
+        self.strict = strict
+        self._validator = Draft202012Validator(schema)
+        self._last_errors: List[str] = []
+
+    def validate(self, data: Any) -> ValidationResult:
+        self._last_errors = []
+        if not isinstance(data, (dict, list)) and self.schema.get("type") == "object":
+            self._last_errors.append("expected object")
+            return ValidationResult(False, list(self._last_errors))
+        try:
+            self._validator.validate(data)
+            return ValidationResult(True, [])
+        except jsonschema.ValidationError as e:
+            self._last_errors.append(e.message)
+            return ValidationResult(False, list(self._last_errors))
+
+    def get_error_details(self) -> Dict[str, Any]:
+        return {"errors": list(self._last_errors)}

package/dashboard/core/schemas/session_schema.py

@@ -0,0 +1,40 @@
+"""JSON Schema fragments for session JSONL lines and sessions.json index."""
+
+# sessions.json 根对象：宽松，兼容 OpenClaw 多版本（仅约束为 object）
+sessions_index_schema: dict = {
+    "$id": "https://openclaw/schemas/sessions-index/v1",
+    "type": "object",
+    "additionalProperties": True,
+    "properties": {
+        "version": {},
+        "schema": {},
+        "entries": {"type": "object", "additionalProperties": True},
+    },
+}
+
+# Line envelope: {"type": "message", "message": {...}, ... }
+session_message_schema: dict = {
+    "$id": "https://openclaw/schemas/session-message/v1",
+    "type": "object",
+    "additionalProperties": True,
+    "properties": {
+        "role": {"type": "string"},
+        "content": {},
+        "timestamp": {"type": ["integer", "number"]},
+        "stopReason": {"type": "string"},
+        "errorMessage": {"type": "string"},
+    },
+    "required": ["role"],
+}
+
+session_envelope_schema: dict = {
+    "$id": "https://openclaw/schemas/session-envelope/v1",
+    "type": "object",
+    "additionalProperties": True,
+    "properties": {
+        "type": {"type": "string"},
+        "message": {"type": "object", "additionalProperties": True},
+        "timestamp": {},
+    },
+    "required": ["type"],
+}

package/dashboard/core/schemas/subagent_schema.py

@@ -0,0 +1,23 @@
+"""JSON Schema for subagents/runs.json root object."""
+
+subagent_runs_root_schema: dict = {
+    "$id": "https://openclaw/schemas/subagent-runs/v1",
+    "type": "object",
+    "additionalProperties": True,
+    "properties": {
+        "version": {"type": ["integer", "number"]},
+        "runs": {"type": "object", "additionalProperties": True},
+    },
+}
+
+subagent_run_record_schema: dict = {
+    "$id": "https://openclaw/schemas/subagent-run-record/v1",
+    "type": "object",
+    "additionalProperties": True,
+    "properties": {
+        "childSessionKey": {"type": "string"},
+        "requesterSessionKey": {"type": "string"},
+        "startedAt": {"type": ["integer", "number"]},
+        "endedAt": {},
+    },
+}

package/dashboard/data/agent_config_manager.py

@@ -8,7 +8,7 @@ import shutil
 from datetime import datetime
 
 from data.config_reader import get_openclaw_root, normalize_openclaw_agent_id, agent_ids_equal
-from data.session_reader import normalize_sessions_index
+from data.session_reader import normalize_sessions_index, _load_sessions_index_file
 
 
 def _backup_config() -> Optional[Path]:
@@ -230,9 +230,11 @@ def get_agent_full_info(agent_id: str) -> Dict[str, Any]:
 
     if session_file.exists():
         try:
-            with open(session_file, 'r', encoding='utf-8') as f:
-                sessions_data = json.load(f)
-            entries = normalize_sessions_index(sessions_data)
+            sessions_data = _load_sessions_index_file(session_file)
+            if not sessions_data:
+                entries = {}
+            else:
+                entries = normalize_sessions_index(sessions_data)
             if entries:
                 latest = max(entries.values(), key=lambda e: e.get('lastMessageAt', 0))
                 last_active = latest.get('lastMessageAt')

package/dashboard/data/chain_reader.py

@@ -28,6 +28,8 @@ from data.config_reader import get_openclaw_root
 
 def _get_agents_config() -> Dict[str, Any]:
     """获取 agents 配置"""
+    from core.error_handler import record_error
+
     config_file = get_openclaw_root() / "openclaw.json"
     if not config_file.exists():
         return {}
@@ -35,7 +37,8 @@ def _get_agents_config() -> Dict[str, Any]:
     try:
         with open(config_file, 'r', encoding='utf-8') as f:
             return json.load(f)
-    except:
+    except (json.JSONDecodeError, OSError) as e:
+        record_error("parsing-error", str(e), "chain_reader:openclaw.json", exc=e)
         return {}
 
 
@@ -68,16 +71,15 @@ def _parse_session_key(session_key: str) -> Dict[str, str]:
 
 
 def _load_runs() -> Dict[str, Any]:
-    """加载 runs.json"""
-    runs_file = get_openclaw_root() / "subagents" / "runs.json"
-    if not runs_file.exists():
-        return {"version": 2, "runs": {}}
+    """加载 runs.json（经 subagent_reader：schema + record_error 与全仓读路径一致）"""
+    from data.subagent_reader import load_subagent_runs
 
-    try:
-        with open(runs_file, 'r', encoding='utf-8') as f:
-            return json.load(f)
-    except:
-        return {"version": 2, "runs": {}}
+    runs: Dict[str, Any] = {}
+    for rec in load_subagent_runs():
+        rid = rec.get("runId") or ""
+        if rid:
+            runs[rid] = rec
+    return {"version": 2, "runs": runs}
 
 
 def _get_workflow_state(project_id: str) -> Dict[str, Any]:
@@ -88,13 +90,15 @@ def _get_workflow_state(project_id: str) -> Dict[str, Any]:
         Path.home() / "vrt-projects" / "projects" / project_id / ".staging" / "workflow_state.json",
     ]
 
+    from core.error_handler import record_error
+
     for path in possible_paths:
         if path.exists():
             try:
                 with open(path, 'r', encoding='utf-8') as f:
                     return json.load(f)
-            except:
-                pass
+            except (json.JSONDecodeError, OSError) as e:
+                record_error("parsing-error", str(e), "chain_reader:workflow_state", exc=e)
 
     return {}
 

package/dashboard/data/error_analyzer.py

@@ -1,7 +1,6 @@
 """
 错误分析器 - 分析 Agent 执行错误，追溯根因
 """
-import json
 import os
 import re
 from pathlib import Path
@@ -32,6 +31,7 @@ class ErrorSeverity(Enum):
 
 
 from data.config_reader import get_openclaw_root, normalize_openclaw_agent_id
+from utils.data_repair import parse_session_jsonl_line
 
 
 # 错误模式匹配规则
@@ -187,11 +187,13 @@ def parse_session_for_errors(session_path: Path) -> List[Dict[str, Any]]:
             turn_index = 0
             for line in f:
                 try:
-                    data = json.loads(line.strip())
-                    if data.get('type') != 'message':
+                    envelope, msg = parse_session_jsonl_line(line)
+                    if (
+                        envelope is None
+                        or envelope.get('type') != 'message'
+                        or msg is None
+                    ):
                         continue
-
-                    msg = data.get('message', {})
                     role = msg.get('role')
                     timestamp = msg.get('timestamp')
                     stop_reason = msg.get('stopReason')
@@ -254,7 +256,7 @@ def parse_session_for_errors(session_path: Path) -> List[Dict[str, Any]]:
 
                     turn_index += 1
 
-                except (json.JSONDecodeError, KeyError):
+                except (KeyError, TypeError, AttributeError):
                     continue
 
     except Exception as e:
@@ -279,11 +281,13 @@ def get_tool_call_chain(session_path: Path, before_turn: int, limit: int = 10) -
                     if turn_index >= before_turn:
                         break
 
-                    data = json.loads(line.strip())
-                    if data.get('type') != 'message':
+                    envelope, msg = parse_session_jsonl_line(line)
+                    if (
+                        envelope is None
+                        or envelope.get('type') != 'message'
+                        or msg is None
+                    ):
                         continue
-
-                    msg = data.get('message', {})
                     role = msg.get('role')
                     timestamp = msg.get('timestamp')
 
@@ -301,7 +305,7 @@ def get_tool_call_chain(session_path: Path, before_turn: int, limit: int = 10) -
 
                     turn_index += 1
 
-                except (json.JSONDecodeError, KeyError):
+                except (KeyError, TypeError, AttributeError):
                     continue
 
     except Exception: