@0dai-dev/cli 4.3.6 → 4.3.8

package/lib/python/json_utils.py

@@ -0,0 +1,193 @@
+"""Shared JSON/JSONL file utilities for 0dai scripts.
+
+Consolidates duplicated _load_json/_save_json/_load_jsonl patterns.
+"""
+from __future__ import annotations
+
+import fcntl
+import gzip
+import json
+import logging
+import os
+import pathlib
+import shutil
+import threading
+import time
+from collections.abc import Iterator
+from typing import Any, Callable
+
+log = logging.getLogger("0dai.json_utils")
+
+_locks: dict[pathlib.Path, threading.Lock] = {}
+_locks_mutex = threading.Lock()
+
+
+def _get_lock(path: pathlib.Path) -> threading.Lock:
+    """Get or create a threading.Lock for a specific file path."""
+    abs_path = path.resolve()
+    with _locks_mutex:
+        if abs_path not in _locks:
+            _locks[abs_path] = threading.Lock()
+        return _locks[abs_path]
+
+
+def load_json(path: pathlib.Path) -> dict:
+    """Load JSON file, return empty dict on missing.
+    Backs up corrupted files before returning {} on invalid JSON.
+    """
+    if not path.is_file():
+        return {}
+
+    try:
+        return json.loads(path.read_text("utf-8"))
+    except json.JSONDecodeError as e:
+        ts = int(time.time())
+        backup = path.with_suffix(f".corrupt-{ts}.json")
+        try:
+            shutil.copy2(path, backup)
+            log.error("load_json: corrupted %s backed up to %s: %s", path, backup, e)
+        except OSError as backup_err:
+            log.critical("load_json: %s corrupted AND backup failed: %s + %s", path, e, backup_err)
+            raise
+        return {}
+    except OSError as e:
+        log.warning("load_json: cannot read %s: %s", path, e)
+        return {}
+
+
+def save_json(path: pathlib.Path, data: dict) -> None:
+    """Atomic JSON write via temp file rename."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(".tmp")
+    tmp.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n", "utf-8")
+    tmp.rename(path)
+
+
+def update_json(path: pathlib.Path, mutator_fn: Callable[[dict[str, Any]], None]) -> dict[str, Any]:
+    """Atomically load, mutate, and save a JSON file under a path-based lock.
+
+    The mutator_fn receives the loaded dict and should modify it in-place.
+    Returns the updated dictionary.
+    """
+    lock = _get_lock(path)
+    with lock:
+        data = load_json(path)
+        mutator_fn(data)
+        save_json(path, data)
+        return data
+
+
+def load_jsonl(path: pathlib.Path) -> list[dict]:
+    """Load JSONL file, skip invalid lines."""
+    if not path.is_file():
+        return []
+    entries = []
+    for line in path.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if line:
+            try:
+                entries.append(json.loads(line))
+            except json.JSONDecodeError:
+                pass
+    return entries
+
+
+def _loads_jsonl_line(raw: bytes) -> dict | None:
+    line = raw.strip()
+    if not line:
+        return None
+    try:
+        value = json.loads(line.decode("utf-8", errors="replace"))
+    except (UnicodeDecodeError, json.JSONDecodeError):
+        return None
+    return value if isinstance(value, dict) else None
+
+
+def iter_jsonl_reverse(path: pathlib.Path, *, chunk_size: int = 64 * 1024) -> Iterator[dict]:
+    """Yield valid JSONL dicts from newest to oldest without loading the full file.
+
+    Large operational logs such as swarm ledgers grow continuously. Status and
+    health probes usually need recent rows only; this keeps those hot paths
+    bounded by tail size instead of total file size.
+    """
+    if chunk_size <= 0:
+        raise ValueError("chunk_size must be positive")
+    if not path.is_file():
+        return
+    try:
+        with path.open("rb") as fh:
+            fh.seek(0, os.SEEK_END)
+            pos = fh.tell()
+            pending = b""
+            while pos > 0:
+                read_size = min(chunk_size, pos)
+                pos -= read_size
+                fh.seek(pos)
+                chunk = fh.read(read_size)
+                parts = (chunk + pending).split(b"\n")
+                pending = parts[0]
+                for raw in reversed(parts[1:]):
+                    parsed = _loads_jsonl_line(raw)
+                    if parsed is not None:
+                        yield parsed
+            parsed = _loads_jsonl_line(pending)
+            if parsed is not None:
+                yield parsed
+    except (OSError, PermissionError):
+        return
+
+
+def tail_jsonl(path: pathlib.Path, max_lines: int) -> list[dict]:
+    """Return up to max_lines valid JSONL dicts in chronological order."""
+    if max_lines <= 0:
+        return []
+    rows: list[dict] = []
+    for row in iter_jsonl_reverse(path):
+        rows.append(row)
+        if len(rows) >= max_lines:
+            break
+    return list(reversed(rows))
+
+
+def append_jsonl(path: pathlib.Path, entry: dict) -> None:
+    """Append one JSON line to a JSONL file (flock-safe for concurrent agents)."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    line = json.dumps(entry, ensure_ascii=False) + "\n"
+    with open(path, "a", encoding="utf-8") as f:
+        fcntl.flock(f, fcntl.LOCK_EX)
+        f.write(line)
+        fcntl.flock(f, fcntl.LOCK_UN)
+
+
+def rotate_jsonl(path: pathlib.Path, max_lines: int = 10_000) -> pathlib.Path | None:
+    """Rotate JSONL file when it exceeds max_lines.
+
+    Archives old content to {stem}-YYYY-MM.jsonl.gz and keeps only
+    the most recent max_lines // 2 entries in the original file.
+    Returns the archive path if rotated, None otherwise.
+    """
+    if not path.is_file():
+        return None
+    lines = path.read_text(encoding="utf-8").splitlines()
+    if len(lines) <= max_lines:
+        return None
+
+    # Archive all but the most recent half
+    keep = max_lines // 2
+    archive_lines = lines[:-keep]
+    keep_lines = lines[-keep:]
+
+    month = time.strftime("%Y-%m", time.gmtime())
+    archive_name = f"{path.stem}-{month}.jsonl.gz"
+    archive_path = path.parent / archive_name
+
+    # Append to existing archive (monthly) or create new
+    with gzip.open(archive_path, "at", encoding="utf-8") as gz:
+        gz.write("\n".join(archive_lines) + "\n")
+
+    # Rewrite current file with recent entries only
+    tmp = path.with_suffix(".tmp")
+    tmp.write_text("\n".join(keep_lines) + "\n", "utf-8")
+    tmp.rename(path)
+
+    return archive_path

package/lib/python/mcp_exposure_check.py

@@ -0,0 +1,247 @@
+#!/usr/bin/env python3
+"""Check whether an agent runtime exposes the expected 0dai MCP tools."""
+from __future__ import annotations
+
+import argparse
+import importlib.util
+import json
+import os
+import pathlib
+import re
+import sys
+from typing import Any
+
+REPO_ROOT = pathlib.Path(
+    os.environ.get("ODAI_REPO_ROOT", str(pathlib.Path(__file__).resolve().parent.parent))
+)
+CONTRACT_PATH = pathlib.Path(
+    os.environ.get(
+        "ODAI_MCP_EXPOSURE_CONTRACT",
+        str(REPO_ROOT / "ai" / "manifest" / "mcp-exposure-contract.json"),
+    )
+)
+
+
+def _read_json(path: pathlib.Path, default: Any) -> Any:
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return default
+
+
+def load_contract(path: pathlib.Path | None = None) -> dict[str, Any]:
+    data = _read_json(path or CONTRACT_PATH, {})
+    return data if isinstance(data, dict) else {}
+
+
+def canonical_tool_name(name: str) -> str:
+    """Normalize tool names across MCP namespace formats."""
+    name = name.strip()
+    if not name:
+        return ""
+    if "." in name:
+        name = name.rsplit(".", 1)[-1]
+    if name.startswith("mcp__"):
+        parts = [part for part in name.split("__") if part]
+        if parts:
+            name = parts[-1]
+    return name.lstrip("_")
+
+
+def _as_list(value: Any) -> list[str]:
+    if isinstance(value, str):
+        return [value]
+    if isinstance(value, list):
+        return [str(item) for item in value if str(item)]
+    return []
+
+
+def _merge_agent_contract(contract: dict[str, Any], agent: str) -> dict[str, list[str]]:
+    agents = contract.get("agents", {})
+    if not isinstance(agents, dict):
+        return {"required": [], "recommended": []}
+    current = agents.get(agent) if isinstance(agents.get(agent), dict) else None
+    if current is None:
+        current = agents.get("default") if isinstance(agents.get("default"), dict) else {}
+    parent: dict[str, Any] = {}
+    inherits = current.get("inherits") if isinstance(current, dict) else ""
+    if isinstance(inherits, str) and isinstance(agents.get(inherits), dict):
+        parent = agents[inherits]
+    required = set(_as_list(parent.get("required"))) | set(_as_list(current.get("required")))
+    recommended = set(_as_list(parent.get("recommended"))) | set(_as_list(current.get("recommended")))
+    return {
+        "required": sorted(canonical_tool_name(item) for item in required if item),
+        "recommended": sorted(canonical_tool_name(item) for item in recommended if item),
+    }
+
+
+def _load_tier_tool_count() -> int | None:
+    scripts_dir = REPO_ROOT / "scripts"
+    config_path = scripts_dir / "mcp_tier_config.py"
+    if not config_path.is_file():
+        return None
+    module_name = f"_odai_mcp_exposure_tier_config_{abs(hash(config_path))}"
+    spec = importlib.util.spec_from_file_location(module_name, config_path)
+    if spec is None or spec.loader is None:
+        return None
+    old_path = list(sys.path)
+    sys.path = [str(scripts_dir), *[item for item in old_path if item != str(scripts_dir)]]
+    try:
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+    except Exception:  # noqa: BLE001 — tier config load failed
+        return None
+    finally:
+        sys.path = old_path
+    return len(set(module.FREE_TOOLS) | set(module.PRO_TOOLS))
+
+
+def parse_observed_tools(values: list[str] | None = None, observed_file: pathlib.Path | None = None) -> set[str]:
+    raw: list[str] = []
+    if values:
+        for value in values:
+            raw.extend([item for item in re.split(r"[\s,]+", value) if item])
+    env_value = os.environ.get("ODAI_EXPOSED_MCP_TOOLS", "")
+    if env_value:
+        raw.extend([item for item in re.split(r"[\s,]+", env_value) if item])
+    if observed_file:
+        data = _read_json(observed_file, [])
+        if isinstance(data, list):
+            raw.extend(str(item) for item in data)
+        elif isinstance(data, dict):
+            for key in ("tools", "tool_names", "observed_tools"):
+                raw.extend(_as_list(data.get(key)))
+    return {name for name in (canonical_tool_name(item) for item in raw) if name}
+
+
+def evaluate_exposure(
+    *,
+    agent: str = "default",
+    observed_tools: set[str] | None = None,
+    contract: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    if contract is None:
+        contract = load_contract()
+    expected = _merge_agent_contract(contract, agent)
+    observed = observed_tools or set()
+    required = set(expected["required"])
+    recommended = set(expected["recommended"])
+    agents = contract.get("agents")
+    contract_available = isinstance(agents, dict) and bool(agents)
+    contract_tool_count = int((contract.get("project_server") or {}).get("expected_tool_count") or 0)
+    expected_count = contract_tool_count
+    tier_count = _load_tier_tool_count()
+    if tier_count:
+        expected_count = tier_count
+
+    if not observed:
+        status = "unknown"
+        anomaly_type = ""
+        blocking = False
+    elif not contract_available:
+        status = "red"
+        anomaly_type = "tooling.mcp_contract_missing"
+        blocking = True
+    else:
+        missing_required = sorted(required - observed)
+        missing_recommended = sorted(recommended - observed)
+        if missing_required:
+            status = "red"
+            anomaly_type = "tooling.mcp_required_missing"
+            blocking = True
+        elif missing_recommended:
+            status = "yellow"
+            anomaly_type = "tooling.mcp_partial_exposure"
+            blocking = False
+        elif expected_count <= 0:
+            status = "yellow"
+            anomaly_type = "tooling.mcp_contract_count_missing"
+            blocking = False
+        else:
+            status = "green"
+            anomaly_type = ""
+            blocking = False
+
+    contract_tier_count_match = True
+    if tier_count is not None:
+        contract_tier_count_match = contract_tool_count == tier_count
+        if not contract_tier_count_match and status != "red":
+            status = "yellow"
+            anomaly_type = "tooling.mcp_contract_tier_drift"
+            blocking = False
+
+    result = {
+        "status": status,
+        "agent": agent,
+        "blocking": blocking,
+        "anomaly_type": anomaly_type,
+        "required": sorted(required),
+        "recommended": sorted(recommended),
+        "observed_count": len(observed),
+        "expected_tool_count": expected_count,
+        "missing_required": sorted(required - observed) if observed else [],
+        "missing_recommended": sorted(recommended - observed) if observed else [],
+        "observed_required": sorted(required & observed),
+        "observed_recommended": sorted(recommended & observed),
+        "contract_tool_count": contract_tool_count,
+        "contract_available": contract_available,
+    }
+    if tier_count is not None:
+        result["tier_config_tool_count"] = tier_count
+        result["contract_tier_count_match"] = contract_tier_count_match
+    return result
+
+
+def evaluate_from_env(agent: str = "") -> dict[str, Any]:
+    return evaluate_exposure(
+        agent=agent or os.environ.get("ODAI_AGENT_KIND") or os.environ.get("ODAI_AGENT") or "default",
+        observed_tools=parse_observed_tools(),
+    )
+
+
+def _emit_anomaly(result: dict[str, Any], *, dry_run: bool) -> dict[str, Any] | None:
+    anomaly_type = result.get("anomaly_type")
+    if not anomaly_type:
+        return None
+    try:
+        import anomaly_alert  # noqa: PLC0415
+    except ImportError:
+        return {"status": "failed", "reason": "anomaly_alert_import_failed"}
+    return anomaly_alert.emit_anomaly(
+        str(anomaly_type),
+        f"MCP exposure check {result['status']} for {result['agent']}",
+        source="mcp_exposure_check",
+        context={
+            "agent": result["agent"],
+            "missing_required": result.get("missing_required", []),
+            "missing_recommended": result.get("missing_recommended", []),
+            "observed_count": result.get("observed_count", 0),
+            "expected_tool_count": result.get("expected_tool_count", 0),
+        },
+        dry_run=dry_run,
+    )
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--agent", default="default")
+    parser.add_argument("--observed-tool", action="append", default=[])
+    parser.add_argument("--observed-file", type=pathlib.Path)
+    parser.add_argument("--emit-anomaly", action="store_true")
+    parser.add_argument("--dry-run", action="store_true")
+    parser.add_argument("--fail-red", action="store_true")
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _build_parser().parse_args(argv)
+    observed = parse_observed_tools(args.observed_tool, args.observed_file)
+    result = evaluate_exposure(agent=args.agent, observed_tools=observed)
+    if args.emit_anomaly:
+        result["emitted_anomaly"] = _emit_anomaly(result, dry_run=args.dry_run)
+    sys.stdout.write(json.dumps(result, ensure_ascii=True, sort_keys=True) + "\n")
+    return 2 if args.fail_red and result["status"] == "red" else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())