@scemoon/cdh 1.0.8 → 1.0.9-beta.2

package/cdh/logging.py

@@ -0,0 +1,314 @@
+"""Structured JSON logging for CDH observability.
+
+Provides a third observability signal (Logs) alongside Traces and Metrics.
+Logs are written as NDJSON (newline-delimited JSON) to ~/.cdh/logs/app.jsonl
+and can be exported in OTLP format.
+
+Usage::
+
+    from cdh.logging import get_logger, LogLevel
+    log = get_logger("cdh.validate")
+    log.info("Running EARS checks", fr_id="WEB-FR-001", passed=True)
+    log.error("Validation failed", error="missing_frontmatter")
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import sys
+import time
+import uuid
+from datetime import datetime, timezone
+from enum import Enum
+from pathlib import Path
+from typing import Any
+
+__all__ = ["get_logger", "LogLevel", "JSONLogHandler", "export_logs"]
+
+DEFAULT_LOG_DIR = Path.home() / ".cdh" / "logs"
+DEFAULT_LOG_FILE = "app.jsonl"
+_OTEL_VERSION = "1.5.0"
+
+
+class LogLevel(Enum):
+    DEBUG = "DEBUG"
+    INFO = "INFO"
+    WARN = "WARN"
+    ERROR = "ERROR"
+    FATAL = "FATAL"
+
+    @property
+    def otel_severity(self) -> int:
+        """Convert to OTLP severity number."""
+        return {
+            "DEBUG": 5,
+            "INFO": 9,
+            "WARN": 13,
+            "ERROR": 17,
+            "FATAL": 21,
+        }[self.value]
+
+    @property
+    def python_level(self) -> int:
+        return {
+            "DEBUG": logging.DEBUG,
+            "INFO": logging.INFO,
+            "WARN": logging.WARNING,
+            "ERROR": logging.ERROR,
+            "FATAL": logging.CRITICAL,
+        }[self.value]
+
+
+_loggers: dict[str, "JSONLogger"] = {}
+_log_dir_initialized = False
+
+
+def _ensure_log_dir() -> Path:
+    global _log_dir_initialized
+    log_dir = Path(os.environ.get("CDH_LOG_DIR", str(DEFAULT_LOG_DIR)))
+    log_dir.mkdir(parents=True, exist_ok=True)
+    _log_dir_initialized = True
+    return log_dir
+
+
+class JSONLogHandler(logging.Handler):
+    """logging.Handler that emits NDJSON records to a file."""
+
+    def __init__(self, log_file: Path | None = None, rotation_hours: int = 24):
+        super().__init__()
+        self._log_file = log_file or (_ensure_log_dir() / DEFAULT_LOG_FILE)
+        self._rotation_hours = rotation_hours
+        self._last_rotation = time.time()
+        self._lock = __import__("threading").Lock()
+
+    def _maybe_rotate(self) -> None:
+        now = time.time()
+        if now - self._last_rotation > self._rotation_hours * 3600:
+            self._rotate()
+
+    def _rotate(self) -> None:
+        if not self._log_file.exists():
+            return
+        ts = datetime.fromtimestamp(self._last_rotation, tz=timezone.utc).strftime("%Y%m%dT%H%M%S")
+        rotated = self._log_file.parent / f"{self._log_file.stem}.{ts}.jsonl"
+        try:
+            self._log_file.rename(rotated)
+        except OSError:
+            pass
+        self._last_rotation = time.time()
+
+    def emit(self, record: logging.LogRecord) -> None:
+        try:
+            self._maybe_rotate()
+            log_entry = self._format_entry(record)
+            with self._lock:
+                with open(self._log_file, "a", encoding="utf-8") as fh:
+                    fh.write(json.dumps(log_entry, default=str) + "\n")
+        except Exception:
+            pass
+
+    def _format_entry(self, record: logging.LogRecord) -> dict[str, Any]:
+        ts_ns = int(record.created * 1e9)
+        trace_id = getattr(record, "cdh_trace_id", "") or ""
+        span_id = getattr(record, "cdh_span_id", "") or ""
+
+        entry: dict[str, Any] = {
+            "timeUnixNano": str(ts_ns),
+            "observedTimeUnixNano": str(int(time.time() * 1e9)),
+            "severityNumber": _OTEL_SEVERITY.get(record.levelno, 9),
+            "severityText": record.levelname,
+            "body": record.getMessage(),
+            "attributes": [
+                {"key": "logger.name", "value": {"stringValue": record.name}},
+                {"key": "process.pid", "value": {"intValue": str(record.process)}},
+                {"key": "thread.id", "value": {"intValue": str(record.thread)}},
+            ],
+        }
+        if trace_id:
+            entry["traceId"] = trace_id
+        if span_id:
+            entry["spanId"] = span_id
+
+        if record.exc_info:
+            entry["attributes"].append({
+                "key": "exception.type",
+                "value": {"stringValue": str(record.exc_info[0].__name__)},
+            })
+            entry["attributes"].append({
+                "key": "exception.message",
+                "value": {"stringValue": str(record.exc_info[1])},
+            })
+
+        if hasattr(record, "fr_id"):
+            entry["attributes"].append({"key": "cdh.fr_id", "value": {"stringValue": str(record.fr_id)}})
+        if hasattr(record, "session_id"):
+            entry["attributes"].append({"key": "cdh.session_id", "value": {"stringValue": str(record.session_id)}})
+        if hasattr(record, "tags"):
+            for k, v in (record.tags or {}).items():
+                entry["attributes"].append({f"cdh.tag.{k}": _otlp_value(v)})
+
+        return entry
+
+
+_OTEL_SEVERITY = {
+    logging.DEBUG: 5,
+    logging.INFO: 9,
+    logging.WARNING: 13,
+    logging.ERROR: 17,
+    logging.CRITICAL: 21,
+}
+
+
+def _otlp_value(v: Any) -> dict[str, Any]:
+    if v is None:
+        return {"stringValue": ""}
+    if isinstance(v, bool):
+        return {"boolValue": v}
+    if isinstance(v, int):
+        return {"intValue": str(v)}
+    if isinstance(v, float):
+        return {"doubleValue": v}
+    return {"stringValue": str(v)}
+
+
+class JSONLogger:
+    """Structured logger that writes NDJSON and supports trace correlation."""
+
+    def __init__(self, name: str, level: LogLevel = LogLevel.INFO):
+        self.name = name
+        self._level = level
+        self._logger = logging.getLogger(name)
+        self._logger.setLevel(level.python_level)
+        self._logger.propagate = False
+        if not self._logger.handlers:
+            self._logger.addHandler(JSONLogHandler())
+
+    def _log(self, level: LogLevel, msg: str, **kwargs: Any) -> None:
+        record = self._logger.makeRecord(
+            self.name,
+            level.python_level,
+            "(unknown)",
+            0,
+            msg,
+            (),
+            None,
+        )
+        for k, v in kwargs.items():
+            setattr(record, k, v)
+        self._logger.handle(record)
+
+    def debug(self, msg: str, **kwargs: Any) -> None:
+        self._log(LogLevel.DEBUG, msg, **kwargs)
+
+    def info(self, msg: str, **kwargs: Any) -> None:
+        self._log(LogLevel.INFO, msg, **kwargs)
+
+    def warn(self, msg: str, **kwargs: Any) -> None:
+        self._log(LogLevel.WARN, msg, **kwargs)
+
+    def error(self, msg: str, **kwargs: Any) -> None:
+        self._log(LogLevel.ERROR, msg, **kwargs)
+
+    def fatal(self, msg: str, **kwargs: Any) -> None:
+        self._log(LogLevel.FATAL, msg, **kwargs)
+
+
+def get_logger(name: str, level: LogLevel = LogLevel.INFO) -> JSONLogger:
+    """Get or create a structured JSON logger."""
+    if name not in _loggers:
+        _loggers[name] = JSONLogger(name, level)
+    return _loggers[name]
+
+
+def export_logs(
+    log_file: Path | None = None,
+    since: datetime | None = None,
+    session_id: str | None = None,
+    min_level: LogLevel = LogLevel.DEBUG,
+    endpoint: str | None = None,
+    dry_run: bool = False,
+) -> dict[str, int]:
+    """Export logs in OTLP format.
+
+    Returns counters: read, exported, failed.
+    """
+    counters = {"read": 0, "exported": 0, "failed": 0}
+    path = log_file or (_ensure_log_dir() / DEFAULT_LOG_FILE)
+
+    if not path.exists():
+        return counters
+
+    if endpoint is None:
+        endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT")
+
+    records: list[dict[str, Any]] = []
+    cutoff_ns = int(since.timestamp() * 1e9) if since else 0
+
+    with open(path, "r", encoding="utf-8", errors="replace") as fh:
+        for line in fh:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                record = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            ts_ns = int(record.get("timeUnixNano", 0))
+            if cutoff_ns and ts_ns < cutoff_ns:
+                continue
+            counters["read"] += 1
+            records.append(record)
+
+    if not records:
+        return counters
+
+    if dry_run or not endpoint:
+        counters["exported"] = len(records)
+        return counters
+
+    payload = _build_otlp_payload(records)
+    status, body = _http_post(endpoint, payload)
+    if 200 <= status < 300:
+        counters["exported"] = len(records)
+    else:
+        counters["failed"] = len(records)
+
+    return counters
+
+
+def _build_otlp_payload(records: list[dict[str, Any]]) -> bytes:
+    resource_logs = [{
+        "resource": {
+            "attributes": [
+                {"key": "service.name", "value": {"stringValue": "cdh"}},
+                {"key": "telemetry.sdk.language", "value": {"stringValue": "python"}},
+                {"key": "telemetry.sdk.version", "value": {"stringValue": _OTEL_VERSION}},
+            ]
+        },
+        "scopeLogs": [{
+            "scope": {"name": "cdh.logging", "version": "1.0.0"},
+            "logRecords": records,
+        }],
+    }]
+    return json.dumps(resource_logs).encode("utf-8")
+
+
+def _http_post(url: str, payload: bytes, timeout: float = 10.0) -> tuple[int, str]:
+    headers = {
+        "Content-Type": "application/json",
+    }
+    env_hdrs = os.environ.get("OTEL_EXPORTER_OTLP_HEADERS", "")
+    for part in env_hdrs.split(","):
+        if "=" in part:
+            k, v = part.split("=", 1)
+            headers[k.strip()] = v.strip()
+
+    try:
+        import urllib.request
+        req = urllib.request.Request(url, data=payload, headers=headers, method="POST")
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return resp.status, resp.read().decode("utf-8", errors="replace")
+    except Exception as e:
+        return 0, str(e)

package/cdh/optimizer/__init__.py

@@ -0,0 +1,7 @@
+from cdh.optimizer.loop import HillclimbLoop, HillclimbState
+from cdh.optimizer.reward import RewardCalculator, SessionMetrics
+from cdh.optimizer.mutation import ConfigMutator, ConfigMutation, AGENT_CONFIG_PATH
+from cdh.optimizer.tracker import OptimizationTracker
+
+__all__ = ["HillclimbLoop", "HillclimbState", "RewardCalculator", "SessionMetrics",
+           "ConfigMutator", "ConfigMutation", "OptimizationTracker", "AGENT_CONFIG_PATH"]

package/cdh/optimizer/loop.py

@@ -0,0 +1,201 @@
+from __future__ import annotations
+
+import logging
+import shutil
+from enum import Enum
+from typing import Any
+
+import yaml
+
+from cdh.event_loop.events import Event, EventTypes
+from cdh.optimizer.mutation import ConfigMutator, ConfigMutation, AGENT_CONFIG_PATH
+from cdh.optimizer.reward import RewardCalculator, SessionMetrics
+from cdh.optimizer.tracker import OptimizationTracker
+
+logger = logging.getLogger("cdh.optimizer.loop")
+
+_VALID_PARAMS: dict[str, tuple[float, float]] = {
+    "temperature": (0.0, 2.0),
+    "max_iterations": (1, 200),
+    "top_p": (0.0, 1.0),
+} 
+
+
+class HillclimbState(Enum):
+    IDLE = "idle"
+    COLLECTING = "collecting"
+    EVALUATING = "evaluating"
+    MUTATING = "mutating"
+    DEPLOYING = "deploying"
+    COMPLETED = "completed"
+    FAILED = "failed"
+
+
+class HillclimbLoop:
+    def __init__(self, min_sessions: int = 3):
+        self.state = HillclimbState.IDLE
+        self._base_min_sessions = min_sessions
+        self.min_sessions = min_sessions
+        self.tracker = OptimizationTracker()
+        self.reward_calc = RewardCalculator()
+        self.mutator = ConfigMutator()
+        self.bus: Any = None
+        self._tool_count: int = 0
+        self._test_pass_count: int = 0
+        self._test_total_count: int = 0
+        self._resetting: bool = False
+
+    def start(self, bus: Any = None) -> None:
+        self.state = HillclimbState.COLLECTING
+        self.bus = bus
+        self.tracker.initialize()
+        logger.info("Hillclimb: started collecting, min_sessions=%d", self.min_sessions)
+
+    def subscribe(self, bus: Any) -> None:
+        bus.subscribe(EventTypes.SESSION_ENDED, self.on_session_ended)
+        bus.subscribe(EventTypes.TOOL_EXECUTED, self._on_tool_executed)
+        bus.subscribe(EventTypes.VERIFICATION_PASSED, self._on_verification_pass)
+        bus.subscribe(EventTypes.VERIFICATION_FAILED, self._on_verification_fail)
+
+    def _on_tool_executed(self, event: Event) -> None:
+        if self.state == HillclimbState.COLLECTING and not self._resetting:
+            self._tool_count += 1
+
+    def _on_verification_pass(self, event: Event) -> None:
+        if self.state == HillclimbState.COLLECTING and not self._resetting:
+            self._test_total_count += 1
+            self._test_pass_count += 1
+
+    def _on_verification_fail(self, event: Event) -> None:
+        if self.state == HillclimbState.COLLECTING and not self._resetting:
+            self._test_total_count += 1
+
+    def on_session_ended(self, event: Event) -> None:
+        if self.state != HillclimbState.COLLECTING:
+            return
+
+        payload = event.payload
+        metrics = payload.get("metrics", {})
+
+        test_pass_rate = (
+            self._test_pass_count / max(self._test_total_count, 1)
+            if self._test_total_count > 0
+            else metrics.get("test_pass_rate", 0.0)
+        )
+        tool_efficiency = (
+            min(1.0, 10.0 / max(self._tool_count, 1))
+            if self._tool_count > 0
+            else metrics.get("tool_efficiency", 0.0)
+        )
+        task_pct = metrics.get("task_completion_pct", None)
+
+        session_metrics = SessionMetrics(
+            session_id=payload.get("session_id", ""),
+            test_pass_rate=test_pass_rate,
+            task_completion_pct=task_pct if task_pct is not None else 0.0,
+            tool_efficiency=tool_efficiency,
+            turn_count=payload.get("turn_count", 0),
+        )
+        self.tracker.record(session_metrics)
+
+        if self.tracker.count() >= self.min_sessions:
+            self._run_optimization_cycle()
+
+    def on_config_changed(self, event: Event) -> None:
+        payload = event.payload
+        mutation_data = payload.get("mutation", {})
+        logger.info("Hillclimb: config changed via mutation %s", mutation_data)
+
+    def _run_optimization_cycle(self) -> None:
+        logger.info("Hillclimb: optimization cycle started (%d sessions)",
+                    self.tracker.count())
+
+        self.state = HillclimbState.EVALUATING
+        all_metrics = self.tracker.get_all()
+        baseline = self.reward_calc.compute_all(all_metrics)
+
+        self.state = HillclimbState.MUTATING
+        mutation = self.mutator.suggest(all_metrics, baseline)
+
+        if mutation:
+            self.state = HillclimbState.DEPLOYING
+            self._apply_mutation(mutation)
+            self.tracker.save_mutation(mutation)
+
+        self._resetting = True
+        self.tracker.clear()
+        self._tool_count = 0
+        self._test_pass_count = 0
+        self._test_total_count = 0
+        self._resetting = False
+        self.min_sessions = min(self._base_min_sessions * 5, self.min_sessions + 2)
+        self.state = HillclimbState.COLLECTING
+
+        if mutation and self.bus:
+            self.bus.publish(Event(
+                type=EventTypes.CONFIG_CHANGED,
+                source="cdh.optimizer",
+                payload={"mutation": mutation.to_dict()},
+            ))
+
+    def _validate_mutation(self, mutation: ConfigMutation) -> list[str]:
+        errors: list[str] = []
+        onecode = mutation.engine_params.get("onecode.dev", {})
+        for key, value in onecode.items():
+            if key in _VALID_PARAMS:
+                lo, hi = _VALID_PARAMS[key]
+                if not (lo <= float(value) <= hi):
+                    errors.append(f"{key}={value} out of range [{lo}, {hi}]")
+            else:
+                logger.debug("Unrecognised engine param in mutation: %s", key)
+        return errors
+
+    def _apply_mutation(self, mutation: ConfigMutation) -> None:
+        errors = self._validate_mutation(mutation)
+        if errors:
+            logger.warning("Mutation rejected: %s", errors)
+            return
+
+        AGENT_CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
+
+        if AGENT_CONFIG_PATH.exists():
+            backup = AGENT_CONFIG_PATH.with_suffix(".yaml.bak")
+            try:
+                shutil.copy(AGENT_CONFIG_PATH, backup)
+            except Exception as e:
+                logger.warning("Failed to backup agent config: %s", e)
+
+        config: dict[str, Any] = {}
+        if AGENT_CONFIG_PATH.exists():
+            try:
+                config = yaml.safe_load(AGENT_CONFIG_PATH.read_text()) or {}
+            except Exception as e:
+                logger.warning("Hillclimb: failed to read agent config: %s", e)
+
+        if "version" not in config:
+            config["version"] = 1
+
+        platform = config.setdefault("platform", {})
+        for key, value in mutation.platform_params.items():
+            self._set_nested(platform, key, value)
+
+        engine = config.setdefault("engine", {})
+        for engine_id, params in mutation.engine_params.items():
+            if engine_id not in engine:
+                engine[engine_id] = {}
+            engine[engine_id].update(params)
+
+        try:
+            AGENT_CONFIG_PATH.write_text(yaml.dump(config, default_flow_style=False))
+            logger.info("Hillclimb: applied mutation to %s", AGENT_CONFIG_PATH)
+        except Exception as e:
+            logger.warning("Hillclimb: failed to write agent config: %s", e)
+
+    def _set_nested(self, config: dict, key: str, value: Any) -> None:
+        parts = key.split(".")
+        d = config
+        for part in parts[:-1]:
+            if part not in d:
+                d[part] = {}
+            d = d[part]
+        d[parts[-1]] = value

package/cdh/optimizer/mutation.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+import random
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from cdh.optimizer.reward import SessionMetrics
+
+AGENT_CONFIG_PATH = Path.home() / ".cdh" / "agent_config.yaml"
+
+
+@dataclass
+class ConfigMutation:
+    platform_params: dict[str, Any] = field(default_factory=dict)
+    engine_params: dict[str, dict[str, Any]] = field(default_factory=dict)
+    timestamp: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
+    parent_reward: float = 0.0
+
+    def to_dict(self) -> dict:
+        return {
+            "platform": self.platform_params,
+            "engine": self.engine_params,
+            "timestamp": self.timestamp,
+            "parent_reward": self.parent_reward,
+        }
+
+
+PLATFORM_PARAMS = [
+    "verification.policy",
+    "verification.gates",
+]
+
+ENGINE_PARAMS = [
+    "onecode.temperature",
+    "onecode.max_iterations",
+    "onecode.plan_gate_mode",
+]
+
+
+class ConfigMutator:
+    def __init__(self, seed: int = 42):
+        self.rng = random.Random(seed)
+
+    def suggest(self, history: list[SessionMetrics],
+                current_reward: float) -> ConfigMutation | None:
+        if not history:
+            return None
+
+        if current_reward < 0.5:
+            return self._random_explore(current_reward)
+        elif current_reward >= 0.8:
+            return None
+        else:
+            return self._random_explore(current_reward)
+
+    def _random_explore(self, current_reward: float) -> ConfigMutation | None:
+        platform_params: dict[str, Any] = {}
+        engine_params: dict[str, dict[str, Any]] = {}
+
+        if self.rng.random() < 0.2:
+            platform_params["verification.policy"] = self.rng.choice(
+                ["conditional", "every_step", "final_only"]
+            )
+
+        onecode: dict[str, Any] = {}
+        if self.rng.random() < 0.3:
+            onecode["temperature"] = round(
+                max(0.0, min(2.0, 0.7 + self.rng.uniform(-0.1, 0.1))), 2
+            )
+        if self.rng.random() < 0.3:
+            onecode["max_iterations"] = max(5, 10 + self.rng.choice([-5, 0, 5]))
+        if onecode:
+            engine_params["onecode.dev"] = onecode
+
+        platform = platform_params if platform_params else None
+        engine = engine_params if engine_params else None
+        if not platform and not engine:
+            return None
+        return ConfigMutation(
+            platform_params=platform_params,
+            engine_params=engine_params,
+            parent_reward=current_reward,
+        )

package/cdh/optimizer/reward.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass
+class SessionMetrics:
+    session_id: str
+    test_pass_rate: float = 0.0
+    task_completion_pct: float = 0.0
+    tool_efficiency: float = 0.0
+    turn_count: int = 0
+    timestamp: str = ""
+    user_feedback: float = 0.0
+
+
+class RewardCalculator:
+    WEIGHTS_FULL = {
+        "test_pass_rate": 0.40,
+        "task_completion": 0.30,
+        "tool_efficiency": 0.20,
+        "user_feedback": 0.10,
+    }
+
+    WEIGHTS_PARTIAL = {
+        "test_pass_rate": 0.55,
+        "tool_efficiency": 0.30,
+        "user_feedback": 0.15,
+    }
+
+    def compute(self, metrics: SessionMetrics) -> float:
+        if metrics.task_completion_pct is not None and metrics.task_completion_pct > 0:
+            w = self.WEIGHTS_FULL
+            return (
+                metrics.test_pass_rate * w["test_pass_rate"]
+                + metrics.task_completion_pct * w["task_completion"]
+                + metrics.tool_efficiency * w["tool_efficiency"]
+                + metrics.user_feedback * w["user_feedback"]
+            )
+        else:
+            w = self.WEIGHTS_PARTIAL
+            return (
+                metrics.test_pass_rate * w["test_pass_rate"]
+                + metrics.tool_efficiency * w["tool_efficiency"]
+                + metrics.user_feedback * w["user_feedback"]
+            )
+
+    def compute_all(self, all_metrics: list[SessionMetrics]) -> float:
+        if not all_metrics:
+            return 0.0
+        return sum(self.compute(m) for m in all_metrics) / len(all_metrics)