devcouncil 0.1.1 → 0.2.0

package/src/devcouncil/llm/router.py

@@ -1,4 +1,4 @@
-from typing import List, Dict, Any, Type, Optional
+from typing import List, Dict, Any, Type, Optional, TypeVar
 import copy
 import json
 import logging
@@ -6,25 +6,138 @@ import asyncio
 from pathlib import Path
 
 from pydantic import BaseModel
-from devcouncil.llm.provider import Provider
+from devcouncil.llm.provider import Provider, LLMResponse
 from devcouncil.llm.cache import LLMCache
 from devcouncil.telemetry.tracker import TelemetryTracker
+from devcouncil.telemetry.traces import TraceLogger
 
 logger = logging.getLogger(__name__)
+StructuredModel = TypeVar("StructuredModel", bound=BaseModel)
+
+
+class StructuredOutputError(RuntimeError):
+    """A model could not produce valid structured output for a role, even after
+    a healing retry. Carries the role/model so the CLI can give actionable advice
+    (usually: switch that role to a more capable model)."""
+
+    def __init__(self, message: str, *, role: str, model: str):
+        super().__init__(message)
+        self.role = role
+        self.model = model
+
 
 class ModelRouter:
-    def __init__(self, provider: Provider, role_config: Dict[str, Dict[str, Any]]):
+    # Independent fresh attempts at producing valid structured output before
+    # giving up. Even capable models occasionally emit malformed JSON; a second
+    # clean attempt usually succeeds. Malformed responses are never cached, so a
+    # retry is genuinely fresh rather than re-serving the same bad JSON.
+    STRUCTURED_ATTEMPTS = 2
+
+    def __init__(
+        self,
+        provider: Provider,
+        role_config: Dict[str, Dict[str, Any]],
+        project_root: Path = Path("."),
+    ):
         self.provider = provider
         self.role_config = role_config
+        self.project_root = project_root
+
+    @staticmethod
+    def _extract_json(content: str) -> str:
+        """Best-effort extraction of a JSON document from a model response.
+
+        Handles the common ways a model wraps valid JSON: triple-backtick fences and
+        surrounding prose ("Here you go: {...} thanks"). Strips fences, returns the
+        whole thing if it already parses, otherwise scans for the first balanced
+        object/array (string- and escape-aware so braces inside string values don't
+        confuse it). Falls back to the de-fenced text so the existing healing path
+        still produces a meaningful error. A strict superset of plain fence-stripping
+        — clean/fenced JSON is returned unchanged."""
+        text = content.strip()
+        if "```json" in text:
+            text = text.split("```json", 1)[1].split("```", 1)[0].strip()
+        elif "```" in text:
+            text = text.split("```", 1)[1].split("```", 1)[0].strip()
+        try:
+            json.loads(text)
+            return text
+        except Exception:
+            pass
+        for opener, closer in (("{", "}"), ("[", "]")):
+            start = text.find(opener)
+            if start == -1:
+                continue
+            depth = 0
+            in_str = False
+            escaped = False
+            for i in range(start, len(text)):
+                ch = text[i]
+                if in_str:
+                    if escaped:
+                        escaped = False
+                    elif ch == "\\":
+                        escaped = True
+                    elif ch == '"':
+                        in_str = False
+                    continue
+                if ch == '"':
+                    in_str = True
+                elif ch == opener:
+                    depth += 1
+                elif ch == closer:
+                    depth -= 1
+                    if depth == 0:
+                        candidate = text[start:i + 1]
+                        try:
+                            json.loads(candidate)
+                            return candidate
+                        except Exception:
+                            break
+        return text
+
+    async def _complete_with_retry(
+        self,
+        *,
+        model: str,
+        messages: List[Dict[str, str]],
+        temperature: float,
+        run_id: Optional[str],
+        attempts: int = 3,
+    ) -> "LLMResponse":
+        """Provider completion with bounded exponential-backoff retry. Used for BOTH the
+        initial call and the healing call so a transient fault in either is retried (and,
+        if still failing, surfaced to the caller's fallback logic) rather than aborting
+        the run."""
+        for attempt in range(attempts):
+            try:
+                return await self.provider.complete(
+                    model=model,
+                    messages=messages,
+                    temperature=temperature,
+                    json_mode=True,
+                    run_id=run_id,
+                )
+            except Exception as exc:
+                if attempt == attempts - 1:
+                    raise
+                logger.warning(
+                    "LLM request failed (attempt %d/%d): %s. Retrying...",
+                    attempt + 1, attempts, exc,
+                )
+                await asyncio.sleep(2 ** attempt)
+        raise RuntimeError("unreachable")  # loop either returns or raises
 
     async def complete_structured(
         self,
         role: str,
         messages: List[Dict[str, str]],
-        schema: Type[BaseModel],
+        schema: Type[StructuredModel],
         temperature: Optional[float] = None,
         run_id: Optional[str] = None,
-    ) -> BaseModel:
+        fallback: Optional[StructuredModel] = None,
+        _attempt: int = 0,
+    ) -> StructuredModel:
         config = self.role_config.get(role)
         if not config:
             raise ValueError(f"No config found for role: {role}")
@@ -51,33 +164,31 @@ class ModelRouter:
 
         logger.info("LLM call: role=%s model=%s run_id=%s", role, model, run_id)
 
-        project_root = Path(".")
-        cache = LLMCache(project_root)
-        tracker = TelemetryTracker(project_root)
+        cache = LLMCache(self.project_root)
+        tracker = TelemetryTracker(self.project_root)
+        traces = TraceLogger(self.project_root)
+
+        # Provider knobs (e.g. Ollama num_ctx / base_url) that change the output for an
+        # identical prompt must be part of the cache key, else raising OLLAMA_NUM_CTX
+        # after a truncated answer would keep serving the stale response.
+        provider_fp = self.provider.cache_fingerprint()
+        # Zero local (Ollama) usage by provider so telemetry matches the cost ledger.
+        provider_local = self.provider.is_local_cost_free()
 
         # Check cache first
-        response = cache.get(model, msgs, temp, True)
+        response = cache.get(model, msgs, temp, True, provider_fp)
         cache_hit = response is not None
 
         if not response:
-            for attempt in range(3):
-                try:
-                    response = await self.provider.complete(
-                        model=model,
-                        messages=msgs,
-                        temperature=temp,
-                        json_mode=True
-                    )
-                    cache.set(model, msgs, temp, True, response)
-                    break
-                except Exception as e:
-                    if attempt == 2:
-                        raise
-                    logger.warning(f"LLM request failed (attempt {attempt+1}): {e}. Retrying...")
-                    await asyncio.sleep(2 ** attempt)
+            response = await self._complete_with_retry(
+                model=model, messages=msgs, temperature=temp, run_id=run_id
+            )
+
+        if response is None:
+            raise RuntimeError(f"LLM request for role {role} did not return a response.")
 
         if not cache_hit:
-            tracker.log_usage(model, response.usage)
+            tracker.log_usage(model, response.usage, local=provider_local)
 
         logger.info(
             "LLM response: role=%s model=%s tokens=%s",
@@ -85,17 +196,27 @@ class ModelRouter:
         )
         
         try:
-            # Attempt to find JSON block if it's wrapped in markdown
-            content = response.content.strip()
-            if "```json" in content:
-                content = content.split("```json")[1].split("```")[0].strip()
-            elif "```" in content:
-                content = content.split("```")[1].split("```")[0].strip()
-                
+            # Extract JSON from fences/surrounding prose (balanced-aware).
+            content = self._extract_json(response.content)
             data = json.loads(content)
-            return schema.model_validate(data)
+            result = schema.model_validate(data)
+            if not cache_hit:
+                cache.set(model, msgs, temp, True, response, provider_fp)  # cache only validated output
+            return result
         except Exception as e:
             logger.warning(f"Initial parse failed for {role}, attempting healing: {e}")
+            traces.log_event(
+                "llm_structured_parse_failed",
+                {
+                    "role": role,
+                    "model": response.model,
+                    "schema": schema.__name__,
+                    "error": str(e),
+                    "content_preview": response.content[:500],
+                },
+                run_id=run_id,
+                summary=f"Structured response parse failed for {role}; attempting repair.",
+            )
             
             # Healing attempt: Ask the model to fix its own JSON
             healing_prompt = f"""
@@ -106,20 +227,84 @@ Content:
 
 Please return the corrected JSON object only. No prose.
 """
-            # We use a lower temperature for healing
-            healed_response = await self.provider.complete(
-                model=model,
-                messages=[{"role": "user", "content": healing_prompt}],
-                temperature=0.0,
-                json_mode=True
-            )
-            
+            # The healing completion runs INSIDE this try (with the same retry/backoff as
+            # the initial call). A transient failure here (429/timeout) must be treated as
+            # "healing failed" so it routes into the fresh-attempt/fallback logic below,
+            # not propagate as a raw provider error that defeats the supplied fallback.
+            healed_response = None
             try:
-                healed_content = healed_response.content.strip()
-                if "```json" in healed_content:
-                    healed_content = healed_content.split("```json")[1].split("```")[0].strip()
+                # We use a lower temperature for healing
+                healed_response = await self._complete_with_retry(
+                    model=model,
+                    messages=[{"role": "user", "content": healing_prompt}],
+                    temperature=0.0,
+                    run_id=run_id,
+                )
+                tracker.log_usage(healed_response.model, healed_response.usage, local=provider_local)
+                healed_content = self._extract_json(healed_response.content)
                 data = json.loads(healed_content)
-                return schema.model_validate(data)
+                result = schema.model_validate(data)
+                cache.set(model, msgs, temp, True, healed_response, provider_fp)
+                return result
             except Exception as final_e:
                 logger.error(f"Healing failed for {role}: {final_e}")
-                raise ValueError(f"Failed to parse or validate LLM response after healing: {final_e}\nContent (truncated): {response.content[:200]}...")
+                traces.log_event(
+                    "llm_structured_parse_repair_failed",
+                    {
+                        "role": role,
+                        "model": healed_response.model if healed_response else model,
+                        "schema": schema.__name__,
+                        "error": str(final_e),
+                        "original_content_preview": response.content[:500],
+                        "healed_content_preview": healed_response.content[:500] if healed_response else "(healing request failed)",
+                    },
+                    run_id=run_id,
+                    summary=f"Structured response repair failed for {role}.",
+                )
+                if _attempt + 1 < self.STRUCTURED_ATTEMPTS:
+                    # A fresh, independent attempt often succeeds where one bad draft
+                    # (plus its repair) failed. The malformed response was never
+                    # cached, so this re-runs the completion rather than re-reading it.
+                    # Prepend a strict JSON-only instruction (on a copy, so the caller's
+                    # messages are untouched) to nudge the retry toward parseable output.
+                    logger.warning(
+                        "Structured output failed for role '%s'; retrying fresh "
+                        "(attempt %d/%d).",
+                        role, _attempt + 2, self.STRUCTURED_ATTEMPTS,
+                    )
+                    strict_messages = copy.deepcopy(messages)
+                    strict_messages.insert(0, {
+                        "role": "system",
+                        "content": (
+                            "Respond with a single valid JSON object only — no prose, no "
+                            "markdown fences, no trailing text. It must parse with a strict "
+                            "JSON parser and match the requested schema."
+                        ),
+                    })
+                    return await self.complete_structured(
+                        role,
+                        strict_messages,
+                        schema,
+                        temperature=temperature,
+                        run_id=run_id,
+                        fallback=fallback,
+                        _attempt=_attempt + 1,
+                    )
+                if fallback is not None:
+                    # Degradable role (e.g. critique/rebuttal/enhancement): keep
+                    # planning alive on weaker models instead of crashing the run.
+                    logger.warning(
+                        "Role '%s' (model '%s') could not produce valid %s; "
+                        "using a safe fallback so planning can continue.",
+                        role, model, schema.__name__,
+                    )
+                    return fallback
+                raise StructuredOutputError(
+                    f"Model '{model}' for role '{role}' could not produce valid "
+                    f"{schema.__name__} JSON, even after a repair attempt. "
+                    f"Use a more capable model for this role "
+                    f"(e.g. 'dev config models --role {role} --model <model>'). "
+                    f"Parser error: {final_e}",
+                    role=role,
+                    model=model,
+                )

package/src/devcouncil/optimization/__init__.py

@@ -0,0 +1 @@
+"""Optimization integrations for DevCouncil prompt and workflow assets."""

package/src/devcouncil/optimization/gepa_agent.py

@@ -0,0 +1,318 @@
+from __future__ import annotations
+
+import importlib
+import json
+import sys
+from collections.abc import Iterable
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Callable
+
+import yaml  # type: ignore[import-untyped]
+
+from devcouncil.executors.agent_registry import (
+    load_agent_profiles,
+    load_cli_agent_specs,
+    normalize_agent_name,
+)
+
+
+DEFAULT_OBJECTIVE = (
+    "Optimize the DevCouncil CLI-agent profile prompt preamble so coding agents stay inside "
+    "planned scope, run the expected verification commands, preserve evidence, avoid destructive "
+    "or out-of-policy changes, and address the observed failures in the evaluation examples."
+)
+
+
+class GepaUnavailableError(RuntimeError):
+    """Raised when the optional GEPA package is not installed."""
+
+
+@dataclass(frozen=True)
+class AgentPromptOptimizationResult:
+    agent: str
+    profile_name: str
+    seed_preamble: str
+    best_preamble: str
+    best_score: float | None
+    artifact_path: Path
+    applied: bool
+
+
+def optimize_agent_profile(
+    *,
+    project_root: Path,
+    agent: str,
+    profile_name: str,
+    evals_path: Path,
+    max_metric_calls: int = 40,
+    objective: str | None = None,
+    apply: bool = False,
+    output_path: Path | None = None,
+) -> AgentPromptOptimizationResult:
+    """Run GEPA over a CLI-agent profile prompt preamble.
+
+    GEPA needs an evaluator. DevCouncil's evaluator consumes an offline JSON/JSONL
+    dataset of observed agent failures and desired prompt behavior, then feeds that
+    context back to GEPA as actionable side information.
+    """
+
+    root = project_root.expanduser().resolve()
+    normalized_agent = normalize_agent_name(agent)
+    if normalized_agent not in load_cli_agent_specs(root):
+        raise ValueError(f"Agent '{agent}' is not registered or supported.")
+
+    profiles = load_agent_profiles(root)
+    profile = profiles.get(profile_name)
+    if profile is None:
+        raise ValueError(f"Profile '{profile_name}' is not configured.")
+
+    resolved_evals_path = evals_path.expanduser()
+    if not resolved_evals_path.is_absolute():
+        resolved_evals_path = root / resolved_evals_path
+    dataset = load_agent_eval_dataset(resolved_evals_path)
+    gepa_module = _load_gepa_optimize_anything()
+    seed_preamble = profile.prompt_preamble or ""
+    effective_objective = objective or DEFAULT_OBJECTIVE
+
+    def evaluator(candidate: Any, example: dict[str, Any] | None = None) -> float:
+        if example is None:
+            scores = [
+                _score_candidate(
+                    candidate,
+                    item,
+                    log=getattr(gepa_module, "log", lambda message: None),
+                )
+                for item in dataset
+            ]
+            return sum(scores) / len(scores)
+        return _score_candidate(
+            candidate,
+            example,
+            log=getattr(gepa_module, "log", lambda message: None),
+        )
+
+    engine = gepa_module.EngineConfig(max_metric_calls=max_metric_calls)
+    config = gepa_module.GEPAConfig(engine=engine)
+    gepa_result = gepa_module.optimize_anything(
+        seed_candidate=seed_preamble,
+        evaluator=evaluator,
+        dataset=dataset,
+        objective=effective_objective,
+        config=config,
+    )
+
+    best_preamble = _candidate_to_text(_best_candidate(gepa_result)).strip()
+    if not best_preamble:
+        best_preamble = seed_preamble
+    best_score = _best_score(gepa_result)
+
+    artifact_path = output_path or _default_artifact_path(root, normalized_agent, profile_name)
+    artifact_path = artifact_path.expanduser()
+    if not artifact_path.is_absolute():
+        artifact_path = root / artifact_path
+    _write_result_artifact(
+        artifact_path,
+        {
+            "optimizer": "gepa.optimize_anything",
+            "agent": normalized_agent,
+            "profile": profile_name,
+            "objective": effective_objective,
+            "evals_path": str(resolved_evals_path),
+            "max_metric_calls": max_metric_calls,
+            "example_count": len(dataset),
+            "seed_preamble": seed_preamble,
+            "best_preamble": best_preamble,
+            "best_score": best_score,
+            "applied": apply,
+            "created_at": datetime.now(timezone.utc).isoformat(),
+        },
+    )
+
+    if apply:
+        _apply_profile_preamble(root, profile_name, best_preamble)
+
+    return AgentPromptOptimizationResult(
+        agent=normalized_agent,
+        profile_name=profile_name,
+        seed_preamble=seed_preamble,
+        best_preamble=best_preamble,
+        best_score=best_score,
+        artifact_path=artifact_path,
+        applied=apply,
+    )
+
+
+def load_agent_eval_dataset(path: Path) -> list[dict[str, Any]]:
+    evals_path = path.expanduser()
+    if not evals_path.exists():
+        raise ValueError(f"GEPA eval dataset not found: {evals_path}")
+
+    if evals_path.suffix.lower() == ".jsonl":
+        examples = [
+            json.loads(line)
+            for line in evals_path.read_text(encoding="utf-8").splitlines()
+            if line.strip()
+        ]
+    else:
+        raw = json.loads(evals_path.read_text(encoding="utf-8"))
+        if isinstance(raw, dict):
+            if isinstance(raw.get("examples"), list):
+                examples = raw["examples"]
+            elif isinstance(raw.get("dataset"), list):
+                examples = raw["dataset"]
+            else:
+                examples = [raw]
+        elif isinstance(raw, list):
+            examples = raw
+        else:
+            raise ValueError("GEPA eval dataset must be a JSON object, JSON array, or JSONL file.")
+
+    normalized: list[dict[str, Any]] = []
+    for index, item in enumerate(examples, start=1):
+        if not isinstance(item, dict):
+            raise ValueError("Every GEPA eval example must be a JSON object.")
+        example = dict(item)
+        example.setdefault("id", f"example-{index}")
+        normalized.append(example)
+
+    if not normalized:
+        raise ValueError("GEPA eval dataset must contain at least one example.")
+    return normalized
+
+
+def _load_gepa_optimize_anything() -> Any:
+    injected_module = sys.modules.get("gepa.optimize_anything")
+    if injected_module is not None:
+        return injected_module
+    try:
+        return importlib.import_module("gepa.optimize_anything")
+    except ImportError as exc:
+        raise GepaUnavailableError(
+            "GEPA is not installed in this environment. Reinstall or sync DevCouncil dependencies, "
+            "then rerun `dev agents optimize`."
+        ) from exc
+
+
+def _score_candidate(candidate: Any, example: dict[str, Any], *, log: Callable[[str], None]) -> float:
+    candidate_text = _candidate_to_text(candidate)
+    lower_candidate = candidate_text.lower()
+    required_terms = _string_list(example, "required_terms", "expected_prompt_fragments", "must_include")
+    forbidden_terms = _string_list(example, "forbidden_terms", "must_avoid")
+
+    log(f"Example {example.get('id')}: {example.get('task', '')}".strip())
+    for key in ("observed_failure", "desired_behavior", "feedback", "rubric"):
+        value = example.get(key)
+        if value:
+            log(f"{key.replace('_', ' ').title()}: {value}")
+    if required_terms:
+        log(f"Required prompt terms: {', '.join(required_terms)}")
+    if forbidden_terms:
+        log(f"Forbidden prompt terms: {', '.join(forbidden_terms)}")
+
+    if not candidate_text.strip():
+        log("Candidate preamble is empty.")
+        return 0.0
+
+    required_hits = [term for term in required_terms if term.lower() in lower_candidate]
+    forbidden_hits = [term for term in forbidden_terms if term.lower() in lower_candidate]
+    missing_terms = [term for term in required_terms if term not in required_hits]
+    if missing_terms:
+        log(f"Missing required terms: {', '.join(missing_terms)}")
+    if forbidden_hits:
+        log(f"Forbidden terms present: {', '.join(forbidden_hits)}")
+
+    required_score = len(required_hits) / len(required_terms) if required_terms else 0.5
+    forbidden_score = 1.0 - (len(forbidden_hits) / len(forbidden_terms)) if forbidden_terms else 1.0
+    length_score = _length_score(candidate_text)
+    score = (required_score * 0.65) + (forbidden_score * 0.25) + (length_score * 0.10)
+    return max(0.0, min(1.0, score))
+
+
+def _candidate_to_text(candidate: Any) -> str:
+    if candidate is None:
+        return ""
+    if isinstance(candidate, str):
+        return candidate
+    if isinstance(candidate, dict):
+        for key in ("prompt_preamble", "preamble", "prompt", "system_prompt", "text"):
+            value = candidate.get(key)
+            if isinstance(value, str):
+                return value
+        return json.dumps(candidate, sort_keys=True)
+    for attr in ("prompt_preamble", "preamble", "prompt", "system_prompt", "text"):
+        value = getattr(candidate, attr, None)
+        if isinstance(value, str):
+            return value
+    return str(candidate)
+
+
+def _best_candidate(result: Any) -> Any:
+    if isinstance(result, dict):
+        for key in ("best_candidate", "candidate", "best"):
+            if key in result:
+                return result[key]
+        return result
+    for attr in ("best_candidate", "candidate", "best"):
+        if hasattr(result, attr):
+            return getattr(result, attr)
+    return result
+
+
+def _best_score(result: Any) -> float | None:
+    raw_score: Any
+    if isinstance(result, dict):
+        raw_score = result.get("best_score", result.get("score"))
+    else:
+        raw_score = getattr(result, "best_score", getattr(result, "score", None))
+    if raw_score is None:
+        return None
+    try:
+        return float(raw_score)
+    except (TypeError, ValueError):
+        return None
+
+
+def _string_list(example: dict[str, Any], *keys: str) -> list[str]:
+    values: list[str] = []
+    for key in keys:
+        raw = example.get(key)
+        if raw is None:
+            continue
+        if isinstance(raw, str):
+            values.append(raw)
+        elif isinstance(raw, Iterable):
+            values.extend(str(item) for item in raw if str(item).strip())
+    return [value.strip() for value in values if value.strip()]
+
+
+def _length_score(candidate_text: str) -> float:
+    words = candidate_text.split()
+    if 10 <= len(words) <= 180:
+        return 1.0
+    if len(words) < 10:
+        return len(words) / 10
+    return max(0.0, 1.0 - ((len(words) - 180) / 180))
+
+
+def _default_artifact_path(project_root: Path, agent: str, profile_name: str) -> Path:
+    timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+    safe_profile = profile_name.replace("/", "-").replace("\\", "-")
+    return project_root / ".devcouncil" / "optimizations" / f"{timestamp}-{agent}-{safe_profile}-gepa.json"
+
+
+def _write_result_artifact(path: Path, payload: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
+
+
+def _apply_profile_preamble(project_root: Path, profile_name: str, best_preamble: str) -> None:
+    config_path = project_root / ".devcouncil" / "config.yaml"
+    raw_config = yaml.safe_load(config_path.read_text(encoding="utf-8")) if config_path.exists() else {}
+    raw_config = raw_config or {}
+    profiles = raw_config.setdefault("integrations", {}).setdefault("cli_agents", {}).setdefault("profiles", {})
+    profile = profiles.setdefault(profile_name, {})
+    profile["prompt_preamble"] = best_preamble
+    config_path.parent.mkdir(parents=True, exist_ok=True)
+    config_path.write_text(yaml.safe_dump(raw_config, sort_keys=False), encoding="utf-8")