devcouncil 0.3.1 → 0.4.1

package/src/devcouncil/llm/router.py

@@ -1,17 +1,21 @@
-from typing import List, Dict, Any, Type, Optional, TypeVar
+from typing import List, Dict, Any, Type, Optional, TypeVar, cast
 import copy
 import functools
 import json
 import logging
 import asyncio
+import os
+import re
 import time
 from pathlib import Path
 
+import httpx
 from pydantic import BaseModel
-from devcouncil.llm.provider import Provider, LLMResponse
+from devcouncil.llm.provider import Provider, LLMResponse, ProviderRequestError
 from devcouncil.llm.cache import LLMCache
 from devcouncil.telemetry.tracker import TelemetryTracker
 from devcouncil.telemetry.traces import TraceLogger
+from devcouncil.telemetry.stages import log_step
 
 logger = logging.getLogger(__name__)
 StructuredModel = TypeVar("StructuredModel", bound=BaseModel)
@@ -33,6 +37,32 @@ class StructuredOutputError(RuntimeError):
         self.model = model
 
 
+def _provider_retry_delay(exc: Exception, attempt: int) -> float:
+    """Seconds to wait before retrying a failed provider call."""
+    if isinstance(exc, ProviderRequestError):
+        if exc.retry_after_seconds is not None:
+            return float(min(120.0, max(1.0, exc.retry_after_seconds)))
+        if exc.status_code == 429:
+            # OpenRouter tiers often cap at ~20 RPM; back off generously.
+            return float(min(90.0, 15.0 * (2 ** attempt)))
+    return min(30.0, float(2 ** attempt))
+
+
+def _rate_limit_retry_budget() -> int:
+    """How many 429 responses to wait out per call (beyond the normal attempts).
+
+    Default 8: with the 429 backoff above that is ~7 minutes of patience —
+    enough to ride out an RPM-window burst, small enough that a hard quota
+    (daily cap) still fails the call in bounded time. Override with
+    DEVCOUNCIL_RATE_LIMIT_RETRIES; 0 disables the separate budget."""
+    raw = os.environ.get("DEVCOUNCIL_RATE_LIMIT_RETRIES", "")
+    try:
+        value = int(raw)
+    except ValueError:
+        return 8
+    return max(0, value)
+
+
 class ModelRouter:
     # Independent fresh attempts at producing valid structured output before
     # giving up. Even capable models occasionally emit malformed JSON; a second
@@ -45,6 +75,7 @@ class ModelRouter:
         provider: Provider,
         role_config: Dict[str, Dict[str, Any]],
         project_root: Path = Path("."),
+        semantic_adapter: Optional[Any] = None,
     ):
         self.provider = provider
         self.role_config = role_config
@@ -55,6 +86,15 @@ class ModelRouter:
         # concurrent-write safe.
         self._cache = LLMCache(self.project_root)
         self._traces = TraceLogger(self.project_root)
+        # Optional semantic cache / routing / compression (config-driven, lazy).
+        if semantic_adapter is not None:
+            self._semantic = semantic_adapter
+        else:
+            from devcouncil.llm.semantic_bridge import load_semantic_adapter
+
+            self._semantic = load_semantic_adapter(project_root)
+        if self._semantic is not None:
+            self._semantic.warm_up()
         # Lazily-built providers for roles that override ``models.provider`` with
         # their own ``provider:`` (e.g. live_reviewer on Ollama while planners run
         # on OpenRouter). Keyed by normalized provider name; the default provider
@@ -95,34 +135,29 @@ class ModelRouter:
             )
         return self._role_providers[normalized]
 
-    @staticmethod
-    def _extract_json(content: str) -> str:
-        """Best-effort extraction of a JSON document from a model response.
+    # Inline reasoning blocks emitted by thinking models when the serving stack does
+    # not split them into a separate channel (local runners with a mismatched chat
+    # template are the common case). Removed before JSON extraction: the reasoning
+    # prose routinely contains JSON-looking examples that would otherwise be picked
+    # up instead of the real answer that follows the block.
+    _THINK_BLOCK_RE = re.compile(
+        r"<(think|thinking|reasoning|thought)>.*?</\1>", re.DOTALL | re.IGNORECASE
+    )
+    # An UNCLOSED reasoning tag (generation cut off or template quirk): everything
+    # from the opener is reasoning; nothing after it to salvage, but text BEFORE a
+    # dangling opener (rare) may hold the answer.
+    _THINK_OPEN_RE = re.compile(r"<(think|thinking|reasoning|thought)>", re.IGNORECASE)
 
-        Handles the common ways a model wraps valid JSON: triple-backtick fences and
-        surrounding prose ("Here you go: {...} thanks"). Strips fences, returns the
-        whole thing if it already parses, otherwise scans for the first balanced
-        object/array (string- and escape-aware so braces inside string values don't
-        confuse it). Falls back to the de-fenced text so the existing healing path
-        still produces a meaningful error. A strict superset of plain fence-stripping
-        — clean/fenced JSON is returned unchanged."""
-        text = content.strip()
-        if "```json" in text:
-            text = text.split("```json", 1)[1].split("```", 1)[0].strip()
-        elif "```" in text:
-            text = text.split("```", 1)[1].split("```", 1)[0].strip()
-        try:
-            json.loads(text)
-            return text
-        except Exception:
-            pass
-        for opener, closer in (("{", "}"), ("[", "]")):
-            start = text.find(opener)
-            if start == -1:
-                continue
+    @staticmethod
+    def _balanced_candidates(text: str, opener: str, closer: str):
+        """Yield every balanced top-level ``opener...closer`` span in ``text``,
+        string- and escape-aware so braces inside string values don't confuse it."""
+        start = text.find(opener)
+        while start != -1:
             depth = 0
             in_str = False
             escaped = False
+            end = -1
             for i in range(start, len(text)):
                 ch = text[i]
                 if in_str:
@@ -140,12 +175,56 @@ class ModelRouter:
                 elif ch == closer:
                     depth -= 1
                     if depth == 0:
-                        candidate = text[start:i + 1]
-                        try:
-                            json.loads(candidate)
-                            return candidate
-                        except Exception:
-                            break
+                        end = i
+                        break
+            if end == -1:
+                return
+            yield text[start:end + 1]
+            start = text.find(opener, end + 1)
+
+    @classmethod
+    def _extract_json(cls, content: str) -> str:
+        """Best-effort extraction of a JSON document from a model response.
+
+        Handles the common ways a model wraps valid JSON: inline ``<think>`` blocks
+        (local/thinking models whose serving stack leaves reasoning in the content),
+        triple-backtick fences, and surrounding prose ("Here you go: {...} thanks").
+        Strips reasoning blocks and fences, returns the whole thing if it already
+        parses, otherwise scans EVERY balanced object/array candidate (string- and
+        escape-aware) and returns the first that parses — a JSON-looking fragment in
+        leading prose no longer masks the real answer that follows it. Falls back to
+        the de-fenced text so the existing healing path still produces a meaningful
+        error. A strict superset of plain fence-stripping — clean/fenced JSON is
+        returned unchanged."""
+        text = content.strip()
+        # Drop closed reasoning blocks; on a dangling opener keep only what precedes it.
+        if "<" in text:
+            stripped = cls._THINK_BLOCK_RE.sub("", text).strip()
+            dangling = cls._THINK_OPEN_RE.search(stripped)
+            if dangling and "</" not in stripped[dangling.start():]:
+                before = stripped[:dangling.start()].strip()
+                after = stripped[dangling.end():].strip()
+                # The answer usually follows the (cut-off) reasoning; prefer whichever
+                # side actually contains a JSON-ish payload.
+                stripped = after if ("{" in after or "[" in after) else before
+            if stripped:
+                text = stripped
+        if "```json" in text:
+            text = text.split("```json", 1)[1].split("```", 1)[0].strip()
+        elif "```" in text:
+            text = text.split("```", 1)[1].split("```", 1)[0].strip()
+        try:
+            json.loads(text)
+            return text
+        except Exception as e:
+            logger.debug("Response is not directly parseable JSON, scanning balanced candidates: %s", e)
+        for opener, closer in (("{", "}"), ("[", "]")):
+            for candidate in cls._balanced_candidates(text, opener, closer):
+                try:
+                    json.loads(candidate)
+                    return cast(str, candidate)
+                except Exception:
+                    continue
         return text
 
     @staticmethod
@@ -173,15 +252,44 @@ class ModelRouter:
         temperature: float,
         run_id: Optional[str],
         provider: Optional[Provider] = None,
-        attempts: int = 3,
+        attempts: int = 5,
+        json_schema: Optional[Dict[str, Any]] = None,
     ) -> "LLMResponse":
         """Provider completion with bounded exponential-backoff retry. Used for BOTH the
         initial call and the healing call so a transient fault in either is retried (and,
         if still failing, surfaced to the caller's fallback logic) rather than aborting
         the run. ``provider`` defaults to the router's default provider but may be a
-        per-role provider for roles that override ``models.provider``."""
+        per-role provider for roles that override ``models.provider``. ``json_schema``
+        flows through to providers that support grammar-constrained structured output
+        (Ollama); others ignore it."""
+        if run_id is None:
+            # Fall back to the orchestrator-declared run so model_calls.jsonl
+            # records stay attributable even when call sites don't thread run_id.
+            from devcouncil.telemetry.context import get_current_run_id
+
+            run_id = get_current_run_id()
         provider = provider or self.provider
-        for attempt in range(attempts):
+        # Only pass json_schema to providers whose ``complete`` accepts it — duck-typed
+        # or third-party Provider implementations may predate the parameter, and a
+        # structured-output OPTIMIZATION must never break them.
+        extra_kwargs: Dict[str, Any] = {}
+        if json_schema is not None:
+            import inspect
+
+            try:
+                if "json_schema" in inspect.signature(provider.complete).parameters:
+                    extra_kwargs["json_schema"] = json_schema
+            except (TypeError, ValueError):
+                pass
+        # Rate limiting (429) gets its OWN, larger budget: it is the provider
+        # telling us exactly when to come back (Retry-After), not a fault in the
+        # request — counting it against the shared ``attempts`` let a busy
+        # endpoint exhaust the budget and kill a run that only needed patience
+        # (observed: benchmark tasks dying blocked on limit_rpm mid-run).
+        max_rate_limit_retries = _rate_limit_retry_budget()
+        attempt = 0
+        rate_limit_retries = 0
+        while True:
             try:
                 return await provider.complete(
                     model=model,
@@ -189,16 +297,57 @@ class ModelRouter:
                     temperature=temperature,
                     json_mode=True,
                     run_id=run_id,
+                    **extra_kwargs,
+                )
+            except httpx.TimeoutException as exc:
+                # A timeout already consumed the provider's ENTIRE request window
+                # (600s by default on a local Ollama host). Retrying the identical
+                # request almost always times out again — and the structured-output
+                # layers above this (healing call + fresh attempts) would multiply
+                # the stall until the whole run is killed from outside (observed:
+                # benchmark arms burning their full 20-minute budget on 2x 600s
+                # timeouts and dying with exit 124 before producing a verdict).
+                # Fail fast with an actionable message instead.
+                raise ProviderRequestError(
+                    f"LLM request to model '{model}' timed out after the provider's "
+                    f"request window ({exc!r}). If this is a local (Ollama) model, "
+                    "generation is too slow for the configured window: raise "
+                    "OLLAMA_TIMEOUT, cap generation with OLLAMA_NUM_PREDICT, reduce "
+                    "thinking with OLLAMA_THINK=low or OLLAMA_THINK=false, or use a "
+                    "smaller/faster model."
+                ) from exc
+            except ProviderRequestError as exc:
+                if exc.status_code == 429 and rate_limit_retries < max_rate_limit_retries:
+                    rate_limit_retries += 1
+                    delay = _provider_retry_delay(exc, rate_limit_retries)
+                    logger.warning(
+                        "LLM provider rate-limited (429, retry %d/%d): %r. Retrying in %.0fs...",
+                        rate_limit_retries, max_rate_limit_retries, exc, delay,
+                    )
+                    await asyncio.sleep(delay)
+                    continue
+                attempt += 1
+                if attempt >= attempts:
+                    raise
+                delay = _provider_retry_delay(exc, attempt - 1)
+                logger.warning(
+                    "LLM provider request failed (attempt %d/%d): %r. Retrying in %.0fs...",
+                    attempt, attempts, exc, delay,
                 )
+                await asyncio.sleep(delay)
             except Exception as exc:
-                if attempt == attempts - 1:
+                attempt += 1
+                if attempt >= attempts:
                     raise
+                delay = _provider_retry_delay(exc, attempt - 1)
+                # %r, not %s: common failures (httpx.ReadTimeout, CancelledError)
+                # stringify to an EMPTY message, which previously logged the useless
+                # "failed (attempt 1/3): ." and made timeouts undiagnosable from logs.
                 logger.warning(
-                    "LLM request failed (attempt %d/%d): %s. Retrying...",
-                    attempt + 1, attempts, exc,
+                    "LLM request failed (attempt %d/%d): %r. Retrying in %.0fs...",
+                    attempt, attempts, exc, delay,
                 )
-                await asyncio.sleep(2 ** attempt)
-        raise RuntimeError("unreachable")  # loop either returns or raises
+                await asyncio.sleep(delay)
 
     async def complete_structured(
         self,
@@ -217,9 +366,22 @@ class ModelRouter:
         model = config["model"]
         temp = temperature if temperature is not None else config.get("temperature", 0.0)
         provider = self._provider_for_role(config)
-        
+        role_provider = config.get("provider")
+
         # Deep-copy to avoid mutating the caller's messages list
         msgs = copy.deepcopy(messages)
+
+        # Optional long-context compression before schema injection.
+        if self._semantic is not None:
+            msgs = await self._semantic.maybe_compress_messages_async(msgs)
+
+        # Optional complexity-based model routing (local Ollama only when enabled).
+        if self._semantic is not None:
+            model = await self._semantic.maybe_route_model_async(
+                msgs,
+                configured_model=model,
+                role_provider=role_provider,
+            )
         
         # Add schema instructions to system or user message. Spell out "instance, not
         # the schema" explicitly: weaker/local models otherwise sometimes echo the schema
@@ -259,12 +421,46 @@ class ModelRouter:
         # Check cache first
         response = cache.get(model, msgs, temp, True, provider_fp)
         cache_hit = response is not None
+        semantic_cache_hit = False
+
+        if not cache_hit and self._semantic is not None:
+            response = await self._semantic.lookup_cache_async(msgs, model=model, role=role)
+            if response is not None:
+                cache_hit = True
+                semantic_cache_hit = True
+
+        # Structured-output schema for providers with grammar-constrained decoding
+        # (Ollama's native ``format: <schema>``): the model cannot emit invalid JSON,
+        # which on weak/local models eliminates most schema echoes and healing rounds.
+        structured_schema = schema.model_json_schema()
 
         started = time.monotonic()
         if not response:
-            response = await self._complete_with_retry(
-                model=model, messages=msgs, temperature=temp, run_id=run_id, provider=provider
-            )
+            try:
+                response = await self._complete_with_retry(
+                    model=model, messages=msgs, temperature=temp, run_id=run_id, provider=provider,
+                    json_schema=structured_schema,
+                )
+            except ProviderRequestError as exc:
+                # A degradable role (caller supplied a fallback) must degrade on a
+                # provider failure exactly as it does on unparseable output: the
+                # fallback exists to keep the run alive on a flaky/slow model, and a
+                # fail-fast timeout (see _complete_with_retry) or exhausted retries
+                # is the same class of "this role produced nothing usable".
+                if fallback is not None:
+                    logger.warning(
+                        "Role '%s' (model '%s') provider request failed (%s); "
+                        "using a safe fallback so the run can continue.",
+                        role, model, exc,
+                    )
+                    traces.log_event(
+                        "llm_provider_request_failed_fallback",
+                        {"role": role, "model": model, "error": str(exc)},
+                        run_id=run_id,
+                        summary=f"Provider request failed for {role}; degraded to fallback.",
+                    )
+                    return fallback
+                raise
         elapsed = time.monotonic() - started
 
         if response is None:
@@ -273,12 +469,26 @@ class ModelRouter:
         if not cache_hit:
             tracker.log_usage(model, response.usage, local=provider_local)
 
+        cache_label = "cache_hit" if cache_hit else f"{elapsed:.1f}s"
+        if semantic_cache_hit:
+            cache_label = "semantic_cache_hit"
+
         # Include latency + cache status: on a slow (e.g. local) model this is what tells
         # you *which* call dominated a multi-minute planning/verification stage.
         logger.info(
             "LLM response: role=%s model=%s tokens=%s %s",
             role, response.model, response.usage,
-            "cache_hit" if cache_hit else f"{elapsed:.1f}s",
+            cache_label,
+        )
+        log_step(
+            f"llm/{role}: {response.model} {cache_label}",
+            project_root=self.project_root,
+            run_id=run_id,
+            role=role,
+            model=response.model,
+            latency_s=round(elapsed, 2) if not cache_hit else 0,
+            cache_hit=cache_hit,
+            semantic_cache_hit=semantic_cache_hit,
         )
         
         try:
@@ -288,6 +498,8 @@ class ModelRouter:
             result = schema.model_validate(data)
             if not cache_hit:
                 cache.set(model, msgs, temp, True, response, provider_fp)  # cache only validated output
+            if self._semantic is not None and not semantic_cache_hit:
+                await self._semantic.store_cache_async(msgs, response, model=model, role=role)
             return result
         except Exception as e:
             logger.warning(f"Initial parse failed for {role}, attempting healing: {e}")
@@ -323,6 +535,16 @@ Content:
 {echo_hint}
 Please return the corrected JSON object only. No prose.
 """
+            # The healing call must SEE the schema. Previously it got only the error
+            # + bad content, so a "Field required" failure asked the model to invent
+            # the missing fields blind — on providers without grammar-constrained
+            # decoding (OpenRouter/Vertex, e.g. gemini-2.5-flash omitting empty
+            # list fields) healing then failed the same way and the whole planning
+            # run crashed. Reuse the same schema instruction as the initial call.
+            healing_messages = [
+                {"role": "system", "content": f"You repair malformed JSON.{instruction}"},
+                {"role": "user", "content": healing_prompt},
+            ]
             # The healing completion runs INSIDE this try (with the same retry/backoff as
             # the initial call). A transient failure here (429/timeout) must be treated as
             # "healing failed" so it routes into the fresh-attempt/fallback logic below,
@@ -332,16 +554,19 @@ Please return the corrected JSON object only. No prose.
                 # We use a lower temperature for healing
                 healed_response = await self._complete_with_retry(
                     model=model,
-                    messages=[{"role": "user", "content": healing_prompt}],
+                    messages=healing_messages,
                     temperature=0.0,
                     run_id=run_id,
                     provider=provider,
+                    json_schema=structured_schema,
                 )
                 tracker.log_usage(healed_response.model, healed_response.usage, local=provider_local)
                 healed_content = self._extract_json(healed_response.content)
                 data = json.loads(healed_content)
                 result = schema.model_validate(data)
                 cache.set(model, msgs, temp, True, healed_response, provider_fp)
+                if self._semantic is not None:
+                    await self._semantic.store_cache_async(msgs, healed_response, model=model, role=role)
                 return result
             except Exception as final_e:
                 logger.error(f"Healing failed for {role}: {final_e}")