flonat-research 0.1.0

package/packages/llm-council/src/llm_council/__main__.py

@@ -0,0 +1,259 @@
+"""CLI entry point for running council deliberations.
+
+Usage:
+    # Run a council
+    llm-council \\
+        --system-prompt "You are a paper reviewer..." \\
+        --user-message "Review this paper: ..." \\
+        --models "anthropic/claude-sonnet-4.5,openai/gpt-4.1,google/gemini-2.5-pro" \\
+        --chairman "anthropic/claude-sonnet-4.5" \\
+        --output result.json
+
+    # Or read prompts from files:
+    llm-council \\
+        --system-prompt-file system.txt \\
+        --user-message-file user.txt
+
+    # Manage models
+    llm-council models                          # show available + defaults
+    llm-council models --pricing                # include OpenRouter pricing
+    llm-council models --set-defaults "m1,m2"   # set default council models
+    llm-council models --set-chairman "m1"      # set default chairman
+    llm-council models --reset                  # revert to built-in defaults
+
+Environment:
+    OPENROUTER_API_KEY  Required for council runs and pricing lookups.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import os
+import sys
+
+from llm_council.client import LLMClient
+from llm_council.config import (
+    AVAILABLE_MODELS,
+    USER_CONFIG_PATH,
+    _BUILTIN_DEFAULT_CHAIRMAN,
+    _BUILTIN_DEFAULT_MODELS,
+    get_chairman_default,
+    get_council_defaults,
+    reset_council_defaults,
+    set_council_defaults,
+)
+
+
+# --- Models subcommand ---
+
+
+async def _models_command(args: argparse.Namespace) -> None:
+    if args.reset:
+        reset_council_defaults()
+        print("Reset to built-in defaults.")
+        print(f"  Council: {', '.join(_BUILTIN_DEFAULT_MODELS)}")
+        print(f"  Chairman: {_BUILTIN_DEFAULT_CHAIRMAN}")
+        return
+
+    if args.set_defaults:
+        models = [m.strip() for m in args.set_defaults.split(",")]
+        known_ids = {m["id"] for m in AVAILABLE_MODELS}
+        unknown = [m for m in models if m not in known_ids]
+        if unknown:
+            print(f"Warning: unknown model(s): {', '.join(unknown)}", file=sys.stderr)
+            print("They will be saved but may not work on OpenRouter.", file=sys.stderr)
+        set_council_defaults(models=models)
+        print(f"Default council models set: {', '.join(models)}")
+
+    if args.set_chairman:
+        set_council_defaults(chairman=args.set_chairman)
+        print(f"Default chairman set: {args.set_chairman}")
+
+    if args.set_defaults or args.set_chairman:
+        return
+
+    # List models
+    council_defaults = get_council_defaults()
+    chairman = get_chairman_default()
+    is_custom = USER_CONFIG_PATH.exists()
+
+    if args.pricing:
+        from llm_council.config import fetch_model_pricing
+
+        models = await fetch_model_pricing()
+    else:
+        models = [m.copy() for m in AVAILABLE_MODELS]
+
+    # Group by provider
+    providers: dict[str, list[dict]] = {}
+    for m in models:
+        provider = m["id"].split("/")[0]
+        providers.setdefault(provider, []).append(m)
+
+    source = "user config" if is_custom else "built-in"
+    print(f"Council defaults ({source}):")
+    print(f"  Models:   {', '.join(council_defaults)}")
+    print(f"  Chairman: {chairman}")
+    if is_custom:
+        print(f"  Config:   {USER_CONFIG_PATH}")
+    print()
+
+    print(f"Available models ({len(models)}):")
+    for provider in ["anthropic", "openai", "google"]:
+        if provider not in providers:
+            continue
+        print(f"\n  {provider.title()}:")
+        for m in providers[provider]:
+            marker = ""
+            if m["id"] in council_defaults and m["id"] == chairman:
+                marker = " [default, chairman]"
+            elif m["id"] in council_defaults:
+                marker = " [default]"
+            elif m["id"] == chairman:
+                marker = " [chairman]"
+
+            price = ""
+            if args.pricing and "input_price" in m:
+                price = f"  ({m['input_price']}/{m['output_price']} per 1M tok)"
+
+            print(f"    {m['id']:45s} {m.get('tier', ''):25s}{marker}{price}")
+
+
+# --- Run subcommand (existing behavior) ---
+
+
+async def _run_command(args: argparse.Namespace) -> None:
+    api_key = os.environ.get("OPENROUTER_API_KEY", "")
+    if not api_key:
+        print("Error: OPENROUTER_API_KEY environment variable not set.", file=sys.stderr)
+        sys.exit(1)
+
+    if args.system_prompt_file:
+        system_prompt = open(args.system_prompt_file).read()
+    else:
+        system_prompt = args.system_prompt or "You are a helpful expert assistant."
+
+    if args.user_message_file:
+        user_msg = open(args.user_message_file).read()
+    else:
+        user_msg = args.user_message or ""
+
+    if not user_msg:
+        print("Error: --user-message or --user-message-file required.", file=sys.stderr)
+        sys.exit(1)
+
+    models = [m.strip() for m in args.models.split(",")]
+    chairman = args.chairman
+
+    from llm_council.council import CouncilService
+
+    llm = LLMClient(api_key=api_key, max_tokens=args.max_tokens)
+    council = CouncilService(llm)
+
+    try:
+        result = await council.run_council(
+            system_prompt=system_prompt,
+            user_msg=user_msg,
+            council_models=models,
+            chairman_model=chairman,
+        )
+
+        output = result.model_dump()
+
+        if args.output:
+            with open(args.output, "w") as f:
+                json.dump(output, f, indent=2)
+            print(f"Result written to {args.output}", file=sys.stderr)
+        else:
+            print(json.dumps(output, indent=2))
+
+    finally:
+        await llm.close()
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Multi-model LLM council via OpenRouter.",
+    )
+    subparsers = parser.add_subparsers(dest="command")
+
+    # --- models subcommand ---
+    models_parser = subparsers.add_parser(
+        "models", help="List available models and manage defaults.",
+    )
+    models_parser.add_argument(
+        "--pricing", action="store_true",
+        help="Fetch and display OpenRouter pricing.",
+    )
+    models_parser.add_argument(
+        "--set-defaults", type=str, metavar="MODELS",
+        help="Set default council models (comma-separated model IDs).",
+    )
+    models_parser.add_argument(
+        "--set-chairman", type=str, metavar="MODEL",
+        help="Set default chairman model.",
+    )
+    models_parser.add_argument(
+        "--reset", action="store_true",
+        help="Remove user config and revert to built-in defaults.",
+    )
+
+    # --- run subcommand (also the default) ---
+    run_parser = subparsers.add_parser(
+        "run", help="Run a council deliberation.",
+    )
+    _add_run_args(run_parser)
+
+    # Also add run args to the main parser for backwards compatibility
+    _add_run_args(parser)
+
+    args = parser.parse_args()
+
+    if args.command == "models":
+        asyncio.run(_models_command(args))
+    else:
+        asyncio.run(_run_command(args))
+
+
+def _add_run_args(parser: argparse.ArgumentParser) -> None:
+    """Add the run-command arguments to a parser."""
+    parser.add_argument(
+        "--system-prompt", type=str, default=None,
+        help="System prompt for Stage 1 assessments.",
+    )
+    parser.add_argument(
+        "--system-prompt-file", type=str, default=None,
+        help="Read system prompt from a file.",
+    )
+    parser.add_argument(
+        "--user-message", type=str, default=None,
+        help="User message for Stage 1 assessments.",
+    )
+    parser.add_argument(
+        "--user-message-file", type=str, default=None,
+        help="Read user message from a file.",
+    )
+    parser.add_argument(
+        "--models", type=str,
+        default=",".join(get_council_defaults()),
+        help="Comma-separated list of OpenRouter model IDs.",
+    )
+    parser.add_argument(
+        "--chairman", type=str,
+        default=get_chairman_default(),
+        help="Model ID for the chairman synthesis.",
+    )
+    parser.add_argument(
+        "--max-tokens", type=int, default=4096,
+        help="Max tokens per LLM response.",
+    )
+    parser.add_argument(
+        "--output", "-o", type=str, default=None,
+        help="Write JSON result to file instead of stdout.",
+    )
+
+
+if __name__ == "__main__":
+    main()

package/packages/llm-council/src/llm_council/checkpoint.py

@@ -0,0 +1,193 @@
+"""Checkpoint and file-based coordination for council runs.
+
+Inspired by:
+- Owlex (agentic-mcp-tools/owlex): session resumption across deliberation rounds
+- agents-council (MrLesk/agents-council): atomic file writes, cursor-based state
+
+Provides:
+- Atomic JSON writes (tmp + fsync + rename) for crash safety
+- Stage checkpointing: save after each stage, resume from last completed stage
+- Pending participant tracking
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import tempfile
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_CHECKPOINT_DIR = ".council"
+
+
+def _atomic_write_json(path: Path, data: dict) -> None:
+    """Write JSON atomically: tmp file -> fsync -> rename.
+
+    If the process crashes mid-write, the original file is preserved.
+    """
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, tmp_path = tempfile.mkstemp(
+        dir=str(path.parent), suffix=".tmp", prefix=".ckpt-",
+    )
+    try:
+        with os.fdopen(fd, "w") as f:
+            json.dump(data, f, indent=2, default=str)
+            f.flush()
+            os.fsync(f.fileno())
+        os.rename(tmp_path, str(path))
+    except BaseException:
+        try:
+            os.unlink(tmp_path)
+        except OSError:
+            pass
+        raise
+
+
+def _read_json(path: Path) -> dict | None:
+    """Read a JSON checkpoint file, returning None if missing or corrupt."""
+    if not path.exists():
+        return None
+    try:
+        return json.loads(path.read_text())
+    except (json.JSONDecodeError, OSError) as exc:
+        logger.warning("Corrupt checkpoint %s: %s", path, exc)
+        return None
+
+
+def _make_run_id() -> str:
+    """Generate a timestamped run ID."""
+    return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+
+
+@dataclass
+class CouncilCheckpointer:
+    """Manages checkpoint state for a council run."""
+
+    checkpoint_dir: Path
+    run_id: str = field(default_factory=_make_run_id)
+
+    def __post_init__(self) -> None:
+        self.checkpoint_dir = Path(self.checkpoint_dir)
+
+    # ---- Save ----
+
+    def save_stage1(self, assessments: list[dict], models: list[str]) -> Path:
+        """Save Stage 1 assessments to checkpoint."""
+        path = self._stage_path(1)
+        data = {
+            "meta": {
+                "run_id": self.run_id,
+                "stage": 1,
+                "timestamp": datetime.now(timezone.utc).isoformat(),
+                "models": models,
+            },
+            "assessments": assessments,
+        }
+        _atomic_write_json(path, data)
+        logger.info("Checkpoint saved: Stage 1 → %s", path)
+        return path
+
+    def save_stage2(
+        self,
+        peer_reviews: list[dict],
+        models: list[str],
+        aggregate_rankings: list[dict] | None = None,
+    ) -> Path:
+        """Save Stage 2 peer reviews to checkpoint."""
+        path = self._stage_path(2)
+        data = {
+            "meta": {
+                "run_id": self.run_id,
+                "stage": 2,
+                "timestamp": datetime.now(timezone.utc).isoformat(),
+                "models": models,
+            },
+            "peer_reviews": peer_reviews,
+            "aggregate_rankings": aggregate_rankings or [],
+        }
+        _atomic_write_json(path, data)
+        logger.info("Checkpoint saved: Stage 2 → %s", path)
+        return path
+
+    def save_stage3(self, synthesis: dict | str, chairman: str) -> Path:
+        """Save Stage 3 synthesis to checkpoint."""
+        path = self._stage_path(3)
+        data = {
+            "meta": {
+                "run_id": self.run_id,
+                "stage": 3,
+                "timestamp": datetime.now(timezone.utc).isoformat(),
+                "chairman": chairman,
+            },
+            "synthesis": synthesis,
+        }
+        _atomic_write_json(path, data)
+        logger.info("Checkpoint saved: Stage 3 → %s", path)
+        return path
+
+    # ---- Load / Resume ----
+
+    def last_completed_stage(self) -> int:
+        """Return the highest completed stage (0 if none)."""
+        for stage in (3, 2, 1):
+            if self._stage_path(stage).exists():
+                return stage
+        return 0
+
+    def load_stage1(self) -> list[dict] | None:
+        """Load Stage 1 assessments from checkpoint."""
+        data = _read_json(self._stage_path(1))
+        if data is None:
+            return None
+        return data.get("assessments")
+
+    def load_stage2(self) -> tuple[list[dict], list[dict]] | None:
+        """Load Stage 2 peer reviews + aggregate rankings from checkpoint."""
+        data = _read_json(self._stage_path(2))
+        if data is None:
+            return None
+        return data.get("peer_reviews", []), data.get("aggregate_rankings", [])
+
+    def load_stage3(self) -> dict | str | None:
+        """Load Stage 3 synthesis from checkpoint."""
+        data = _read_json(self._stage_path(3))
+        if data is None:
+            return None
+        return data.get("synthesis")
+
+    # ---- Pending participants ----
+
+    def pending_participants(
+        self, all_models: list[str], responded: list[str],
+    ) -> list[str]:
+        """Return models that haven't responded yet."""
+        return [m for m in all_models if m not in set(responded)]
+
+    # ---- Discovery ----
+
+    def find_latest_run(self) -> str | None:
+        """Find the most recent run_id in the checkpoint directory."""
+        if not self.checkpoint_dir.exists():
+            return None
+        stage1_files = sorted(self.checkpoint_dir.glob("*-stage1.json"), reverse=True)
+        if not stage1_files:
+            return None
+        return stage1_files[0].stem.replace("-stage1", "")
+
+    def clean(self) -> None:
+        """Remove all checkpoint files for this run."""
+        for stage in (1, 2, 3):
+            path = self._stage_path(stage)
+            if path.exists():
+                path.unlink()
+                logger.info("Removed checkpoint: %s", path)
+
+    # ---- Internal ----
+
+    def _stage_path(self, stage: int) -> Path:
+        return self.checkpoint_dir / f"{self.run_id}-stage{stage}.json"

package/packages/llm-council/src/llm_council/client.py

@@ -0,0 +1,253 @@
+"""Generic OpenRouter LLM client.
+
+OpenRouter wrapper with JSON parsing, retry logic, and error handling.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+
+import openai
+from openai import AsyncOpenAI
+
+logger = logging.getLogger(__name__)
+
+OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1"
+OPENROUTER_CREDITS_URL = "https://openrouter.ai/credits"
+
+
+class LLMResponseFormatError(RuntimeError):
+    """Raised when an LLM response cannot be parsed into valid JSON."""
+
+
+class LLMServiceError(Exception):
+    """User-friendly error from the LLM service."""
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        help_url: str | None = None,
+        detail: str | None = None,
+    ) -> None:
+        super().__init__(message)
+        self.help_url = help_url
+        self.detail = detail
+
+
+def _handle_openai_error(exc: Exception) -> LLMServiceError:
+    """Convert openai SDK exceptions into user-friendly LLMServiceError."""
+    if isinstance(exc, openai.AuthenticationError):
+        return LLMServiceError(
+            "Invalid OpenRouter API key. Please check your OPENROUTER_API_KEY.",
+            help_url=OPENROUTER_CREDITS_URL,
+            detail=str(exc),
+        )
+    if isinstance(exc, openai.RateLimitError):
+        return LLMServiceError(
+            "OpenRouter rate limit reached. Please wait a moment and try again.",
+            help_url=OPENROUTER_CREDITS_URL,
+            detail=str(exc),
+        )
+    if isinstance(exc, openai.APIStatusError):
+        status = getattr(exc, "status_code", None)
+        if status == 402:
+            return LLMServiceError(
+                "Insufficient OpenRouter credits. Please top up your account.",
+                help_url=OPENROUTER_CREDITS_URL,
+                detail=str(exc),
+            )
+        if status == 503:
+            return LLMServiceError(
+                "The selected LLM model is temporarily unavailable. Try a different model.",
+                detail=str(exc),
+            )
+        return LLMServiceError(
+            f"OpenRouter API error (HTTP {status}). Please try again.",
+            help_url=OPENROUTER_CREDITS_URL,
+            detail=str(exc),
+        )
+    if isinstance(exc, openai.APIConnectionError):
+        return LLMServiceError(
+            "Could not connect to OpenRouter. Please check your internet connection.",
+            detail=str(exc),
+        )
+    return LLMServiceError(
+        "LLM request failed unexpectedly. Please try again.",
+        detail=str(exc),
+    )
+
+
+class LLMClient:
+    """Generic async LLM client via OpenRouter.
+
+    Provides structured JSON and raw text chat methods with retry logic.
+    Consumers extend this with domain-specific workflow methods.
+    """
+
+    def __init__(
+        self,
+        api_key: str,
+        model: str = "anthropic/claude-sonnet-4.5",
+        max_tokens: int = 4096,
+        json_retry_attempts: int = 2,
+    ) -> None:
+        self.client = AsyncOpenAI(
+            api_key=api_key,
+            base_url=OPENROUTER_BASE_URL,
+        )
+        self.model = model
+        self.max_tokens = max_tokens
+        self.json_retry_attempts = max(1, json_retry_attempts)
+
+    async def chat_json(
+        self,
+        system: str,
+        user_msg: str,
+        *,
+        model: str | None = None,
+        max_tokens: int | None = None,
+    ) -> dict:
+        """Send a message and parse a JSON-object response with retries.
+
+        Parameters
+        ----------
+        max_tokens:
+            Override the client's default ``max_tokens`` for this call.
+            Useful when the expected JSON response is larger than usual
+            (e.g. council synthesis of a discovery workflow).
+        """
+        effective_model = model or self.model
+        effective_max_tokens = max_tokens or self.max_tokens
+        prompt = user_msg + "\n\nRespond ONLY with valid JSON."
+        raw_text = ""
+        parse_error: Exception | None = None
+
+        for attempt in range(1, self.json_retry_attempts + 1):
+            try:
+                response = await self.client.chat.completions.create(
+                    model=effective_model,
+                    max_tokens=effective_max_tokens,
+                    messages=[
+                        {"role": "system", "content": system},
+                        {"role": "user", "content": prompt},
+                    ],
+                )
+            except (openai.APIError, openai.APIConnectionError) as api_exc:
+                raise _handle_openai_error(api_exc) from api_exc
+
+            raw_text = (response.choices[0].message.content or "").strip()
+
+            try:
+                return self._parse_json_response(raw_text)
+            except LLMResponseFormatError as exc:
+                parse_error = exc
+                logger.warning(
+                    "LLM JSON parse failed (attempt %d/%d): %s",
+                    attempt, self.json_retry_attempts, exc,
+                )
+                if attempt == self.json_retry_attempts:
+                    break
+                prompt = (
+                    "Your previous response was not valid JSON.\n"
+                    "Return ONLY a valid JSON object matching the schema in the system prompt.\n"
+                    "Do not include markdown fences or commentary.\n\n"
+                    "Previous invalid response:\n"
+                    f"{raw_text[:8000]}"
+                )
+
+        snippet = raw_text[:240].replace("\n", " ")
+        raise LLMResponseFormatError(
+            f"Failed to parse LLM JSON response after {self.json_retry_attempts} attempts. "
+            f"Last parse error: {parse_error}. Response snippet: {snippet!r}"
+        )
+
+    async def chat_text(
+        self,
+        system: str,
+        user_msg: str,
+        *,
+        model: str | None = None,
+        max_tokens: int | None = None,
+    ) -> str:
+        """Query the LLM and return the raw text response (no JSON parsing).
+
+        Used for Stage 2 peer review which returns free-form text with a
+        FINAL RANKING section.
+
+        Parameters
+        ----------
+        max_tokens:
+            Override the client's default ``max_tokens`` for this call.
+        """
+        effective_model = model or self.model
+        effective_max_tokens = max_tokens or self.max_tokens
+        try:
+            response = await self.client.chat.completions.create(
+                model=effective_model,
+                max_tokens=effective_max_tokens,
+                messages=[
+                    {"role": "system", "content": system},
+                    {"role": "user", "content": user_msg},
+                ],
+            )
+        except (openai.APIError, openai.APIConnectionError) as api_exc:
+            raise _handle_openai_error(api_exc) from api_exc
+        return (response.choices[0].message.content or "").strip()
+
+    async def close(self) -> None:
+        await self.client.close()
+
+    # ------------------------------------------------------------------
+    # JSON parsing utilities
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _extract_json_candidates(text: str) -> list[str]:
+        stripped = text.strip()
+        candidates: list[str] = []
+
+        if stripped:
+            candidates.append(stripped)
+
+        fenced_blocks = re.findall(
+            r"```(?:json)?\s*(.*?)```", stripped, flags=re.IGNORECASE | re.DOTALL,
+        )
+        for block in fenced_blocks:
+            block = block.strip()
+            if block:
+                candidates.append(block)
+
+        first_brace = stripped.find("{")
+        last_brace = stripped.rfind("}")
+        if first_brace != -1 and last_brace != -1 and last_brace > first_brace:
+            candidates.append(stripped[first_brace : last_brace + 1].strip())
+
+        deduped: list[str] = []
+        seen: set[str] = set()
+        for candidate in candidates:
+            if candidate not in seen:
+                seen.add(candidate)
+                deduped.append(candidate)
+        return deduped
+
+    @classmethod
+    def _parse_json_response(cls, text: str) -> dict:
+        errors: list[str] = []
+        for candidate in cls._extract_json_candidates(text):
+            try:
+                parsed = json.loads(candidate)
+            except json.JSONDecodeError as exc:
+                errors.append(f"{exc.msg} (line {exc.lineno}, col {exc.colno})")
+                continue
+            if isinstance(parsed, dict):
+                return parsed
+            errors.append(f"Expected JSON object, got {type(parsed).__name__}")
+
+        snippet = text[:240].replace("\n", " ")
+        raise LLMResponseFormatError(
+            f"Unable to parse JSON object from response. Snippet: {snippet!r}. "
+            f"Errors: {errors[:2]}"
+        )