flonat-research 0.1.0

package/packages/cli-council/src/cli_council/council.py

@@ -0,0 +1,391 @@
+"""3-stage council orchestration via local CLI tools.
+
+Stage 1 — Independent assessments (parallel subprocess calls)
+Stage 2 — Anonymised peer review (each backend reviews all assessments)
+Stage 3 — Chairman synthesis (single backend reads everything, produces final)
+
+Supports checkpoint-based session resumption (inspired by Owlex) and
+atomic file-based state (inspired by agents-council).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import re
+from collections import defaultdict
+from pathlib import Path
+from time import perf_counter
+
+from cli_council.backends.base import CLIBackend
+from cli_council.checkpoint import CouncilCheckpointer
+from cli_council.config import DEFAULT_CHAIRMAN, DEFAULT_COUNCIL_BACKENDS, STAGE_TIMEOUT
+from cli_council.models import Assessment, CouncilMeta, CouncilResult, PeerReview
+
+logger = logging.getLogger(__name__)
+
+
+def _get_backend(name: str) -> CLIBackend:
+    """Instantiate a backend by name."""
+    from cli_council.backends.claude import ClaudeBackend
+    from cli_council.backends.codex import CodexBackend
+    from cli_council.backends.gemini import GeminiBackend
+
+    registry: dict[str, type[CLIBackend]] = {
+        "gemini": GeminiBackend,
+        "codex": CodexBackend,
+        "claude": ClaudeBackend,
+    }
+    cls = registry.get(name)
+    if cls is None:
+        raise ValueError(f"Unknown backend: {name!r}. Available: {list(registry)}")
+    return cls()
+
+
+class CouncilRunner:
+    """Orchestrates a multi-model council using local CLI tools."""
+
+    def __init__(
+        self,
+        backends: list[str] | None = None,
+        chairman: str | None = None,
+        timeout: int = STAGE_TIMEOUT,
+        cwd: str | None = None,
+        checkpoint_dir: str | Path | None = None,
+    ) -> None:
+        self.backend_names = backends or DEFAULT_COUNCIL_BACKENDS
+        self.chairman_name = chairman or DEFAULT_CHAIRMAN
+        self.timeout = timeout
+        self.cwd = cwd
+        self._checkpoint_dir = Path(checkpoint_dir) if checkpoint_dir else None
+
+        # Instantiate and validate backends
+        self.backends: dict[str, CLIBackend] = {}
+        for name in self.backend_names:
+            backend = _get_backend(name)
+            if not backend.is_available():
+                logger.warning("Backend %s not found on PATH — skipping", name)
+                continue
+            self.backends[name] = backend
+
+        if self.chairman_name not in self.backends:
+            chairman_backend = _get_backend(self.chairman_name)
+            if chairman_backend.is_available():
+                self.backends[self.chairman_name] = chairman_backend
+
+    async def run(
+        self,
+        prompt: str,
+        *,
+        system_context: str = "",
+        resume: bool = False,
+    ) -> CouncilResult:
+        """Run the full 3-stage council process.
+
+        Parameters
+        ----------
+        prompt:
+            The main question or task for the council.
+        system_context:
+            Optional context prepended to all prompts (project description,
+            file contents, constraints, etc.).
+        resume:
+            If True and checkpoint_dir was provided, resume from the last
+            completed stage instead of starting fresh.
+        """
+        errors: list[str] = []
+        t_total = perf_counter()
+
+        full_prompt = f"{system_context}\n\n{prompt}".strip() if system_context else prompt
+
+        # Set up checkpointing
+        ckpt = None
+        resume_from = 0
+        if self._checkpoint_dir:
+            if resume:
+                # Find the latest run and resume from it
+                probe = CouncilCheckpointer(self._checkpoint_dir)
+                latest_run = probe.find_latest_run()
+                if latest_run:
+                    ckpt = CouncilCheckpointer(self._checkpoint_dir, run_id=latest_run)
+                    resume_from = ckpt.last_completed_stage()
+                    if resume_from > 0:
+                        logger.info(
+                            "Resuming run %s from stage %d",
+                            latest_run, resume_from + 1,
+                        )
+                    else:
+                        ckpt = CouncilCheckpointer(self._checkpoint_dir)
+                else:
+                    ckpt = CouncilCheckpointer(self._checkpoint_dir)
+            else:
+                ckpt = CouncilCheckpointer(self._checkpoint_dir)
+
+        # Stage 1: Independent assessments
+        stage1_ms = 0
+        if resume_from >= 1 and ckpt:
+            # Resume: load from checkpoint
+            saved = ckpt.load_stage1()
+            if saved:
+                assessments = [Assessment(**a) for a in saved]
+                logger.info("Stage 1: loaded %d assessments from checkpoint", len(assessments))
+            else:
+                t1 = perf_counter()
+                assessments = await self._stage1(full_prompt, errors)
+                stage1_ms = int((perf_counter() - t1) * 1000)
+        else:
+            t1 = perf_counter()
+            assessments = await self._stage1(full_prompt, errors)
+            stage1_ms = int((perf_counter() - t1) * 1000)
+
+        if len(assessments) < 2:
+            errors.append(f"Only {len(assessments)} backend(s) responded — need at least 2 for peer review")
+            return CouncilResult(
+                synthesis=assessments[0].text if assessments else "",
+                assessments=assessments,
+                peer_reviews=[],
+                meta=CouncilMeta(
+                    backends_used=[a.backend for a in assessments],
+                    stage1_ms=stage1_ms,
+                    total_ms=int((perf_counter() - t_total) * 1000),
+                    chairman_backend=self.chairman_name,
+                    errors=errors,
+                ),
+            )
+
+        # Assign anonymised labels
+        for i, a in enumerate(assessments):
+            a.label = f"Assessment {chr(65 + i)}"
+
+        # Checkpoint Stage 1
+        if ckpt and resume_from < 1:
+            ckpt.save_stage1(
+                [a.model_dump() for a in assessments],
+                [a.backend for a in assessments],
+            )
+            pending = ckpt.pending_participants(
+                self.backend_names,
+                [a.backend for a in assessments],
+            )
+            if pending:
+                logger.warning("Stage 1: pending backends: %s", pending)
+
+        # Stage 2: Peer review
+        stage2_ms = 0
+        if resume_from >= 2 and ckpt:
+            saved = ckpt.load_stage2()
+            if saved:
+                reviews_data, _ = saved
+                peer_reviews = [PeerReview(**r) for r in reviews_data]
+                logger.info("Stage 2: loaded %d reviews from checkpoint", len(peer_reviews))
+            else:
+                t2 = perf_counter()
+                peer_reviews = await self._stage2(prompt, assessments, errors)
+                stage2_ms = int((perf_counter() - t2) * 1000)
+        else:
+            t2 = perf_counter()
+            peer_reviews = await self._stage2(prompt, assessments, errors)
+            stage2_ms = int((perf_counter() - t2) * 1000)
+
+        # Checkpoint Stage 2
+        if ckpt and resume_from < 2:
+            ckpt.save_stage2(
+                [r.model_dump() for r in peer_reviews],
+                [r.backend for r in peer_reviews],
+            )
+
+        # Stage 3: Chairman synthesis
+        t3 = perf_counter()
+        synthesis = await self._stage3(prompt, assessments, peer_reviews, errors)
+        stage3_ms = int((perf_counter() - t3) * 1000)
+
+        # Checkpoint Stage 3
+        if ckpt:
+            ckpt.save_stage3(synthesis, self.chairman_name)
+
+        total_ms = int((perf_counter() - t_total) * 1000)
+
+        return CouncilResult(
+            synthesis=synthesis,
+            assessments=assessments,
+            peer_reviews=peer_reviews,
+            meta=CouncilMeta(
+                backends_used=[a.backend for a in assessments],
+                stage1_ms=stage1_ms,
+                stage2_ms=stage2_ms,
+                stage3_ms=stage3_ms,
+                total_ms=total_ms,
+                chairman_backend=self.chairman_name,
+                errors=errors,
+            ),
+        )
+
+    # ------------------------------------------------------------------
+    # Stage 1: Independent Assessments
+    # ------------------------------------------------------------------
+
+    async def _stage1(
+        self, prompt: str, errors: list[str],
+    ) -> list[Assessment]:
+        async def _query(name: str, backend: CLIBackend) -> Assessment | None:
+            try:
+                text, elapsed = await backend.run(
+                    prompt, timeout=self.timeout, cwd=self.cwd,
+                )
+                return Assessment(
+                    backend=name,
+                    model=backend.default_model,
+                    text=text,
+                    elapsed_ms=elapsed,
+                )
+            except Exception as exc:
+                msg = f"Stage 1: {name} failed — {exc}"
+                logger.warning(msg)
+                errors.append(msg)
+                return None
+
+        tasks = [_query(name, b) for name, b in self.backends.items()]
+        results = await asyncio.gather(*tasks)
+        assessments = [r for r in results if r is not None]
+
+        logger.info(
+            "Stage 1: %d/%d backends responded",
+            len(assessments), len(self.backends),
+        )
+        return assessments
+
+    # ------------------------------------------------------------------
+    # Stage 2: Anonymised Peer Review
+    # ------------------------------------------------------------------
+
+    async def _stage2(
+        self,
+        original_prompt: str,
+        assessments: list[Assessment],
+        errors: list[str],
+    ) -> list[PeerReview]:
+        assessments_block = "\n\n---\n\n".join(
+            f"**{a.label}:**\n{a.text}" for a in assessments
+        )
+
+        review_prompt = f"""You are reviewing multiple independent assessments of the same question.
+
+ORIGINAL QUESTION:
+{original_prompt}
+
+ASSESSMENTS:
+
+{assessments_block}
+
+YOUR TASK:
+1. Evaluate each assessment: strengths, weaknesses, gaps.
+2. Identify areas of AGREEMENT across assessments.
+3. Identify areas of DISAGREEMENT — which position is more convincing and why.
+4. Provide a final ranking from best to worst.
+
+IMPORTANT: End your review with "FINAL RANKING:" followed by a numbered list.
+Each line: number, period, space, then the assessment label (e.g. "1. Assessment A")."""
+
+        async def _review(name: str, backend: CLIBackend) -> PeerReview | None:
+            try:
+                text, elapsed = await backend.run(
+                    review_prompt, timeout=self.timeout, cwd=self.cwd,
+                )
+                ranking = self._parse_ranking(text)
+                return PeerReview(
+                    backend=name,
+                    model=backend.default_model,
+                    review_text=text,
+                    parsed_ranking=ranking,
+                    elapsed_ms=elapsed,
+                )
+            except Exception as exc:
+                msg = f"Stage 2: {name} failed — {exc}"
+                logger.warning(msg)
+                errors.append(msg)
+                return None
+
+        tasks = [_review(name, b) for name, b in self.backends.items()]
+        results = await asyncio.gather(*tasks)
+        reviews = [r for r in results if r is not None]
+
+        logger.info(
+            "Stage 2: %d/%d backends reviewed",
+            len(reviews), len(self.backends),
+        )
+        return reviews
+
+    # ------------------------------------------------------------------
+    # Stage 3: Chairman Synthesis
+    # ------------------------------------------------------------------
+
+    async def _stage3(
+        self,
+        original_prompt: str,
+        assessments: list[Assessment],
+        peer_reviews: list[PeerReview],
+        errors: list[str],
+    ) -> str:
+        assessments_block = "\n\n".join(
+            f"**{a.label}** (by {a.backend}):\n{a.text}" for a in assessments
+        )
+        reviews_block = "\n\n".join(
+            f"**Review by {r.backend}:**\n{r.review_text}" for r in peer_reviews
+        )
+
+        chairman_prompt = f"""You are the Chairman of a multi-model council. Multiple AI systems have independently assessed the same question, then peer-reviewed each other's assessments.
+
+ORIGINAL QUESTION:
+{original_prompt}
+
+STAGE 1 — Individual Assessments:
+{assessments_block}
+
+STAGE 2 — Peer Reviews:
+{reviews_block}
+
+YOUR TASK AS CHAIRMAN:
+1. Consider all individual assessments and their insights.
+2. Consider the peer reviews and what they reveal about quality and disagreements.
+3. Identify areas of strong consensus vs. genuine disagreement.
+4. Synthesise a SINGLE, comprehensive answer that represents the council's collective wisdom.
+
+Where the council agrees, reflect that consensus. Where they disagree, use your judgement to select the most well-reasoned position and explain why."""
+
+        chairman_backend = self.backends.get(self.chairman_name)
+        if chairman_backend is None:
+            errors.append(f"Chairman backend {self.chairman_name!r} not available")
+            return assessments[0].text if assessments else ""
+
+        try:
+            text, _ = await chairman_backend.run(
+                chairman_prompt, timeout=self.timeout * 2, cwd=self.cwd,
+            )
+            return text
+        except Exception as exc:
+            msg = f"Stage 3: chairman ({self.chairman_name}) failed — {exc}"
+            logger.warning(msg)
+            errors.append(msg)
+            return assessments[0].text if assessments else ""
+
+    # ------------------------------------------------------------------
+    # Utilities
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _parse_ranking(text: str) -> list[str]:
+        """Extract assessment ranking from review text."""
+        if "FINAL RANKING:" in text:
+            section = text.split("FINAL RANKING:", 1)[1]
+            numbered = re.findall(r"\d+\.\s*Assessment [A-Z]", section)
+            if numbered:
+                return [
+                    re.search(r"Assessment [A-Z]", m).group()
+                    for m in numbered
+                ]
+            return re.findall(r"Assessment [A-Z]", section)
+        return re.findall(r"Assessment [A-Z]", text)
+
+    def available_backends(self) -> list[str]:
+        """Return names of backends that are installed and available."""
+        return list(self.backends.keys())

package/packages/cli-council/src/cli_council/models.py

@@ -0,0 +1,46 @@
+"""Pydantic models for CLI council results."""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field
+
+
+class Assessment(BaseModel):
+    """One council member's individual assessment (Stage 1)."""
+
+    backend: str        # "gemini", "codex", "claude"
+    model: str          # model name/ID used by the backend
+    text: str           # raw text response
+    label: str = ""     # anonymised label, e.g. "Assessment A"
+    elapsed_ms: int = 0
+
+
+class PeerReview(BaseModel):
+    """One council member's peer review of all assessments (Stage 2)."""
+
+    backend: str
+    model: str
+    review_text: str
+    parsed_ranking: list[str] = Field(default_factory=list)
+    elapsed_ms: int = 0
+
+
+class CouncilMeta(BaseModel):
+    """Metadata for a council run."""
+
+    backends_used: list[str]
+    stage1_ms: int = 0
+    stage2_ms: int = 0
+    stage3_ms: int = 0
+    total_ms: int = 0
+    chairman_backend: str = "claude"
+    errors: list[str] = Field(default_factory=list)
+
+
+class CouncilResult(BaseModel):
+    """Full council deliberation result."""
+
+    synthesis: str
+    assessments: list[Assessment]
+    peer_reviews: list[PeerReview]
+    meta: CouncilMeta

package/packages/llm-council/.github/workflows/claude-code-review.yml

@@ -0,0 +1,44 @@
+name: Claude Code Review
+
+on:
+  pull_request:
+    types: [opened, synchronize, ready_for_review, reopened]
+    # Optional: Only run on specific file changes
+    # paths:
+    #   - "src/**/*.ts"
+    #   - "src/**/*.tsx"
+    #   - "src/**/*.js"
+    #   - "src/**/*.jsx"
+
+jobs:
+  claude-review:
+    # Optional: Filter by PR author
+    # if: |
+    #   github.event.pull_request.user.login == 'external-contributor' ||
+    #   github.event.pull_request.user.login == 'new-developer' ||
+    #   github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR'
+
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: read
+      issues: read
+      id-token: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Run Claude Code Review
+        id: claude-review
+        uses: anthropics/claude-code-action@v1
+        with:
+          claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
+          plugin_marketplaces: 'https://github.com/anthropics/claude-code.git'
+          plugins: 'code-review@claude-code-plugins'
+          prompt: '/code-review:code-review ${{ github.repository }}/pull/${{ github.event.pull_request.number }}'
+          # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
+          # or https://code.claude.com/docs/en/cli-reference for available options
+

package/packages/llm-council/.github/workflows/claude.yml

@@ -0,0 +1,50 @@
+name: Claude Code
+
+on:
+  issue_comment:
+    types: [created]
+  pull_request_review_comment:
+    types: [created]
+  issues:
+    types: [opened, assigned]
+  pull_request_review:
+    types: [submitted]
+
+jobs:
+  claude:
+    if: |
+      (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
+      (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
+      (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
+      (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: read
+      issues: read
+      id-token: write
+      actions: read # Required for Claude to read CI results on PRs
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Run Claude Code
+        id: claude
+        uses: anthropics/claude-code-action@v1
+        with:
+          claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
+
+          # This is an optional setting that allows Claude to read CI results on PRs
+          additional_permissions: |
+            actions: read
+
+          # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it.
+          # prompt: 'Update the pull request description to include a summary of changes.'
+
+          # Optional: Add claude_args to customize behavior and configuration
+          # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
+          # or https://code.claude.com/docs/en/cli-reference for available options
+          # claude_args: '--allowed-tools Bash(gh pr:*)'
+