@pennyfarthing/core 11.0.0 → 11.1.1

package/pennyfarthing_scripts/tests/test_108_2_remove_handoff_fallback.py

@@ -0,0 +1,339 @@
+"""Tests for removing handoff subagents and inline fallback — Story 108-2.
+
+Epic: 108 (Full Migration & Cleanup)
+Story: 108-2 — Remove handoff subagents and inline fallback
+
+Tests the removal of deprecated handoff subagents and the gate.type
+fallback code path, leaving only gate.file-based resolution.
+
+Acceptance Criteria:
+- [AC1] pennyfarthing-dist/agents/handoff.md deleted
+- [AC2] pennyfarthing-dist/agents/sm-handoff.md deleted
+- [AC3] gate.type fallback removed from resolve-gate — file-only gates resolved
+- [AC4] checkGate() in Cyclist updated or removed (see TS test)
+- [AC5] No remaining references to handoff/sm-handoff subagents in agent files
+- [AC6] All existing tests pass
+- [AC7] Single code path: all gated phases in phased workflows have gate.file
+"""
+
+from __future__ import annotations
+
+import re
+import textwrap
+from pathlib import Path
+
+import pytest
+import yaml
+
+from pennyfarthing_scripts.handoff.resolve_gate import resolve_gate
+
+# ---------------------------------------------------------------------------
+# Paths
+# ---------------------------------------------------------------------------
+
+DIST_DIR = Path(__file__).resolve().parents[2] / "pennyfarthing-dist"
+AGENTS_DIR = DIST_DIR / "agents"
+WORKFLOWS_DIR = DIST_DIR / "workflows"
+GATES_DIR = DIST_DIR / "gates"
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+SESSION_WITH_ASSESSMENT = textwrap.dedent("""\
+    # Story 108-2: Remove handoff subagents and inline fallback
+
+    **Story ID:** 108-2
+    **Workflow:** tdd
+    **Phase:** green
+    **Phase Started:** 2026-02-15T10:00:00Z
+
+    ## TEA Assessment
+
+    **Tests Written:** 5 tests
+    **Status:** RED confirmed
+""")
+
+SESSION_WITHOUT_ASSESSMENT = textwrap.dedent("""\
+    # Story 108-2: Remove handoff subagents and inline fallback
+
+    **Story ID:** 108-2
+    **Workflow:** tdd
+    **Phase:** green
+    **Phase Started:** 2026-02-15T10:00:00Z
+""")
+
+WORKFLOW_FILE_ONLY_GATE = {
+    "workflow": {
+        "name": "file-only-test",
+        "phases": [
+            {"name": "setup", "agent": "sm"},
+            {
+                "name": "green",
+                "agent": "dev",
+                "gate": {
+                    "file": "gates/tests-pass",
+                    "condition": "All tests passing",
+                },
+            },
+            {"name": "finish", "agent": "sm"},
+        ],
+    }
+}
+
+
+@pytest.fixture
+def project(tmp_path: Path) -> Path:
+    """Create a minimal project with workflow YAMLs and session."""
+    workflows_dir = tmp_path / ".pennyfarthing" / "workflows"
+    workflows_dir.mkdir(parents=True)
+    (tmp_path / ".session").mkdir()
+
+    # Write the file-only-gate workflow
+    (workflows_dir / "file-only-test.yaml").write_text(
+        yaml.dump(WORKFLOW_FILE_ONLY_GATE, default_flow_style=False)
+    )
+
+    # Copy real tdd.yaml for integration tests
+    tdd_src = WORKFLOWS_DIR / "tdd.yaml"
+    if tdd_src.exists():
+        (workflows_dir / "tdd.yaml").write_text(tdd_src.read_text())
+
+    return tmp_path
+
+
+# ===========================================================================
+# AC1: pennyfarthing-dist/agents/handoff.md deleted
+# ===========================================================================
+
+
+class TestHandoffMdDeleted:
+    """AC1: handoff.md must not exist in pennyfarthing-dist/agents/."""
+
+    def test_handoff_md_does_not_exist(self) -> None:
+        """AC1: handoff.md should be deleted from agents directory."""
+        assert not (AGENTS_DIR / "handoff.md").exists(), (
+            "handoff.md still exists — should be deleted in 108-2"
+        )
+
+
+# ===========================================================================
+# AC2: pennyfarthing-dist/agents/sm-handoff.md deleted
+# ===========================================================================
+
+
+class TestSmHandoffMdDeleted:
+    """AC2: sm-handoff.md must not exist in pennyfarthing-dist/agents/."""
+
+    def test_sm_handoff_md_does_not_exist(self) -> None:
+        """AC2: sm-handoff.md should be deleted from agents directory."""
+        assert not (AGENTS_DIR / "sm-handoff.md").exists(), (
+            "sm-handoff.md still exists — should be deleted in 108-2"
+        )
+
+
+# ===========================================================================
+# AC3: gate.type fallback removed — file-only gates resolve properly
+# ===========================================================================
+
+
+class TestGateTypeFallbackRemoved:
+    """AC3: gate with file but no type should resolve via assessment, not skip."""
+
+    def test_no_gate_at_all_still_returns_skip(
+        self, project: Path
+    ) -> None:
+        """AC3: Phase with no gate (setup/finish) should still return 'skip'."""
+        session = project / ".session" / "108-2-session.md"
+        session.write_text(SESSION_WITH_ASSESSMENT)
+
+        result = resolve_gate(
+            "108-2", "tdd", "setup", project_root=project
+        )
+        assert result["status"] == "skip", (
+            f"No-gate phase should skip, got: {result['status']}"
+        )
+
+    def test_gate_file_only_returns_ready_with_assessment(
+        self, project: Path
+    ) -> None:
+        """AC3: gate with file but no type + assessment → 'ready' (not 'skip').
+
+        Before 108-2: gate_type is None → skip (fallback).
+        After 108-2: gate.file present → check assessment → ready.
+        """
+        session = project / ".session" / "108-2-session.md"
+        session.write_text(SESSION_WITH_ASSESSMENT)
+
+        result = resolve_gate(
+            "108-2", "file-only-test", "green", project_root=project
+        )
+        assert result["status"] == "ready", (
+            f"File-only gate with assessment should be 'ready', got: "
+            f"'{result['status']}' — gate.type fallback still active?"
+        )
+
+    def test_gate_file_only_returns_blocked_without_assessment(
+        self, project: Path
+    ) -> None:
+        """AC3: gate with file but no type + NO assessment → 'blocked'.
+
+        Before 108-2: gate_type is None → skip (ignores missing assessment).
+        After 108-2: gate.file present → check assessment → blocked.
+        """
+        session = project / ".session" / "108-2-session.md"
+        session.write_text(SESSION_WITHOUT_ASSESSMENT)
+
+        result = resolve_gate(
+            "108-2", "file-only-test", "green", project_root=project
+        )
+        assert result["status"] == "blocked", (
+            f"File-only gate without assessment should be 'blocked', got: "
+            f"'{result['status']}' — gate.type fallback still active?"
+        )
+
+    def test_gate_file_only_populates_gate_file_field(
+        self, project: Path
+    ) -> None:
+        """AC3: gate_file field should be populated for file-only gates."""
+        session = project / ".session" / "108-2-session.md"
+        session.write_text(SESSION_WITH_ASSESSMENT)
+
+        result = resolve_gate(
+            "108-2", "file-only-test", "green", project_root=project
+        )
+        assert result["gate_file"] == "gates/tests-pass", (
+            f"Expected gate_file='gates/tests-pass', got: {result['gate_file']}"
+        )
+
+    def test_gate_with_both_file_and_type_still_ready(
+        self, project: Path
+    ) -> None:
+        """AC3: gate with both file and type should still return 'ready'."""
+        session = project / ".session" / "108-2-session.md"
+        session.write_text(SESSION_WITH_ASSESSMENT)
+
+        result = resolve_gate(
+            "108-2", "tdd", "green", project_root=project
+        )
+        assert result["status"] == "ready"
+        assert result["gate_file"] == "gates/tests-pass"
+        assert result["gate_type"] == "tests_pass"
+
+
+# ===========================================================================
+# AC5: No remaining references to handoff/sm-handoff subagents
+# ===========================================================================
+
+
+class TestNoSubagentReferences:
+    """AC5: No agent files should reference the deleted subagents."""
+
+    def _scan_agent_files(self, pattern: str) -> list[tuple[str, str]]:
+        """Scan all agent .md files for a regex pattern. Returns (file, line) pairs."""
+        hits = []
+        for md_file in AGENTS_DIR.glob("*.md"):
+            # Skip the files being deleted (they'll be gone)
+            if md_file.name in ("handoff.md", "sm-handoff.md"):
+                continue
+            content = md_file.read_text()
+            for line in content.splitlines():
+                if re.search(pattern, line, re.IGNORECASE):
+                    hits.append((md_file.name, line.strip()))
+        return hits
+
+    def test_no_agent_references_handoff_subagent(self) -> None:
+        """AC5: No agent .md files should spawn or reference 'handoff' subagent.
+
+        Matches patterns like:
+        - subagent_type: handoff
+        - spawn handoff
+        - sm-handoff subagent
+        But NOT: 'pf handoff' (the CLI) or 'handoff.ts' (the module)
+        """
+        # Look for subagent spawn references to the deprecated handoff subagent
+        hits = self._scan_agent_files(
+            r'(?:subagent.*handoff\.md|sm-handoff\.md|"sm-handoff"|"handoff"'
+            r"|handoff subagent|sm-handoff subagent)"
+        )
+        assert len(hits) == 0, (
+            f"Found {len(hits)} references to deprecated handoff subagents:\n"
+            + "\n".join(f"  {f}: {line}" for f, line in hits)
+        )
+
+    def test_agents_readme_no_deprecated_entries(self) -> None:
+        """AC5: README.md should not list handoff.md or sm-handoff.md."""
+        readme = AGENTS_DIR / "README.md"
+        if not readme.exists():
+            pytest.skip("No agents README.md")
+
+        content = readme.read_text()
+        # Should not have the deprecated entries
+        assert "handoff.md" not in content, (
+            "README.md still references handoff.md"
+        )
+        assert "sm-handoff.md" not in content, (
+            "README.md still references sm-handoff.md"
+        )
+
+
+# ===========================================================================
+# AC7: Single code path — all gated phases have gate.file
+# ===========================================================================
+
+# Phased workflows that should have gate.file on every gated phase
+PHASED_WORKFLOWS = ["tdd", "trivial", "bdd", "bdd-tandem", "tdd-tandem",
+                     "2party-tdd", "agent-docs", "patch"]
+
+
+class TestAllGatedPhasesHaveGateFile:
+    """AC7: Every gated phase in phased workflows must have gate.file."""
+
+    def _load_workflow(self, name: str) -> dict | None:
+        """Load a workflow YAML, return None if not found."""
+        path = WORKFLOWS_DIR / f"{name}.yaml"
+        if not path.exists():
+            return None
+        return yaml.safe_load(path.read_text())
+
+    def _gated_phases_missing_file(
+        self, workflow_data: dict
+    ) -> list[tuple[str, str]]:
+        """Find phases that have a gate but no gate.file.
+
+        Returns list of (phase_name, gate_type) tuples.
+        Manual gates are excluded — they skip resolution entirely.
+        """
+        missing = []
+        for phase in workflow_data["workflow"]["phases"]:
+            gate = phase.get("gate")
+            if not gate:
+                continue  # No gate at all — fine (setup, finish)
+            gate_type = gate.get("type")
+            if gate_type == "manual":
+                continue  # Manual gates always skip
+            if not gate.get("file"):
+                missing.append((phase["name"], gate_type or "(no type)"))
+        return missing
+
+    @pytest.mark.parametrize("workflow_name", PHASED_WORKFLOWS)
+    def test_workflow_gated_phases_have_gate_file(
+        self, workflow_name: str
+    ) -> None:
+        """AC7: Every non-manual gated phase must have gate.file defined.
+
+        This ensures the single code path — resolve-gate only needs
+        gate.file, not gate.type, for resolution.
+        """
+        wf = self._load_workflow(workflow_name)
+        if wf is None:
+            pytest.skip(f"Workflow {workflow_name} not found")
+
+        missing = self._gated_phases_missing_file(wf)
+        assert len(missing) == 0, (
+            f"Workflow '{workflow_name}' has gated phases without gate.file:\n"
+            + "\n".join(
+                f"  - {name} (type={gtype})" for name, gtype in missing
+            )
+            + "\nAll gated phases must have gate.file for single code path."
+        )

package/pennyfarthing_scripts/tests/test_confidence_sm_evaluation.py

@@ -0,0 +1,253 @@
+"""Tests for SM confidence gate evaluation results — Story 90-3.
+
+Epic: 90 (Confidence Circuit Breaker via Gate)
+Story: 90-3 — Evaluate and document results
+
+Tests the evaluation document that assesses the SM confidence gate's
+effectiveness after deployment. The gate (confidence-sm.md) was shipped
+in story 90-2. This story evaluates trigger frequency, wrong-approach
+reduction, user experience, and makes a rollout recommendation.
+
+Acceptance Criteria:
+- [AC1] Evaluate trigger frequency after 1 sprint
+- [AC2] Document whether it reduced wrong-approach incidents
+- [AC3] Assess if the gate was annoying (user overrode or dismissed)
+- [AC4] Document findings in a results file
+- [AC5] Decide on rollout recommendation to other agents
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+import pytest
+
+# ---------------------------------------------------------------------------
+# Path resolution
+# ---------------------------------------------------------------------------
+
+_THIS_DIR = Path(__file__).resolve().parent
+_SCRIPTS_DIR = _THIS_DIR.parent  # pennyfarthing_scripts/
+_FRAMEWORK_ROOT = _SCRIPTS_DIR.parent  # pennyfarthing/
+_EVAL_FILE = (
+    _FRAMEWORK_ROOT
+    / "pennyfarthing-dist"
+    / "gates"
+    / "evaluations"
+    / "confidence-sm.md"
+)
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+
+@pytest.fixture
+def eval_path() -> Path:
+    """Return the expected path to the evaluation results file."""
+    return _EVAL_FILE
+
+
+@pytest.fixture
+def eval_content(eval_path: Path) -> str:
+    """Read and return the evaluation file content. Fails if file missing."""
+    assert eval_path.is_file(), f"Evaluation file not found: {eval_path}"
+    return eval_path.read_text()
+
+
+# ===========================================================================
+# AC1: Evaluate trigger frequency after 1 sprint
+# ===========================================================================
+
+
+class TestTriggerFrequency:
+    """AC1: Evaluation documents how often the gate triggered."""
+
+    def test_eval_has_trigger_section(self, eval_content: str) -> None:
+        """AC1: Evaluation has a section about trigger frequency."""
+        assert re.search(
+            r"(?i)trigger.*frequen|frequen.*trigger|how often|activation.*rate",
+            eval_content,
+        ), "Missing trigger frequency section"
+
+    def test_trigger_section_has_data(self, eval_content: str) -> None:
+        """AC1: Trigger section includes specific data or baseline note."""
+        content_lower = eval_content.lower()
+        assert any(
+            term in content_lower
+            for term in [
+                "triggered",
+                "fired",
+                "activated",
+                "invoked",
+                "n/a",
+                "baseline",
+                "times",
+                "occurrences",
+            ]
+        ), "Trigger section lacks frequency data or baseline acknowledgment"
+
+
+# ===========================================================================
+# AC2: Document whether it reduced wrong-approach incidents
+# ===========================================================================
+
+
+class TestWrongApproachReduction:
+    """AC2: Evaluation documents impact on wrong-approach incidents."""
+
+    def test_eval_has_wrong_approach_section(self, eval_content: str) -> None:
+        """AC2: Evaluation has a section about wrong-approach incidents."""
+        content_lower = eval_content.lower()
+        assert any(
+            term in content_lower
+            for term in [
+                "wrong-approach",
+                "wrong approach",
+                "incident",
+                "misroute",
+                "incorrect action",
+                "ambiguous",
+            ]
+        ), "Missing wrong-approach reduction section"
+
+    def test_wrong_approach_has_comparison(self, eval_content: str) -> None:
+        """AC2: Wrong-approach section includes comparison or baseline note."""
+        content_lower = eval_content.lower()
+        assert any(
+            term in content_lower
+            for term in [
+                "before",
+                "after",
+                "baseline",
+                "reduced",
+                "comparison",
+                "prior",
+                "improvement",
+                "no data yet",
+            ]
+        ), "Wrong-approach section lacks before/after comparison or baseline"
+
+
+# ===========================================================================
+# AC3: Assess if the gate was annoying
+# ===========================================================================
+
+
+class TestUserExperience:
+    """AC3: Evaluation assesses user experience with the gate."""
+
+    def test_eval_has_ux_section(self, eval_content: str) -> None:
+        """AC3: Evaluation has a section about user experience."""
+        content_lower = eval_content.lower()
+        assert any(
+            term in content_lower
+            for term in [
+                "user experience",
+                "annoying",
+                "friction",
+                "override",
+                "dismiss",
+                "usability",
+            ]
+        ), "Missing user experience assessment section"
+
+    def test_ux_mentions_override_behavior(self, eval_content: str) -> None:
+        """AC3: UX section addresses override/dismissal behavior."""
+        content_lower = eval_content.lower()
+        assert any(
+            term in content_lower
+            for term in ["override", "dismiss", "bypass", "skip", "ignored"]
+        ), "UX section doesn't address override/dismissal behavior"
+
+
+# ===========================================================================
+# AC4: Document findings in a results file
+# ===========================================================================
+
+
+class TestResultsFile:
+    """AC4: Evaluation results file exists with proper structure."""
+
+    def test_eval_file_exists(self, eval_path: Path) -> None:
+        """AC4: Evaluation results file exists at expected location."""
+        assert eval_path.is_file(), f"Evaluation file not found: {eval_path}"
+
+    def test_eval_file_not_empty(self, eval_path: Path) -> None:
+        """AC4: Evaluation file is not empty."""
+        assert eval_path.is_file(), f"File not found: {eval_path}"
+        content = eval_path.read_text()
+        assert len(content.strip()) > 0, "Evaluation file is empty"
+
+    def test_eval_has_title(self, eval_content: str) -> None:
+        """AC4: Evaluation has a title heading."""
+        assert re.search(r"^#\s+", eval_content, re.MULTILINE), (
+            "Evaluation file missing title heading"
+        )
+
+    def test_eval_has_structured_sections(self, eval_content: str) -> None:
+        """AC4: Evaluation has multiple sections (at least 3 headings)."""
+        headings = re.findall(r"^##\s+", eval_content, re.MULTILINE)
+        assert len(headings) >= 3, (
+            f"Expected at least 3 sections, found {len(headings)}"
+        )
+
+    def test_eval_references_gate(self, eval_content: str) -> None:
+        """AC4: Evaluation references the confidence-sm gate."""
+        assert "confidence-sm" in eval_content, (
+            "Evaluation doesn't reference the confidence-sm gate"
+        )
+
+
+# ===========================================================================
+# AC5: Decide on rollout recommendation to other agents
+# ===========================================================================
+
+
+class TestRolloutRecommendation:
+    """AC5: Evaluation includes a rollout recommendation."""
+
+    def test_eval_has_recommendation_section(self, eval_content: str) -> None:
+        """AC5: Evaluation has a rollout recommendation section."""
+        content_lower = eval_content.lower()
+        assert any(
+            term in content_lower
+            for term in ["recommendation", "rollout", "next steps", "decision"]
+        ), "Missing rollout recommendation section"
+
+    def test_recommendation_is_actionable(self, eval_content: str) -> None:
+        """AC5: Recommendation states a clear actionable decision."""
+        content_lower = eval_content.lower()
+        assert any(
+            term in content_lower
+            for term in [
+                "expand",
+                "hold",
+                "remove",
+                "proceed",
+                "defer",
+                "recommend",
+                "adopt",
+                "extend",
+            ]
+        ), "Recommendation lacks clear actionable decision"
+
+    def test_recommendation_mentions_other_agents(
+        self, eval_content: str
+    ) -> None:
+        """AC5: Recommendation addresses rollout to other agents."""
+        content_lower = eval_content.lower()
+        assert any(
+            term in content_lower
+            for term in [
+                "other agent",
+                "additional agent",
+                "tea",
+                "dev",
+                "reviewer",
+                "all agents",
+                "agent-specific",
+                "per-agent",
+            ]
+        ), "Recommendation doesn't address rollout to other agents"