claude-dev-env 2.9.0 → 2.11.0

package/skills/e-code-review/scripts/effort_evaluation.py

@@ -0,0 +1,362 @@
+"""Machine-readable Opus effort evaluation for e-code-review.
+
+Freezes easy / medium / demanding fixtures, validates evaluation rows, and
+publishes a recommendation that cites completed rows. Live paid runs are
+optional — offline tests feed synthetic rows through the same functions::
+
+    load fixtures → validate rows → recommend lowest effort that holds quality
+
+Thinking stays enabled; effort is the only planned cost and latency lever.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from typing import Mapping, MutableMapping, Sequence
+
+_SCRIPTS_DIRECTORY = Path(__file__).resolve().parent
+if str(_SCRIPTS_DIRECTORY / "config") not in sys.path:
+    sys.path.insert(0, str(_SCRIPTS_DIRECTORY / "config"))
+if str(_SCRIPTS_DIRECTORY) not in sys.path:
+    sys.path.insert(0, str(_SCRIPTS_DIRECTORY))
+
+from e_code_review_effort_constants import (
+    ALL_EFFORT_LEVELS,
+    ALL_EFFORT_RANK_BY_NAME,
+    ALL_FIXTURE_BANDS,
+    ALL_REQUIRED_ROW_KEYS,
+    ALL_SCORE_ROW_KEYS,
+    ALL_SKILL_EFFORT_FOR_EVALUATION_EFFORT,
+    ALL_SKILL_EFFORT_LEVELS,
+    COST_LATENCY_LEVER,
+    EVALUATION_EVIDENCE_FILENAME,
+    EVALUATION_SCHEMA_VERSION,
+    FIXTURES_DIRECTORY_NAME,
+    JSON_SUFFIX,
+    LATENCY_MS_ROW_KEY,
+    MINIMUM_QUALITY_HOLD_SCORE,
+    THINKING_ENABLED_DEFAULT,
+    VISIBLE_TOKENS_ROW_KEY,
+    WORKFLOW_FAMILY_E_CODE_REVIEW,
+)
+
+
+def fixtures_directory() -> Path:
+    """Return the frozen fixture directory next to this module.
+
+    Returns:
+        Absolute path to the fixtures directory.
+    """
+    return _SCRIPTS_DIRECTORY / FIXTURES_DIRECTORY_NAME
+
+
+def load_fixtures() -> list[dict[str, object]]:
+    """Load every JSON fixture under the fixtures directory.
+
+    Returns:
+        Fixture dicts sorted by band then fixture_id.
+
+    Raises:
+        ValueError: When a fixture file is not a JSON object.
+        OSError: When a fixture file cannot be read.
+        json.JSONDecodeError: When a fixture file is not valid JSON.
+    """
+    directory = fixtures_directory()
+    all_fixtures: list[dict[str, object]] = []
+    for each_path in sorted(directory.glob(f"*{JSON_SUFFIX}")):
+        loaded = json.loads(each_path.read_text(encoding="utf-8"))
+        if not isinstance(loaded, dict):
+            raise ValueError(f"Fixture must be an object: {each_path}")
+        all_fixtures.append(loaded)
+    all_fixtures.sort(
+        key=lambda each_fixture: (
+            str(each_fixture.get("band", "")),
+            str(each_fixture.get("fixture_id", "")),
+        )
+    )
+    return all_fixtures
+
+
+def validate_evaluation_row(all_row_fields: Mapping[str, object]) -> list[str]:
+    """Return human-readable problems for one evaluation row.
+
+    Args:
+        all_row_fields: Candidate evaluation row mapping.
+
+    Returns:
+        Problem strings; empty when the row is valid.
+    """
+    all_problems: list[str] = []
+    for each_key in ALL_REQUIRED_ROW_KEYS:
+        if each_key not in all_row_fields:
+            all_problems.append(f"missing key: {each_key}")
+    if all_problems:
+        return all_problems
+    effort = all_row_fields["effort"]
+    if effort not in ALL_EFFORT_LEVELS:
+        all_problems.append(f"unknown effort: {effort!r}")
+    fixture_band = all_row_fields["fixture_band"]
+    if fixture_band not in ALL_FIXTURE_BANDS:
+        all_problems.append(f"unknown fixture_band: {fixture_band!r}")
+    if all_row_fields["thinking_enabled"] is not True:
+        all_problems.append("thinking_enabled must be true on Opus paths")
+    for each_score_key in ALL_SCORE_ROW_KEYS:
+        score_amount = all_row_fields[each_score_key]
+        if isinstance(score_amount, bool) or not isinstance(score_amount, (int, float)):
+            all_problems.append(f"{each_score_key} must be numeric")
+        elif not 0.0 <= float(score_amount) <= 1.0:
+            all_problems.append(f"{each_score_key} out of range [0, 1]")
+    for each_count_key in (VISIBLE_TOKENS_ROW_KEY, LATENCY_MS_ROW_KEY):
+        count_amount = all_row_fields[each_count_key]
+        if (
+            isinstance(count_amount, bool)
+            or not isinstance(count_amount, (int, float))
+            or float(count_amount) < 0
+        ):
+            all_problems.append(f"{each_count_key} must be a non-negative number")
+    return all_problems
+
+
+def quality_holds(
+    all_row_fields: Mapping[str, object],
+    minimum_quality: float = MINIMUM_QUALITY_HOLD_SCORE,
+) -> bool:
+    """Return True when quality, recall, and precision all meet the floor.
+
+    Args:
+        all_row_fields: Evaluation row mapping.
+        minimum_quality: Floor for quality, recall, and precision.
+
+    Returns:
+        Whether the row holds quality at the floor.
+    """
+    return all(
+        float(all_row_fields[each_score_key]) >= minimum_quality
+        for each_score_key in ALL_SCORE_ROW_KEYS
+    )
+
+
+def _lowest_holding_row(
+    all_band_rows: Sequence[Mapping[str, object]],
+) -> Mapping[str, object]:
+    return min(
+        all_band_rows,
+        key=lambda each_row: ALL_EFFORT_RANK_BY_NAME[str(each_row["effort"])],
+    )
+
+
+def recommend_effort_by_band(
+    all_rows: Sequence[Mapping[str, object]],
+    minimum_quality: float = MINIMUM_QUALITY_HOLD_SCORE,
+) -> dict[str, object]:
+    """Pick the lowest effort that holds quality for each fixture band.
+
+    Every recommendation entry cites the evaluation row that justified it.
+    Bands with no holding row stay null with an explicit blocker note.
+
+    Args:
+        all_rows: Completed evaluation rows (already validated by the caller).
+        minimum_quality: Floor for quality, recall, and precision.
+
+    Returns:
+        Machine-readable recommendation document.
+    """
+    recommendation_by_band: MutableMapping[str, object] = {}
+    for each_band in ALL_FIXTURE_BANDS:
+        all_holding_rows = [
+            each_row
+            for each_row in all_rows
+            if each_row["fixture_band"] == each_band
+            and quality_holds(each_row, minimum_quality)
+        ]
+        if not all_holding_rows:
+            recommendation_by_band[each_band] = {
+                "recommended_effort": None,
+                "cited_row": None,
+                "blocker": "no completed row meets the quality floor",
+            }
+            continue
+        chosen_row = _lowest_holding_row(all_holding_rows)
+        recommendation_by_band[each_band] = {
+            "recommended_effort": chosen_row["effort"],
+            "cited_row": dict(chosen_row),
+            "blocker": None,
+        }
+    return {
+        "schema_version": EVALUATION_SCHEMA_VERSION,
+        "thinking_enabled": THINKING_ENABLED_DEFAULT,
+        "cost_latency_lever": COST_LATENCY_LEVER,
+        "defaults_unchanged": True,
+        "minimum_quality": minimum_quality,
+        "recommendation_by_band": dict(recommendation_by_band),
+    }
+
+
+def build_synthetic_row(
+    fixture_id: str,
+    fixture_band: str,
+    effort: str,
+    quality_score: float,
+    finding_recall: float,
+    finding_precision: float,
+    visible_tokens: int,
+    latency_ms: int,
+) -> dict[str, object]:
+    """Build one evaluation row with thinking enabled.
+
+    Args:
+        fixture_id: Fixture identifier.
+        fixture_band: Band name (easy / medium / demanding).
+        effort: Effort level name.
+        quality_score: Overall quality in [0, 1].
+        finding_recall: Recall in [0, 1].
+        finding_precision: Precision in [0, 1].
+        visible_tokens: Visible token count.
+        latency_ms: Latency in milliseconds.
+
+    Returns:
+        Complete evaluation row mapping.
+    """
+    return {
+        "fixture_id": fixture_id,
+        "fixture_band": fixture_band,
+        "effort": effort,
+        "quality_score": quality_score,
+        "finding_recall": finding_recall,
+        "finding_precision": finding_precision,
+        VISIBLE_TOKENS_ROW_KEY: visible_tokens,
+        LATENCY_MS_ROW_KEY: latency_ms,
+        "thinking_enabled": THINKING_ENABLED_DEFAULT,
+    }
+
+
+def evaluation_evidence_path() -> Path:
+    """Return the committed evaluation evidence file path.
+
+    Returns:
+        Path to ``effort_defaults_evidence.json`` beside this module.
+    """
+    return _SCRIPTS_DIRECTORY / EVALUATION_EVIDENCE_FILENAME
+
+
+def load_evaluation_evidence() -> dict[str, object]:
+    """Load the committed evaluation evidence document.
+
+    Returns:
+        Parsed evidence mapping (rows, recommendation, skill defaults).
+
+    Raises:
+        OSError: When the evidence file cannot be read.
+        json.JSONDecodeError: When the evidence file is not valid JSON.
+    """
+    return json.loads(evaluation_evidence_path().read_text(encoding="utf-8"))
+
+
+def map_evaluation_effort_to_skill_level(evaluation_effort: str) -> str:
+    """Map a full evaluation effort name to an e-code-review skill level.
+
+    Args:
+        evaluation_effort: One of low / medium / high / xhigh / max.
+
+    Returns:
+        One of low / medium / xhigh.
+
+    Raises:
+        ValueError: When the evaluation effort is unknown.
+    """
+    skill_effort = ALL_SKILL_EFFORT_FOR_EVALUATION_EFFORT.get(evaluation_effort)
+    if skill_effort is None:
+        raise ValueError(f"unknown evaluation effort: {evaluation_effort!r}")
+    return skill_effort
+
+
+def skill_defaults_from_recommendation(
+    all_recommendation_fields: Mapping[str, object],
+) -> dict[str, object]:
+    """Build e-code-review skill defaults that each cite a recommendation row.
+
+    Args:
+        all_recommendation_fields: Output of ``recommend_effort_by_band``.
+
+    Returns:
+        Skill-family defaults document with cited rows per band.
+
+    Raises:
+        TypeError: When recommendation structure is not mapping-shaped.
+        ValueError: When a mapped skill effort leaves the skill surface.
+    """
+    recommendation_by_band = all_recommendation_fields["recommendation_by_band"]
+    if not isinstance(recommendation_by_band, Mapping):
+        raise TypeError("recommendation_by_band must be a mapping")
+    default_by_band: MutableMapping[str, object] = {}
+    for each_band in ALL_FIXTURE_BANDS:
+        band_entry = recommendation_by_band[each_band]
+        if not isinstance(band_entry, Mapping):
+            raise TypeError(f"band entry for {each_band} must be a mapping")
+        evaluation_effort = band_entry.get("recommended_effort")
+        cited_row = band_entry.get("cited_row")
+        if evaluation_effort is None or cited_row is None:
+            default_by_band[each_band] = {
+                "skill_effort": None,
+                "cited_row": None,
+                "blocker": band_entry.get("blocker"),
+            }
+            continue
+        skill_effort = map_evaluation_effort_to_skill_level(str(evaluation_effort))
+        if skill_effort not in ALL_SKILL_EFFORT_LEVELS:
+            raise ValueError(f"skill effort out of surface: {skill_effort!r}")
+        default_by_band[each_band] = {
+            "skill_effort": skill_effort,
+            "evaluation_effort": evaluation_effort,
+            "cited_row": dict(cited_row) if isinstance(cited_row, Mapping) else cited_row,
+            "blocker": None,
+        }
+    return {
+        "workflow_family": WORKFLOW_FAMILY_E_CODE_REVIEW,
+        "thinking_enabled": THINKING_ENABLED_DEFAULT,
+        "cost_latency_lever": COST_LATENCY_LEVER,
+        "skill_levels": list(ALL_SKILL_EFFORT_LEVELS),
+        "default_by_band": dict(default_by_band),
+    }
+
+
+def resolve_skill_effort_for_band(fixture_band: str) -> str:
+    """Return the evaluation-backed skill effort for one fixture band.
+
+    Args:
+        fixture_band: easy / medium / demanding.
+
+    Returns:
+        Skill effort level (low / medium / xhigh).
+
+    Raises:
+        ValueError: When the band is unknown or has no holding recommendation.
+        KeyError: When the evidence document lacks the band.
+    """
+    if fixture_band not in ALL_FIXTURE_BANDS:
+        raise ValueError(f"unknown fixture band: {fixture_band!r}")
+    evidence = load_evaluation_evidence()
+    skill_defaults = evidence["skill_defaults"]
+    if not isinstance(skill_defaults, Mapping):
+        raise TypeError("skill_defaults must be a mapping")
+    default_by_band = skill_defaults["default_by_band"]
+    if not isinstance(default_by_band, Mapping):
+        raise TypeError("default_by_band must be a mapping")
+    band_default = default_by_band[fixture_band]
+    if not isinstance(band_default, Mapping):
+        raise TypeError(f"default for {fixture_band} must be a mapping")
+    skill_effort = band_default.get("skill_effort")
+    if skill_effort is None:
+        raise ValueError(
+            f"no evaluation-backed skill effort for band {fixture_band!r}: "
+            f"{band_default.get('blocker')}"
+        )
+    skill_effort_name = str(skill_effort)
+    if skill_effort_name not in ALL_SKILL_EFFORT_LEVELS:
+        raise ValueError(
+            f"skill effort out of surface for band {fixture_band!r}: "
+            f"{skill_effort_name!r}"
+        )
+    return skill_effort_name

package/skills/e-code-review/scripts/finding_pipeline.py

@@ -0,0 +1,140 @@
+"""Collect every real review finding; filter severity only later.
+
+::
+
+    collection = collect_findings(all_seeded)
+        ok: low and nit findings remain in the collection record
+    filtered = filter_findings_by_severity(collection, minimum_severity="medium")
+        ok: consumer stage drops lower severities; collection is unchanged
+
+Collection never drops a real finding by severity. A later consumer stage
+reads the complete collection and may filter for action or display.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from e_code_review_scripts_constants.finding_pipeline_constants import (
+    ALL_COLLECTION_SEVERITIES,
+    COLLECTION_STAGE_NAME,
+    FILTER_STAGE_NAME,
+    FINDING_FIELD_CATEGORY,
+    FINDING_FIELD_EVIDENCE,
+    FINDING_FIELD_FILE,
+    FINDING_FIELD_LINE,
+    FINDING_FIELD_SEVERITY,
+    ALL_SEVERITY_RANK_BY_TOKEN,
+)
+
+
+@dataclass(frozen=True)
+class CollectedFinding:
+    """One real finding retained during the collection stage."""
+
+    file_path: str
+    line_number: int
+    severity: str
+    category: str
+    evidence: str
+
+    def as_mapping(self) -> dict[str, str | int]:
+        """Return the stable field map every retained finding keeps.
+
+        Returns:
+            Mapping with file, line, severity, category, and evidence.
+        """
+        return {
+            FINDING_FIELD_FILE: self.file_path,
+            FINDING_FIELD_LINE: self.line_number,
+            FINDING_FIELD_SEVERITY: self.severity,
+            FINDING_FIELD_CATEGORY: self.category,
+            FINDING_FIELD_EVIDENCE: self.evidence,
+        }
+
+
+@dataclass(frozen=True)
+class FindingCollection:
+    """Unfiltered collection record of every real finding."""
+
+    stage_name: str
+    all_findings: tuple[CollectedFinding, ...]
+
+
+@dataclass(frozen=True)
+class SeverityFilterView:
+    """Severity-filtered view produced by a separate consumer stage."""
+
+    stage_name: str
+    minimum_severity: str
+    all_findings: tuple[CollectedFinding, ...]
+
+
+def collect_findings(
+    all_findings: list[CollectedFinding] | tuple[CollectedFinding, ...],
+) -> FindingCollection:
+    """Retain every real finding with its file, line, evidence, and category.
+
+    Args:
+        all_findings: Seeded real findings of any legal severity.
+
+    Returns:
+        Collection stage record holding every input finding.
+
+    Raises:
+        ValueError: When a finding lacks a required field or legal severity.
+    """
+    all_retained: list[CollectedFinding] = []
+    for each_finding in all_findings:
+        if not each_finding.file_path:
+            raise ValueError("finding requires a non-empty file path")
+        if each_finding.line_number < 1:
+            raise ValueError("finding requires a positive line number")
+        if not each_finding.category:
+            raise ValueError("finding requires a non-empty category")
+        if not each_finding.evidence:
+            raise ValueError("finding requires non-empty evidence")
+        if each_finding.severity not in ALL_COLLECTION_SEVERITIES:
+            raise ValueError(
+                f"unknown collection severity: {each_finding.severity!r}"
+            )
+        all_retained.append(each_finding)
+    return FindingCollection(
+        stage_name=COLLECTION_STAGE_NAME,
+        all_findings=tuple(all_retained),
+    )
+
+
+def filter_findings_by_severity(
+    collection: FindingCollection,
+    *,
+    minimum_severity: str,
+) -> SeverityFilterView:
+    """Filter a complete collection by severity in a separate consumer stage.
+
+    Args:
+        collection: Unfiltered collection record from ``collect_findings``.
+        minimum_severity: Lowest severity to keep (inclusive).
+
+    Returns:
+        Filter-stage view. The input collection record is not mutated.
+
+    Raises:
+        ValueError: When ``minimum_severity`` or a finding severity is unknown.
+    """
+    if minimum_severity not in ALL_SEVERITY_RANK_BY_TOKEN:
+        raise ValueError(f"unknown minimum severity: {minimum_severity!r}")
+    minimum_rank = ALL_SEVERITY_RANK_BY_TOKEN[minimum_severity]
+    all_kept: list[CollectedFinding] = []
+    for each_finding in collection.all_findings:
+        if each_finding.severity not in ALL_SEVERITY_RANK_BY_TOKEN:
+            raise ValueError(
+                f"unknown collection severity: {each_finding.severity!r}"
+            )
+        if ALL_SEVERITY_RANK_BY_TOKEN[each_finding.severity] >= minimum_rank:
+            all_kept.append(each_finding)
+    return SeverityFilterView(
+        stage_name=FILTER_STAGE_NAME,
+        minimum_severity=minimum_severity,
+        all_findings=tuple(all_kept),
+    )

package/skills/e-code-review/scripts/fixtures/demanding.json

@@ -0,0 +1,26 @@
+{
+  "fixture_id": "demanding-multi-surface",
+  "band": "demanding",
+  "description": "Cross-file review with three seeded findings across hooks and tests.",
+  "target_summary": "Hook change drifts docstring, drops a paired test path, and uses a banned identifier.",
+  "seeded_findings": [
+    {
+      "file": "hooks/blocking/sample_gate.py",
+      "line": 88,
+      "category": "docstring-vs-impl",
+      "severity": "high"
+    },
+    {
+      "file": "hooks/blocking/test_sample_gate.py",
+      "line": 1,
+      "category": "paired-test-coverage",
+      "severity": "blocker"
+    },
+    {
+      "file": "hooks/blocking/sample_gate.py",
+      "line": 15,
+      "category": "banned-identifier",
+      "severity": "medium"
+    }
+  ]
+}

package/skills/e-code-review/scripts/fixtures/easy.json

@@ -0,0 +1,14 @@
+{
+  "fixture_id": "easy-comment-preservation",
+  "band": "easy",
+  "description": "Single-file comment-preservation miss with one seeded finding.",
+  "target_summary": "Production helper drops an existing inline comment on an unchanged line.",
+  "seeded_findings": [
+    {
+      "file": "sample_helper.py",
+      "line": 12,
+      "category": "comment-preservation",
+      "severity": "medium"
+    }
+  ]
+}

package/skills/e-code-review/scripts/fixtures/medium.json

@@ -0,0 +1,20 @@
+{
+  "fixture_id": "medium-constants-and-types",
+  "band": "medium",
+  "description": "Two seeded findings: bare type-ignore and a magic timeout literal.",
+  "target_summary": "Service module adds a bare type ignore and a hard-coded timeout.",
+  "seeded_findings": [
+    {
+      "file": "order_service.py",
+      "line": 24,
+      "category": "type-ignore-justification",
+      "severity": "high"
+    },
+    {
+      "file": "order_service.py",
+      "line": 41,
+      "category": "magic-value",
+      "severity": "medium"
+    }
+  ]
+}

package/skills/e-code-review/scripts/grok_code_review.py

@@ -197,21 +197,30 @@ def run_medium_review(
             batch.is_rejected = True
             batch.rejection_reason = "finder_head_mismatch"
             return batch
-    all_angles = [each.angle for each in all_finder_candidates]
+    all_angles = sorted({each.angle for each in all_finder_candidates})
+    finder_count = len(all_angles)
+    all_identities = {
+        (
+            each.angle,
+            each.leader_socket,
+            each.advisor_session_id,
+            each.worktree_path,
+        )
+        for each in all_finder_candidates
+    }
     all_sockets = {each.leader_socket for each in all_finder_candidates}
     all_sessions = {each.advisor_session_id for each in all_finder_candidates}
     all_worktrees = {each.worktree_path for each in all_finder_candidates}
     if (
-        len(all_sockets) != len(all_finder_candidates)
-        or len(all_sessions) != len(all_finder_candidates)
-        or len(all_worktrees) != len(all_finder_candidates)
+        len(all_identities) != finder_count
+        or len(all_sockets) != finder_count
+        or len(all_sessions) != finder_count
+        or len(all_worktrees) != finder_count
     ):
         batch.is_rejected = True
         batch.rejection_reason = "non_unique_finder_identity"
         return batch
-    require_exact_finder_set(tuple(sorted(set(all_angles))))
-    if len(set(all_angles)) != MEDIUM_REVIEW_FINDER_COUNT:
-        raise ValueError("each finder angle must run exactly once")
+    require_exact_finder_set(tuple(all_angles))
     deduped = deduplicate_candidates(all_finder_candidates)
     batch.all_retained_findings = retain_verified_findings(
         all_candidates=deduped,