eduevidence 5.2.0 → 6.2.0

package/engine/autoevolve/agent_view.py

@@ -0,0 +1,167 @@
+"""Sanitized mutation view for Skill Autoresearch agents.
+
+This prevents accidental holdout/gold exposure by default: the mutation agent
+sees a tracked-file copy without .git and without evaluator/holdout/adversarial
+benchmark material. `benchmarks/questions.jsonl` and gold annotations are
+filtered to DEV ids only. It is context isolation, not an OS security sandbox;
+a promotion evaluator must independently attest stronger holdout isolation.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import shutil
+import subprocess
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+
+def _sha(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def _tracked_files(worktree: Path) -> list[str]:
+    output = subprocess.check_output(
+        ["git", "ls-files", "-z"], cwd=worktree, text=False
+    )
+    return [item.decode("utf-8") for item in output.split(b"\0") if item]
+
+
+def _dev_ids(worktree: Path) -> set[str]:
+    path = worktree / "benchmarks" / "partitions.json"
+    if not path.is_file():
+        return set()
+    value = json.loads(path.read_text(encoding="utf-8"))
+    return {str(item) for item in value.get("dev", [])}
+
+
+def _copy_filtered_benchmark(source: Path, dest: Path, rel: str, dev_ids: set[str]) -> bool:
+    if rel == "benchmarks/partitions.json":
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(source, dest)
+        return True
+    if rel == "benchmarks/questions.jsonl":
+        rows = []
+        for line in source.read_text(encoding="utf-8").splitlines():
+            if not line.strip():
+                continue
+            row = json.loads(line)
+            if row.get("id") in dev_ids:
+                rows.append(json.dumps(row, ensure_ascii=False))
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        dest.write_text("\n".join(rows) + ("\n" if rows else ""), encoding="utf-8")
+        return True
+    if rel.startswith("benchmarks/annotations/gold-"):
+        qid = Path(rel).stem.removeprefix("gold-")
+        if qid in dev_ids:
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copy2(source, dest)
+            return True
+    if rel.startswith("benchmarks/dev/"):
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(source, dest)
+        return True
+    return False
+
+
+@dataclass
+class AgentMutationView:
+    path: Path
+    baseline_hashes: dict[str, str]
+
+    @classmethod
+    def create(
+        cls,
+        worktree: str | Path,
+        *,
+        session_context: dict[str, Any] | None = None,
+    ) -> "AgentMutationView":
+        worktree = Path(worktree).resolve()
+        root = Path(tempfile.mkdtemp(prefix="eduevidence-autoevolve-agent-"))
+        dev_ids = _dev_ids(worktree)
+        for rel in _tracked_files(worktree):
+            source = worktree / rel
+            if not source.is_file() or source.is_symlink():
+                continue
+            dest = root / rel
+            if rel.startswith("benchmarks/"):
+                _copy_filtered_benchmark(source, dest, rel, dev_ids)
+                continue
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copy2(source, dest)
+        if session_context is not None:
+            context_path = root / "autoevolve" / "session-context.json"
+            context_path.parent.mkdir(parents=True, exist_ok=True)
+            context_path.write_text(
+                json.dumps(session_context, ensure_ascii=False, indent=2) + "\n",
+                encoding="utf-8",
+            )
+        baseline = {
+            path.relative_to(root).as_posix(): _sha(path)
+            for path in root.rglob("*")
+            if path.is_file() and not path.is_symlink()
+        }
+        return cls(root, baseline)
+
+    def changed_files(self) -> list[str]:
+        current = {
+            path.relative_to(self.path).as_posix(): _sha(path)
+            for path in self.path.rglob("*")
+            if path.is_file() and not path.is_symlink()
+        }
+        changed = {
+            rel
+            for rel in set(self.baseline_hashes) | set(current)
+            if self.baseline_hashes.get(rel) != current.get(rel)
+        }
+        changed.update(
+            path.relative_to(self.path).as_posix()
+            for path in self.path.rglob("*")
+            if path.is_symlink()
+        )
+        return sorted(changed)
+
+    def sync_to(self, worktree: str | Path, changed: list[str]) -> None:
+        worktree = Path(worktree).resolve()
+        for rel in changed:
+            source = self.path / rel
+            target = (worktree / rel).resolve()
+            if worktree not in target.parents and target != worktree:
+                raise ValueError(f"unsafe mutation path: {rel}")
+            if source.is_symlink():
+                raise ValueError(f"symlink mutations are forbidden: {rel}")
+            if source.is_file():
+                target.parent.mkdir(parents=True, exist_ok=True)
+                shutil.copy2(source, target)
+            elif target.exists():
+                if target.is_dir():
+                    shutil.rmtree(target)
+                else:
+                    target.unlink()
+
+    def export_changes(self, destination: str | Path, changed: list[str]) -> None:
+        """Preserve candidate files outside the worktree before a revert."""
+        destination = Path(destination)
+        destination.mkdir(parents=True, exist_ok=True)
+        manifest = []
+        for rel in changed:
+            source = self.path / rel
+            if source.is_symlink():
+                manifest.append({"path": rel, "status": "symlink_forbidden"})
+                continue
+            if source.is_file():
+                target = destination / rel
+                target.parent.mkdir(parents=True, exist_ok=True)
+                shutil.copy2(source, target)
+                manifest.append({"path": rel, "status": "present"})
+            else:
+                manifest.append({"path": rel, "status": "deleted"})
+        (destination / "manifest.json").write_text(
+            json.dumps(manifest, ensure_ascii=False, indent=2) + "\n",
+            encoding="utf-8",
+        )
+
+    def cleanup(self) -> None:
+        shutil.rmtree(self.path, ignore_errors=True)

package/engine/autoevolve/core.py

@@ -0,0 +1,357 @@
+from __future__ import annotations
+
+import fnmatch
+import hashlib
+import json
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+
+PROTECTED_DEFAULTS = (
+    "autoevolve/protected.manifest.yaml",
+    "benchmarks/annotations/**",
+    "benchmarks/holdout/**",
+    "benchmarks/evaluator/**",
+    "schemas/**",
+    "references/scientific-invariants.md",
+    "scripts/pre_verdict_gate.py",
+    "scripts/compute_confidence.py",
+    "scripts/check_autoresearch_invariants.py",
+    "engine/graph_store.py",
+    "engine/study_design.py",
+    "engine/autoevolve/**",
+    ".github/workflows/autoresearch-gates.yml",
+)
+SAFE_DEFAULTS = (
+    "skill/workflows/**",
+    "skill/agents/**",
+    "skill/sub-skills/**",
+    "retrieval/**",
+    "references/presentation/**",
+)
+CONTROLLED_DEFAULTS = (
+    "engine/semantics.py",
+    "engine/gaps.py",
+    "scripts/complexity_gate.py",
+    "engine/orchestration.py",
+)
+
+
+@dataclass(frozen=True)
+class EvalSnapshot:
+    eval_id: str
+    hard_gates_passed: bool
+    science_score: float
+    research_score: float
+    robustness: float
+    cost: float
+    latency: float
+    complexity: float
+    repeats: int = 1
+    noise_floor: float = 0.0
+    dev_passed: bool = False
+    holdout_passed: bool = False
+    adversarial_passed: bool = False
+    holdout_isolation_verified: bool = False
+    eval_suite_hash: str = ""
+
+
+@dataclass
+class SkillExperiment:
+    experiment_id: str
+    session_id: str
+    parent_skill_revision: str
+    hypothesis: str
+    mutation_scope: tuple[str, ...]
+    changed_files: list[str] = field(default_factory=list)
+    candidate_commit: str | None = None
+    baseline_eval_id: str | None = None
+    candidate_eval_id: str | None = None
+    protected_hash_before: str | None = None
+    protected_hash_after: str | None = None
+    status: str = "created"
+    promotion_reason: str = ""
+    complexity_delta: float = 0.0
+
+
+def _matches(path: str, patterns: tuple[str, ...]) -> bool:
+    normalized = path.replace("\\", "/")
+    return any(
+        fnmatch.fnmatch(normalized, pattern)
+        or (pattern.endswith("/**") and normalized.startswith(pattern[:-3]))
+        for pattern in patterns
+    )
+
+
+def _parse_manifest(path: Path) -> tuple[tuple[str, ...], tuple[str, ...], tuple[str, ...]]:
+    """Parse the intentionally tiny manifest subset without a YAML dependency."""
+    protected: list[str] = []
+    safe: list[str] = []
+    controlled: list[str] = []
+    section = ""
+    subsection = ""
+    for raw in path.read_text(encoding="utf-8").splitlines():
+        line = raw.split("#", 1)[0].rstrip()
+        if not line.strip():
+            continue
+        stripped = line.strip()
+        indent = len(line) - len(line.lstrip(" "))
+        if indent == 0 and stripped.endswith(":"):
+            section = stripped[:-1]
+            subsection = ""
+            continue
+        if section == "mutable" and indent == 2 and stripped.endswith(":"):
+            subsection = stripped[:-1]
+            continue
+        if stripped.startswith("- "):
+            value = stripped[2:].strip().strip('"\'')
+            if section == "protected":
+                protected.append(value)
+            elif section == "mutable" and subsection == "safe":
+                safe.append(value)
+            elif section == "mutable" and subsection == "controlled":
+                controlled.append(value)
+    if not protected or not safe:
+        raise ValueError(f"invalid autoresearch manifest: {path}")
+    return tuple(protected), tuple(safe), tuple(controlled)
+
+
+class ProtectedManifest:
+    def __init__(
+        self,
+        patterns=PROTECTED_DEFAULTS,
+        *,
+        safe_patterns=SAFE_DEFAULTS,
+        controlled_patterns=CONTROLLED_DEFAULTS,
+    ):
+        self.patterns = tuple(patterns)
+        self.safe_patterns = tuple(safe_patterns)
+        self.controlled_patterns = tuple(controlled_patterns)
+
+    @classmethod
+    def from_repo(cls, root: str | Path) -> "ProtectedManifest":
+        path = Path(root) / "autoevolve" / "protected.manifest.yaml"
+        if not path.is_file():
+            return cls()
+        protected, safe, controlled = _parse_manifest(path)
+        merged_protected = tuple(dict.fromkeys((*protected, *PROTECTED_DEFAULTS)))
+        return cls(
+            merged_protected,
+            safe_patterns=safe,
+            controlled_patterns=controlled,
+        )
+
+    def is_protected(self, path: str) -> bool:
+        return _matches(path, self.patterns)
+
+    def classify(self, path: str) -> str:
+        if self.is_protected(path):
+            return "protected"
+        if _matches(path, self.safe_patterns):
+            return "safe"
+        if _matches(path, self.controlled_patterns):
+            return "controlled"
+        return "unknown"
+
+    def validate_changes(self, changed: list[str]) -> tuple[bool, list[str]]:
+        bad = [path for path in changed if self.is_protected(path)]
+        return (not bad, bad)
+
+    def validate_mutation_scope(
+        self,
+        changed: list[str],
+        *,
+        mutation_tiers: tuple[str, ...] | list[str],
+        allow_controlled: bool,
+    ) -> tuple[bool, list[str]]:
+        tiers = set(mutation_tiers)
+        bad: list[str] = []
+        for path in changed:
+            kind = self.classify(path)
+            allowed = kind == "safe" and "safe" in tiers
+            allowed = allowed or (
+                kind == "controlled" and allow_controlled and "controlled" in tiers
+            )
+            if not allowed:
+                bad.append(path)
+        return (not bad, bad)
+
+    def hash_tree(self, root: str | Path) -> str:
+        root = Path(root)
+        digest = hashlib.sha256()
+        files = []
+        for file in root.rglob("*"):
+            if file.is_file() and self.is_protected(file.relative_to(root).as_posix()):
+                files.append(file)
+        for file in sorted(files):
+            rel = file.relative_to(root).as_posix()
+            digest.update(rel.encode())
+            digest.update(b"\0")
+            digest.update(file.read_bytes())
+            digest.update(b"\0")
+        return digest.hexdigest()
+
+
+def _promotion_evidence_ready(baseline: EvalSnapshot, candidate: EvalSnapshot) -> tuple[bool, str]:
+    if not baseline.eval_suite_hash or baseline.eval_suite_hash != candidate.eval_suite_hash:
+        return False, "baseline/candidate eval suite hash missing or mismatched"
+    required = (
+        baseline.dev_passed,
+        baseline.holdout_passed,
+        baseline.adversarial_passed,
+        baseline.holdout_isolation_verified,
+        candidate.dev_passed,
+        candidate.holdout_passed,
+        candidate.adversarial_passed,
+        candidate.holdout_isolation_verified,
+    )
+    if not all(required):
+        return False, "DEV/HOLDOUT/adversarial gates and holdout isolation are required for automatic KEEP"
+    return True, ""
+
+
+def promote(
+    baseline: EvalSnapshot,
+    candidate: EvalSnapshot,
+    *,
+    simplicity_tolerance: float = 0.0,
+    efficiency_tolerance: float = 0.05,
+    minimum_repeats: int = 3,
+) -> tuple[str, str]:
+    """Constraint-first promotion; automatic KEEP is deliberately conservative."""
+    if not candidate.hard_gates_passed:
+        return "REJECT", "L0 hard gate failed"
+    if candidate.science_score < baseline.science_score:
+        return "REJECT", "scientific correctness regressed"
+    if min(baseline.repeats, candidate.repeats) < minimum_repeats:
+        return "RETEST", f"automatic promotion requires >= {minimum_repeats} repeated runs"
+
+    delta = candidate.research_score - baseline.research_score
+    noise = max(baseline.noise_floor, candidate.noise_floor)
+    if delta < -noise:
+        return "REJECT", "research-quality regression exceeds empirical noise floor"
+
+    regressions = []
+    if candidate.robustness < baseline.robustness:
+        regressions.append("robustness")
+    if baseline.cost > 0 and candidate.cost > baseline.cost * (1 + efficiency_tolerance):
+        regressions.append("cost")
+    if baseline.latency > 0 and candidate.latency > baseline.latency * (1 + efficiency_tolerance):
+        regressions.append("latency")
+    if candidate.complexity > baseline.complexity + simplicity_tolerance:
+        regressions.append("complexity")
+
+    if abs(delta) <= noise:
+        if regressions:
+            return "REJECT", "within noise floor with regression: " + ",".join(regressions)
+        if candidate.complexity < baseline.complexity - simplicity_tolerance:
+            ready, why = _promotion_evidence_ready(baseline, candidate)
+            if not ready:
+                return "HUMAN_REVIEW", why
+            return "KEEP", "equivalent research quality with simpler implementation"
+        return "RETEST", "candidate delta is within empirical noise floor"
+
+    if delta > noise and not regressions:
+        ready, why = _promotion_evidence_ready(baseline, candidate)
+        if not ready:
+            return "HUMAN_REVIEW", why
+        return "KEEP", "material research-quality improvement without Pareto regression"
+    return "HUMAN_REVIEW", "Pareto trade-off: " + ",".join(regressions or ["mixed metrics"])
+
+
+class ExperimentLog:
+    HEADER = (
+        "experiment_id",
+        "parent_revision",
+        "candidate_commit",
+        "scope",
+        "hypothesis",
+        "eval_suite_hash",
+        "repeats",
+        "dev_passed",
+        "holdout_passed",
+        "adversarial_passed",
+        "holdout_isolation_verified",
+        "hard_gates",
+        "science_score",
+        "research_score",
+        "robustness",
+        "cost",
+        "latency",
+        "complexity_delta",
+        "status",
+        "description",
+    )
+
+    def __init__(self, root: str | Path):
+        self.root = Path(root)
+        self.root.mkdir(parents=True, exist_ok=True)
+        self.tsv = self.root / "results.tsv"
+        self.jsonl = self.root / "experiments.jsonl"
+        if not self.tsv.exists():
+            self.tsv.write_text("\t".join(self.HEADER) + "\n", encoding="utf-8")
+
+    def append(
+        self,
+        experiment: SkillExperiment,
+        *,
+        candidate: EvalSnapshot | None = None,
+        description: str = "",
+    ) -> None:
+        row = [
+            experiment.experiment_id,
+            experiment.parent_skill_revision,
+            experiment.candidate_commit or "",
+            ",".join(experiment.mutation_scope),
+            experiment.hypothesis,
+            str(candidate.eval_suite_hash if candidate else ""),
+            str(candidate.repeats if candidate else ""),
+            str(candidate.dev_passed if candidate else ""),
+            str(candidate.holdout_passed if candidate else ""),
+            str(candidate.adversarial_passed if candidate else ""),
+            str(candidate.holdout_isolation_verified if candidate else ""),
+            str(candidate.hard_gates_passed if candidate else ""),
+            str(candidate.science_score if candidate else ""),
+            str(candidate.research_score if candidate else ""),
+            str(candidate.robustness if candidate else ""),
+            str(candidate.cost if candidate else ""),
+            str(candidate.latency if candidate else ""),
+            str(experiment.complexity_delta),
+            experiment.status,
+            description,
+        ]
+        with self.tsv.open("a", encoding="utf-8") as handle:
+            handle.write("\t".join(value.replace("\t", " ") for value in row) + "\n")
+        with self.jsonl.open("a", encoding="utf-8") as handle:
+            handle.write(json.dumps(asdict(experiment), ensure_ascii=False, sort_keys=True) + "\n")
+
+
+class PlateauTracker:
+    def __init__(self, limit: int = 5):
+        self.limit = limit
+
+    def plateau(self, statuses: list[str]) -> bool:
+        valid = [status for status in statuses if status not in {"CRASH", "INVALID", "RETEST"}]
+        return len(valid) >= self.limit and all(status != "KEEP" for status in valid[-self.limit :])
+
+
+@dataclass(frozen=True)
+class DailyProfile:
+    max_experiments: int = 20
+    max_cost_usd: float = 5.0
+    max_wall_minutes: int = 180
+    mutation_tiers: tuple[str, ...] = ("safe",)
+    allow_controlled: bool = False
+    promotion: str = "branch_only"
+
+    def validate(self) -> None:
+        if not 1 <= self.max_experiments <= 50:
+            raise ValueError("daily max_experiments must be 1..50")
+        if self.max_cost_usd <= 0 or self.max_wall_minutes <= 0:
+            raise ValueError("daily budget must be positive")
+        if self.promotion != "branch_only":
+            raise ValueError("daily mode is branch_only")
+        unknown = set(self.mutation_tiers) - {"safe", "controlled"}
+        if unknown:
+            raise ValueError(f"unknown mutation tiers: {sorted(unknown)}")
+        if "controlled" in self.mutation_tiers and not self.allow_controlled:
+            raise ValueError("controlled mutation tier requires allow_controlled=true")

package/engine/autoevolve/events.py

@@ -0,0 +1,11 @@
+SKILL_AUTORESEARCH_EVENTS = (
+    "autoevolve.session.started",
+    "autoevolve.experiment.created",
+    "autoevolve.candidate.built",
+    "autoevolve.eval.completed",
+    "autoevolve.candidate.kept",
+    "autoevolve.candidate.rejected",
+    "autoevolve.candidate.retest",
+    "autoevolve.plateau",
+    "autoevolve.session.completed",
+)

package/engine/autoevolve/git_workspace.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+import re
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+
+
+def _run(repo: Path, *args: str, capture: bool = False) -> str:
+    completed = subprocess.run(
+        ["git", *args], cwd=repo, check=True, text=True, capture_output=capture
+    )
+    return completed.stdout.rstrip("\n") if capture else ""
+
+
+def safe_tag(tag: str) -> str:
+    clean = re.sub(r"[^A-Za-z0-9._-]+", "-", tag).strip("-")
+    if not clean or clean in {".", ".."}:
+        raise ValueError("invalid run tag")
+    return clean[:80]
+
+
+@dataclass(frozen=True)
+class GitWorkspace:
+    repo: Path
+    path: Path
+    branch: str
+
+    @classmethod
+    def create(cls, repo: str | Path, tag: str):
+        repo = Path(repo).resolve()
+        tag = safe_tag(tag)
+        branch = f"autoresearch/{tag}"
+        current = _run(repo, "branch", "--show-current", capture=True)
+        if current.startswith("autoresearch/"):
+            raise ValueError("create the session from a non-autoresearch base branch")
+        path = repo / ".autoevolve-worktrees" / tag
+        path.parent.mkdir(parents=True, exist_ok=True)
+        if path.exists():
+            raise FileExistsError(path)
+        _run(repo, "worktree", "add", "-b", branch, str(path), "HEAD")
+        return cls(repo, path, branch)
+
+    def head(self) -> str:
+        return _run(self.path, "rev-parse", "HEAD", capture=True)
+
+    def diff(self) -> str:
+        return _run(self.path, "diff", "--binary", "HEAD", capture=True)
+
+    def changed_files(self) -> list[str]:
+        output = _run(self.path, "status", "--porcelain", capture=True)
+        return [line[3:] for line in output.splitlines() if line.strip()]
+
+    def restore(self) -> None:
+        output = _run(self.path, "status", "--porcelain", capture=True)
+        untracked = [line[3:] for line in output.splitlines() if line.startswith("?? ")]
+        _run(self.path, "restore", "--staged", "--worktree", ".")
+        for relative in untracked:
+            target = (self.path / relative).resolve()
+            if self.path not in target.parents and target != self.path:
+                raise ValueError("unsafe untracked path")
+            if target.is_file() or target.is_symlink():
+                target.unlink(missing_ok=True)
+            elif target.is_dir():
+                import shutil
+                shutil.rmtree(target)
+
+    def commit(self, message: str) -> str:
+        _run(self.path, "add", "-A")
+        _run(self.path, "commit", "-m", message)
+        return _run(self.path, "rev-parse", "HEAD", capture=True)
+
+    def push(self, remote: str = "origin") -> None:
+        """Push only this experiment branch. Never force-push or merge."""
+        if not self.branch.startswith("autoresearch/"):
+            raise ValueError("only autoresearch branches may be pushed by autoevolve")
+        _run(self.path, "push", "-u", remote, self.branch)

package/engine/autoevolve/projection.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+from typing import Any
+from .core import PlateauTracker
+
+
+def skill_evolution_projection(
+    *,
+    baseline: dict[str, Any] | None,
+    best: dict[str, Any] | None,
+    experiments: list[dict[str, Any]],
+    protected_integrity: bool = True,
+) -> dict[str, Any]:
+    """Projection-only developer state; never mutates repository or user projects."""
+    statuses = [str(item.get("status", "")) for item in experiments]
+    return {
+        "baseline": baseline,
+        "best": best,
+        "experiments": experiments,
+        "experiment_count": len(experiments),
+        "plateau": PlateauTracker().plateau(statuses),
+        "protected_integrity": protected_integrity,
+        "promotion": "branch_only",
+    }