eduevidence 5.2.0 → 6.2.0

package/scripts/check_protocol_alignment.py

@@ -0,0 +1,375 @@
+#!/usr/bin/env python3
+"""check_protocol_alignment.py — Protocol five-way alignment gate.
+
+Every scientific contract in this repository is declared in more than one
+place: the workflow registry (engine/workflows.py), the capability registry
+(engine/capabilities.py), the role registry (skill/roles/registry.yaml), the
+role prompts (skill/agents/*.md), the stage briefs (skill/task-briefs/*.md),
+the sub-skill recipes (skill/sub-skills/*/SKILL.md), the routing requirements
+(integrations/agent_mcp.py) and the packaging manifest (packaging/*).
+
+Drift between them is silent: a role can lose its brief, a capability can
+exist with no recipe, a version can be bumped in one file only. This gate
+makes that drift fail loudly. Stdlib only; a non-zero exit blocks CI.
+
+Usage:
+    python3 scripts/check_protocol_alignment.py
+"""
+from __future__ import annotations
+
+import json
+import re
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+
+from engine.capabilities import capability_registry  # noqa: E402
+from engine.versions import ENGINE_VERSION  # noqa: E402
+from engine.workflows import execution_stages, workflow_registry  # noqa: E402
+
+FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n", re.DOTALL)
+
+#: Projection-layer capabilities sit outside the scientific stage model
+#: (engine/workflows.py: Projection is not a scientific stage), so they are
+#: owned by the projection brief rather than by a scientific role.
+PROJECTION_CAPABILITIES = {"report_projection", "report_rendering"}
+
+
+def list_domains_from_registry() -> list[dict]:
+    """Registered domains (evidencecore is the registry owner)."""
+    from engine.evidencecore import list_domains
+
+    return list_domains()
+
+
+def _frontmatter(path: Path) -> dict[str, str]:
+    """Parse the flat key: value frontmatter used by skills and role prompts."""
+    match = FRONTMATTER_RE.match(path.read_text(encoding="utf-8"))
+    if not match:
+        return {}
+    fields: dict[str, str] = {}
+    for line in match.group(1).splitlines():
+        if line.strip().startswith("#") or ":" not in line:
+            continue
+        key, _, value = line.partition(":")
+        fields[key.strip()] = value.split("#", 1)[0].strip()
+    return fields
+
+
+def _registry_roles() -> dict[str, dict]:
+    """The role registry is a small fixed-shape YAML file; parse it narrowly."""
+    text = (ROOT / "skill" / "roles" / "registry.yaml").read_text(encoding="utf-8")
+    roles: dict[str, dict] = {}
+    current: str | None = None
+    for raw in text.splitlines():
+        if not raw.strip() or raw.strip().startswith("#"):
+            continue
+        if raw.startswith("roles:"):
+            continue
+        if raw.startswith("execution:"):
+            break
+        if re.match(r"^  [A-Za-z0-9_-]+:\s*$", raw):
+            current = raw.strip().rstrip(":")
+            roles[current] = {}
+            continue
+        if current and raw.strip().startswith("stages:"):
+            stages = raw.split(":", 1)[1].strip().strip("[]")
+            roles[current]["stages"] = [s.strip() for s in stages.split(",") if s.strip()]
+        elif current and raw.strip().startswith("capabilities:"):
+            caps = raw.split(":", 1)[1].strip().strip("[]")
+            roles[current]["capabilities"] = [c.strip() for c in caps.split(",") if c.strip()]
+        elif current and ":" in raw.strip():
+            key, _, value = raw.strip().partition(":")
+            roles[current][key.strip()] = value.strip()
+    return roles
+
+
+def _merge_role(roles: dict[str, dict], name: str, stage: str, capability: str,
+                critical: bool | None = None, independence: bool | None = None) -> None:
+    entry = roles.setdefault(name, {"stages": [], "capabilities": []})
+    if stage not in entry["stages"]:
+        entry["stages"].append(stage)
+    for cap in capability.split("+"):
+        cap = cap.strip()
+        if cap and cap not in entry["capabilities"]:
+            entry["capabilities"].append(cap)
+    if critical is not None:
+        entry["critical_path"] = "true" if critical else "false"
+    if independence:
+        entry["independence_required"] = "true"
+
+
+def check() -> list[str]:
+    errors: list[str] = []
+
+    stages = list(execution_stages())
+    scientific_stages = [s for s in stages if s != "projection"]
+
+    # ---------------------------------------------------------------- briefs
+    brief_dir = ROOT / "skill" / "task-briefs"
+    briefs = {p.stem for p in brief_dir.glob("*.md")}
+    for stage in stages:
+        if stage not in briefs:
+            errors.append(f"stage {stage!r} has no task brief in skill/task-briefs/")
+    for extra in sorted(briefs - set(stages) - {"present"}):
+        errors.append(f"task brief {extra!r} does not map to a canonical stage")
+
+    # ---------------------------------------------------------------- roles
+    registry = _registry_roles()
+    if not registry:
+        errors.append("skill/roles/registry.yaml declares no roles")
+    agent_files = {p.stem: p for p in (ROOT / "skill" / "agents").glob("*.md")}
+    if set(registry) != set(agent_files):
+        errors.append("role registry and skill/agents/*.md disagree: "
+                      f"registry-only={sorted(set(registry) - set(agent_files))} "
+                      f"agent-only={sorted(set(agent_files) - set(registry))}")
+
+    stage_owner: dict[str, str] = {}
+    for role, entry in registry.items():
+        for stage in entry.get("stages", []):
+            if stage in stage_owner:
+                errors.append(f"stage {stage!r} is owned by both {stage_owner[stage]!r} and {role!r}")
+            stage_owner[stage] = role
+    for stage in scientific_stages:
+        if stage not in stage_owner:
+            errors.append(f"stage {stage!r} has no owning role in the registry")
+
+    # Registry capabilities must be engine capability IDs: a free-text label
+    # here silently detaches the role from the capability it claims to run.
+    for role, entry in registry.items():
+        for cap in entry.get("capabilities", []):
+            if cap not in capability_registry():
+                errors.append(f"registry role {role!r} declares capability {cap!r}, "
+                              "which is not in engine/capabilities.py")
+
+    # Independence is graded: the skeptic must come from a different model
+    # family, the method reviewer must be separated from the content judgement.
+    independence = {role: entry.get("independence_required")
+                    for role, entry in registry.items() if entry.get("independence_required")}
+    if independence != {"skeptic": "different-model-family",
+                        "method-reviewer": "role-separation"}:
+        expected = {"skeptic": "different-model-family", "method-reviewer": "role-separation"}
+        errors.append("independence_required must be "
+                      f"{expected}, found {independence}")
+
+    # ------------------------------------------------- role prompt frontmatter
+    for role, path in agent_files.items():
+        fields = _frontmatter(path)
+        if fields.get("name") != role:
+            errors.append(f"{path.name}: frontmatter name {fields.get('name')!r} != filename {role!r}")
+        if fields.get("role_id") != role:
+            errors.append(f"{path.name}: missing role_id: {role}")
+        if not fields.get("capabilities"):
+            errors.append(f"{path.name}: missing capabilities declaration")
+        if not fields.get("output_contracts"):
+            errors.append(f"{path.name}: missing output_contracts declaration")
+        for banned in ("default_cli", "default_model"):
+            if banned in fields:
+                errors.append(f"{path.name}: {banned} must not be bound in the role prompt "
+                              "(model/CLI choice is a user-confirmed routing decision)")
+        for token in ("claude-", "gpt-", "deepseek-", "glm-", "kimi-"):
+            if token in fields.get("recommended_reasoning", ""):
+                errors.append(f"{path.name}: recommended_reasoning must not contain a model name")
+        if role in registry:
+            declared = set(registry[role].get("capabilities", []))
+            prompt_caps = {c.strip() for c in fields.get("capabilities", "").split(",") if c.strip()}
+            unknown = {c for c in prompt_caps if c not in capability_registry()}
+            unmapped = {c for c in unknown if not c.startswith("(")}
+            if unmapped:
+                errors.append(f"{path.name}: capabilities not in the capability registry: {sorted(unmapped)}")
+            missing = {c for c in declared if c in capability_registry()} - prompt_caps
+            if missing:
+                errors.append(f"{path.name}: registry capabilities absent from the prompt: "
+                              f"{sorted(missing)}")
+        if registry.get(role, {}).get("critical_path") == "true":
+            if fields.get("critical_path") != "true":
+                errors.append(f"{path.name}: registry marks this role critical_path but the "
+                              "prompt does not declare critical_path: true")
+
+    # ----------------------------------------------- routing-side requirements
+    from integrations.agent_mcp import ROLE_REQUIREMENTS  # noqa: E402
+    if set(ROLE_REQUIREMENTS) != set(registry):
+        errors.append("integrations.agent_mcp.ROLE_REQUIREMENTS and the role registry disagree: "
+                      f"routing-only={sorted(set(ROLE_REQUIREMENTS) - set(registry))} "
+                      f"registry-only={sorted(set(registry) - set(ROLE_REQUIREMENTS))}")
+    for role, reqs in ROLE_REQUIREMENTS.items():
+        for banned in ("default_cli", "default_model", "model", "cli"):
+            if banned in reqs:
+                errors.append(f"ROLE_REQUIREMENTS[{role!r}] must not bind {banned!r}")
+        wants_family = registry.get(role, {}).get("independence_required") == "different-model-family"
+        if wants_family and reqs.get("independence") != "different-model-family":
+            errors.append(f"ROLE_REQUIREMENTS[{role!r}] must require a different model family")
+
+    # ------------------------------------------------------------ capabilities
+    capabilities = capability_registry()
+    sub_skills = sorted(p for p in (ROOT / "skill" / "sub-skills").iterdir()
+                        if p.is_dir() and not p.name.startswith("."))
+    if len(sub_skills) < 5:
+        errors.append(f"expected at least 5 sub-skills, found {len(sub_skills)}")
+    mapped: set[str] = set()
+    for skill_dir in sub_skills:
+        path = skill_dir / "SKILL.md"
+        if not path.is_file():
+            errors.append(f"sub-skill {skill_dir.name} has no SKILL.md")
+            continue
+        fields = _frontmatter(path)
+        if fields.get("name") != skill_dir.name:
+            errors.append(f"{skill_dir.name}/SKILL.md: name {fields.get('name')!r} != directory")
+        declared = fields.get("capability", "")
+        if not declared:
+            errors.append(f"{skill_dir.name}/SKILL.md: missing capability declaration")
+            continue
+        for cap in declared.split("+"):
+            cap = cap.strip().split()[0] if cap.strip() else ""
+            if cap and not cap.startswith("("):
+                mapped.add(cap)
+                if cap not in capabilities:
+                    errors.append(f"{skill_dir.name}/SKILL.md: capability {cap!r} is not in "
+                                  "engine/capabilities.py")
+    # Every registered capability must be owned by at least one role, and every
+    # recipe capability must be one the engine actually registers.
+    owned: set[str] = set()
+    for entry in registry.values():
+        owned.update(entry.get("capabilities", []))
+    for cap in capabilities:
+        if cap not in owned and cap not in PROJECTION_CAPABILITIES:
+            errors.append(f"capability {cap!r} is registered in engine/capabilities.py "
+                          "but no role owns it")
+    for cap in sorted(mapped):
+        if cap in capabilities and cap not in owned and cap not in PROJECTION_CAPABILITIES:
+            errors.append(f"sub-skill capability {cap!r} is owned by no role")
+
+    # ---------------------------------------------------------------- workflows
+    workflow_dir = ROOT / "skill" / "workflows"
+    workflow_files = {p.stem for p in workflow_dir.glob("*.md")}
+    registry_workflows = workflow_registry()
+    public = {w for w in registry_workflows if w != "full_research_cycle"}
+    normalised = {name.replace("_", "-") for name in public}
+    if not normalised <= workflow_files:
+        errors.append("workflows missing a runbook: "
+                      f"{sorted(normalised - workflow_files)}")
+    skill_text = (ROOT / "SKILL.md").read_text(encoding="utf-8")
+    for name in public:
+        reference = f"skill/workflows/{name.replace('_', '-')}.md"
+        if reference not in skill_text:
+            errors.append(f"SKILL.md does not route to {reference}")
+
+    skill_md = root_skill_text = skill_text  # alias for readability
+    for stage in scientific_stages:
+        if f"| {stage.capitalize()} " not in skill_md and stage not in skill_md:
+            errors.append(f"SKILL.md does not mention stage {stage!r}")
+
+    # ------------------------------------------------------------- taxonomy
+    # The registry is the authority for outcome tokens and their categories;
+    # the JSON Schemas carry static enums because JSON Schema cannot read a
+    # file at validation time. This dimension is what keeps the static enums
+    # honest: drift between a schema enum and the registry fails the gate.
+    from engine.taxonomy import (
+        all_tokens_ordered,
+        categories as taxonomy_categories,
+    )
+
+    registered_tokens = set(all_tokens_ordered())
+    evidence_path = ROOT / "schemas" / "evidence.schema.json"
+    if evidence_path.is_file():
+        evidence_schema = json.loads(evidence_path.read_text(encoding="utf-8"))
+        enum = (evidence_schema.get("properties", {})
+                .get("outcome_type", {}).get("enum"))
+        if not isinstance(enum, list) or not enum:
+            errors.append("schemas/evidence.schema.json declares no outcome_type enum")
+        else:
+            missing = sorted(registered_tokens - set(enum))
+            extra = sorted(set(enum) - registered_tokens)
+            if missing:
+                errors.append(
+                    "outcome_type enum is missing registered token(s): " + repr(missing))
+            if extra:
+                errors.append(
+                    "outcome_type enum declares unregistered token(s): " + repr(extra))
+
+    # Every domain's ADOPT-gate categories must be categories it declares.
+    from engine.tribunal import primary_effect_categories
+
+    for domain_id in (d["id"] for d in list_domains_from_registry()):
+        declared = set(taxonomy_categories(domain_id))
+        try:
+            primary = primary_effect_categories(domain_id)
+        except ValueError as exc:
+            errors.append(f"domain {domain_id!r}: ADOPT gate misconfigured: {exc}")
+            continue
+        for category in primary:
+            if category not in declared:
+                errors.append(
+                    f"domain {domain_id!r}: ADOPT gate names undeclared category "
+                    + repr(category))
+
+    # Every V2 outcome bucket must be a category some domain declares.
+    v2_outcome = ROOT / "schemas" / "v2" / "outcome.schema.json"
+    if v2_outcome.is_file():
+        v2_schema = json.loads(v2_outcome.read_text(encoding="utf-8"))
+        buckets = (v2_schema.get("properties", {})
+                   .get("outcome_type", {}).get("enum"))
+        if isinstance(buckets, list) and buckets:
+            every = set()
+            for domain_id in (d["id"] for d in list_domains_from_registry()):
+                every.update(taxonomy_categories(domain_id))
+            orphan = sorted(set(buckets) - every)
+            # Reverse direction too: a category a domain declares but the V2
+            # contract omits would reject that domain's outcomes at the graph
+            # layer, which is exactly how policy was blocked.
+            missing_bucket = sorted(every - set(buckets))
+            if missing_bucket:
+                errors.append(
+                    "schemas/v2/outcome.schema.json is missing category bucket(s) "
+                    "that domains declare: " + repr(missing_bucket))
+            if orphan:
+                errors.append(
+                    "schemas/v2/outcome.schema.json declares bucket(s) no domain "
+                    "registers: " + repr(orphan))
+
+    # ---------------------------------------------------------------- versions
+    for relative in ("packaging/scp-manifest.json",):
+        path = ROOT / relative
+        if not path.is_file():
+            continue
+        data = json.loads(path.read_text(encoding="utf-8"))
+        declared = (data.get("skill") or {}).get("version")
+        if declared != ENGINE_VERSION:
+            errors.append(f"{relative}: skill.version {declared!r} != ENGINE_VERSION {ENGINE_VERSION!r}")
+    start_here = ROOT / "packaging" / "START-HERE.md"
+    if start_here.is_file():
+        text = start_here.read_text(encoding="utf-8")
+        major_minor = ".".join(ENGINE_VERSION.split(".")[:2])
+        if f"EduEvidence {major_minor}" not in text:
+            errors.append(f"packaging/START-HERE.md does not state EduEvidence {major_minor}")
+
+    # ------------------------------------------------------- docs must not drift
+    for doc in ("docs/architecture.md", "README.zh-CN.md", "docs/install-guide.md"):
+        path = ROOT / doc
+        if not path.is_file():
+            continue
+        text = path.read_text(encoding="utf-8")
+        for stale in ("752 个测试", "752 tests"):
+            if stale in text:
+                errors.append(f"{doc}: stale test count {stale!r}; use docs/metrics.json")
+
+    return errors
+
+
+def main() -> int:
+    print("[*] Checking protocol alignment across workflows, roles, capabilities and packaging...")
+    errors = check()
+    if errors:
+        print(f"[-] Protocol alignment FAILED with {len(errors)} error(s):", file=sys.stderr)
+        for error in errors:
+            print(f"    • {error}", file=sys.stderr)
+        return 1
+    print("[+] Protocol alignment PASSED: stages, briefs, roles, prompts, capabilities, "
+          "sub-skills, workflows and packaging agree.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/check_versioned_schemas.py

@@ -0,0 +1,254 @@
+#!/usr/bin/env python3
+"""check_versioned_schemas.py - data-level validation for v3 / v4 / vNext.
+
+CI validated this contract family by parsing the JSON only, so a schema could
+drift arbitrarily (renamed required fields, changed enums, wrong types) and
+stay green. Each family is exercised here by building a record from its own
+dataclass or builder and validating that record against its schema, which is
+the check that would have caught a real mismatch.
+
+Stdlib only; exit 1 on any invalid record.
+"""
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT))
+sys.path.insert(0, str(ROOT / 'scripts'))
+
+
+def _validate(record, schema_rel, label):
+    from validate_schema import SchemaError, Validator
+
+    path = ROOT / schema_rel
+    schema = json.loads(path.read_text(encoding='utf-8'))
+    validator = Validator(schema, base_dir=path.parent)
+    try:
+        validator.validate(record, schema, label)
+    except SchemaError as exc:
+        return f'{label}: {exc}'
+    return None
+
+
+def main() -> int:
+    problems = []
+    checked = 0
+
+    # --- vNext: records built from their own dataclasses -------------------
+    try:
+        from engine.autoresearch.contracts import NegativeSearchRecord
+
+        record = NegativeSearchRecord(
+            negative_search_id='NSR-1', research_iteration_id='RI-1',
+            gap_id='GAP-1', queries=('a', 'b'), providers=('openalex',),
+            candidate_count=0, fetched_count=0, eligible_count=0)
+        payload = record.__dict__ if hasattr(record, '__dict__') else record
+        # JSON round-trip puts tuples back into their serialized form, which is
+        # what actually gets persisted; validate that shape.
+        payload = json.loads(json.dumps(payload, default=str))
+        err = _validate(payload, 'schemas/vNext/negative-search-record.schema.json',
+                        'negative-search-record')
+        if err:
+            problems.append(err)
+        checked += 1
+    except Exception as exc:  # import or construction failure is itself a defect
+        problems.append(f'negative-search-record: could not build record: {exc}')
+
+    # --- vNext: autoresearch lifecycle records ---------------------------
+    # Every schema in this family is exercised against the dataclass that
+    # actually produces it, so a renamed field, retyped value or narrowed
+    # enum fails CI instead of drifting silently. Only `negative-search-record`
+    # was wired before; the rest parsed as JSON and nothing more.
+    try:
+        import dataclasses
+
+        from engine.autoresearch.contracts import (ResearchBudget,
+                                                   ResearchExperimentType,
+                                                   ResearchIteration,
+                                                   ResearchStrategy,
+                                                   IterationStatus)
+        from engine.autoresearch.gap_priority import GapPriority
+
+        strategy = ResearchStrategy(
+            strategy_id='RST-1',
+            experiment_type=ResearchExperimentType.TARGETED_RETRIEVAL,
+            hypothesis='a targeted retrieval closes the retention gap',
+            expected_gain='one or more direct retention findings',
+            budget=ResearchBudget())
+        iteration = ResearchIteration(
+            iteration_id='RI-1', project_id='PRJ-1', base_graph_revision=1,
+            gap_id='GAP-1', strategy=strategy)
+        iteration.complete(IterationStatus.SEARCH_SATURATED)
+        iteration_payload = json.loads(json.dumps(iteration.as_dict(), default=str))
+        strategy_payload = json.loads(json.dumps(dataclasses.asdict(strategy), default=str))
+        strategy_payload['experiment_type'] = strategy.experiment_type.value
+        priority = GapPriority(
+            gap_id='GAP-1', dvi_band='HIGH', cost_band='LOW',
+            decision_material=True, drivers=('missing_retention',),
+            # The enum is the controller's own vocabulary (gap_priority.py and
+            # controller.py emit these three); it is not the experiment-type
+            # vocabulary. Using a plausible-looking but wrong value here is
+            # exactly the drift this check exists to catch.
+            next_research_mode='secondary_evidence_search', score=1)
+        priority_payload = json.loads(json.dumps(dataclasses.asdict(priority), default=str))
+
+        for label, payload, schema_rel in (
+            ('research-strategy', strategy_payload, 'schemas/vNext/research-strategy.schema.json'),
+            ('research-iteration', iteration_payload, 'schemas/vNext/research-iteration.schema.json'),
+            ('gap-priority', priority_payload, 'schemas/vNext/gap-priority.schema.json'),
+        ):
+            err = _validate(payload, schema_rel, label)
+            if err:
+                problems.append(err)
+            checked += 1
+    except Exception as exc:
+        problems.append(f'autoresearch lifecycle records: could not build record: {exc}')
+
+    # --- vNext: orchestration plan and task contracts ---------------------
+    try:
+        import dataclasses as _dc
+
+        from engine.orchestration import ExecutionPlanner, ExecutionMode
+
+        # A delegated plan needs a run id and a base revision; without them
+        # TaskSpec.validate_for_dispatch refuses to describe a dispatchable task.
+        plan = ExecutionPlanner().plan('M', run_id='RUN-1', base_revision=1)
+        plan_payload = json.loads(json.dumps(_dc.asdict(plan), default=str))
+        plan_payload['complexity'] = str(getattr(plan.complexity, 'value', plan.complexity))
+        err = _validate(plan_payload, 'schemas/vNext/execution-plan.schema.json',
+                        'execution-plan')
+        if err:
+            problems.append(err)
+        checked += 1
+
+        delegated = [t for t in plan.tasks if t.execution_mode is ExecutionMode.DELEGATED]
+        if not delegated:
+            problems.append('execution plan has no delegated task to exercise task-spec')
+        else:
+            task_payload = json.loads(json.dumps(delegated[0].to_dict(), default=str))
+            err = _validate(task_payload, 'schemas/vNext/task-spec.schema.json', 'task-spec')
+            if err:
+                problems.append(err)
+            checked += 1
+
+            # A WorkerResult is built only by the lead process, which ignores
+            # worker self-attestation; exercise that real path, not a hand-built dict.
+            from engine.worker_result import validate_worker_output
+            result = validate_worker_output(delegated[0], {
+                'task_id': delegated[0].task_id,
+                'status': 'completed',
+                'staging_artifacts': [{'artifact_type': delegated[0].expected_outputs[0],
+                                       'summary': 'staged'}],
+                'validated': True,
+                'metrics': {},
+                'summary': 'ok',
+            })
+            err = _validate(json.loads(json.dumps(result.to_dict(), default=str)),
+                            'schemas/vNext/worker-result.schema.json', 'worker-result')
+            if err:
+                problems.append(err)
+            checked += 1
+    except Exception as exc:
+        problems.append(f'orchestration records: could not build record: {exc}')
+
+    # --- vNext: autoevolve session and experiment records -----------------
+    try:
+        import dataclasses as _dc2
+
+        from engine.autoevolve.core import EvalSnapshot, SkillExperiment
+
+        snapshot = EvalSnapshot(
+            eval_id='EVAL-1', hard_gates_passed=True, science_score=0.8,
+            research_score=0.7, robustness=0.9, cost=0.0,
+            latency=12.5, complexity=0.5, repeats=3, noise_floor=0.01,
+            dev_passed=True, holdout_passed=True, adversarial_passed=True,
+            holdout_isolation_verified=False, eval_suite_hash='deadbeef')
+        snapshot_payload = json.loads(json.dumps(_dc2.asdict(snapshot), default=str))
+        err = _validate(snapshot_payload, 'schemas/vNext/eval-snapshot.schema.json',
+                        'eval-snapshot')
+        if err:
+            problems.append(err)
+        checked += 1
+
+        experiment = SkillExperiment(
+            experiment_id='EXP-1', session_id='session-1',
+            parent_skill_revision='rev-1', hypothesis='a narrower prompt scores higher',
+            mutation_scope=('safe',), changed_files=('skill/agents/skeptic.md',),
+            candidate_commit='deadbeef', baseline_eval_id='EVAL-1',
+            candidate_eval_id='EVAL-2', protected_hash_before='h1',
+            protected_hash_after='h1', status='REJECT',
+            promotion_reason='no significant improvement')
+        experiment_payload = json.loads(json.dumps(_dc2.asdict(experiment), default=str))
+        err = _validate(experiment_payload, 'schemas/vNext/skill-experiment.schema.json',
+                        'skill-experiment')
+        if err:
+            problems.append(err)
+        checked += 1
+
+        # The session report is the real runner payload (runner.py writes
+        # daily-report.json); validate that documented shape directly.
+        session_report = {
+            'run_tag': 'session-1', 'branch': 'autoevolve/session-1',
+            'experiments': 1, 'statuses': ['REJECT'], 'best_experiment_id': None,
+            'best_candidate_commit': None, 'cost': 0.0, 'wall_minutes': 1.0,
+            'plateau': False, 'stop_reason': 'completed',
+            'promotion': 'branch_only', 'branch_push_requested': False,
+            'branch_pushed': False, 'mutation_view': 'dev_only_context_isolation',
+            'holdout_isolation_verified': False, 'isolation_provider': 'none',
+            'isolation_reason': 'no os isolation provider available',
+            'eval_suite_hash': 'deadbeef', 'security_note': 'branch only',
+            'candidate_artifacts': 'local session state only; never auto-pushed',
+        }
+        err = _validate(session_report, 'schemas/vNext/autoevolve-session.schema.json',
+                        'autoevolve-session')
+        if err:
+            problems.append(err)
+        checked += 1
+    except Exception as exc:
+        problems.append(f'autoevolve records: could not build record: {exc}')
+
+    # --- v3: the BENCHMARK run manifest (scripts/benchmark_v3.py) ---------
+    # Note: schemas/v3/run-manifest.schema.json describes the empirical
+    # benchmark harness record, not the run-workspace manifest. Validating the
+    # wrong producer is how a contract silently stops describing reality.
+    try:
+        fixture = sorted((ROOT / 'benchmarks' / 'empirical').glob('*/manifest.json'))
+        if fixture:
+            payload = json.loads(fixture[0].read_text(encoding='utf-8'))
+            err = _validate(payload, 'schemas/v3/run-manifest.schema.json',
+                            f'run-manifest ({fixture[0].parent.name})')
+            if err:
+                problems.append(err)
+            checked += 1
+        else:
+            # benchmarks/empirical/ holds local empirical runs and is excluded
+            # from the submission package on purpose. Its absence means there is
+            # nothing to check in this checkout, not that the contract is broken.
+            print('note: no benchmarks/empirical/*/manifest.json in this checkout; '
+                  'skipping the v3 run-manifest data check')
+    except Exception as exc:
+        problems.append(f'v3 run-manifest: {exc}')
+
+    # --- sanity: every declared schema still parses -----------------------
+    for family in ('v3', 'v4', 'vNext'):
+        for path in sorted((ROOT / 'schemas' / family).glob('*.json')):
+            try:
+                json.loads(path.read_text(encoding='utf-8'))
+                checked += 1
+            except (OSError, json.JSONDecodeError) as exc:
+                problems.append(f'{path.relative_to(ROOT)}: {exc}')
+
+    if problems:
+        print('ERROR: versioned schema validation failed', file=sys.stderr)
+        for item in problems:
+            print(f'  {item}', file=sys.stderr)
+        return 1
+    print(f'versioned schemas OK ({checked} checks across v3 / v4 / vNext)')
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

package/scripts/claim_audit.py

@@ -27,14 +27,19 @@ from pathlib import Path
 
 from evidence_semantics import claim_relation
 
-SUPPORTED_OUTCOMES = {
-    "knowledge_gain", "concept_understanding", "retention", "transfer",
-    "independent_problem_solving", "completion_time", "accuracy",
-    "code_quality", "assignment_score", "engagement", "motivation",
-    "cognitive_load", "help_seeking", "metacognition", "ai_dependency",
-    "over_reliance", "reduced_effort", "reduced_transfer",
-    "academic_integrity_risk", "false_confidence",
-}
+def _supported_outcomes() -> set[str]:
+    """Every registered outcome token, read from the domain registry.
+
+    This was a hand-copied 20-token education list in three separate files;
+    it silently rejected policy tokens such as policy_effectiveness. The
+    registry (domains/<id>/outcome_taxonomy.json) is the single authority.
+    """
+    from engine.taxonomy import all_tokens_ordered
+
+    return set(all_tokens_ordered())
+
+
+SUPPORTED_OUTCOMES = _supported_outcomes()
 
 
 def load_records(path: Path) -> list[dict]:

package/scripts/compute_confidence.py

@@ -40,6 +40,16 @@ import json
 import sys
 from pathlib import Path
 
+# This module is also documented as a standalone command (SKILL.md), so it
+# has to find the repository root on its own path; the orchestrator sets
+# sys.path for it, which hid the missing import from that caller.
+import sys as _sys
+from pathlib import Path as _Path
+_ROOT = _Path(__file__).resolve().parent.parent
+if str(_ROOT) not in _sys.path:
+    _sys.path.insert(0, str(_ROOT))
+
+
 from evidence_score import (CONFIDENCE_POLICY_VERSION,
                             decision_consistency_score, directness_score,
                             independent_samples, independent_studies)