eduevidence 5.2.0 → 6.2.0

package/scripts/pre_verdict_gate.py

@@ -7,7 +7,7 @@ reads ONLY the run workspace — no model call, no network.
 
 Checklist:
 
-  1.  research_frame_valid       frame.json validates against education-frame.schema.json
+  1.  research_frame_valid       frame.json validates against the run domain frame schema
   2.  sources_valid              sources.jsonl non-empty and schema-valid
   3.  evidence_schema_valid      evidence.jsonl non-empty and schema-valid
   4.  source_dedupe              no duplicate sources remain (dedupe applied)
@@ -58,7 +58,7 @@ for _p in (str(ROOT), str(ROOT / "scripts")):
 
 from validate_schema import SchemaError, Validator  # noqa: E402
 from evidence_score import independent_samples, independent_studies  # noqa: E402
-from evidence_semantics import claim_relation  # noqa: E402
+from evidence_semantics import claim_relation, decision_relation  # noqa: E402
 from run_workspace import utc_now  # noqa: E402
 
 GATE_VERSION = "2026-08-13.v1"
@@ -66,29 +66,52 @@ GATE_VERSION = "2026-08-13.v1"
 CONFIDENCE_RANK = {"Insufficient": 0, "Low": 1, "Moderate": 2, "High": 3}
 
 #: Advisory taxonomy for verdict outcome keys (shared with claim_audit).
-SUPPORTED_OUTCOMES = {
-    "knowledge_gain", "concept_understanding", "retention", "transfer",
-    "independent_problem_solving", "completion_time", "accuracy",
-    "code_quality", "assignment_score", "engagement", "motivation",
-    "cognitive_load", "help_seeking", "metacognition", "ai_dependency",
-    "over_reliance", "reduced_effort", "reduced_transfer",
-    "academic_integrity_risk", "false_confidence",
-}
+def _supported_outcomes() -> set[str]:
+    """Every registered outcome token, read from the domain registry.
+
+    This was a hand-copied 20-token education list in three separate files;
+    it silently rejected policy tokens such as policy_effectiveness. The
+    registry (domains/<id>/outcome_taxonomy.json) is the single authority.
+    """
+    from engine.taxonomy import all_tokens_ordered
+
+    return set(all_tokens_ordered())
+
+
+SUPPORTED_OUTCOMES = _supported_outcomes()
 
 _CLAIM_ID_RE = re.compile(r"\b[A-Z]{1,3}-\d{2,4}\b")
 
 _SCHEMA_CACHE: dict[str, dict[str, Any]] = {}
 
 
+def _schema_path(name: str) -> Path:
+    """Resolve a schema name to a file.
+
+    Accepts a bare name in schemas/ as well as a repository-relative path,
+    because a domain may own its frame contract outside schemas/ (policy does:
+    domains/policy/frame.schema.json).
+    """
+    candidate = ROOT / "schemas" / name
+    if candidate.is_file():
+        return candidate
+    owned = ROOT / name
+    if owned.is_file():
+        return owned
+    return candidate  # missing: the caller reports it
+
+
 def _schema(name: str) -> dict[str, Any]:
-    if name not in _SCHEMA_CACHE:
-        _SCHEMA_CACHE[name] = json.loads((ROOT / "schemas" / name).read_text(encoding="utf-8"))
-    return _SCHEMA_CACHE[name]
+    path = _schema_path(name)
+    key = str(path)
+    if key not in _SCHEMA_CACHE:
+        _SCHEMA_CACHE[key] = json.loads(path.read_text(encoding="utf-8"))
+    return _SCHEMA_CACHE[key]
 
 
 def _validate_records(records: list[dict[str, Any]], schema_name: str, path: str) -> list[str]:
     schema = _schema(schema_name)
-    validator = Validator(schema, base_dir=(ROOT / "schemas"))
+    validator = Validator(schema, base_dir=_schema_path(schema_name).parent)
     errors = []
     for idx, record in enumerate(records):
         try:
@@ -133,11 +156,43 @@ def _item_res(status: str, detail: str, *, blocks_high: bool | None = None) -> d
     return res
 
 
+def workspace_domain(ws: Path) -> str:
+    """The domain this run/pack registered; defaults to education when absent.
+
+    A run workspace records it in run_manifest.json, but an example pack has
+    no manifest: reading only that file made a policy pack validate against the
+    education frame schema and fail item 1. Fall back to the same declared
+    places the read model uses, in the same order.
+    """
+    manifest = _load_ws_json(ws, "run_manifest.json")
+    if manifest.get("domain"):
+        return str(manifest["domain"])
+    frame = _load_ws_json(ws, "frame.json")
+    declared = (frame.get("extensions") or {}).get("domain")
+    if declared:
+        return str(declared)
+    result_meta = (_load_ws_json(ws, "result.json").get("meta") or {})
+    if result_meta.get("domain"):
+        return str(result_meta["domain"])
+    return "education"
+
+
+def frame_schema_name(domain: str) -> str:
+    """Registered frame schema for a domain (repository-relative path)."""
+    from engine.evidencecore import load_domain
+
+    return str(load_domain(domain)["frame_schema"])
+
+
 def check_research_frame(ws: Path) -> dict[str, str]:
     frame = _load_ws_json(ws, "frame.json")
     if not frame:
         return _item_res("fail", "frame.json missing or empty (research question not framed)")
-    errors = _validate_records([frame], "education-frame.schema.json", "frame")
+    try:
+        schema_name = frame_schema_name(workspace_domain(ws))
+    except KeyError as exc:
+        return _item_res("fail", f"run declares an unknown domain: {exc}")
+    errors = _validate_records([frame], schema_name, "frame")
     if errors:
         return _item_res("fail", f"frame.json schema invalid: {errors[0]}")
     return _item_res("pass", f"frame.json valid (question={frame.get('question', '')[:80]})")
@@ -182,11 +237,41 @@ def check_counter_evidence(ws: Path) -> dict[str, str]:
         return _item_res("fail", "skeptic.json missing or empty (counter-evidence search not performed)")
     if skeptic.get("search_performed") is not True:
         return _item_res("fail", "skeptic.json lacks search_performed=true")
-    contradictions = skeptic.get("contradictions", []) or []
-    null_results = skeptic.get("null_results", []) or []
-    detail = f"search_performed=true; contradictions={len(contradictions)}, null_results={len(null_results)}"
-    if skeptic.get("no_contradictory_evidence_found"):
-        detail += "; no contradictory evidence found"
+
+    # The nine fixed checks are the contract (skill/task-briefs/challenge.md).
+    # A two-key shell used to pass this gate, which made the counter-evidence
+    # check cosmetic on every deterministic path.
+    required_checks = (
+        "1_null_result", "2_negative_result", "3_contradictory_evidence",
+        "4_alternative_explanation", "5_measurement_mismatch", "6_sampling_bias",
+        "7_novelty_effect", "8_ai_dependency", "9_scope_overreach",
+    )
+    findings = skeptic.get("skeptic_findings")
+    if not isinstance(findings, list) or not findings:
+        return _item_res(
+            "fail", "skeptic.json lacks skeptic_findings[] (nine fixed checks not run)")
+    present = {str(f.get("check")) for f in findings if isinstance(f, dict)}
+    missing = [c for c in required_checks if c not in present]
+    if missing:
+        return _item_res("fail", f"skeptic findings incomplete; missing {missing}")
+    for item in findings:
+        if not isinstance(item, dict) or item.get("status") not in ("found", "not_found"):
+            return _item_res(
+                "fail", f"skeptic finding {item.get('check')!r} lacks a found/not_found status")
+
+    found = bool(skeptic.get("contradictory_evidence_found"))
+    statement = skeptic.get("no_contradictory_evidence_statement")
+    if not found and not statement:
+        # Absence of counter-evidence must be asserted, not left implicit.
+        return _item_res(
+            "fail", "no contradictory evidence found but no statement recorded")
+    if found and statement:
+        return _item_res(
+            "fail", "contradictory_evidence_found=true together with a no-evidence statement")
+
+    found_checks = sum(1 for f in findings if f.get("status") == "found")
+    detail = (f"search_performed=true; 9/9 checks run; findings={found_checks}; "
+              f"contradictory_evidence_found={found}")
     return _item_res("pass", detail)
 
 
@@ -270,6 +355,96 @@ def check_claim_evidence(ws: Path) -> dict[str, str]:
     return _item_res("pass", "all verdict claims bind to existing evidence with consistent categories")
 
 
+def _primary_evidence_summary(ws: Path) -> dict[str, Any]:
+    """Re-derive ADOPT eligibility from the pack's own evidence records.
+
+    The gate audits an artifact another party wrote, so it cannot trust the
+    verdict about itself: primary-result directness is read back from the
+    corpus (frame primary outcomes + per-record D5 Directness), using the same
+    domain registry and directness threshold the tribunal imports.
+    """
+    from engine.decision_policy import (
+        ADOPT_DIRECTNESS,
+        outcome_category,
+        primary_effect_categories,
+    )
+
+    domain = workspace_domain(ws)
+    try:
+        primary = primary_effect_categories(domain)
+    except (KeyError, ValueError) as exc:
+        return {"domain": domain, "primary": (), "direct": False, "error": str(exc)}
+
+    frame = _load_ws_json(ws, "frame.json")
+    declared_primary = [
+        str(token) for token in ((frame.get("outcomes") or {}).get("primary") or [])
+    ]
+    evidence = _load_ws_jsonl(ws, "evidence.jsonl")
+    direct_hits: list[str] = []
+    for ev in evidence:
+        token = str(ev.get("outcome_type") or "")
+        category = outcome_category(domain, token, primary)
+        if category is None or category not in primary:
+            continue
+        dims = ev.get("quality_dimensions") or {}
+        d5 = dims.get("D5_directness")
+        if isinstance(d5, bool) or not isinstance(d5, int):
+            continue
+        if d5 >= ADOPT_DIRECTNESS:
+            direct_hits.append(str(ev.get("evidence_id") or "?"))
+    return {
+        "domain": domain,
+        "primary": primary,
+        "declared_primary": declared_primary,
+        "direct": bool(direct_hits),
+        "direct_evidence_ids": direct_hits,
+    }
+
+
+def check_decision_action(ws: Path) -> dict[str, str]:
+    """The stated action must be the action the evidence supports.
+
+    A verdict cannot hand itself ADOPT: the gate requires High confidence, a
+    supporting decisive relation, and primary-outcome evidence at directness 2.
+    Anything less is capped to pilot - the conservative bound - because the
+    underlying evidence may still justify a bounded trial.
+    """
+    verdict = _verdict_for_audit(ws)
+    if not verdict:
+        return _item_res("warn", "no verdict artifact yet; action not audited")
+    action = str(verdict.get("recommended_action") or "").lower()
+    if action != "adopt":
+        label = action or "unset"
+        return _item_res("pass", "action=" + label + " is within the conservative bound")
+
+    from engine.decision_policy import ADOPT_REQUIRED_LABEL, decision_action
+
+    evidence = _load_ws_jsonl(ws, "evidence.jsonl")
+    relations = [decision_relation(ev) for ev in evidence]
+    decisive = {str(index): rel for index, rel in enumerate(relations)
+                if rel in ("support_adoption", "oppose_adoption")}
+    label = str(verdict.get("confidence") or "")
+    summary = _primary_evidence_summary(ws)
+    expected = decision_action(
+        confidence_label=label,
+        decisive_relations=decisive,
+        has_direct_primary_evidence=bool(summary.get("direct")),
+    )
+    if expected == "ADOPT":
+        detail = "ADOPT is supported: " + label + " confidence with direct primary-outcome evidence"
+        return _item_res("pass", detail)
+    reasons: list[str] = []
+    if label != ADOPT_REQUIRED_LABEL:
+        reasons.append("confidence=" + (label or "unset") + " (needs " + ADOPT_REQUIRED_LABEL + ")")
+    if "support_adoption" not in relations:
+        reasons.append("no decisive supporting evidence")
+    if not summary.get("direct"):
+        reasons.append("no primary-outcome evidence at directness 2")
+    detail = ("recommended_action=adopt is not supported (" + "; ".join(reasons)
+              + "); the evidence bounds this decision to pilot")
+    return _item_res("fail", detail)
+
+
 def check_outcome_mapping(ws: Path) -> dict[str, str]:
     verdict = _verdict_for_audit(ws)
     frame = _load_ws_json(ws, "frame.json")
@@ -282,18 +457,35 @@ def check_outcome_mapping(ws: Path) -> dict[str, str]:
         issues.append(f"unknown outcome key(s) in verdict: {', '.join(sorted(unknown))}")
 
     evidence_outcomes = {e.get("outcome_type") for e in _load_ws_jsonl(ws, "evidence.jsonl")}
+    frame_outcomes = frame.get("outcomes", {}) or {}
     declared = set()
     for group in ("primary", "secondary", "risk"):
-        declared.update((frame.get("outcomes", {}) or {}).get(group, []) or [])
+        declared.update(frame_outcomes.get(group, []) or [])
     if declared:
         missing = sorted(d for d in declared if d and d not in evidence_outcomes)
         if missing:
-            notes.append(f"frame-declared outcomes without evidence: {', '.join(missing)}")
+            # Missing evidence is not a zero effect, and a secondary or risk
+            # outcome the frame named but the corpus never measured does not
+            # contaminate the decision. Only a PRIMARY outcome with no
+            # evidence at all means the decision rests on the wrong construct,
+            # so only that case blocks High confidence.
+            primary_missing = sorted(
+                d for d in (frame_outcomes.get("primary", []) or [])
+                if d and d not in evidence_outcomes)
+            note = f"frame-declared outcomes without evidence: {', '.join(missing)}"
+            if primary_missing:
+                return _item_res(
+                    "warn",
+                    note + f"; primary outcomes unmeasured: {', '.join(primary_missing)}",
+                    blocks_high=True)
+            notes.append(note)
+        else:
+            notes.append("all frame-declared outcomes are covered by evidence")
 
     if issues:
         return _item_res("fail", "; ".join(issues))
-    if notes:
-        return _item_res("warn", "outcome keys known; " + notes[0])
+    if notes and "without evidence" in notes[0]:
+        return _item_res("warn", "outcome keys known; " + notes[0], blocks_high=False)
     if not declared:
         return _item_res("warn", "outcome keys known; frame declares no outcomes to map")
     return _item_res("pass", f"outcome mapping complete ({len(evidence_outcomes)} outcome type(s) covered)")
@@ -375,13 +567,15 @@ GATE_ITEMS: list[dict[str, Any]] = [
     {"id": "claim_evidence_audit", "title": "Claim-Evidence Audit", "critical": True,
      "blocks_high": False, "check": check_claim_evidence},
     {"id": "outcome_mapping", "title": "Outcome mapping", "critical": False,
-     "blocks_high": False, "check": check_outcome_mapping},
+     "blocks_high": True, "check": check_outcome_mapping},
     {"id": "scope_calibration", "title": "Scope calibration", "critical": False,
-     "blocks_high": False, "check": check_scope_calibration},
+     "blocks_high": True, "check": check_scope_calibration},
     {"id": "independent_study_count", "title": "Independent study-sample count", "critical": True,
      "blocks_high": True, "check": check_study_count},
     {"id": "deterministic_confidence", "title": "Deterministic confidence", "critical": True,
      "blocks_high": True, "check": None},
+    {"id": "decision_action_consistency", "title": "Decision action consistency",
+     "critical": True, "blocks_high": False, "check": check_decision_action},
 ]
 
 
@@ -437,6 +631,7 @@ def apply_enforcement(verdict: dict[str, Any], gate: dict[str, Any]) -> dict[str
 
     - gate failed            -> confidence at most Low; adopt downgraded to pilot
     - gate passed but High blocked -> confidence at most Moderate
+    - stated adopt the evidence does not support -> downgraded to pilot
     Always records the enforcement inside verdict.extensions.gate_enforcement.
     """
     import copy
@@ -448,6 +643,9 @@ def apply_enforcement(verdict: dict[str, Any], gate: dict[str, Any]) -> dict[str
         out["confidence"] = cap
     if not gate.get("passed", False) and out.get("recommended_action") == "adopt":
         out["recommended_action"] = "pilot"
+    action_item = (gate.get("items") or {}).get("decision_action_consistency") or {}
+    if action_item.get("status") == "fail" and out.get("recommended_action") == "adopt":
+        out["recommended_action"] = "pilot"
     extensions = out.setdefault("extensions", {})
     if not isinstance(extensions, dict):
         extensions = {}

package/scripts/quickstart.py

@@ -40,8 +40,23 @@ STAGE_BRIEFS = {
 }
 
 
+def runs_root() -> Path:
+    """Runs directory, honouring EDUEVIDENCE_RUNS_DIR like the orchestrator does.
+
+    Without this the quickstart wrote into the repository even when the user had
+    pointed the runs directory elsewhere.
+    """
+    import os
+
+    return Path(os.environ.get("EDUEVIDENCE_RUNS_DIR") or (ROOT / "runs"))
+
+
 def newest_run_dir() -> Path:
-    runs_dir = ROOT / "runs"
+    runs_dir = runs_root()
+    if not runs_dir.is_dir():
+        raise SystemExit(
+            f"no runs directory at {runs_dir}; run `eduevidence run --question ...` first "
+            "(or set EDUEVIDENCE_RUNS_DIR)")
     candidates = sorted((p for p in runs_dir.iterdir() if p.is_dir()),
                         key=lambda p: p.stat().st_mtime, reverse=True)
     if not candidates:
@@ -86,7 +101,8 @@ def build_next_steps(run_dir: Path, question: str, depth: str) -> str:
         "",
         "## 可信度自检",
         "",
-        "- 引用逐条核验报告：`benchmarks/doi-audit/report.md` 与包内 `citation_check.md`",
+        "- 引用逐条核验报告：包内 `citation_check.md`"
+        "（源码仓库另见 `benchmarks/doi-audit/report.md`）",
         "- 报告头徽章标注 data_origin；synthetic 演示不得当作实证引用",
         ""]
     return "\n".join(lines)

package/scripts/rebake_all_5themes.py

@@ -10,8 +10,7 @@ from pathlib import Path
 ROOT = Path(__file__).resolve().parent.parent
 
 PROJECTS = [
-    "examples/highschool-math-ai-tutor",
-    "examples/esl-academic-writing-ai",
+    "examples/workplace-ai-assistant",
     "examples/ai-coding-assistant-evidence"
 ]