eduevidence 5.2.0

package/scripts/complexity_gate.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+"""complexity_gate.py — Deterministic complexity grading gate (plan section 9, Phase 6).
+
+Decides whether an EduEvidence task should run in single-agent mode (S),
+with one independent check (M), or enter the full 8-role workflow (L), and
+whether delegation to Agent MCP is warranted at all.
+
+Pure local heuristics — zero tokens, no spawn. The result is a suggestion:
+the main agent's judgment of the actual task always wins ("宁可不拆").
+
+Rules (per EduEvidence 实施方案 §9 and agent-mcp 编排 skill):
+- S: single question, single outcome, few sources, no obvious conflict
+  -> Frame → Retrieve → Extract → Verify → Answer (0 spawn)
+- M: multiple studies, 2-3 outcomes, partial conflict, one independent check
+  -> Primary Analysis + Independent Check (<=2-3 roles)
+- L: multiple outcomes, multiple learner groups, strong conflict,
+  needs teaching deployment plan -> full 8-role workflow
+
+Do-NOT-delegate list (hit any -> delegate=False):
+  quick Q&A, single-source check, formatting-only task, trivial edit,
+  strong sequential dependency chain.
+
+Usage:
+    python scripts/complexity_gate.py --question "..." --depth standard \\
+        --target teaching_decision --outcomes retention transfer \\
+        --multi-learner --needs-pilot
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import re
+from typing import Any
+
+# Signals that push complexity up (from plan §9 / §13)
+L_SIGNALS = [
+    "长期", "迁移", "保持", "retention", "transfer", "长期效果",
+    "多学习者", "多群体", "不同学生", "全面部署", "全校", "试点", "pilot",
+    "风险", "依赖", "over-reliance", "ai dependency", "冲突", "矛盾",
+]
+M_SIGNALS = [
+    "是否有效", "提高", "影响", "比较", "对比", "vs", "与", "不同研究",
+    "证据", "研究", "结论",
+]
+
+# Do-not-delegate triggers
+NO_DELEGATE_SIGNALS = [
+    "一句话", "简要", "快速回答", "解释一下", "这个文件", "格式化",
+    "重命名", "改个名字", "简单回答",
+]
+
+
+def _signal_count(text: str, signals: list[str]) -> int:
+    low = text.lower()
+    return sum(1 for s in signals if s.lower() in low)
+
+
+def _count_outcomes(outcomes: list[str] | None) -> int:
+    if not outcomes:
+        return 0
+    return len([o for o in outcomes if o])
+
+
+def grade(
+    question: str,
+    *,
+    depth: str = "standard",
+    target: str = "evidence_review",
+    outcomes: list[str] | None = None,
+    multi_learner: bool = False,
+    needs_pilot: bool = False,
+    needs_evaluation: bool = False,
+    conflict_hint: bool = False,
+) -> dict[str, Any]:
+    """Grade a task S/M/L and decide whether delegation is warranted.
+
+    Returns {level, rationale, delegate, suggestion}.
+    """
+    question = question or ""
+    l_score = _signal_count(question, L_SIGNALS)
+    m_score = _signal_count(question, M_SIGNALS)
+    n_outcomes = _count_outcomes(outcomes)
+    no_delegate = _signal_count(question, NO_DELEGATE_SIGNALS) > 0
+
+    # --- do-not-delegate gate (hit any -> never spawn) ---
+    if no_delegate:
+        return {
+            "level": "S",
+            "rationale": "do-not-delegate signal in question (quick/simple/formulaic task)",
+            "delegate": False,
+            "suggestion": "主 Agent 直接执行，禁止 spawn。",
+        }
+
+    # --- explicit inputs push complexity ---
+    score = 0
+    reasons: list[str] = []
+
+    if depth == "deep":
+        score += 2
+        reasons.append("depth=deep")
+    elif depth == "standard":
+        score += 1
+        reasons.append("depth=standard")
+
+    if target in ("teaching_decision", "pilot_design", "evaluation_design"):
+        score += 1
+        reasons.append(f"target={target}")
+    if multi_learner:
+        score += 1
+        reasons.append("multi-learner")
+    if needs_pilot or needs_evaluation:
+        score += 1
+        reasons.append("needs pilot/evaluation")
+    if conflict_hint:
+        score += 1
+        reasons.append("conflict hinted")
+    if n_outcomes >= 4:
+        score += 2
+        reasons.append(f"{n_outcomes} outcomes")
+    elif n_outcomes >= 2:
+        score += 1
+        reasons.append(f"{n_outcomes} outcomes")
+
+    # --- question-language signals ---
+    score += min(2, l_score)
+    score += min(1, m_score)
+    if l_score:
+        reasons.append(f"L-signals x{l_score}")
+    if m_score:
+        reasons.append(f"M-signals x{m_score}")
+
+    if score >= 4:
+        level = "L"
+    elif score >= 2:
+        level = "M"
+    else:
+        level = "S"
+
+    delegate = level in ("M", "L") and not no_delegate
+    suggestion = {
+        "S": "Frame → Retrieve → Extract → Verify → Answer（单 Agent 串行，0 spawn）",
+        "M": "Primary Analysis + Independent Check（增强模式 ≤2-3 角色）",
+        "L": "完整 8 角色工作流（Planner/Retriever/Analyst/Skeptic/Method Reviewer/Judge/Intervention Designer/Evaluation Designer）",
+    }[level]
+
+    rationale = f"complexity score {score} ({', '.join(reasons) if reasons else 'no signals'})"
+    return {"level": level, "rationale": rationale, "delegate": delegate, "suggestion": suggestion}
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="EduEvidence deterministic complexity gate")
+    parser.add_argument("--question", required=True, help="education question text")
+    parser.add_argument("--depth", choices=["quick", "standard", "deep"], default="standard")
+    parser.add_argument("--target", choices=["evidence_review", "teaching_decision",
+                                            "pilot_design", "evaluation_design"],
+                        default="evidence_review")
+    parser.add_argument("--outcomes", nargs="*", default=None)
+    parser.add_argument("--multi-learner", action="store_true")
+    parser.add_argument("--needs-pilot", action="store_true")
+    parser.add_argument("--needs-evaluation", action="store_true")
+    parser.add_argument("--conflict-hint", action="store_true")
+    args = parser.parse_args()
+
+    result = grade(
+        args.question,
+        depth=args.depth,
+        target=args.target,
+        outcomes=args.outcomes,
+        multi_learner=args.multi_learner,
+        needs_pilot=args.needs_pilot,
+        needs_evaluation=args.needs_evaluation,
+        conflict_hint=args.conflict_hint,
+    )
+    print(json.dumps(result, ensure_ascii=False, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    import sys
+    sys.exit(main())

package/scripts/compute_confidence.py

@@ -0,0 +1,176 @@
+#!/usr/bin/env python3
+"""compute_confidence.py — Deterministic confidence computation for final verdicts.
+
+Pipeline entry for the Evidence Integrity Report (P0-05): reads the raw model
+verdict plus the evidence registry and OVERRIDES the model's confidence values
+with the deterministic rule-based computation, so the final verdict's
+confidence is reproducible and auditable rather than model-generated.
+
+Formula (identical policy to evidence_score.confidence, v2 — D-1/D-2):
+
+    score = 0.30 * Evidence Quality + 0.25 * Consistency + 0.20 * Directness
+            + 0.25 * Evidence Count (independent-study weighted)
+            - Conflict Penalty - Unsupported Penalty
+
+where Evidence Count = min(1.0, independent_studies / 4) — independent_samples
+is reported separately instead of being added to the count term —,
+Consistency is computed over decision_relation (support_adoption /
+oppose_adoption / conditional / neutral) rather than relation_to_claim,
+Conflict Penalty = 0.15 if any evidence opposes adoption, and
+Unsupported Penalty = min(0.20, 0.05 * n_unsupported).
+
+IMPORTANT: confidence_score is a rule-based index in [0, 1], NOT a
+probability. It must never be presented as a percentage probability (e.g. "85%
+confidence"); consumers should phrase it as an index/band (High | Moderate |
+Low | Insufficient) with the policy version.
+
+Usage:
+    python3 scripts/compute_confidence.py --verdict raw_verdict.json \
+        --evidence evidence.jsonl --out final_verdict.json
+
+Output (final_verdict.json) carries the deterministic fields:
+confidence / confidence_score / confidence_policy_version /
+independent_studies / independent_samples / confidence_breakdown, plus the
+raw model values preserved as raw_model_confidence* for audit comparison.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from evidence_score import (CONFIDENCE_POLICY_VERSION,
+                            decision_consistency_score, directness_score,
+                            independent_samples, independent_studies)
+from evidence_semantics import decision_relation
+
+
+def compute_confidence(evidence_list: list[dict], *, target_outcome: str | None = None) -> dict:
+    """Deterministic confidence over an evidence list (independent-study weighted).
+
+    Returns a breakdown dict plus the final label (High | Moderate | Low |
+    Insufficient). This is the P0-05 policy engine; the legacy
+    evidence_score.confidence() remains for backward-compatible callers.
+    """
+    if not evidence_list:
+        return {"confidence": "Insufficient",
+                "confidence_policy_version": CONFIDENCE_POLICY_VERSION,
+                "independent_studies": 0,
+                "independent_samples": 0,
+                "confidence_breakdown": {"evidence_count": 0}}
+
+    # 1. Evidence Quality (mean of quality scores, scaled to 0-1)
+    quality_values = [e.get("quality_score") for e in evidence_list]
+    numeric = [q for q in quality_values if isinstance(q, (int, float))]
+    avg_quality = sum(numeric) / len(numeric) if numeric else 0.0
+    quality_term = avg_quality / 10.0
+
+    # 2. Consistency (decision_relation based, D-2): claim-level evidence is
+    #    usually extracted to support its claim, so relation_to_claim would
+    #    overstate agreement about the final teaching decision.
+    decisions = [decision_relation(e) for e in evidence_list]
+    consistency = decision_consistency_score(decisions)
+
+    # 3. Directness (0-2 -> 0-1)
+    directness = directness_score(evidence_list) / 2.0
+
+    # 4. Evidence count weighted by independent studies only (D-1): a typical
+    #    study is 1 study + 1 sample, so adding independent_samples would count
+    #    the same study twice. independent_samples is reported separately.
+    studies = independent_studies(evidence_list)
+    samples = independent_samples(evidence_list)
+    count_term = min(1.0, studies / 4.0)
+
+    # 5. Conflict penalty (0.15 when any evidence opposes adoption)
+    conflict_penalty = 0.15 if "oppose_adoption" in decisions else 0.0
+
+    # 6. Unsupported penalty (capped at 0.20)
+    unsupported = [e for e in evidence_list if e.get("status") in ("UNSUPPORTED", "DOWNGRADE_CONFIDENCE")]
+    unsupported_penalty = min(0.20, len(unsupported) * 0.05)
+
+    score = (0.30 * quality_term + 0.25 * consistency + 0.20 * directness
+             + 0.25 * count_term - conflict_penalty - unsupported_penalty)
+    score = max(0.0, min(1.0, score))
+
+    if target_outcome:
+        relevant = [e for e in evidence_list if e.get("outcome_type") == target_outcome]
+        if relevant:
+            score *= 0.5 + 0.5 * (len(relevant) / max(1, len(evidence_list)))
+
+    label = _confidence_label(score)
+    return {
+        "confidence": label,
+        "confidence_policy_version": CONFIDENCE_POLICY_VERSION,
+        "independent_studies": studies,
+        "independent_samples": samples,
+        "confidence_breakdown": {
+            "score": round(score, 3),
+            "evidence_quality": round(quality_term, 3),
+            "consistency": round(consistency, 3),
+            "directness": round(directness, 3),
+            "evidence_count": len(evidence_list),
+            "independent_studies": studies,
+            "independent_samples": samples,
+            "count_term": round(count_term, 3),
+            "conflict_penalty": conflict_penalty,
+            "unsupported_penalty": round(unsupported_penalty, 3),
+        },
+    }
+
+
+def _confidence_label(score: float) -> str:
+    if score >= 0.72:
+        return "High"
+    if score >= 0.45:
+        return "Moderate"
+    if score >= 0.2:
+        return "Low"
+    return "Insufficient"
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Compute deterministic confidence and merge it into the final verdict")
+    parser.add_argument("--verdict", required=True, help="Raw (model) verdict JSON file")
+    parser.add_argument("--evidence", required=True, help="Evidence JSONL file")
+    parser.add_argument("--out", required=True, help="Output final verdict JSON path")
+    args = parser.parse_args()
+
+    raw_verdict = json.loads(Path(args.verdict).read_text(encoding="utf-8"))
+    evidence_list = []
+    for lineno, line in enumerate(Path(args.evidence).read_text(encoding="utf-8").splitlines(), 1):
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            evidence_list.append(json.loads(line))
+        except json.JSONDecodeError as exc:
+            print(f"{args.evidence}:{lineno}: invalid JSON line: {exc}", file=sys.stderr)
+            return 1
+
+    target_outcome = raw_verdict.get("target_outcome")
+    computed = compute_confidence(evidence_list, target_outcome=target_outcome)
+
+    # Override model-generated confidence fields with the deterministic values.
+    final = dict(raw_verdict)
+    final["confidence"] = computed["confidence"]
+    final["confidence_score"] = computed["confidence_breakdown"].get("score")
+    final["confidence_policy_version"] = computed["confidence_policy_version"]
+    final["independent_studies"] = computed["independent_studies"]
+    final["independent_samples"] = computed["independent_samples"]
+    final["confidence_breakdown"] = computed["confidence_breakdown"]
+    # 保留原始模型输出，供审计比对（模型值被覆盖而非丢弃）。
+    final["raw_model_confidence"] = raw_verdict.get("confidence")
+    final["raw_model_confidence_breakdown"] = raw_verdict.get("confidence_breakdown") or {}
+
+    out_path = Path(args.out)
+    out_path.write_text(json.dumps(final, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    print(f"confidence={computed['confidence']} score={computed['confidence_breakdown']['score']} "
+          f"studies={computed['independent_studies']} samples={computed['independent_samples']} "
+          f"policy={CONFIDENCE_POLICY_VERSION} -> {out_path}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())