eduevidence 6.0.0 → 6.2.0

package/examples/ai-coding-assistant-evidence/artifact_manifest.json

@@ -1,8 +1,8 @@
 {
-  "result_sha256": "b498e00a88080c0645a9f0c2f4f513dc916f1ec4a2c6c69592363c99adb5dc17",
-  "result_zh_sha256": "0b14bf748bf92008a7813cc1aa7db3e30b5ad726e85b5f408b2cebc2dcb94b1c",
+  "result_sha256": "3bce7a28f588643a3064c3d82eb749e3999cb18cb812b85c9ad6ecc2a731c6a6",
+  "result_zh_sha256": "4d58876e31d6f095e733c674217bbdaf2204dd66097fe5e5ccf4545a397583f8",
   "renderer_version": "1.0.0",
-  "git_commit": "0db3403087cf3708bfb7fb0597fa8a289d4fd382",
+  "git_commit": "56f5dc3edc6e805c2614767208a83009b0f0a097",
   "evidence_count": 12,
   "source_count": 8,
   "themes": [

package/examples/ai-coding-assistant-evidence/citation_check.json

@@ -1,5 +1,5 @@
 {
-  "generated_at": "2026-08-24T12:39:14+0800",
+  "generated_at": "2026-09-13T11:17:51+0800",
   "checked": 8,
   "bad": 0,
   "results": [

package/examples/ai-coding-assistant-evidence/final_verdict.json

@@ -0,0 +1,107 @@
+{
+  "decision_question": "大一 C 语言课程是否应该允许学生使用生成式 AI 编程助手？",
+  "target_population": "university first-year computer science students learning C programming for the first time",
+  "target_context": "16-week lecture-lab course, 60 students, TA support, offline",
+  "supported_claims": [
+    "AI coding assistants reliably increase task performance during training (completion speed, correctness) — E-001, E-006.",
+    "Unguarded generative AI access can harm independent problem solving when access is removed — E-004.",
+    "Guardrail design (hints instead of answers) substantially mitigates the negative learning effect — E-005.",
+    "Task performance gains do not automatically imply learning gains — E-004 vs E-006 (within-study contrast).",
+    "Tool capability is substantial: Codex solves roughly half to three-quarters of CS1 exam-style questions — E-010.",
+    "Professional-developer RCT shows ~55% faster task completion with Copilot; directness limited by professional population — E-008.",
+    "LLM code explanations rate comparable to student-authored explanations, viable as scaffold material — E-011."
+  ],
+  "uncertain_claims": [
+    "Whether AI coding assistants improve or preserve actual programming learning in university novices — no direct university-level RCT in reviewed set [无直接证据]",
+    "Whether one-week neutral retention (Kazemitabaar 2023) extends to a semester — E-003.",
+    "Whether benchmark quality findings (E-009) and explanation-quality ratings (E-011) translate into classroom learning gains.",
+    "How comprehension/ownership difficulties documented in usability studies (E-012) behave over a full semester with guardrails."
+  ],
+  "contradicted_claims": [
+    "The claim 'AI tools always improve learning' is contradicted by E-004 (unguarded access, -17% independent exam).",
+    "The claim 'speed gains equal learning gains' is contradicted by the task-vs-learning separation across E-001/E-006/E-008 vs E-004."
+  ],
+  "reason_for_disagreement": "Disagreement comes from outcome separation (task vs learning), tool design (guarded vs unguarded), and population (K-12 / professionals vs university). Task-performance evidence is consistently positive across randomized and benchmark studies; the only study measuring independent performance after AI removal shows harm without guardrails; usability and artifact studies add dependence and quality caveats rather than resolving the learning question.",
+  "methodology_summary": "Eight real sources: three randomized experiments (Kazemitabaar 2023 n=69 K-12; Bastani 2025 n≈950 high-school mathematics; Peng 2023 n=95 professional developers, preprint), one ESL writing mixed-methods study (Marzuki 2024), plus benchmark/capability/usability studies (Yetistiren 2023; Finnie-Ansley 2022; explanation-comparison 2023; Vaithilingam 2022). No direct RCT in university programming courses. Internal validity of the core RCTs is strong; directness to first-year university C programming is weak. All sources carry registry-verified DOIs (see benchmarks/doi-audit/report.md).",
+  "outcome_specific_findings": {
+    "completion_time": "positive during training and professional tasks (E-001, E-008)",
+    "independent_problem_solving": "neutral-to-negative without guardrails (E-002, E-004)",
+    "retention": "neutral over 1 week (E-003)",
+    "assignment_score": "positive during practice, negative on closed-book exam (E-004, E-006); tool itself scores passing-level on CS1 questions (E-010)",
+    "code_quality": "mixed on benchmarks; security concerns documented (E-009)",
+    "metacognition": "LLM explanations compare well (E-011) while novice ownership/debugging difficulties persist (E-012)",
+    "ai_dependency": "documented crutch behavior with unguarded tool (E-004, E-005, E-012)"
+  },
+  "short_term_effect": "Task performance reliably increases; learning effect null-to-negative without guardrails.",
+  "long_term_effect": "No evidence beyond one week; long-term learning effect unknown.",
+  "transfer_effect": "No full transfer evidence; manual code-modification not harmed in one small study (E-002).",
+  "risk_effect": "AI dependency and over-reliance risk is real and documented for unguarded usage (E-004) and foreshadowed by usability findings (E-012).",
+  "applicability": {
+    "suitable_for": "pilot in first-year C course with guardrailed usage policy",
+    "not_suitable_for": "unrestricted AI adoption without usage policy",
+    "required_conditions": [
+      "guardrailed AI usage policy (hints not answers, modeled on GPT Tutor arm)",
+      "no-AI transfer assessment",
+      "TA support"
+    ]
+  },
+  "confidence": "Moderate",
+  "confidence_breakdown": {
+    "score": 0.586,
+    "evidence_quality": 0.758,
+    "consistency": 0.667,
+    "directness": 0.458,
+    "evidence_count": 12,
+    "independent_studies": 8,
+    "independent_samples": 8,
+    "count_term": 1.0,
+    "conflict_penalty": 0.15,
+    "unsupported_penalty": 0.0
+  },
+  "what_can_be_claimed": [
+    "AI coding assistants raise task performance for novices during training.",
+    "Unguarded access carries a real risk of hurting independent problem solving.",
+    "Guardrail design can mitigate that risk.",
+    "Direct evidence for university C programming learning is missing.",
+    "Tool capability headroom is large (CS1 question pass rates; professional speed RCT)."
+  ],
+  "what_cannot_be_claimed": [
+    "AI coding assistants improve (or even preserve) university students' programming learning.",
+    "Any long-term or retention benefit.",
+    "Any claim about which students benefit, based on university samples.",
+    "That benchmark or usability findings substitute for classroom learning outcomes."
+  ],
+  "missing_evidence": [
+    "RCT of AI coding assistants in university programming courses with retention and no-AI transfer tests.",
+    "Studies varying AI usage policy within the same course.",
+    "Longitudinal data on AI dependency beyond one course.",
+    "Peer-reviewed replication of the professional speed RCT (Peng et al. remains a preprint)."
+  ],
+  "recommended_action": "pilot",
+  "decision_rationale": "Positive task-performance evidence plus documented unguarded-access risk, mixed quality/usability signals, and missing university-level learning evidence → bounded, guardrailed pilot with evaluation, not full adoption.",
+  "exceeds_evidence_boundary": [
+    "Claiming 'AI coding assistants improve learning' exceeds the boundary: direct learning-effect evidence is missing.",
+    "Claiming 'AI works for everyone' exceeds the boundary: population and subject mismatch."
+  ],
+  "confidence_score": 0.586,
+  "confidence_policy_version": "2026-08-12.v3",
+  "independent_studies": 8,
+  "independent_samples": 8,
+  "raw_model_confidence": "Moderate",
+  "raw_model_confidence_breakdown": {
+    "score": 0.5,
+    "evidence_quality": 0.7,
+    "consistency": 0.6,
+    "directness": 0.4,
+    "evidence_count": 12,
+    "independent_studies": 8,
+    "independent_samples": 8,
+    "count_term": 1.0,
+    "conflict_penalty": 0.0,
+    "unsupported_penalty": 0.0
+  },
+  "strongest_support": "AI coding assistants reliably speed up practice work: completion rate 1.15x and time 0.57x in a randomised trial of 69 novices.",
+  "key_uncertainty": "No university-level RCT measures learning directly, and the one large trial that did - unguarded GPT-4 - saw independent exam scores fall 17%.",
+  "main_risk": "Unguarded access can raise practice performance while lowering independent exam performance, and learners may not notice the gap.",
+  "next_action": "Run a phased CS1 pilot with hints-not-answers guardrails, weekly lab use, and a no-AI transfer exam that can stop the pilot."
+}

package/examples/ai-coding-assistant-evidence/gate_report.json

@@ -0,0 +1,101 @@
+{
+  "gate_version": "2026-08-13.v1",
+  "checked_at": "2026-09-13T03:17:13.667942+00:00",
+  "workspace": "examples/ai-coding-assistant-evidence",
+  "items": {
+    "research_frame_valid": {
+      "title": "Research Frame valid",
+      "status": "pass",
+      "detail": "frame.json valid (question=我们准备在大学一年级 C 语言课程中允许学生使用生成式 AI 编程助手。它到底会不会提高学习效果？应该怎样引入？)",
+      "critical": true,
+      "blocks_high": true
+    },
+    "sources_valid": {
+      "title": "Sources valid",
+      "status": "pass",
+      "detail": "8 source record(s) schema-valid",
+      "critical": true,
+      "blocks_high": true
+    },
+    "evidence_schema_valid": {
+      "title": "Evidence Schema valid",
+      "status": "pass",
+      "detail": "12 evidence record(s) schema-valid",
+      "critical": true,
+      "blocks_high": true
+    },
+    "source_dedupe": {
+      "title": "Source dedupe",
+      "status": "pass",
+      "detail": "8 unique source(s), no duplicates",
+      "critical": true,
+      "blocks_high": true
+    },
+    "counter_evidence_search": {
+      "title": "Counter-evidence search",
+      "status": "pass",
+      "detail": "search_performed=true; 9/9 checks run; findings=9; contradictory_evidence_found=True",
+      "critical": true,
+      "blocks_high": true
+    },
+    "methodology_audit": {
+      "title": "Methodology audit",
+      "status": "pass",
+      "detail": "methodology verdict=CONCERN; task/learning separated",
+      "critical": true,
+      "blocks_high": true
+    },
+    "claim_evidence_audit": {
+      "title": "Claim-Evidence Audit",
+      "status": "warn",
+      "detail": "supported claim cites contradicting evidence E-004 (may be an intentional negative finding); supported claim cites contradicting evidence E-004 (may be an intentional negative finding) (+3 more)",
+      "critical": true,
+      "blocks_high": false
+    },
+    "outcome_mapping": {
+      "title": "Outcome mapping",
+      "status": "warn",
+      "detail": "outcome keys known; frame-declared outcomes without evidence: ai_dependency, reduced_transfer",
+      "critical": false,
+      "blocks_high": false
+    },
+    "scope_calibration": {
+      "title": "Scope calibration",
+      "status": "pass",
+      "detail": "claims bounded: can=5, cannot=4, exceeds_boundary=2",
+      "critical": false,
+      "blocks_high": true
+    },
+    "independent_study_count": {
+      "title": "Independent study-sample count",
+      "status": "pass",
+      "detail": "independent studies=8, samples=8",
+      "critical": true,
+      "blocks_high": true
+    },
+    "deterministic_confidence": {
+      "title": "Deterministic confidence",
+      "status": "pass",
+      "detail": "deterministic confidence=Moderate (score=0.586, policy=2026-08-12.v3)",
+      "critical": true,
+      "blocks_high": true
+    },
+    "decision_action_consistency": {
+      "title": "Decision action consistency",
+      "status": "pass",
+      "detail": "action=pilot is within the conservative bound",
+      "critical": true,
+      "blocks_high": false
+    }
+  },
+  "passed": true,
+  "critical_failures": [],
+  "high_confidence_allowed": true,
+  "max_confidence": "High",
+  "enforcement": {
+    "rule": "confidence capped at {max}; High confidence requires a fully passing gate and >= 2 independent studies",
+    "max_confidence": "High",
+    "requires_action_change": false,
+    "action_override": null
+  }
+}

package/examples/ai-coding-assistant-evidence/report_spec.json

@@ -108,30 +108,41 @@
         "catalog_ref": "L9 Bubble Almanac",
         "source": "evidence.year_x_dimension"
       },
+      {
+        "chart_id": "lieflat-matrix-heat.svg",
+        "type": "matrix_heat",
+        "catalog_ref": "L16 Matrix Heat",
+        "source": "evidence.year_x_outcome_counts"
+      },
       {
         "chart_id": "lieflat-tick-rows.svg",
         "type": "tick_rows",
         "catalog_ref": "F5 Tick Rows",
         "source": "outcomes.direction_counts"
-      }
-    ],
-    "suppressed": [
+      },
+      {
+        "chart_id": "lieflat-paired-rungs.svg",
+        "type": "paired_rungs",
+        "catalog_ref": "F6 Paired Rungs",
+        "source": "outcomes.paired_counts"
+      },
       {
-        "chart_id": "lieflat-forest-plot.svg",
-        "type": "forest_plot",
-        "catalog_ref": "FOREST-PLOT (publication figure)",
-        "reason": "fewer than 3 studies with numeric effect size (got 0)"
+        "chart_id": "lieflat-brand-spectrum.svg",
+        "type": "brand_spectrum",
+        "catalog_ref": "L7 Brand Spectrum",
+        "source": "outcomes.bipolar_axes"
       },
       {
-        "chart_id": "lieflat-dot-cascade.svg",
-        "type": "dot_cascade",
-        "catalog_ref": "L2 Dot Cascade",
-        "reason": "fewer than 3 studies with numeric effect size (got 0)"
+        "chart_id": "lieflat-hundred-field.svg",
+        "type": "hundred_field",
+        "catalog_ref": "L14 Hundred Field",
+        "source": "evidence.study_type_composition"
       }
     ],
+    "suppressed": [],
     "rejected": [],
     "warnings": [
-      "visual_layout missing or fully invalid — using deterministic safe combination (forest_plot + dot_cascade + bubble_almanac + tick_rows)"
+      "visual_layout missing or fully invalid — data-driven fallback selected bubble_almanac, matrix_heat, tick_rows, paired_rungs, brand_spectrum, hundred_field"
     ]
   },
   "charts": [