eduevidence 5.2.0 → 6.2.0

package/examples/ai-coding-assistant-evidence/raw_verdict.json

@@ -0,0 +1,86 @@
+{
+  "decision_question": "大一 C 语言课程是否应该允许学生使用生成式 AI 编程助手？",
+  "target_population": "university first-year computer science students learning C programming for the first time",
+  "target_context": "16-week lecture-lab course, 60 students, TA support, offline",
+  "supported_claims": [
+    "AI coding assistants reliably increase task performance during training (completion speed, correctness) — E-001, E-006.",
+    "Unguarded generative AI access can harm independent problem solving when access is removed — E-004.",
+    "Guardrail design (hints instead of answers) substantially mitigates the negative learning effect — E-005.",
+    "Task performance gains do not automatically imply learning gains — E-004 vs E-006 (within-study contrast).",
+    "Tool capability is substantial: Codex solves roughly half to three-quarters of CS1 exam-style questions — E-010.",
+    "Professional-developer RCT shows ~55% faster task completion with Copilot; directness limited by professional population — E-008.",
+    "LLM code explanations rate comparable to student-authored explanations, viable as scaffold material — E-011."
+  ],
+  "uncertain_claims": [
+    "Whether AI coding assistants improve or preserve actual programming learning in university novices — no direct university-level RCT in reviewed set [无直接证据]",
+    "Whether one-week neutral retention (Kazemitabaar 2023) extends to a semester — E-003.",
+    "Whether benchmark quality findings (E-009) and explanation-quality ratings (E-011) translate into classroom learning gains.",
+    "How comprehension/ownership difficulties documented in usability studies (E-012) behave over a full semester with guardrails."
+  ],
+  "contradicted_claims": [
+    "The claim 'AI tools always improve learning' is contradicted by E-004 (unguarded access, -17% independent exam).",
+    "The claim 'speed gains equal learning gains' is contradicted by the task-vs-learning separation across E-001/E-006/E-008 vs E-004."
+  ],
+  "reason_for_disagreement": "Disagreement comes from outcome separation (task vs learning), tool design (guarded vs unguarded), and population (K-12 / professionals vs university). Task-performance evidence is consistently positive across randomized and benchmark studies; the only study measuring independent performance after AI removal shows harm without guardrails; usability and artifact studies add dependence and quality caveats rather than resolving the learning question.",
+  "methodology_summary": "Eight real sources: three randomized experiments (Kazemitabaar 2023 n=69 K-12; Bastani 2025 n≈950 high-school mathematics; Peng 2023 n=95 professional developers, preprint), one ESL writing mixed-methods study (Marzuki 2024), plus benchmark/capability/usability studies (Yetistiren 2023; Finnie-Ansley 2022; explanation-comparison 2023; Vaithilingam 2022). No direct RCT in university programming courses. Internal validity of the core RCTs is strong; directness to first-year university C programming is weak. All sources carry registry-verified DOIs (see benchmarks/doi-audit/report.md).",
+  "outcome_specific_findings": {
+    "completion_time": "positive during training and professional tasks (E-001, E-008)",
+    "independent_problem_solving": "neutral-to-negative without guardrails (E-002, E-004)",
+    "retention": "neutral over 1 week (E-003)",
+    "assignment_score": "positive during practice, negative on closed-book exam (E-004, E-006); tool itself scores passing-level on CS1 questions (E-010)",
+    "code_quality": "mixed on benchmarks; security concerns documented (E-009)",
+    "metacognition": "LLM explanations compare well (E-011) while novice ownership/debugging difficulties persist (E-012)",
+    "ai_dependency": "documented crutch behavior with unguarded tool (E-004, E-005, E-012)"
+  },
+  "short_term_effect": "Task performance reliably increases; learning effect null-to-negative without guardrails.",
+  "long_term_effect": "No evidence beyond one week; long-term learning effect unknown.",
+  "transfer_effect": "No full transfer evidence; manual code-modification not harmed in one small study (E-002).",
+  "risk_effect": "AI dependency and over-reliance risk is real and documented for unguarded usage (E-004) and foreshadowed by usability findings (E-012).",
+  "applicability": {
+    "suitable_for": "pilot in first-year C course with guardrailed usage policy",
+    "not_suitable_for": "unrestricted AI adoption without usage policy",
+    "required_conditions": [
+      "guardrailed AI usage policy (hints not answers, modeled on GPT Tutor arm)",
+      "no-AI transfer assessment",
+      "TA support"
+    ]
+  },
+  "confidence": "Moderate",
+  "confidence_breakdown": {
+    "score": 0.5,
+    "evidence_quality": 0.7,
+    "consistency": 0.6,
+    "directness": 0.4,
+    "evidence_count": 12,
+    "independent_studies": 8,
+    "independent_samples": 8,
+    "count_term": 1.0,
+    "conflict_penalty": 0.0,
+    "unsupported_penalty": 0.0
+  },
+  "what_can_be_claimed": [
+    "AI coding assistants raise task performance for novices during training.",
+    "Unguarded access carries a real risk of hurting independent problem solving.",
+    "Guardrail design can mitigate that risk.",
+    "Direct evidence for university C programming learning is missing.",
+    "Tool capability headroom is large (CS1 question pass rates; professional speed RCT)."
+  ],
+  "what_cannot_be_claimed": [
+    "AI coding assistants improve (or even preserve) university students' programming learning.",
+    "Any long-term or retention benefit.",
+    "Any claim about which students benefit, based on university samples.",
+    "That benchmark or usability findings substitute for classroom learning outcomes."
+  ],
+  "missing_evidence": [
+    "RCT of AI coding assistants in university programming courses with retention and no-AI transfer tests.",
+    "Studies varying AI usage policy within the same course.",
+    "Longitudinal data on AI dependency beyond one course.",
+    "Peer-reviewed replication of the professional speed RCT (Peng et al. remains a preprint)."
+  ],
+  "recommended_action": "pilot",
+  "decision_rationale": "Positive task-performance evidence plus documented unguarded-access risk, mixed quality/usability signals, and missing university-level learning evidence → bounded, guardrailed pilot with evaluation, not full adoption.",
+  "exceeds_evidence_boundary": [
+    "Claiming 'AI coding assistants improve learning' exceeds the boundary: direct learning-effect evidence is missing.",
+    "Claiming 'AI works for everyone' exceeds the boundary: population and subject mismatch."
+  ]
+}

package/examples/ai-coding-assistant-evidence/report_spec.json

@@ -0,0 +1,230 @@
+{
+  "generated_by": "build_report.py",
+  "source": "result.json + result.zh.json",
+  "question": "我们准备在大学一年级 C 语言课程中允许学生使用生成式 AI 编程助手。它到底会不会提高学习效果？应该怎样引入？",
+  "theme_selected": "claude",
+  "theme_display": "Claude Research [Light]",
+  "theme_available": [
+    "claude",
+    "academic",
+    "datalab",
+    "datalab-dark",
+    "presentation"
+  ],
+  "theme_selection": "generation_time",
+  "lang_default": "zh",
+  "lang_switchable": [
+    "zh",
+    "en"
+  ],
+  "report_pages": [
+    "visual_brief",
+    "full_report"
+  ],
+  "full_report_outline": {
+    "chapter_count": 6,
+    "source": "safe_fallback",
+    "chapters": [
+      {
+        "key": "decision",
+        "title_zh": "结论、裁决与研究边界",
+        "title_en": "Decision, Adjudication & Research Boundary",
+        "modules": [
+          "decision",
+          "scope"
+        ]
+      },
+      {
+        "key": "evidence",
+        "title_zh": "关键证据与结果分离",
+        "title_en": "Key Evidence & Outcome Separation",
+        "modules": [
+          "retrieval",
+          "outcomes",
+          "evidence"
+        ]
+      },
+      {
+        "key": "quality",
+        "title_zh": "证据可信度、反证与方法审计",
+        "title_en": "Evidence Quality, Counterevidence & Method Audit",
+        "modules": [
+          "quality",
+          "conflicts",
+          "trace"
+        ]
+      },
+      {
+        "key": "action",
+        "title_zh": "适用范围与教学行动",
+        "title_en": "Applicability & Teaching Action",
+        "modules": [
+          "applicability",
+          "intervention"
+        ]
+      },
+      {
+        "key": "evaluation",
+        "title_zh": "试点设计、评估与停止条件",
+        "title_en": "Pilot, Evaluation & Stop Conditions",
+        "modules": [
+          "evaluation"
+        ]
+      },
+      {
+        "key": "sources",
+        "title_zh": "来源、溯源与附录",
+        "title_en": "Sources, Traceability & Appendix",
+        "modules": [
+          "sources"
+        ]
+      }
+    ]
+  },
+  "visualization_decisions": {
+    "outcome_separation": {
+      "render": true,
+      "reason": "multiple outcome constructs present"
+    },
+    "outcome_evidence_balance": {
+      "render": true,
+      "reason": "sufficient effect-direction density"
+    },
+    "claim_trace": {
+      "render": true,
+      "reason": "claim-evidence-source relationships present"
+    },
+    "benchmark": {
+      "render": false,
+      "reason": "suppressed: absent, simulated, or fewer than two baselines"
+    }
+  },
+  "lieflat_gallery": {
+    "layout_source": "deterministic_fallback",
+    "selected": [
+      {
+        "chart_id": "lieflat-bubble-almanac.svg",
+        "type": "bubble_almanac",
+        "catalog_ref": "L9 Bubble Almanac",
+        "source": "evidence.year_x_dimension"
+      },
+      {
+        "chart_id": "lieflat-matrix-heat.svg",
+        "type": "matrix_heat",
+        "catalog_ref": "L16 Matrix Heat",
+        "source": "evidence.year_x_outcome_counts"
+      },
+      {
+        "chart_id": "lieflat-tick-rows.svg",
+        "type": "tick_rows",
+        "catalog_ref": "F5 Tick Rows",
+        "source": "outcomes.direction_counts"
+      },
+      {
+        "chart_id": "lieflat-paired-rungs.svg",
+        "type": "paired_rungs",
+        "catalog_ref": "F6 Paired Rungs",
+        "source": "outcomes.paired_counts"
+      },
+      {
+        "chart_id": "lieflat-brand-spectrum.svg",
+        "type": "brand_spectrum",
+        "catalog_ref": "L7 Brand Spectrum",
+        "source": "outcomes.bipolar_axes"
+      },
+      {
+        "chart_id": "lieflat-hundred-field.svg",
+        "type": "hundred_field",
+        "catalog_ref": "L14 Hundred Field",
+        "source": "evidence.study_type_composition"
+      }
+    ],
+    "suppressed": [],
+    "rejected": [],
+    "warnings": [
+      "visual_layout missing or fully invalid — data-driven fallback selected bubble_almanac, matrix_heat, tick_rows, paired_rungs, brand_spectrum, hundred_field"
+    ]
+  },
+  "charts": [
+    {
+      "chart_id": "outcome-evidence-overview",
+      "purpose": "interactive_analysis",
+      "engine": "echarts",
+      "data_ref": null,
+      "title": "Outcome Evidence Overview",
+      "integrity": {
+        "numbers_match_result": "NOT_CHECKED",
+        "no_axis_distortion": "NOT_CHECKED",
+        "no_false_precision": "NOT_CHECKED",
+        "colorblind_safe": "NOT_CHECKED"
+      }
+    },
+    {
+      "chart_id": "claim-evidence-trace",
+      "purpose": "interactive_analysis",
+      "engine": "echarts",
+      "data_ref": null,
+      "title": "Claim-Evidence Trace",
+      "integrity": {
+        "numbers_match_result": "NOT_CHECKED",
+        "no_axis_distortion": "NOT_CHECKED",
+        "no_false_precision": "NOT_CHECKED",
+        "colorblind_safe": "NOT_CHECKED"
+      }
+    }
+  ],
+  "infographics": [
+    {
+      "chart_id": "workflow",
+      "purpose": "process_or_story",
+      "engine": "antv_infographic",
+      "title": ""
+    },
+    {
+      "chart_id": "tribunal",
+      "purpose": "process_or_story",
+      "engine": "antv_infographic",
+      "title": ""
+    },
+    {
+      "chart_id": "intervention",
+      "purpose": "process_or_story",
+      "engine": "antv_infographic",
+      "title": ""
+    },
+    {
+      "chart_id": "evaluation",
+      "purpose": "process_or_story",
+      "engine": "antv_infographic",
+      "title": ""
+    }
+  ],
+  "academic_figures": [
+    {
+      "chart_id": "outcome-comparison",
+      "purpose": "statistical_publication",
+      "engine": "academic_figure",
+      "caption": "Fig. 1. Counts of positive / negative / null effects per outcome type (based on effect_direction; publication figure, theme-independent). Source: EduEvidence result.json."
+    }
+  ],
+  "integrity_gate": {
+    "status": "PASS",
+    "contract_valid": "PASS",
+    "claims_bound": "PASS",
+    "evidence_bound": 12,
+    "sources_resolved": 8,
+    "numbers_match_result": "PASS",
+    "bilingual_structure_match": "PASS",
+    "language_match": "PASS",
+    "no_false_precision": "PASS",
+    "lieflat_data_bound": "PASS",
+    "no_axis_distortion": "NOT_CHECKED",
+    "colorblind_safe": "NOT_CHECKED",
+    "langs": [
+      "zh",
+      "en"
+    ],
+    "generated_by": "build_report.py",
+    "source": "result.json + result.zh.json"
+  }
+}