eduevidence 5.2.0 → 6.2.0

package/examples/ai-coding-assistant-evidence/result.json

@@ -0,0 +1,1457 @@
+{
+  "meta": {
+    "skill": "eduevidence",
+    "version": "6.2.0",
+    "generated_at": "2026-08-24T04:39:14.576951+00:00",
+    "mode": "platform_native",
+    "question": "我们准备在大学一年级 C 语言课程中允许学生使用生成式 AI 编程助手。它到底会不会提高学习效果？应该怎样引入？",
+    "data_origin": "manual_curated"
+  },
+  "execution": {
+    "complexity": "M",
+    "mode": "platform_native",
+    "agents": [],
+    "usage": {
+      "measurement_status": "NOT_CAPTURED",
+      "input_tokens": null,
+      "output_tokens": null,
+      "cost_usd": null,
+      "latency_s": null
+    }
+  },
+  "research_frame": {
+    "question": "我们准备在大学一年级 C 语言课程中允许学生使用生成式 AI 编程助手。它到底会不会提高学习效果？应该怎样引入？",
+    "decision_target": "teaching_decision",
+    "learner": {
+      "education_level": "undergraduate_year_1",
+      "major": "computer_science",
+      "prior_knowledge": "first_programming_course_no_prior_text_based_programming",
+      "special_characteristics": "mixed_ability_large_class_60_students"
+    },
+    "course": {
+      "subject": "C_programming",
+      "course_type": "lecture_lab",
+      "duration": "16_weeks_one_semester"
+    },
+    "intervention": {
+      "teaching_method": "lecture_with_lab_exercises",
+      "ai_tool": "generative_ai_coding_assistant",
+      "allowed_usage": "under_design_pending_evidence_review",
+      "frequency": "weekly_lab_sessions",
+      "duration": "one_semester"
+    },
+    "comparison": "no_ai_coding_assistant_control",
+    "outcomes": {
+      "primary": [
+        "independent_problem_solving",
+        "code_quality"
+      ],
+      "secondary": [
+        "completion_time",
+        "retention",
+        "knowledge_gain"
+      ],
+      "risk": [
+        "ai_dependency",
+        "over_reliance",
+        "reduced_transfer"
+      ]
+    },
+    "context": {
+      "teacher_support": "TA_supported_two_TAs",
+      "class_size": "60_students",
+      "online_or_offline": "offline"
+    },
+    "scope": {
+      "time_range": "2021-2026",
+      "geography": "worldwide",
+      "study_types": [
+        "rct",
+        "quasi_experimental",
+        "observational"
+      ]
+    },
+    "inclusion_criteria": [
+      "studies_of_generative_AI_coding_tools_in_learning_to_program",
+      "outcomes_measuring_learning_not_only_task_speed",
+      "university_or_novice_programming_populations"
+    ],
+    "exclusion_criteria": [
+      "practitioner_anecdotes_without_data",
+      "industry_professional_populations_only"
+    ],
+    "success_condition": "independent problem solving and code quality improve (or do not decline) while AI dependency risk stays controlled; evidence base supports a bounded pilot."
+  },
+  "decision": {
+    "decision_question": "大一 C 语言课程是否应该允许学生使用生成式 AI 编程助手？",
+    "target_population": "university first-year computer science students learning C programming for the first time",
+    "target_context": "16-week lecture-lab course, 60 students, TA support, offline",
+    "supported_claims": [
+      "AI coding assistants reliably increase task performance during training (completion speed, correctness) — E-001, E-006.",
+      "Unguarded generative AI access can harm independent problem solving when access is removed — E-004.",
+      "Guardrail design (hints instead of answers) substantially mitigates the negative learning effect — E-005.",
+      "Task performance gains do not automatically imply learning gains — E-004 vs E-006 (within-study contrast).",
+      "Tool capability is substantial: Codex solves roughly half to three-quarters of CS1 exam-style questions — E-010.",
+      "Professional-developer RCT shows ~55% faster task completion with Copilot; directness limited by professional population — E-008.",
+      "LLM code explanations rate comparable to student-authored explanations, viable as scaffold material — E-011."
+    ],
+    "uncertain_claims": [
+      "Whether AI coding assistants improve or preserve actual programming learning in university novices — no direct university-level RCT in reviewed set [无直接证据]",
+      "Whether one-week neutral retention (Kazemitabaar 2023) extends to a semester — E-003.",
+      "Whether benchmark quality findings (E-009) and explanation-quality ratings (E-011) translate into classroom learning gains.",
+      "How comprehension/ownership difficulties documented in usability studies (E-012) behave over a full semester with guardrails."
+    ],
+    "contradicted_claims": [
+      "The claim 'AI tools always improve learning' is contradicted by E-004 (unguarded access, -17% independent exam).",
+      "The claim 'speed gains equal learning gains' is contradicted by the task-vs-learning separation across E-001/E-006/E-008 vs E-004."
+    ],
+    "reason_for_disagreement": "Disagreement comes from outcome separation (task vs learning), tool design (guarded vs unguarded), and population (K-12 / professionals vs university). Task-performance evidence is consistently positive across randomized and benchmark studies; the only study measuring independent performance after AI removal shows harm without guardrails; usability and artifact studies add dependence and quality caveats rather than resolving the learning question.",
+    "methodology_summary": "Eight real sources: three randomized experiments (Kazemitabaar 2023 n=69 K-12; Bastani 2025 n≈950 high-school mathematics; Peng 2023 n=95 professional developers, preprint), one ESL writing mixed-methods study (Marzuki 2024), plus benchmark/capability/usability studies (Yetistiren 2023; Finnie-Ansley 2022; explanation-comparison 2023; Vaithilingam 2022). No direct RCT in university programming courses. Internal validity of the core RCTs is strong; directness to first-year university C programming is weak. All sources carry registry-verified DOIs (see benchmarks/doi-audit/report.md).",
+    "outcome_specific_findings": {
+      "completion_time": "positive during training and professional tasks (E-001, E-008)",
+      "independent_problem_solving": "neutral-to-negative without guardrails (E-002, E-004)",
+      "retention": "neutral over 1 week (E-003)",
+      "assignment_score": "positive during practice, negative on closed-book exam (E-004, E-006); tool itself scores passing-level on CS1 questions (E-010)",
+      "code_quality": "mixed on benchmarks; security concerns documented (E-009)",
+      "metacognition": "LLM explanations compare well (E-011) while novice ownership/debugging difficulties persist (E-012)",
+      "ai_dependency": "documented crutch behavior with unguarded tool (E-004, E-005, E-012)"
+    },
+    "short_term_effect": "Task performance reliably increases; learning effect null-to-negative without guardrails.",
+    "long_term_effect": "No evidence beyond one week; long-term learning effect unknown.",
+    "transfer_effect": "No full transfer evidence; manual code-modification not harmed in one small study (E-002).",
+    "risk_effect": "AI dependency and over-reliance risk is real and documented for unguarded usage (E-004) and foreshadowed by usability findings (E-012).",
+    "applicability": {
+      "suitable_for": "pilot in first-year C course with guardrailed usage policy",
+      "not_suitable_for": "unrestricted AI adoption without usage policy",
+      "required_conditions": [
+        "guardrailed AI usage policy (hints not answers, modeled on GPT Tutor arm)",
+        "no-AI transfer assessment",
+        "TA support"
+      ]
+    },
+    "confidence": "Moderate",
+    "confidence_breakdown": {
+      "score": 0.586,
+      "evidence_quality": 0.758,
+      "consistency": 0.667,
+      "directness": 0.458,
+      "evidence_count": 12,
+      "independent_studies": 8,
+      "independent_samples": 8,
+      "count_term": 1.0,
+      "conflict_penalty": 0.15,
+      "unsupported_penalty": 0.0
+    },
+    "what_can_be_claimed": [
+      "AI coding assistants raise task performance for novices during training.",
+      "Unguarded access carries a real risk of hurting independent problem solving.",
+      "Guardrail design can mitigate that risk.",
+      "Direct evidence for university C programming learning is missing.",
+      "Tool capability headroom is large (CS1 question pass rates; professional speed RCT)."
+    ],
+    "what_cannot_be_claimed": [
+      "AI coding assistants improve (or even preserve) university students' programming learning.",
+      "Any long-term or retention benefit.",
+      "Any claim about which students benefit, based on university samples.",
+      "That benchmark or usability findings substitute for classroom learning outcomes."
+    ],
+    "missing_evidence": [
+      "RCT of AI coding assistants in university programming courses with retention and no-AI transfer tests.",
+      "Studies varying AI usage policy within the same course.",
+      "Longitudinal data on AI dependency beyond one course.",
+      "Peer-reviewed replication of the professional speed RCT (Peng et al. remains a preprint)."
+    ],
+    "recommended_action": "pilot",
+    "decision_rationale": "Positive task-performance evidence plus documented unguarded-access risk, mixed quality/usability signals, and missing university-level learning evidence → bounded, guardrailed pilot with evaluation, not full adoption.",
+    "exceeds_evidence_boundary": [
+      "Claiming 'AI coding assistants improve learning' exceeds the boundary: direct learning-effect evidence is missing.",
+      "Claiming 'AI works for everyone' exceeds the boundary: population and subject mismatch."
+    ],
+    "confidence_score": 0.586,
+    "confidence_policy_version": "2026-08-12.v3",
+    "independent_studies": 8,
+    "independent_samples": 8,
+    "raw_model_confidence": "Moderate",
+    "raw_model_confidence_breakdown": {
+      "score": 0.586,
+      "evidence_quality": 0.758,
+      "consistency": 0.667,
+      "directness": 0.458,
+      "evidence_count": 12,
+      "independent_studies": 8,
+      "independent_samples": 8,
+      "count_term": 1.0,
+      "conflict_penalty": 0.15,
+      "unsupported_penalty": 0.0
+    },
+    "strongest_support": "AI coding assistants reliably speed up practice work: completion rate 1.15x and time 0.57x in a randomised trial of 69 novices.",
+    "key_uncertainty": "No university-level RCT measures learning directly, and the one large trial that did - unguarded GPT-4 - saw independent exam scores fall 17%.",
+    "main_risk": "Unguarded access can raise practice performance while lowering independent exam performance, and learners may not notice the gap.",
+    "next_action": "Run a phased CS1 pilot with hints-not-answers guardrails, weekly lab use, and a no-AI transfer exam that can stop the pilot."
+  },
+  "outcomes": [
+    {
+      "outcome_type": "knowledge_gain",
+      "positive_count": 1,
+      "negative_count": 0,
+      "null_count": 0,
+      "evidence_ids": [
+        "E-007"
+      ]
+    },
+    {
+      "outcome_type": "retention",
+      "positive_count": 0,
+      "negative_count": 0,
+      "null_count": 1,
+      "evidence_ids": [
+        "E-003"
+      ]
+    },
+    {
+      "outcome_type": "independent_problem_solving",
+      "positive_count": 0,
+      "negative_count": 1,
+      "null_count": 2,
+      "evidence_ids": [
+        "E-002",
+        "E-004",
+        "E-005"
+      ]
+    },
+    {
+      "outcome_type": "completion_time",
+      "positive_count": 2,
+      "negative_count": 0,
+      "null_count": 0,
+      "evidence_ids": [
+        "E-001",
+        "E-008"
+      ]
+    },
+    {
+      "outcome_type": "code_quality",
+      "positive_count": 0,
+      "negative_count": 0,
+      "null_count": 1,
+      "evidence_ids": [
+        "E-009"
+      ]
+    },
+    {
+      "outcome_type": "assignment_score",
+      "positive_count": 2,
+      "negative_count": 0,
+      "null_count": 0,
+      "evidence_ids": [
+        "E-006",
+        "E-010"
+      ]
+    },
+    {
+      "outcome_type": "metacognition",
+      "positive_count": 1,
+      "negative_count": 0,
+      "null_count": 0,
+      "evidence_ids": [
+        "E-011"
+      ]
+    },
+    {
+      "outcome_type": "over_reliance",
+      "positive_count": 0,
+      "negative_count": 1,
+      "null_count": 0,
+      "evidence_ids": [
+        "E-012"
+      ]
+    }
+  ],
+  "outcome_mapping": {
+    "entries": [
+      {
+        "outcome_type": "ai_dependency",
+        "declared_in_frame": true,
+        "status": "no_evidence",
+        "support_count": 0,
+        "contradict_count": 0,
+        "neutral_count": 0,
+        "evidence_ids": []
+      },
+      {
+        "outcome_type": "assignment_score",
+        "declared_in_frame": false,
+        "status": "supported",
+        "support_count": 2,
+        "contradict_count": 0,
+        "neutral_count": 0,
+        "evidence_ids": [
+          "E-006",
+          "E-010"
+        ]
+      },
+      {
+        "outcome_type": "code_quality",
+        "declared_in_frame": true,
+        "status": "null_evidence_only",
+        "support_count": 0,
+        "contradict_count": 0,
+        "neutral_count": 1,
+        "evidence_ids": [
+          "E-009"
+        ]
+      },
+      {
+        "outcome_type": "completion_time",
+        "declared_in_frame": true,
+        "status": "supported",
+        "support_count": 2,
+        "contradict_count": 0,
+        "neutral_count": 0,
+        "evidence_ids": [
+          "E-001",
+          "E-008"
+        ]
+      },
+      {
+        "outcome_type": "independent_problem_solving",
+        "declared_in_frame": true,
+        "status": "contested",
+        "support_count": 1,
+        "contradict_count": 1,
+        "neutral_count": 1,
+        "evidence_ids": [
+          "E-002",
+          "E-004",
+          "E-005"
+        ]
+      },
+      {
+        "outcome_type": "knowledge_gain",
+        "declared_in_frame": true,
+        "status": "supported",
+        "support_count": 1,
+        "contradict_count": 0,
+        "neutral_count": 0,
+        "evidence_ids": [
+          "E-007"
+        ]
+      },
+      {
+        "outcome_type": "metacognition",
+        "declared_in_frame": false,
+        "status": "supported",
+        "support_count": 1,
+        "contradict_count": 0,
+        "neutral_count": 0,
+        "evidence_ids": [
+          "E-011"
+        ]
+      },
+      {
+        "outcome_type": "over_reliance",
+        "declared_in_frame": true,
+        "status": "contradicted",
+        "support_count": 0,
+        "contradict_count": 1,
+        "neutral_count": 0,
+        "evidence_ids": [
+          "E-012"
+        ]
+      },
+      {
+        "outcome_type": "reduced_transfer",
+        "declared_in_frame": true,
+        "status": "no_evidence",
+        "support_count": 0,
+        "contradict_count": 0,
+        "neutral_count": 0,
+        "evidence_ids": []
+      },
+      {
+        "outcome_type": "retention",
+        "declared_in_frame": true,
+        "status": "null_evidence_only",
+        "support_count": 0,
+        "contradict_count": 0,
+        "neutral_count": 1,
+        "evidence_ids": [
+          "E-003"
+        ]
+      }
+    ],
+    "declared_without_evidence": [
+      "ai_dependency",
+      "reduced_transfer"
+    ]
+  },
+  "claims": [
+    {
+      "claim": "AI coding assistants significantly increase task completion speed and completion rate during training.",
+      "outcome_type": "completion_time",
+      "claim_id": "C-001",
+      "evidence_ids": [
+        "E-001"
+      ],
+      "status": "SUPPORTED"
+    },
+    {
+      "claim": "Access to AI code generation did not decrease performance on manual code-modification tasks.",
+      "outcome_type": "independent_problem_solving",
+      "claim_id": "C-002",
+      "evidence_ids": [
+        "E-002"
+      ],
+      "status": "SUPPORTED"
+    },
+    {
+      "claim": "One week after training, retention differences between Codex and baseline groups did not reach statistical significance.",
+      "outcome_type": "retention",
+      "claim_id": "C-003",
+      "evidence_ids": [
+        "E-003"
+      ],
+      "status": "SUPPORTED"
+    },
+    {
+      "claim": "Students with unguarded GPT-4 access performed 17% worse on the independent exam than the control group, despite higher practice performance.",
+      "outcome_type": "independent_problem_solving",
+      "claim_id": "C-004",
+      "evidence_ids": [
+        "E-004"
+      ],
+      "status": "SUPPORTED"
+    },
+    {
+      "claim": "Guardrail design of the AI tutor (hints instead of answers, teacher-informed prompts) largely eliminated the negative learning effect.",
+      "outcome_type": "independent_problem_solving",
+      "claim_id": "C-005",
+      "evidence_ids": [
+        "E-005"
+      ],
+      "status": "SUPPORTED"
+    },
+    {
+      "claim": "Access to GPT-4 during practice improves task performance (48% for GPT Base, 127% for GPT Tutor) — but this task performance does not transfer to independent exam performance.",
+      "outcome_type": "assignment_score",
+      "claim_id": "C-006",
+      "evidence_ids": [
+        "E-006"
+      ],
+      "status": "SUPPORTED"
+    },
+    {
+      "claim": "ChatGPT as a formative feedback tool produced a significant positive impact on students' academic writing skills with positive student perceptions.",
+      "outcome_type": "knowledge_gain",
+      "claim_id": "C-007",
+      "evidence_ids": [
+        "E-007"
+      ],
+      "status": "SUPPORTED"
+    },
+    {
+      "claim": "Professional developers with Copilot access completed a standardized coding task about 55% faster than the control group (RCT, n=95).",
+      "outcome_type": "completion_time",
+      "claim_id": "C-008",
+      "evidence_ids": [
+        "E-008"
+      ],
+      "status": "SUPPORTED"
+    },
+    {
+      "claim": "Systematic benchmark evaluation reports mixed quality results for Copilot-generated code relative to human code: correctness competitive on parts of the benchmark while security-relevant defects are documented.",
+      "outcome_type": "code_quality",
+      "claim_id": "C-009",
+      "evidence_ids": [
+        "E-009"
+      ],
+      "status": "SUPPORTED"
+    },
+    {
+      "claim": "Codex produced passing-level solutions for roughly half to three-quarters of CS1 exam-style questions depending on the dataset, indicating substantial task-capability headroom available to novices.",
+      "outcome_type": "assignment_score",
+      "claim_id": "C-010",
+      "evidence_ids": [
+        "E-010"
+      ],
+      "status": "SUPPORTED"
+    },
+    {
+      "claim": "Controlled comparisons find LLM-generated code explanations comparable to (in places better than) student-authored explanations, suggesting viability as explanatory scaffold material rather than as a replacement for student explanation practice.",
+      "outcome_type": "metacognition",
+      "claim_id": "C-011",
+      "evidence_ids": [
+        "E-011"
+      ],
+      "status": "SUPPORTED"
+    },
+    {
+      "claim": "Despite faster first-task completion, participants struggled to understand and debug AI-generated solutions and reported low ownership of the final program - documenting metacognitive and dependence risks that pure speed metrics miss.",
+      "outcome_type": "over_reliance",
+      "claim_id": "C-012",
+      "evidence_ids": [
+        "E-012"
+      ],
+      "status": "CONTRADICT"
+    }
+  ],
+  "sources": [
+    {
+      "source_id": "S-2023-kazemitabaar",
+      "title": "Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming",
+      "year": 2023,
+      "doi": "10.1145/3544548.3580919",
+      "canonical_url": "https://dl.acm.org/doi/10.1145/3544548.3580919",
+      "authority_level": "tier1_paper_doi",
+      "source_location": "https://dl.acm.org/doi/10.1145/3544548.3580919",
+      "doi_verified": true,
+      "retracted": false
+    },
+    {
+      "source_id": "S-2025-bastani",
+      "title": "Generative AI without guardrails can harm learning: Evidence from high school mathematics",
+      "year": 2025,
+      "doi": "10.1073/pnas.2422633122",
+      "canonical_url": "https://www.pnas.org/doi/10.1073/pnas.2422633122",
+      "authority_level": "tier1_paper_doi",
+      "source_location": "https://www.pnas.org/doi/10.1073/pnas.2422633122",
+      "doi_verified": true,
+      "retracted": false
+    },
+    {
+      "source_id": "S-2024-marzuki",
+      "title": "Impact of ChatGPT on ESL students' academic writing skills",
+      "year": 2024,
+      "doi": "10.1186/s40561-024-00295-9",
+      "canonical_url": "https://link.springer.com/article/10.1186/s40561-024-00295-9",
+      "authority_level": "tier1_paper_doi",
+      "source_location": "https://link.springer.com/article/10.1186/s40561-024-00295-9",
+      "doi_verified": true,
+      "retracted": false
+    },
+    {
+      "source_id": "S-2023-peng",
+      "title": "The Impact of AI on Developer Productivity: Evidence from GitHub Copilot",
+      "year": 2023,
+      "doi": "10.48550/arXiv.2302.06590",
+      "canonical_url": "https://doi.org/10.48550/arXiv.2302.06590",
+      "authority_level": "tier2_academic_database",
+      "source_location": "https://arxiv.org/abs/2302.06590",
+      "doi_verified": true,
+      "retracted": false
+    },
+    {
+      "source_id": "S-2023-yetistiren",
+      "title": "GitHub Copilot AI Pair Programmer: Asset or Liability?",
+      "year": 2023,
+      "doi": "10.1016/j.jss.2023.111734",
+      "canonical_url": "https://doi.org/10.1016/j.jss.2023.111734",
+      "authority_level": "tier1_paper_doi",
+      "source_location": "https://doi.org/10.1016/j.jss.2023.111734",
+      "doi_verified": true,
+      "retracted": false
+    },
+    {
+      "source_id": "S-2022-finnie-ansley",
+      "title": "Using GitHub Copilot to Solve Introductory Programming Problems",
+      "year": 2022,
+      "doi": "10.1145/3545945.3569830",
+      "canonical_url": "https://dl.acm.org/doi/10.1145/3545945.3569830",
+      "authority_level": "tier1_paper_doi",
+      "source_location": "https://dl.acm.org/doi/10.1145/3545945.3569830",
+      "doi_verified": true,
+      "retracted": false
+    },
+    {
+      "source_id": "S-2023-explanations-compare",
+      "title": "Comparing Code Explanations Created by Students and Large Language Models",
+      "year": 2023,
+      "doi": "10.1145/3587102.3588785",
+      "canonical_url": "https://dl.acm.org/doi/10.1145/3587102.3588785",
+      "authority_level": "tier1_paper_doi",
+      "source_location": "https://dl.acm.org/doi/10.1145/3587102.3588785",
+      "doi_verified": true,
+      "retracted": false
+    },
+    {
+      "source_id": "S-2022-vaithilingam",
+      "title": "Expectation vs. Experience: Evaluating the Usability of Code Generation Tools",
+      "year": 2022,
+      "doi": "10.1145/3491101.3519665",
+      "canonical_url": "https://dl.acm.org/doi/10.1145/3491101.3519665",
+      "authority_level": "tier1_paper_doi",
+      "source_location": "https://dl.acm.org/doi/10.1145/3491101.3519665",
+      "doi_verified": true,
+      "retracted": false
+    }
+  ],
+  "evidence": [
+    {
+      "evidence_id": "E-001",
+      "source_id": "S-2023-kazemitabaar",
+      "title": "Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming",
+      "year": 2023,
+      "study_type": "rct",
+      "education_level": "k12_ages_10_17",
+      "subject": "introductory_python",
+      "population": "69 novices ages 10-17 with no prior text-based programming experience",
+      "sample_size": 69,
+      "intervention": "access_to_openai_codex_ai_coding_assistant_during_training",
+      "comparison": "baseline_group_without_ai_coding_assistant",
+      "outcome_type": "completion_time",
+      "outcome_measure": "code_authoring_task_progress_and_time",
+      "claim": "AI coding assistants significantly increase task completion speed and completion rate during training.",
+      "direction": "support",
+      "relation_to_claim": "support",
+      "effect_direction": "positive",
+      "study_id": "STUDY-KAZEMITABAAR-2023",
+      "sample_id": "SMPL-KAZEMITABAAR-2023-N69",
+      "effect": "1.15x completion rate, 0.57x time, 1.8x correctness",
+      "duration": "3_weeks_training",
+      "method": "controlled experiment with random assignment, immediate post-test and 1-week retention test",
+      "strengths": [
+        "randomized_controlled_design",
+        "immediate_post_test_and_retention_test",
+        "code_modification_task_guard"
+      ],
+      "limitations": [
+        "non_university_population_ages_10_17",
+        "small_sample_69",
+        "self-paced environment differs from classroom"
+      ],
+      "confounders": [
+        "prior_programming_competency_interaction"
+      ],
+      "source_location": "https://dl.acm.org/doi/10.1145/3544548.3580919",
+      "quality_dimensions": {
+        "D1_study_design": 2,
+        "D2_sample_quality": 2,
+        "D3_measurement_validity": 2,
+        "D4_temporal_strength": 2,
+        "D5_directness": 1
+      },
+      "quality_score": 9.0,
+      "evidence_level": "strong",
+      "applicability": {
+        "learner_match": "partial_novice_programmers_but_younger",
+        "subject_match": "introductory_programming",
+        "tool_match": "codex_like_generative_ai",
+        "scope": "task_performance_during_training"
+      },
+      "confidence": 0.7,
+      "status": "SUPPORTED",
+      "decision_relation": "support_adoption",
+      "claim_id": "C-001"
+    },
+    {
+      "evidence_id": "E-002",
+      "source_id": "S-2023-kazemitabaar",
+      "title": "Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming",
+      "year": 2023,
+      "study_type": "rct",
+      "education_level": "k12_ages_10_17",
+      "subject": "introductory_python",
+      "population": "69 novices ages 10-17 with no prior text-based programming experience",
+      "sample_size": 69,
+      "intervention": "access_to_openai_codex_ai_coding_assistant_during_training",
+      "comparison": "baseline_group_without_ai_coding_assistant",
+      "outcome_type": "independent_problem_solving",
+      "outcome_measure": "manual code-modification tasks during training",
+      "claim": "Access to AI code generation did not decrease performance on manual code-modification tasks.",
+      "direction": "neutral",
+      "relation_to_claim": "neutral",
+      "effect_direction": "null",
+      "study_id": "STUDY-KAZEMITABAAR-2023",
+      "sample_id": "SMPL-KAZEMITABAAR-2023-N69",
+      "effect": "no significant difference between groups",
+      "duration": "3_weeks_training",
+      "method": "controlled experiment, code-modification task followed each code-authoring task",
+      "strengths": [
+        "direct_test_of_transfer-adjacent_skill",
+        "same_session_measurement"
+      ],
+      "limitations": [
+        "code modification is not full independent problem solving",
+        "non_university population"
+      ],
+      "confounders": [
+        "practice_effect"
+      ],
+      "source_location": "https://dl.acm.org/doi/10.1145/3544548.3580919",
+      "quality_dimensions": {
+        "D1_study_design": 2,
+        "D2_sample_quality": 2,
+        "D3_measurement_validity": 1,
+        "D4_temporal_strength": 1,
+        "D5_directness": 1
+      },
+      "quality_score": 7.0,
+      "evidence_level": "moderate",
+      "applicability": {
+        "learner_match": "partial",
+        "subject_match": "introductory_programming",
+        "tool_match": "codex",
+        "scope": "short-term manual code modification"
+      },
+      "confidence": 0.5,
+      "status": "SUPPORTED",
+      "decision_relation": "neutral",
+      "claim_id": "C-002"
+    },
+    {
+      "evidence_id": "E-003",
+      "source_id": "S-2023-kazemitabaar",
+      "title": "Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming",
+      "year": 2023,
+      "study_type": "rct",
+      "education_level": "k12_ages_10_17",
+      "subject": "introductory_python",
+      "population": "69 novices ages 10-17 with no prior text-based programming experience",
+      "sample_size": 69,
+      "intervention": "access_to_openai_codex_ai_coding_assistant_during_training",
+      "comparison": "baseline_group_without_ai_coding_assistant",
+      "outcome_type": "retention",
+      "outcome_measure": "retention post-test one week after training",
+      "claim": "One week after training, retention differences between Codex and baseline groups did not reach statistical significance.",
+      "direction": "neutral",
+      "relation_to_claim": "neutral",
+      "effect_direction": "null",
+      "study_id": "STUDY-KAZEMITABAAR-2023",
+      "sample_id": "SMPL-KAZEMITABAAR-2023-N69",
+      "effect": "slightly better for Codex group but not significant",
+      "duration": "3_weeks_training_plus_1_week_retention",
+      "method": "controlled experiment with delayed retention test",
+      "strengths": [
+        "delayed_test_included"
+      ],
+      "limitations": [
+        "1-week retention window is short",
+        "small sample",
+        "non-university population"
+      ],
+      "confounders": [
+        "prior_competency"
+      ],
+      "source_location": "https://dl.acm.org/doi/10.1145/3544548.3580919",
+      "quality_dimensions": {
+        "D1_study_design": 2,
+        "D2_sample_quality": 2,
+        "D3_measurement_validity": 2,
+        "D4_temporal_strength": 2,
+        "D5_directness": 1
+      },
+      "quality_score": 9.0,
+      "evidence_level": "strong",
+      "applicability": {
+        "learner_match": "partial",
+        "subject_match": "introductory_programming",
+        "tool_match": "codex",
+        "scope": "retention over one week"
+      },
+      "confidence": 0.5,
+      "status": "SUPPORTED",
+      "decision_relation": "neutral",
+      "claim_id": "C-003"
+    },
+    {
+      "evidence_id": "E-004",
+      "source_id": "S-2025-bastani",
+      "title": "Generative AI without guardrails can harm learning: Evidence from high school mathematics",
+      "year": 2025,
+      "study_type": "rct",
+      "education_level": "high_school",
+      "subject": "mathematics",
+      "population": "nearly a thousand high school math students in Turkey",
+      "sample_size": 950,
+      "intervention": "gpt4_based_tutor_gpt_base_unguarded",
+      "comparison": "no_generative_ai_control",
+      "outcome_type": "independent_problem_solving",
+      "outcome_measure": "exam without access to AI resources after practice phase",
+      "claim": "Students with unguarded GPT-4 access performed 17% worse on the independent exam than the control group, despite higher practice performance.",
+      "direction": "contradict",
+      "relation_to_claim": "contradict",
+      "effect_direction": "negative",
+      "study_id": "STUDY-BASTANI-2025",
+      "sample_id": "SMPL-BASTANI-2025-N950",
+      "effect": "negative_17_percent_on_independent_exam",
+      "duration": "in_class_study_sessions",
+      "method": "large-scale randomized controlled trial, practice phase then closed-book exam",
+      "strengths": [
+        "large_scale_rct",
+        "independent_exam_without_ai",
+        "arm_wise_design"
+      ],
+      "limitations": [
+        "high_school_mathematics_not_university_programming",
+        "single_country"
+      ],
+      "confounders": [
+        "tool_design_difference"
+      ],
+      "source_location": "https://www.pnas.org/doi/10.1073/pnas.2422633122",
+      "quality_dimensions": {
+        "D1_study_design": 2,
+        "D2_sample_quality": 2,
+        "D3_measurement_validity": 2,
+        "D4_temporal_strength": 1,
+        "D5_directness": 1
+      },
+      "quality_score": 8.0,
+      "evidence_level": "strong",
+      "applicability": {
+        "learner_match": "partial_same_age_band_different_subject",
+        "subject_match": "no_mathematics_vs_programming",
+        "tool_match": "gpt4_chat_interface",
+        "scope": "unguarded_general_chat_interface"
+      },
+      "confidence": 0.75,
+      "status": "SUPPORTED",
+      "decision_relation": "oppose_adoption",
+      "claim_id": "C-004"
+    },
+    {
+      "evidence_id": "E-005",
+      "source_id": "S-2025-bastani",
+      "title": "Generative AI without guardrails can harm learning: Evidence from high school mathematics",
+      "year": 2025,
+      "study_type": "rct",
+      "education_level": "high_school",
+      "subject": "mathematics",
+      "population": "nearly a thousand high school math students in Turkey",
+      "sample_size": 950,
+      "intervention": "gpt4_tutor_with_teacher_designed_guardrails",
+      "comparison": "no_generative_ai_control",
+      "outcome_type": "independent_problem_solving",
+      "outcome_measure": "exam without access to AI resources after practice phase",
+      "claim": "Guardrail design of the AI tutor (hints instead of answers, teacher-informed prompts) largely eliminated the negative learning effect.",
+      "direction": "support",
+      "relation_to_claim": "support",
+      "effect_direction": "null",
+      "study_id": "STUDY-BASTANI-2025",
+      "sample_id": "SMPL-BASTANI-2025-N950",
+      "effect": "negative effect essentially eradicated, no positive effect observed",
+      "duration": "in_class_study_sessions",
+      "method": "large-scale randomized controlled trial, three arms",
+      "strengths": [
+        "direct_manipulation_of_tool_design",
+        "large_sample"
+      ],
+      "limitations": [
+        "no_positive_learning_gain_even_with_guardrails",
+        "subject_mismatch"
+      ],
+      "confounders": [
+        "prompt_engineering_effort"
+      ],
+      "source_location": "https://www.pnas.org/doi/10.1073/pnas.2422633122",
+      "quality_dimensions": {
+        "D1_study_design": 2,
+        "D2_sample_quality": 2,
+        "D3_measurement_validity": 2,
+        "D4_temporal_strength": 1,
+        "D5_directness": 1
+      },
+      "quality_score": 8.0,
+      "evidence_level": "strong",
+      "applicability": {
+        "learner_match": "partial",
+        "subject_match": "no",
+        "tool_match": "guardrailed_tutor_design",
+        "scope": "guardrail_design_principle_transferable"
+      },
+      "confidence": 0.75,
+      "status": "SUPPORTED",
+      "decision_relation": "conditional",
+      "claim_id": "C-005"
+    },
+    {
+      "evidence_id": "E-006",
+      "source_id": "S-2025-bastani",
+      "title": "Generative AI without guardrails can harm learning: Evidence from high school mathematics",
+      "year": 2025,
+      "study_type": "rct",
+      "education_level": "high_school",
+      "subject": "mathematics",
+      "population": "nearly a thousand high school math students in Turkey",
+      "sample_size": 950,
+      "intervention": "gpt4_tutor_access_during_practice",
+      "comparison": "no_generative_ai_control",
+      "outcome_type": "assignment_score",
+      "outcome_measure": "practice problem performance during study sessions",
+      "claim": "Access to GPT-4 during practice improves task performance (48% for GPT Base, 127% for GPT Tutor) — but this task performance does not transfer to independent exam performance.",
+      "direction": "support",
+      "relation_to_claim": "support",
+      "effect_direction": "positive",
+      "study_id": "STUDY-BASTANI-2025",
+      "sample_id": "SMPL-BASTANI-2025-N950",
+      "effect": "48-127 percent improvement on practice problems",
+      "duration": "in_class_study_sessions",
+      "method": "randomized controlled trial with practice and closed-book exam phases",
+      "strengths": [
+        "same_study_compares_task_and_learning",
+        "large_sample"
+      ],
+      "limitations": [
+        "subject_mismatch_mathematics"
+      ],
+      "confounders": [
+        "task_familiarity"
+      ],
+      "source_location": "https://www.pnas.org/doi/10.1073/pnas.2422633122",
+      "quality_dimensions": {
+        "D1_study_design": 2,
+        "D2_sample_quality": 2,
+        "D3_measurement_validity": 2,
+        "D4_temporal_strength": 1,
+        "D5_directness": 1
+      },
+      "quality_score": 8.0,
+      "evidence_level": "strong",
+      "applicability": {
+        "learner_match": "partial",
+        "subject_match": "no",
+        "tool_match": "gpt4",
+        "scope": "task_performance_vs_learning_separation"
+      },
+      "confidence": 0.75,
+      "status": "SUPPORTED",
+      "decision_relation": "conditional",
+      "claim_id": "C-006"
+    },
+    {
+      "evidence_id": "E-007",
+      "source_id": "S-2024-marzuki",
+      "title": "Impact of ChatGPT on ESL students' academic writing skills",
+      "year": 2024,
+      "study_type": "mixed_methods",
+      "education_level": "undergraduate",
+      "subject": "academic_writing_esl",
+      "population": "undergraduate ESL students at an Indian university",
+      "sample_size": 72,
+      "intervention": "chatgpt_as_formative_feedback_tool",
+      "comparison": "traditional_instruction_control",
+      "outcome_type": "knowledge_gain",
+      "outcome_measure": "writing tests with pre-post-delayed design",
+      "claim": "ChatGPT as a formative feedback tool produced a significant positive impact on students' academic writing skills with positive student perceptions.",
+      "direction": "support",
+      "relation_to_claim": "support",
+      "effect_direction": "positive",
+      "study_id": "STUDY-MARZUKI-2024",
+      "sample_id": "SMPL-MARZUKI-2024-N72",
+      "effect": "significant positive impact on writing skills",
+      "duration": "6_hours_intervention",
+      "method": "mixed methods intervention study, pre/post/delayed tests and focus groups",
+      "strengths": [
+        "delayed_post_test",
+        "mixed_methods_triangulation"
+      ],
+      "limitations": [
+        "short_intervention_6_hours",
+        "single_institution",
+        "elite_private_university"
+      ],
+      "confounders": [
+        "self_selection_consent"
+      ],
+      "source_location": "https://link.springer.com/article/10.1186/s40561-024-00295-9",
+      "quality_dimensions": {
+        "D1_study_design": 1,
+        "D2_sample_quality": 1,
+        "D3_measurement_validity": 2,
+        "D4_temporal_strength": 2,
+        "D5_directness": 0
+      },
+      "quality_score": 6.0,
+      "evidence_level": "moderate",
+      "applicability": {
+        "learner_match": "yes_undergraduate",
+        "subject_match": "no_writing_not_programming",
+        "tool_match": "chatgpt",
+        "scope": "formative_feedback_writing"
+      },
+      "confidence": 0.55,
+      "status": "SUPPORTED",
+      "decision_relation": "support_adoption",
+      "claim_id": "C-007"
+    },
+    {
+      "evidence_id": "E-008",
+      "source_id": "S-2023-peng",
+      "title": "The Impact of AI on Developer Productivity: Evidence from GitHub Copilot",
+      "year": 2023,
+      "study_type": "rct",
+      "education_level": "professional_developers_not_students",
+      "subject": "standardized_javascript_http_server_task",
+      "population": "95 recruited professional developers completing a standardized coding task on a freelance platform",
+      "sample_size": 95,
+      "intervention": "access_to_github_copilot_during_task",
+      "comparison": "control_group_without_copilot",
+      "outcome_type": "completion_time",
+      "outcome_measure": "time_to_complete_http_server_implementation",
+      "claim": "Professional developers with Copilot access completed a standardized coding task about 55% faster than the control group (RCT, n=95).",
+      "direction": "support",
+      "relation_to_claim": "support",
+      "effect_direction": "positive",
+      "study_id": "STUDY-PENG-2023",
+      "sample_id": "SMPL-PENG-2023-N95",
+      "effect": "~55.8% faster task completion in Copilot group",
+      "duration": "single_task_session",
+      "method": "online randomized controlled experiment with objective completion-time metric",
+      "strengths": [
+        "randomized_controlled_design",
+        "objective_completion_time_metric"
+      ],
+      "limitations": [
+        "professional_population_not_students",
+        "single_task_ecology",
+        "preprint_not_peer_reviewed"
+      ],
+      "confounders": [
+        "task_familiarity",
+        "platform_recruitment_self_selection"
+      ],
+      "source_location": "https://doi.org/10.48550/arXiv.2302.06590",
+      "quality_dimensions": {
+        "D1_study_design": 2,
+        "D2_sample_quality": 2,
+        "D3_measurement_validity": 2,
+        "D4_temporal_strength": 1,
+        "D5_directness": 1
+      },
+      "quality_score": 8.0,
+      "evidence_level": "moderate",
+      "applicability": {
+        "learner_match": "mismatch_professional_developers",
+        "subject_match": "adjacent_web_development_task",
+        "tool_match": "copilot_like_generative_ai",
+        "scope": "task_performance_only_no_learning_outcome"
+      },
+      "confidence": 0.6,
+      "status": "SUPPORTED",
+      "decision_relation": "conditional",
+      "claim_id": "C-008"
+    },
+    {
+      "evidence_id": "E-009",
+      "source_id": "S-2023-yetistiren",
+      "title": "GitHub Copilot AI Pair Programmer: Asset or Liability?",
+      "year": 2023,
+      "study_type": "observational",
+      "education_level": "not_applicable_code_artifacts",
+      "subject": "code_generation_benchmarks",
+      "population": "Copilot-generated and human-written programs drawn from published benchmark datasets",
+      "sample_size": null,
+      "intervention": "copilot_generated_programs",
+      "comparison": "human_written_programs_on_same_benchmarks",
+      "outcome_type": "code_quality",
+      "outcome_measure": "correctness_security_maintainability_metrics_on_benchmarks",
+      "claim": "Systematic benchmark evaluation reports mixed quality results for Copilot-generated code relative to human code: correctness competitive on parts of the benchmark while security-relevant defects are documented.",
+      "direction": "neutral",
+      "relation_to_claim": "neutral",
+      "effect_direction": "null",
+      "study_id": "STUDY-YETISTIREN-2023",
+      "sample_id": "SMPL-YETISTIREN-2023-BENCH",
+      "effect": "mixed quality profile; no single-direction summary",
+      "duration": "not_applicable_artifact_study",
+      "method": "systematic empirical evaluation of generated code against human baselines on public benchmarks",
+      "strengths": [
+        "multi_dimensional_quality_metrics",
+        "reproducible_benchmark_protocol"
+      ],
+      "limitations": [
+        "artifact_benchmark_not_classroom",
+        "no_learning_outcome",
+        "tool_version_from_2023"
+      ],
+      "confounders": [
+        "benchmark_task_distribution"
+      ],
+      "source_location": "https://doi.org/10.1016/j.jss.2023.111734",
+      "quality_dimensions": {
+        "D1_study_design": 1,
+        "D2_sample_quality": 2,
+        "D3_measurement_validity": 2,
+        "D4_temporal_strength": 1,
+        "D5_directness": 1
+      },
+      "quality_score": 7.0,
+      "evidence_level": "moderate",
+      "applicability": {
+        "learner_match": "mismatch_no_learners_in_study",
+        "subject_match": "introductory_adjacent_code_tasks",
+        "tool_match": "copilot_like_generative_ai",
+        "scope": "output_quality_only"
+      },
+      "confidence": 0.55,
+      "status": "SUPPORTED",
+      "decision_relation": "conditional",
+      "claim_id": "C-009"
+    },
+    {
+      "evidence_id": "E-010",
+      "source_id": "S-2022-finnie-ansley",
+      "title": "Using GitHub Copilot to Solve Introductory Programming Problems",
+      "year": 2022,
+      "study_type": "observational",
+      "education_level": "university_year_1_question_sets",
+      "subject": "introductory_python",
+      "population": "CS1 exam-style question sets answered by Codex and compared against published student score distributions",
+      "sample_size": null,
+      "intervention": "codex_answer_generation_on_cs1_questions",
+      "comparison": "published_student_cohort_score_distributions",
+      "outcome_type": "assignment_score",
+      "outcome_measure": "pass_rate_on_cs1_exam_style_questions",
+      "claim": "Codex produced passing-level solutions for roughly half to three-quarters of CS1 exam-style questions depending on the dataset, indicating substantial task-capability headroom available to novices.",
+      "direction": "support",
+      "relation_to_claim": "support",
+      "effect_direction": "positive",
+      "study_id": "STUDY-FINNIEANSLEY-2022",
+      "sample_id": "SMPL-FINNIEANSLEY-2022-QSETS",
+      "effect": "passing solutions on ~50-75% of questions across datasets",
+      "duration": "not_applicable_capability_probe",
+      "method": "capability benchmark against published student distributions; reproducible question sets",
+      "strengths": [
+        "public_reproducible_question_sets",
+        "directly_relevant_task_domain"
+      ],
+      "limitations": [
+        "tool_solves_task_does_not_equate_student_learning",
+        "codex_2021_model_version_outdated"
+      ],
+      "confounders": [
+        "question_leakage_into_training_data_possible"
+      ],
+      "source_location": "https://dl.acm.org/doi/10.1145/3545945.3569830",
+      "quality_dimensions": {
+        "D1_study_design": 1,
+        "D2_sample_quality": 2,
+        "D3_measurement_validity": 2,
+        "D4_temporal_strength": 1,
+        "D5_directness": 1
+      },
+      "quality_score": 7.0,
+      "evidence_level": "moderate",
+      "applicability": {
+        "learner_match": "partial_measures_tool_not_students",
+        "subject_match": "introductory_programming",
+        "tool_match": "copilot_like_generative_ai",
+        "scope": "tool_capability_headroom"
+      },
+      "confidence": 0.55,
+      "status": "SUPPORTED",
+      "decision_relation": "conditional",
+      "claim_id": "C-010"
+    },
+    {
+      "evidence_id": "E-011",
+      "source_id": "S-2023-explanations-compare",
+      "title": "Comparing Code Explanations Created by Students and Large Language Models",
+      "year": 2023,
+      "study_type": "observational",
+      "education_level": "university_introductory",
+      "subject": "code_explanation_scaffolding",
+      "population": "Student-produced versus LLM-produced explanations of short programs under controlled comparison",
+      "sample_size": null,
+      "intervention": "llm_generated_code_explanations",
+      "comparison": "student_generated_explanations_of_same_programs",
+      "outcome_type": "metacognition",
+      "outcome_measure": "rated_explanation_quality_and_comprehensibility",
+      "claim": "Controlled comparisons find LLM-generated code explanations comparable to (in places better than) student-authored explanations, suggesting viability as explanatory scaffold material rather than as a replacement for student explanation practice.",
+      "direction": "support",
+      "relation_to_claim": "support",
+      "effect_direction": "positive",
+      "study_id": "STUDY-EXPLCOMP-2023",
+      "sample_id": "SMPL-EXPLCOMP-2023-RATINGS",
+      "effect": "comparable-or-better rated quality vs student explanations",
+      "duration": "single_session_ratings",
+      "method": "controlled comparison with blind rating of explanation pairs",
+      "strengths": [
+        "controlled_pairwise_comparison",
+        "learning_process_relevant_construct"
+      ],
+      "limitations": [
+        "short_term_ratings_not_learning_gains",
+        "small_program_snippets_ecology"
+      ],
+      "confounders": [
+        "rating_criteria_subjectivity"
+      ],
+      "source_location": "https://dl.acm.org/doi/10.1145/3587102.3588785",
+      "quality_dimensions": {
+        "D1_study_design": 1,
+        "D2_sample_quality": 2,
+        "D3_measurement_validity": 2,
+        "D4_temporal_strength": 1,
+        "D5_directness": 1
+      },
+      "quality_score": 7.0,
+      "evidence_level": "moderate",
+      "applicability": {
+        "learner_match": "partial_scaffold_material_only",
+        "subject_match": "introductory_programming",
+        "tool_match": "llm_explanations",
+        "scope": "scaffold_quality_not_effectiveness"
+      },
+      "confidence": 0.55,
+      "status": "SUPPORTED",
+      "decision_relation": "conditional",
+      "claim_id": "C-011"
+    },
+    {
+      "evidence_id": "E-012",
+      "source_id": "S-2022-vaithilingam",
+      "title": "Expectation vs. Experience: Evaluating the Usability of Code Generation Tools",
+      "year": 2022,
+      "study_type": "qualitative",
+      "education_level": "mixed_cs_students_and_professionals",
+      "subject": "programmer_usability_of_codegen_tools",
+      "population": "24 participants in a within-subjects usability study of Copilot-style tools",
+      "sample_size": 24,
+      "intervention": "copilot_assisted_program_writing",
+      "comparison": "within_subject_baseline_without_tool",
+      "outcome_type": "over_reliance",
+      "outcome_measure": "understanding_ownership_and_debugging_reports",
+      "claim": "Despite faster first-task completion, participants struggled to understand and debug AI-generated solutions and reported low ownership of the final program - documenting metacognitive and dependence risks that pure speed metrics miss.",
+      "direction": "contradict",
+      "relation_to_claim": "contradict",
+      "effect_direction": "negative",
+      "study_id": "STUDY-VAITHILINGAM-2022",
+      "sample_id": "SMPL-VAITHILINGAM-2022-N24",
+      "effect": "documented comprehension/ownership difficulties despite speed gain",
+      "duration": "single_session",
+      "method": "within-subject usability study with tasks, observation and interviews",
+      "strengths": [
+        "rich_qualitative_process_data",
+        "constructs_missed_by_speed_metrics"
+      ],
+      "limitations": [
+        "small_n_24",
+        "single_session",
+        "self_reported_understanding"
+      ],
+      "confounders": [
+        "participant_ai_familiarity"
+      ],
+      "source_location": "https://dl.acm.org/doi/10.1145/3491101.3519665",
+      "quality_dimensions": {
+        "D1_study_design": 1,
+        "D2_sample_quality": 2,
+        "D3_measurement_validity": 2,
+        "D4_temporal_strength": 1,
+        "D5_directness": 1
+      },
+      "quality_score": 7.0,
+      "evidence_level": "moderate",
+      "applicability": {
+        "learner_match": "partial_includes_cs_students",
+        "subject_match": "programming_adjacent",
+        "tool_match": "copilot_like_generative_ai",
+        "scope": "risk_identification"
+      },
+      "confidence": 0.55,
+      "status": "CONTRADICT",
+      "decision_relation": "conditional",
+      "claim_id": "C-012"
+    }
+  ],
+  "methodology_reviews": [
+    {
+      "target": "overall",
+      "audit_items": {
+        "control_group": {
+          "status": "met",
+          "note": "All three studies include a no-AI control group."
+        },
+        "randomization": {
+          "status": "met",
+          "note": "Kazemitabaar 2023 and Bastani 2025 use randomized assignment."
+        },
+        "pre_test": {
+          "status": "met",
+          "note": "Kazemitabaar 2023 has a pre-study evaluation; Bastani 2025 measures baseline covariates."
+        },
+        "post_test": {
+          "status": "met",
+          "note": "Immediate post-tests present in all studies."
+        },
+        "retention_test": {
+          "status": "partial",
+          "note": "Kazemitabaar 2023 has 1-week retention; Bastani 2025 has no delayed test; Marzuki 2024 has delayed test."
+        },
+        "transfer_test": {
+          "status": "partial",
+          "note": "Kazemitabaar 2023 code-modification task is transfer-adjacent; no full no-AI transfer task."
+        },
+        "sample_bias": {
+          "status": "met",
+          "note": "Bastani 2025 nearly 1000 students; Kazemitabaar 2023 small (69) young sample."
+        },
+        "self_selection": {
+          "status": "partial",
+          "note": "Marzuki 2024 consent-based participation risks self-selection."
+        },
+        "measurement_validity": {
+          "status": "partial",
+          "note": "Practice/task performance is not equated to learning; independent exams present in Bastani 2025 only."
+        },
+        "confounders": {
+          "status": "partial",
+          "note": "Prior programming competency interacts with AI benefit in Kazemitabaar 2023."
+        },
+        "instructor_effect": {
+          "status": "not_applicable",
+          "note": "Kazemitabaar 2023 is self-paced; classroom studies may carry instructor effects."
+        },
+        "novelty_effect": {
+          "status": "partial",
+          "note": "Short interventions likely inflate engagement; none of the studies controlled for novelty."
+        },
+        "tool_version_effect": {
+          "status": "not_applicable",
+          "note": "Single tool versions studied; rapid tool change limits durability."
+        },
+        "ai_usage_policy": {
+          "status": "partial",
+          "note": "Bastani 2025 explicitly contrasts unguarded vs guardrailed usage policies."
+        },
+        "dropout": {
+          "status": "partial",
+          "note": "Marzuki 2024 reports attrition; others not detailed."
+        }
+      },
+      "task_vs_learning_guard": {
+        "measured_construct": "practice/task performance vs independent exam",
+        "equates_task_with_learning": false,
+        "note": "Bastani 2025 demonstrates the danger of equating the two: +48-127% practice performance coexisted with -17% independent exam performance."
+      },
+      "verdict": "CONCERN",
+      "limitations": [
+        "No randomized study of AI coding assistants in university C programming courses was found in the reviewed set.",
+        "Task performance gains are consistently large, learning-effect estimates are null-to-negative and subject-mismatched.",
+        "Retention evidence exists for only one week (Kazemitabaar 2023)."
+      ],
+      "suggestions": [
+        "Treat all learning-effect conclusions as requiring direct university-level studies.",
+        "Design the pilot with guardrails modeled on Bastani 2025's GPT Tutor (hints, not answers).",
+        "Include a no-AI transfer task in the evaluation plan."
+      ]
+    }
+  ],
+  "conflicts": [
+    {
+      "reason_for_disagreement": "Disagreement comes from outcome separation (task vs learning), tool design (guarded vs unguarded), and population (K-12 / professionals vs university). Task-performance evidence is consistently positive across randomized and benchmark studies; the only study measuring independent performance after AI removal shows harm without guardrails; usability and artifact studies add dependence and quality caveats rather than resolving the learning question."
+    }
+  ],
+  "applicability": {
+    "suitable_for": "pilot in first-year C course with guardrailed usage policy",
+    "not_suitable_for": "unrestricted AI adoption without usage policy",
+    "required_conditions": [
+      "guardrailed AI usage policy (hints not answers, modeled on GPT Tutor arm)",
+      "no-AI transfer assessment",
+      "TA support"
+    ]
+  },
+  "intervention": {
+    "decision": "pilot",
+    "target_learners": "university first-year C programming students (60-student lecture-lab class)",
+    "learning_goals": [
+      "independently write and debug small C programs without AI",
+      "understand core concepts: variables, conditionals, loops, arrays, pointers",
+      "use AI tools critically as explanation and debugging aids, not answer machines"
+    ],
+    "pilot_duration": "8_weeks",
+    "phase_1": {
+      "name": "Phase 1 — Independent Foundation",
+      "activities": [
+        "baseline assessment",
+        "first 2 weeks of assignments completed without any AI code generation"
+      ],
+      "ai_usage_rule": "full code generation forbidden; AI allowed only for conceptual explanations",
+      "outcome_check": "baseline task-performance and independent-problem-solving measures"
+    },
+    "phase_2": {
+      "name": "Phase 2 — Explain, Don't Solve",
+      "activities": [
+        "weeks 3-4: AI allowed to explain errors, concepts and debugging approaches"
+      ],
+      "ai_usage_rule": "AI may explain but must not produce full solutions",
+      "outcome_check": "mid-pilot quiz without AI"
+    },
+    "phase_3": {
+      "name": "Phase 3 — Structured Collaboration",
+      "activities": [
+        "weeks 5-7: AI allowed to generate partial code; students must explain every AI-generated block in their own words"
+      ],
+      "ai_usage_rule": "partial code generation allowed; mandatory written explanation of key logic; submission requires reasoning trace",
+      "outcome_check": "weekly lab completion and code-quality rubric"
+    },
+    "phase_4": {
+      "name": "Phase 4 — Transfer Check",
+      "activities": [
+        "week 8: new programming task completed in a no-AI environment"
+      ],
+      "ai_usage_rule": "no AI access during transfer assessment",
+      "outcome_check": "transfer test score, independent problem solving"
+    },
+    "ai_usage_policy": "AI usage is allowed in three explicitly graded modes (explain / collaborate / no-AI-transfer). Copying unexamined AI output is an academic integrity violation and is assessed via the reasoning-trace requirement.",
+    "teacher_role": "design guardrailed prompts and rubrics; monitor usage logs; hold weekly reflection debriefs",
+    "student_role": "complete assignments in the allowed mode; submit reasoning traces; reflect on when AI helped vs. when it masked understanding",
+    "reflection_requirement": "students must explain key AI-generated logic in their own words; one written reflection per phase",
+    "assessment": "no-AI baseline quiz, mid-pilot quiz, no-AI transfer task, code-quality rubric, AI-usage self-report",
+    "risk_control": [
+      "guardrailed usage policy modeled on Bastani 2025 GPT Tutor (hints, not answers)",
+      "no-AI transfer assessment prevents grade inflation from task-performance gains",
+      "usage logs reviewed weekly to detect crutch behavior"
+    ],
+    "stop_conditions": [
+      "transfer-test scores drop significantly below baseline cohort expectations",
+      "widespread integrity violations in reasoning traces",
+      "AI dependency signals exceed threshold in risk metrics",
+      "TA/teacher workload becomes unsustainable"
+    ],
+    "evidence_alignment": [
+      "E-004",
+      "E-005",
+      "E-006"
+    ]
+  },
+  "evaluation": {
+    "research_question": "In a first-year C programming course, does a guardrailed AI coding assistant (explain-only → structured collaboration) improve independent problem solving without increasing AI dependency, compared to no-AI instruction?",
+    "groups": {
+      "treatment": "two lab sections using the 4-phase guardrailed AI policy",
+      "comparison": "two matched lab sections without AI access (same instructor and materials)"
+    },
+    "baseline": "week-1 no-AI programming quiz (independent problem solving, completion time)",
+    "post_test": "week-8 no-AI programming quiz (independent problem solving, code quality)",
+    "retention_test": "final exam (week 16) — delayed measure 8 weeks after pilot",
+    "transfer_test": "week-8 novel programming task in a strictly no-AI environment",
+    "process_metrics": [
+      "weekly lab completion rate",
+      "AI usage logs: prompts submitted, copied blocks, reasoning traces",
+      "help-seeking behavior counts"
+    ],
+    "learning_metrics": [
+      "independent problem solving (no-AI quizzes)",
+      "code quality rubric",
+      "retention on final exam",
+      "transfer task score"
+    ],
+    "risk_metrics": [
+      "AI dependency index (usage without understanding, from reasoning-trace quality)",
+      "academic integrity violations",
+      "self-reported over-reliance",
+      "false confidence (post-quiz confidence vs actual score)"
+    ],
+    "analysis_plan": "pre-registered comparison of treatment vs comparison sections on baseline-adjusted learning metrics (ANCOVA); task-performance metrics reported separately from learning metrics; subgroup analysis by prior programming competency; stop-condition monitoring at weeks 3, 5, 7.",
+    "success_threshold": "treatment group shows non-inferior independent problem solving (delta within 5%) AND superior or equal retention AND AI dependency index below threshold; if independent problem solving declines >10%, the pilot is judged unsuccessful regardless of task-performance gains.",
+    "stop_conditions": [
+      "transfer-test decline >10% vs comparison",
+      "integrity violations in >20% of submissions",
+      "AI dependency index exceeding pre-set threshold for two consecutive weeks"
+    ]
+  },
+  "benchmark": {},
+  "provenance": {
+    "search_provider": "n/a"
+  }
+}