eduevidence 6.2.0 → 6.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/CHANGELOG.md +395 -0
  2. package/README.md +22 -13
  3. package/README.zh-CN.md +15 -8
  4. package/SKILL.md +10 -9
  5. package/benchmarks/evidence-library.json +277 -1
  6. package/docs/architecture.md +6 -3
  7. package/docs/j-ev-experimental.md +250 -0
  8. package/docs/reproducibility.md +138 -0
  9. package/domains/_neutral/copy/few_shots.json +21 -0
  10. package/domains/_neutral/copy/framing_lexicon.json +19 -0
  11. package/domains/_neutral/copy/module_labels.json +5 -0
  12. package/domains/_neutral/copy/module_labels_footer.json +102 -0
  13. package/domains/_neutral/copy/module_labels_modules.json +204 -0
  14. package/domains/_neutral/copy/module_labels_nav.json +126 -0
  15. package/domains/_neutral/copy/module_labels_summary.json +98 -0
  16. package/domains/_neutral/copy/module_labels_tables.json +164 -0
  17. package/domains/_neutral/copy/module_labels_v2.json +90 -0
  18. package/domains/_neutral/copy/risk_constructs.json +20 -0
  19. package/domains/_neutral/copy/section_titles.json +66 -0
  20. package/domains/_neutral/copy/terminology.json +11 -0
  21. package/domains/check_copy_packs.py +103 -0
  22. package/domains/education/copy/few_shots.json +22 -0
  23. package/domains/education/copy/framing_enums.json +167 -0
  24. package/domains/education/copy/framing_lexicon.json +166 -0
  25. package/domains/education/copy/module_labels.json +169 -0
  26. package/domains/education/copy/risk_constructs.json +48 -0
  27. package/domains/education/copy/section_titles.json +186 -0
  28. package/domains/education/copy/terminology.json +70 -0
  29. package/domains/education/manifest.json +1 -1
  30. package/domains/education/outcome_taxonomy.json +2 -2
  31. package/domains/manifest.json +1 -1
  32. package/domains/policy/copy/few_shots.json +22 -0
  33. package/domains/policy/copy/framing_enums.json +94 -0
  34. package/domains/policy/copy/framing_lexicon.json +174 -0
  35. package/domains/policy/copy/module_labels.json +168 -0
  36. package/domains/policy/copy/risk_constructs.json +33 -0
  37. package/domains/policy/copy/section_titles.json +186 -0
  38. package/domains/policy/copy/terminology.json +64 -0
  39. package/engine/capabilities.py +57 -5
  40. package/engine/decision_policy.py +88 -17
  41. package/engine/library_builtin.py +7 -4
  42. package/engine/tribunal.py +17 -23
  43. package/engine/versions.py +1 -1
  44. package/examples/ai-coding-assistant-evidence/EduEvidence_Report.html +4 -4
  45. package/examples/ai-coding-assistant-evidence/reports-5themes/EduEvidence_Report_academic.html +4 -4
  46. package/examples/ai-coding-assistant-evidence/reports-5themes/EduEvidence_Report_claude.html +4 -4
  47. package/examples/ai-coding-assistant-evidence/reports-5themes/EduEvidence_Report_datalab-dark.html +4 -4
  48. package/examples/ai-coding-assistant-evidence/reports-5themes/EduEvidence_Report_datalab.html +4 -4
  49. package/examples/ai-coding-assistant-evidence/reports-5themes/EduEvidence_Report_presentation.html +4 -4
  50. package/examples/spaced-retrieval-practice/EduEvidence_Report.html +2728 -0
  51. package/examples/spaced-retrieval-practice/report.html +2522 -0
  52. package/examples/spaced-retrieval-practice/reports-5themes/EduEvidence_Report_academic.html +4 -4
  53. package/examples/spaced-retrieval-practice/reports-5themes/EduEvidence_Report_claude.html +4 -4
  54. package/examples/spaced-retrieval-practice/reports-5themes/EduEvidence_Report_datalab-dark.html +4 -4
  55. package/examples/spaced-retrieval-practice/reports-5themes/EduEvidence_Report_datalab.html +4 -4
  56. package/examples/spaced-retrieval-practice/reports-5themes/EduEvidence_Report_presentation.html +4 -4
  57. package/examples/workplace-ai-assistant/EduEvidence_Report.html +2814 -0
  58. package/examples/workplace-ai-assistant/reports-5themes/EduEvidence_Report_academic.html +36 -36
  59. package/examples/workplace-ai-assistant/reports-5themes/EduEvidence_Report_claude.html +36 -36
  60. package/examples/workplace-ai-assistant/reports-5themes/EduEvidence_Report_datalab-dark.html +36 -36
  61. package/examples/workplace-ai-assistant/reports-5themes/EduEvidence_Report_datalab.html +36 -36
  62. package/examples/workplace-ai-assistant/reports-5themes/EduEvidence_Report_presentation.html +36 -36
  63. package/integrations/jev/__init__.py +115 -0
  64. package/integrations/jev/approval.py +212 -0
  65. package/integrations/jev/cli.py +84 -0
  66. package/integrations/jev/config.py +112 -0
  67. package/integrations/jev/gateway.py +128 -0
  68. package/integrations/jev/modes.py +38 -0
  69. package/integrations/jev/tools_classify.py +88 -0
  70. package/integrations/jev/tools_extract.py +111 -0
  71. package/integrations/jev/tools_rerank.py +71 -0
  72. package/integrations/jev/tools_screen.py +87 -0
  73. package/integrations/jev/tools_verify.py +95 -0
  74. package/integrations/jev_mcp.py +22 -0
  75. package/integrations/semantic_decide.py +286 -0
  76. package/integrations/semdecide_cli.py +55 -0
  77. package/package.json +9 -1
  78. package/pyproject.toml +1 -1
  79. package/references/report-copy-style.md +43 -3
  80. package/schemas/v2/decision-snapshot.schema.json +20 -9
  81. package/schemas/v2/intake.schema.json +191 -0
  82. package/scripts/build_evidence_library.py +15 -5
  83. package/scripts/dashboard_server.py +13 -2
  84. package/scripts/intake/__init__.py +31 -0
  85. package/scripts/intake/__main__.py +18 -0
  86. package/scripts/intake/background.py +78 -0
  87. package/scripts/intake/browser.py +79 -0
  88. package/scripts/intake/cli.py +57 -0
  89. package/scripts/intake/constants.py +57 -0
  90. package/scripts/intake/depth.py +53 -0
  91. package/scripts/intake/enhancements.py +106 -0
  92. package/scripts/intake/hooks.py +90 -0
  93. package/scripts/intake/prefs.py +76 -0
  94. package/scripts/intake/prompts.py +85 -0
  95. package/scripts/intake/session.py +152 -0
  96. package/scripts/lint_file_layers.py +126 -0
  97. package/scripts/orchestrator.py +68 -17
  98. package/scripts/pre_verdict_gate.py +21 -7
  99. package/scripts/skill_lint.py +11 -1
  100. package/scripts/skill_payload.py +3 -3
  101. package/scripts/test_adversarial_empirical.py +70 -6
  102. package/skill/agents/evidence-judge.md +49 -7
  103. package/skill/workflows/experimental-jev.md +170 -0
  104. package/skill/workflows/intake.md +120 -0
  105. package/visualization/eduevidence-report/scripts/build_infographics.py +32 -14
  106. package/visualization/eduevidence-report/scripts/build_report.py +75 -662
  107. package/visualization/eduevidence-report/scripts/report_copy_pack.py +296 -0
  108. package/visualization/eduevidence-report/scripts/report_copy_policy_guard.py +47 -0
  109. package/visualization/eduevidence-report/scripts/zh_labels.py +61 -0
  110. package/scripts/build_esl_artifacts.py +0 -1921
  111. package/scripts/build_killer_demo.py +0 -295
  112. package/scripts/enrich_projects_human_and_lieflat.py +0 -315
  113. package/scripts/generate_new_projects.py +0 -686
  114. package/scripts/sync_killer_demo_report.py +0 -270
package/examples/ai-coding-assistant-evidence/reports-5themes/EduEvidence_Report_datalab-dark.html CHANGED
@@ -2219,7 +2219,7 @@ select:focus-visible,
2219
2219
  <p><strong>成功阈值：</strong>实验班独立问题解决非劣（差异在 5% 以内）且保持相当或更优、AI 依赖指数低于阈值；若独立问题解决下滑超过 10%，无论任务收益如何，试点均判为失败。</p>
2220
2220
  <p><strong>分析计划：</strong>预登记的实验班 vs 对照班基线调整学习指标比较（ANCOVA）；任务表现指标与学习指标分开报告；按先验编程能力做亚组分析；第 3、5、7 周监测停止条件。</p>
2221
2221
  <h3>评价设计信息图</h3>
2222
- <svg viewBox="0 0 720 260" xmlns="http://www.w3.org/2000/svg" role="img" aria-label="评价设计流程"><title>评价设计流程</title><desc>基线、后测、保持测试与迁移测试的评价流程；完整指标与分析计划见评估章节。</desc><defs><marker id="arr-zh-full-evaluation" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto"><path d="M0,0 L6,3 L0,6 Z" fill="#8A867E"/></marker></defs><rect width="720" height="260" fill="#FCFAF6" rx="12"/><text x="24" y="34" font-size="16" font-weight="700" fill="#3A3833">评价设计流程</text><rect x="30" y="110" width="150" height="56" rx="8" fill="#5E8A6A"/><text x="105.0" y="138.0" text-anchor="middle" fill="#FFFFFF" font-size="12" font-weight="600"><tspan x="105.0" y="138.0">基线</tspan></text><text x="105.0" y="148" text-anchor="middle" font-size="9" fill="#fff" opacity="0.95">前测</text><line x1="180" y1="138" x2="192" y2="138" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-zh-full-evaluation)"/><rect x="192" y="110" width="150" height="56" rx="8" fill="#5E8A6A"/><text x="267.0" y="138.0" text-anchor="middle" fill="#FFFFFF" font-size="12" font-weight="600"><tspan x="267.0" y="138.0">后测</tspan></text><text x="267.0" y="148" text-anchor="middle" font-size="9" fill="#fff" opacity="0.95">后测</text><line x1="342" y1="138" x2="354" y2="138" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-zh-full-evaluation)"/><rect x="354" y="110" width="150" height="56" rx="8" fill="#5E8A6A"/><text x="429.0" y="138.0" text-anchor="middle" fill="#FFFFFF" font-size="12" font-weight="600"><tspan x="429.0" y="138.0">保持</tspan></text><text x="429.0" y="148" text-anchor="middle" font-size="9" fill="#fff" opacity="0.95">保持测试</text><line x1="504" y1="138" x2="516" y2="138" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-zh-full-evaluation)"/><rect x="516" y="110" width="150" height="56" rx="8" fill="#5E8A6A"/><text x="591.0" y="138.0" text-anchor="middle" fill="#FFFFFF" font-size="12" font-weight="600"><tspan x="591.0" y="138.0">迁移</tspan></text><text x="591.0" y="148" text-anchor="middle" font-size="9" fill="#fff" opacity="0.95">迁移测试（无 AI）</text></svg><div class="visual-suppressed"><strong>基准图已抑制</strong><p>result.json 未携带 benchmark.baselines，本图不绘制；基准表现见独立基准报告。</p></div></div></section><section id="full-06-sources" class="full-chapter" data-full-chapter><header class="full-chapter-header"><h2>06 来源、溯源与附录</h2><p class="full-chapter-lead">保留原始来源、URL、证据 ID 和获取信息，确保可回查。</p></header><div class="full-chapter-body"><h3>来源列表</h3><div class='table-wrap'><table class='data-table source-table'><thead><tr><th>ID</th><th>标题</th><th>年份</th><th>权威级别</th><th>可验证位置</th></tr></thead><tbody><tr><td><code>S-2023-kazemitabaar</code></td><td class='cell-main source-title-cell'>Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming <span class="cite-badge cite-ok">DOI ✓</span><details class="source-expander detail-expander"><summary>查看来源与溯源</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>原文标题</dt><dd>Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming</dd></div><div class="source-detail-row"><dt>年份</dt><dd>2023</dd></div><div class="source-detail-row"><dt>权威级别</dt><dd>T1 DOI 可验证论文</dd></div><div class="source-detail-row"><dt>来源位置</dt><dd>https://dl.acm.org/doi/10.1145/3544548.3580919</dd></div><div class="source-detail-row"><dt>可验证链接</dt><dd><a href="https://dl.acm.org/doi/10.1145/3544548.3580919">https://dl.acm.org/doi/10.1145/3544548.3580919</a></dd></div></dl></details></td><td>2023</td><td>T1 DOI 可验证论文</td><td class='cell-main'><a href='https://dl.acm.org/doi/10.1145/3544548.3580919'>https://dl.acm.org/doi/10.1145/3544548.3580919</a></td></tr><tr><td><code>S-2025-bastani</code></td><td class='cell-main source-title-cell'>Generative AI without guardrails can harm learning: Evidence from high school mathematics <span class="cite-badge cite-ok">DOI ✓</span><details class="source-expander detail-expander"><summary>查看来源与溯源</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>原文标题</dt><dd>Generative AI without guardrails can harm learning: Evidence from high school mathematics</dd></div><div class="source-detail-row"><dt>年份</dt><dd>2025</dd></div><div class="source-detail-row"><dt>权威级别</dt><dd>T1 DOI 可验证论文</dd></div><div class="source-detail-row"><dt>来源位置</dt><dd>https://www.pnas.org/doi/10.1073/pnas.2422633122</dd></div><div class="source-detail-row"><dt>可验证链接</dt><dd><a href="https://www.pnas.org/doi/10.1073/pnas.2422633122">https://www.pnas.org/doi/10.1073/pnas.2422633122</a></dd></div></dl></details></td><td>2025</td><td>T1 DOI 可验证论文</td><td class='cell-main'><a href='https://www.pnas.org/doi/10.1073/pnas.2422633122'>https://www.pnas.org/doi/10.1073/pnas.2422633122</a></td></tr><tr><td><code>S-2024-marzuki</code></td><td class='cell-main source-title-cell'>Impact of ChatGPT on ESL students&#x27; academic writing skills <span class="cite-badge cite-ok">DOI ✓</span><details class="source-expander detail-expander"><summary>查看来源与溯源</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>原文标题</dt><dd>Impact of ChatGPT on ESL students&#x27; academic writing skills</dd></div><div class="source-detail-row"><dt>年份</dt><dd>2024</dd></div><div class="source-detail-row"><dt>权威级别</dt><dd>T1 DOI 可验证论文</dd></div><div class="source-detail-row"><dt>来源位置</dt><dd>https://link.springer.com/article/10.1186/s40561-024-00295-9</dd></div><div class="source-detail-row"><dt>可验证链接</dt><dd><a href="https://link.springer.com/article/10.1186/s40561-024-00295-9">https://link.springer.com/article/10.1186/s40561-024-00295-9</a></dd></div></dl></details></td><td>2024</td><td>T1 DOI 可验证论文</td><td class='cell-main'><a href='https://link.springer.com/article/10.1186/s40561-024-00295-9'>https://link.springer.com/article/10.1186/s40561-024-00295-9</a></td></tr><tr><td><code>S-2023-peng</code></td><td class='cell-main source-title-cell'>The Impact of AI on Developer Productivity: Evidence from GitHub Copilot <span class="cite-badge cite-ok">DOI ✓</span><details class="source-expander detail-expander"><summary>查看来源与溯源</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>原文标题</dt><dd>The Impact of AI on Developer Productivity: Evidence from GitHub Copilot</dd></div><div class="source-detail-row"><dt>年份</dt><dd>2023</dd></div><div class="source-detail-row"><dt>权威级别</dt><dd>Tier2 Academic Database</dd></div><div class="source-detail-row"><dt>来源位置</dt><dd>https://arxiv.org/abs/2302.06590</dd></div><div class="source-detail-row"><dt>可验证链接</dt><dd><a href="https://doi.org/10.48550/arXiv.2302.06590">https://doi.org/10.48550/arXiv.2302.06590</a></dd></div></dl></details></td><td>2023</td><td>Tier2 Academic Database</td><td class='cell-main'><a href='https://doi.org/10.48550/arXiv.2302.06590'>https://doi.org/10.48550/arXiv.2302.06590</a></td></tr><tr><td><code>S-2023-yetistiren</code></td><td class='cell-main source-title-cell'>GitHub Copilot AI Pair Programmer: Asset or Liability? <span class="cite-badge cite-ok">DOI ✓</span><details class="source-expander detail-expander"><summary>查看来源与溯源</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>原文标题</dt><dd>GitHub Copilot AI Pair Programmer: Asset or Liability?</dd></div><div class="source-detail-row"><dt>年份</dt><dd>2023</dd></div><div class="source-detail-row"><dt>权威级别</dt><dd>T1 DOI 可验证论文</dd></div><div class="source-detail-row"><dt>来源位置</dt><dd>https://doi.org/10.1016/j.jss.2023.111734</dd></div><div class="source-detail-row"><dt>可验证链接</dt><dd><a href="https://doi.org/10.1016/j.jss.2023.111734">https://doi.org/10.1016/j.jss.2023.111734</a></dd></div></dl></details></td><td>2023</td><td>T1 DOI 可验证论文</td><td class='cell-main'><a href='https://doi.org/10.1016/j.jss.2023.111734'>https://doi.org/10.1016/j.jss.2023.111734</a></td></tr><tr><td><code>S-2022-finnie-ansley</code></td><td class='cell-main source-title-cell'>Using GitHub Copilot to Solve Introductory Programming Problems <span class="cite-badge cite-ok">DOI ✓</span><details class="source-expander detail-expander"><summary>查看来源与溯源</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>原文标题</dt><dd>Using GitHub Copilot to Solve Introductory Programming Problems</dd></div><div class="source-detail-row"><dt>年份</dt><dd>2022</dd></div><div class="source-detail-row"><dt>权威级别</dt><dd>T1 DOI 可验证论文</dd></div><div class="source-detail-row"><dt>来源位置</dt><dd>https://dl.acm.org/doi/10.1145/3545945.3569830</dd></div><div class="source-detail-row"><dt>可验证链接</dt><dd><a href="https://dl.acm.org/doi/10.1145/3545945.3569830">https://dl.acm.org/doi/10.1145/3545945.3569830</a></dd></div></dl></details></td><td>2022</td><td>T1 DOI 可验证论文</td><td class='cell-main'><a href='https://dl.acm.org/doi/10.1145/3545945.3569830'>https://dl.acm.org/doi/10.1145/3545945.3569830</a></td></tr><tr><td><code>S-2023-explanations-compare</code></td><td class='cell-main source-title-cell'>Comparing Code Explanations Created by Students and Large Language Models <span class="cite-badge cite-ok">DOI ✓</span><details class="source-expander detail-expander"><summary>查看来源与溯源</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>原文标题</dt><dd>Comparing Code Explanations Created by Students and Large Language Models</dd></div><div class="source-detail-row"><dt>年份</dt><dd>2023</dd></div><div class="source-detail-row"><dt>权威级别</dt><dd>T1 DOI 可验证论文</dd></div><div class="source-detail-row"><dt>来源位置</dt><dd>https://dl.acm.org/doi/10.1145/3587102.3588785</dd></div><div class="source-detail-row"><dt>可验证链接</dt><dd><a href="https://dl.acm.org/doi/10.1145/3587102.3588785">https://dl.acm.org/doi/10.1145/3587102.3588785</a></dd></div></dl></details></td><td>2023</td><td>T1 DOI 可验证论文</td><td class='cell-main'><a href='https://dl.acm.org/doi/10.1145/3587102.3588785'>https://dl.acm.org/doi/10.1145/3587102.3588785</a></td></tr><tr><td><code>S-2022-vaithilingam</code></td><td class='cell-main source-title-cell'>Expectation vs. Experience: Evaluating the Usability of Code Generation Tools <span class="cite-badge cite-ok">DOI ✓</span><details class="source-expander detail-expander"><summary>查看来源与溯源</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>原文标题</dt><dd>Expectation vs. Experience: Evaluating the Usability of Code Generation Tools</dd></div><div class="source-detail-row"><dt>年份</dt><dd>2022</dd></div><div class="source-detail-row"><dt>权威级别</dt><dd>T1 DOI 可验证论文</dd></div><div class="source-detail-row"><dt>来源位置</dt><dd>https://dl.acm.org/doi/10.1145/3491101.3519665</dd></div><div class="source-detail-row"><dt>可验证链接</dt><dd><a href="https://dl.acm.org/doi/10.1145/3491101.3519665">https://dl.acm.org/doi/10.1145/3491101.3519665</a></dd></div></dl></details></td><td>2022</td><td>T1 DOI 可验证论文</td><td class='cell-main'><a href='https://dl.acm.org/doi/10.1145/3491101.3519665'>https://dl.acm.org/doi/10.1145/3491101.3519665</a></td></tr></tbody></table></div><h3>Fetch 溯源</h3><p class="provenance-summary">搜索提供方：n/a</p><p class='provenance-empty'>无逐条 fetch 记录（来源由研究管线直接提供）。</p></div></section></main></div>
2222
+ <svg viewBox="0 0 720 260" xmlns="http://www.w3.org/2000/svg" role="img" aria-label="评价设计流程"><title>评价设计流程</title><desc>基线、后测、保持测试与迁移测试的评价流程；完整指标与分析计划见评估章节。</desc><defs><marker id="arr-zh-full-evaluation" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto"><path d="M0,0 L6,3 L0,6 Z" fill="#8A867E"/></marker></defs><rect width="720" height="260" fill="#FCFAF6" rx="12"/><text x="24" y="34" font-size="16" font-weight="700" fill="#3A3833">评价设计流程</text><rect x="30" y="110" width="150" height="56" rx="8" fill="#5E8A6A"/><text x="105.0" y="138.0" text-anchor="middle" fill="#FFFFFF" font-size="12" font-weight="600"><tspan x="105.0" y="138.0">基线</tspan></text><text x="105.0" y="148" text-anchor="middle" font-size="9" fill="#fff" opacity="0.95">前测</text><line x1="180" y1="138" x2="192" y2="138" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-zh-full-evaluation)"/><rect x="192" y="110" width="150" height="56" rx="8" fill="#5E8A6A"/><text x="267.0" y="138.0" text-anchor="middle" fill="#FFFFFF" font-size="12" font-weight="600"><tspan x="267.0" y="138.0">后测</tspan></text><text x="267.0" y="148" text-anchor="middle" font-size="9" fill="#fff" opacity="0.95">后测</text><line x1="342" y1="138" x2="354" y2="138" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-zh-full-evaluation)"/><rect x="354" y="110" width="150" height="56" rx="8" fill="#5E8A6A"/><text x="429.0" y="138.0" text-anchor="middle" fill="#FFFFFF" font-size="12" font-weight="600"><tspan x="429.0" y="138.0">保持</tspan></text><text x="429.0" y="148" text-anchor="middle" font-size="9" fill="#fff" opacity="0.95">保持测试</text><line x1="504" y1="138" x2="516" y2="138" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-zh-full-evaluation)"/><rect x="516" y="110" width="150" height="56" rx="8" fill="#5E8A6A"/><text x="591.0" y="138.0" text-anchor="middle" fill="#FFFFFF" font-size="12" font-weight="600"><tspan x="591.0" y="138.0">迁移</tspan></text><text x="591.0" y="148" text-anchor="middle" font-size="9" fill="#fff" opacity="0.95">迁移测试</text></svg><div class="visual-suppressed"><strong>基准图已抑制</strong><p>result.json 未携带 benchmark.baselines，本图不绘制；基准表现见独立基准报告。</p></div></div></section><section id="full-06-sources" class="full-chapter" data-full-chapter><header class="full-chapter-header"><h2>06 来源、溯源与附录</h2><p class="full-chapter-lead">保留原始来源、URL、证据 ID 和获取信息，确保可回查。</p></header><div class="full-chapter-body"><h3>来源列表</h3><div class='table-wrap'><table class='data-table source-table'><thead><tr><th>ID</th><th>标题</th><th>年份</th><th>权威级别</th><th>可验证位置</th></tr></thead><tbody><tr><td><code>S-2023-kazemitabaar</code></td><td class='cell-main source-title-cell'>Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming <span class="cite-badge cite-ok">DOI 可验证</span><details class="source-expander detail-expander"><summary>查看来源与溯源</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>原文标题</dt><dd>Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming</dd></div><div class="source-detail-row"><dt>年份</dt><dd>2023</dd></div><div class="source-detail-row"><dt>权威级别</dt><dd>T1 DOI 可验证论文</dd></div><div class="source-detail-row"><dt>来源位置</dt><dd>https://dl.acm.org/doi/10.1145/3544548.3580919</dd></div><div class="source-detail-row"><dt>可验证链接</dt><dd><a href="https://dl.acm.org/doi/10.1145/3544548.3580919">https://dl.acm.org/doi/10.1145/3544548.3580919</a></dd></div></dl></details></td><td>2023</td><td>T1 DOI 可验证论文</td><td class='cell-main'><a href='https://dl.acm.org/doi/10.1145/3544548.3580919'>https://dl.acm.org/doi/10.1145/3544548.3580919</a></td></tr><tr><td><code>S-2025-bastani</code></td><td class='cell-main source-title-cell'>Generative AI without guardrails can harm learning: Evidence from high school mathematics <span class="cite-badge cite-ok">DOI 可验证</span><details class="source-expander detail-expander"><summary>查看来源与溯源</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>原文标题</dt><dd>Generative AI without guardrails can harm learning: Evidence from high school mathematics</dd></div><div class="source-detail-row"><dt>年份</dt><dd>2025</dd></div><div class="source-detail-row"><dt>权威级别</dt><dd>T1 DOI 可验证论文</dd></div><div class="source-detail-row"><dt>来源位置</dt><dd>https://www.pnas.org/doi/10.1073/pnas.2422633122</dd></div><div class="source-detail-row"><dt>可验证链接</dt><dd><a href="https://www.pnas.org/doi/10.1073/pnas.2422633122">https://www.pnas.org/doi/10.1073/pnas.2422633122</a></dd></div></dl></details></td><td>2025</td><td>T1 DOI 可验证论文</td><td class='cell-main'><a href='https://www.pnas.org/doi/10.1073/pnas.2422633122'>https://www.pnas.org/doi/10.1073/pnas.2422633122</a></td></tr><tr><td><code>S-2024-marzuki</code></td><td class='cell-main source-title-cell'>Impact of ChatGPT on ESL students&#x27; academic writing skills <span class="cite-badge cite-ok">DOI 可验证</span><details class="source-expander detail-expander"><summary>查看来源与溯源</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>原文标题</dt><dd>Impact of ChatGPT on ESL students&#x27; academic writing skills</dd></div><div class="source-detail-row"><dt>年份</dt><dd>2024</dd></div><div class="source-detail-row"><dt>权威级别</dt><dd>T1 DOI 可验证论文</dd></div><div class="source-detail-row"><dt>来源位置</dt><dd>https://link.springer.com/article/10.1186/s40561-024-00295-9</dd></div><div class="source-detail-row"><dt>可验证链接</dt><dd><a href="https://link.springer.com/article/10.1186/s40561-024-00295-9">https://link.springer.com/article/10.1186/s40561-024-00295-9</a></dd></div></dl></details></td><td>2024</td><td>T1 DOI 可验证论文</td><td class='cell-main'><a href='https://link.springer.com/article/10.1186/s40561-024-00295-9'>https://link.springer.com/article/10.1186/s40561-024-00295-9</a></td></tr><tr><td><code>S-2023-peng</code></td><td class='cell-main source-title-cell'>The Impact of AI on Developer Productivity: Evidence from GitHub Copilot <span class="cite-badge cite-ok">DOI 可验证</span><details class="source-expander detail-expander"><summary>查看来源与溯源</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>原文标题</dt><dd>The Impact of AI on Developer Productivity: Evidence from GitHub Copilot</dd></div><div class="source-detail-row"><dt>年份</dt><dd>2023</dd></div><div class="source-detail-row"><dt>权威级别</dt><dd>Tier2 Academic Database</dd></div><div class="source-detail-row"><dt>来源位置</dt><dd>https://arxiv.org/abs/2302.06590</dd></div><div class="source-detail-row"><dt>可验证链接</dt><dd><a href="https://doi.org/10.48550/arXiv.2302.06590">https://doi.org/10.48550/arXiv.2302.06590</a></dd></div></dl></details></td><td>2023</td><td>Tier2 Academic Database</td><td class='cell-main'><a href='https://doi.org/10.48550/arXiv.2302.06590'>https://doi.org/10.48550/arXiv.2302.06590</a></td></tr><tr><td><code>S-2023-yetistiren</code></td><td class='cell-main source-title-cell'>GitHub Copilot AI Pair Programmer: Asset or Liability? <span class="cite-badge cite-ok">DOI 可验证</span><details class="source-expander detail-expander"><summary>查看来源与溯源</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>原文标题</dt><dd>GitHub Copilot AI Pair Programmer: Asset or Liability?</dd></div><div class="source-detail-row"><dt>年份</dt><dd>2023</dd></div><div class="source-detail-row"><dt>权威级别</dt><dd>T1 DOI 可验证论文</dd></div><div class="source-detail-row"><dt>来源位置</dt><dd>https://doi.org/10.1016/j.jss.2023.111734</dd></div><div class="source-detail-row"><dt>可验证链接</dt><dd><a href="https://doi.org/10.1016/j.jss.2023.111734">https://doi.org/10.1016/j.jss.2023.111734</a></dd></div></dl></details></td><td>2023</td><td>T1 DOI 可验证论文</td><td class='cell-main'><a href='https://doi.org/10.1016/j.jss.2023.111734'>https://doi.org/10.1016/j.jss.2023.111734</a></td></tr><tr><td><code>S-2022-finnie-ansley</code></td><td class='cell-main source-title-cell'>Using GitHub Copilot to Solve Introductory Programming Problems <span class="cite-badge cite-ok">DOI 可验证</span><details class="source-expander detail-expander"><summary>查看来源与溯源</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>原文标题</dt><dd>Using GitHub Copilot to Solve Introductory Programming Problems</dd></div><div class="source-detail-row"><dt>年份</dt><dd>2022</dd></div><div class="source-detail-row"><dt>权威级别</dt><dd>T1 DOI 可验证论文</dd></div><div class="source-detail-row"><dt>来源位置</dt><dd>https://dl.acm.org/doi/10.1145/3545945.3569830</dd></div><div class="source-detail-row"><dt>可验证链接</dt><dd><a href="https://dl.acm.org/doi/10.1145/3545945.3569830">https://dl.acm.org/doi/10.1145/3545945.3569830</a></dd></div></dl></details></td><td>2022</td><td>T1 DOI 可验证论文</td><td class='cell-main'><a href='https://dl.acm.org/doi/10.1145/3545945.3569830'>https://dl.acm.org/doi/10.1145/3545945.3569830</a></td></tr><tr><td><code>S-2023-explanations-compare</code></td><td class='cell-main source-title-cell'>Comparing Code Explanations Created by Students and Large Language Models <span class="cite-badge cite-ok">DOI 可验证</span><details class="source-expander detail-expander"><summary>查看来源与溯源</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>原文标题</dt><dd>Comparing Code Explanations Created by Students and Large Language Models</dd></div><div class="source-detail-row"><dt>年份</dt><dd>2023</dd></div><div class="source-detail-row"><dt>权威级别</dt><dd>T1 DOI 可验证论文</dd></div><div class="source-detail-row"><dt>来源位置</dt><dd>https://dl.acm.org/doi/10.1145/3587102.3588785</dd></div><div class="source-detail-row"><dt>可验证链接</dt><dd><a href="https://dl.acm.org/doi/10.1145/3587102.3588785">https://dl.acm.org/doi/10.1145/3587102.3588785</a></dd></div></dl></details></td><td>2023</td><td>T1 DOI 可验证论文</td><td class='cell-main'><a href='https://dl.acm.org/doi/10.1145/3587102.3588785'>https://dl.acm.org/doi/10.1145/3587102.3588785</a></td></tr><tr><td><code>S-2022-vaithilingam</code></td><td class='cell-main source-title-cell'>Expectation vs. Experience: Evaluating the Usability of Code Generation Tools <span class="cite-badge cite-ok">DOI 可验证</span><details class="source-expander detail-expander"><summary>查看来源与溯源</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>原文标题</dt><dd>Expectation vs. Experience: Evaluating the Usability of Code Generation Tools</dd></div><div class="source-detail-row"><dt>年份</dt><dd>2022</dd></div><div class="source-detail-row"><dt>权威级别</dt><dd>T1 DOI 可验证论文</dd></div><div class="source-detail-row"><dt>来源位置</dt><dd>https://dl.acm.org/doi/10.1145/3491101.3519665</dd></div><div class="source-detail-row"><dt>可验证链接</dt><dd><a href="https://dl.acm.org/doi/10.1145/3491101.3519665">https://dl.acm.org/doi/10.1145/3491101.3519665</a></dd></div></dl></details></td><td>2022</td><td>T1 DOI 可验证论文</td><td class='cell-main'><a href='https://dl.acm.org/doi/10.1145/3491101.3519665'>https://dl.acm.org/doi/10.1145/3491101.3519665</a></td></tr></tbody></table></div><h3>Fetch 溯源</h3><p class="provenance-summary">搜索提供方：n/a</p><p class='provenance-empty'>无逐条 fetch 记录（来源由研究管线直接提供）。</p></div></section></main></div>
2223
2223
  </div>
2224
2224
  <footer class="report-footer"><p>EduEvidence 证据报告 · Schema PASS · Claim Binding PASS · Numeric Consistency PASS · Bilingual Structure PASS · 语言人话化 PASS · 无伪精度 PASS · Lieflat 数据溯源 PASS · 坐标轴无失真 NOT_CHECKED · 色盲安全 NOT_CHECKED · 单文件离线可打开 · 数据源：result.json</p></footer>
2225
2225
  </div>
@@ -2506,7 +2506,7 @@ select:focus-visible,
2506
2506
  <article class="hero-insight next"><span>Next action</span><p class="hero-insight-text">Run a phased CS1 pilot with hints-not-answers guardrails, weekly lab use, and a no-AI transfer exam that can stop the pilot.</p></article>
2507
2507
  </div>
2508
2508
  <p class="hero-provenance"><span>Evidence / sources</span> · 12 / 8</p>
2509
- </div><div class="scope-grid"><article class="scope-card"><h3>Research question</h3><p class="scope-text">我们准备在大学一年级 C 语言课程中允许学生使用生成式 AI 编程助手。它到底会不会提高学习效果？应该怎样引入？</p></article><article class="scope-card"><h3>Target learners</h3><p class="scope-text">Education level: First-year undergraduate; Major: Computer science; Prior knowledge: First programming course, no prior text-based programming; Learner characteristics: Mixed-ability large class (60 students)</p></article><article class="scope-card"><h3>Course context</h3><p class="scope-text">Subject: C programming; Course type: Lecture + lab; Duration: 16 weeks (one semester)</p></article><article class="scope-card"><h3>AI intervention</h3><p class="scope-text">Method: Lecture with lab exercises; AI tool: Generative AI coding assistant; Allowed usage: Under design (pending evidence review); Frequency: Weekly lab sessions; Duration: One semester</p></article><article class="scope-card"><h3>Comparison</h3><p class="scope-text">No AI Coding Assistant Control</p></article><article class="scope-card"><h3>Outcome constructs</h3><p class="scope-text">Primary outcomes: Independent problem solving, Code quality; Secondary outcomes: Completion time, Retention, Knowledge gain; Risk outcomes: AI dependency, Over-reliance, Reduced transfer</p></article><article class="scope-card"><h3>Research scope</h3><p class="scope-text">Time range: 2021 2026; Geography: Worldwide; Study designs: Randomized controlled trial, Quasi-experimental, Observational</p></article><article class="scope-card"><h3>Decision success condition</h3><p class="scope-text">independent problem solving and code quality improve (or do not decline) while AI dependency risk stays controlled; evidence base supports a bounded pilot.</p></article></div></div></section><section id="full-02-evidence-en" class="full-chapter" data-full-chapter><header class="full-chapter-header"><h2>02 Key Evidence &amp; Outcome Separation</h2><p class="full-chapter-lead">Place task performance, actual learning, retention and risk on one evidence map without conflating them.</p></header><div class="full-chapter-body"><div class="retrieval-grid"><article><h3>Inclusion criteria</h3><ul><li>Studies Of Generative AI Coding Tools In Learning To Program</li><li>Outcomes Measuring Learning Not Only Task Speed</li><li>University Or Novice Programming Populations</li></ul></article><article><h3>Exclusion criteria</h3><ul><li>Practitioner Anecdotes Without Data</li><li>Industry Professional Populations Only</li></ul></article></div><div class="retrieval-coverage"><h3>Source coverage</h3><p><code>S-2023-kazemitabaar</code> <code>S-2025-bastani</code> <code>S-2024-marzuki</code> <code>S-2023-peng</code> <code>S-2023-yetistiren</code> <code>S-2022-finnie-ansley</code> <code>S-2023-explanations-compare</code> <code>S-2022-vaithilingam</code></p><p class="retrieval-note">This report shows only retrieval metadata present in result; it does not fabricate PRISMA/funnel counts when none exist.</p></div><div class="outcome-separation" data-visual="outcome-separation"><div class="visual-heading"><h3>Outcome Separation · Task performance ≠ learning</h3><p>Outcomes are adjudicated separately so faster training performance is not silently treated as evidence of learning.</p><p class="semantic-note">Effect direction comes from evidence.effect_direction; supporting a claim does not imply a positive outcome.</p></div><div class="outcome-groups"><article class="outcome-group outcome-task"><h3>Task / proximal performance</h3><ul><li><strong>Completion time</strong><span class="outcome-states"><span class="dir pos">Positive effect 2</span></span></li><li><strong>Code quality</strong><span class="outcome-states"><span class="dir neu">Null effect 1</span></span></li><li><strong>Assignment score</strong><span class="outcome-states"><span class="dir pos">Positive effect 2</span></span></li></ul></article><article class="outcome-group outcome-learning"><h3>Learning / retention / transfer</h3><ul><li><strong>Knowledge gain</strong><span class="outcome-states"><span class="dir pos">Positive effect 1</span></span></li><li><strong>Retention</strong><span class="outcome-states"><span class="dir neu">Null effect 1</span></span></li><li><strong>Independent problem solving</strong><span class="outcome-states"><span class="dir neg">Negative effect 1</span><span class="dir neu">Null effect 2</span></span></li></ul></article><article class="outcome-group outcome-risk"><h3>Risk / dependency</h3><ul><li><strong>Over-reliance</strong><span class="outcome-states"><span class="dir neg">Negative effect 1</span></span></li></ul></article><article class="outcome-group outcome-other"><h3>Other outcomes</h3><ul><li><strong>Metacognition</strong><span class="outcome-states"><span class="dir pos">Positive effect 1</span></span></li></ul></article></div></div><div class='table-wrap outcome-table'><table class='data-table'><thead><tr><th>Outcome</th><th>Positive effect</th><th>Negative effect</th><th>Null effect</th><th>Evidence</th></tr></thead><tbody><tr><td><strong>Knowledge gain</strong><span class='raw-tag' title='raw id'>knowledge_gain</span></td><td class='num'>1</td><td class='num'>0</td><td class='num'>0</td><td><code>E-007</code> </td></tr><tr><td><strong>Retention</strong><span class='raw-tag' title='raw id'>retention</span></td><td class='num'>0</td><td class='num'>0</td><td class='num'>1</td><td><code>E-003</code> </td></tr><tr><td><strong>Independent problem solving</strong><span class='raw-tag' title='raw id'>independent_problem_solving</span></td><td class='num'>0</td><td class='num'>1</td><td class='num'>2</td><td><code>E-002</code> <code>E-004</code> <code>E-005</code> </td></tr><tr><td><strong>Completion time</strong><span class='raw-tag' title='raw id'>completion_time</span></td><td class='num'>2</td><td class='num'>0</td><td class='num'>0</td><td><code>E-001</code> <code>E-008</code> </td></tr><tr><td><strong>Code quality</strong><span class='raw-tag' title='raw id'>code_quality</span></td><td class='num'>0</td><td class='num'>0</td><td class='num'>1</td><td><code>E-009</code> </td></tr><tr><td><strong>Assignment score</strong><span class='raw-tag' title='raw id'>assignment_score</span></td><td class='num'>2</td><td class='num'>0</td><td class='num'>0</td><td><code>E-006</code> <code>E-010</code> </td></tr><tr><td><strong>Metacognition</strong><span class='raw-tag' title='raw id'>metacognition</span></td><td class='num'>1</td><td class='num'>0</td><td class='num'>0</td><td><code>E-011</code> </td></tr><tr><td><strong>Over-reliance</strong><span class='raw-tag' title='raw id'>over_reliance</span></td><td class='num'>0</td><td class='num'>1</td><td class='num'>0</td><td><code>E-012</code> </td></tr></tbody></table></div><div class="visual-surface" data-visual="outcome-evidence-balance"><svg viewBox="0 0 720 300" xmlns="http://www.w3.org/2000/svg" role="img" aria-label="Outcome evidence effect balance"><title>Outcome evidence effect balance</title><desc>Positive / negative / null effect counts per outcome (based on effect_direction, not claim support).</desc><rect x="0" y="0" width="720" height="300" fill="#FFFFFF"/><line x1="439.0" y1="46" x2="439.0" y2="182" stroke="#999" stroke-width="1" stroke-dasharray="3,3"/><text x="190" y="58.0" text-anchor="end" font-size="11" fill="#333">Knowledge gain</text><rect x="439.0" y="49.8" width="120.5" height="9.4" fill="#5E8A6A"/><text x="499.2" y="58.0" text-anchor="middle" font-size="9" fill="#fff">1</text><text x="190" y="75.0" text-anchor="end" font-size="11" fill="#333">Retention</text><text x="190" y="92.0" text-anchor="end" font-size="11" fill="#333">Independent problem solving</text><rect x="318.5" y="83.8" width="120.5" height="9.4" fill="#A85B53"/><text x="378.8" y="92.0" text-anchor="middle" font-size="9" fill="#fff">-1</text><text x="190" y="109.0" text-anchor="end" font-size="11" fill="#333">Completion time</text><rect x="439.0" y="100.8" width="241.0" height="9.4" fill="#5E8A6A"/><text x="559.5" y="109.0" text-anchor="middle" font-size="9" fill="#fff">2</text><text x="190" y="126.0" text-anchor="end" font-size="11" fill="#333">Code quality</text><text x="190" y="143.0" text-anchor="end" font-size="11" fill="#333">Assignment score</text><rect x="439.0" y="134.8" width="241.0" height="9.4" fill="#5E8A6A"/><text x="559.5" y="143.0" text-anchor="middle" font-size="9" fill="#fff">2</text><text x="190" y="160.0" text-anchor="end" font-size="11" fill="#333">Metacognition</text><rect x="439.0" y="151.8" width="120.5" height="9.4" fill="#5E8A6A"/><text x="499.2" y="160.0" text-anchor="middle" font-size="9" fill="#fff">1</text><text x="190" y="177.0" text-anchor="end" font-size="11" fill="#333">Over-reliance</text><rect x="318.5" y="168.8" width="120.5" height="9.4" fill="#A85B53"/><text x="378.8" y="177.0" text-anchor="middle" font-size="9" fill="#fff">-1</text><rect x="439.0" y="206.1" width="8.0" height="6" fill="#C99A4A"/><rect x="439.0" y="214.9" width="8.0" height="6" fill="#C99A4A"/><rect x="439.0" y="232.4" width="8.0" height="6" fill="#C99A4A"/><rect x="198" y="14" width="10" height="10" fill="#5E8A6A"/><text x="212" y="23" font-size="10" fill="#333">Positive effect</text><rect x="395" y="14" width="10" height="10" fill="#A85B53"/><text x="409" y="23" font-size="10" fill="#333">Negative effect</text><rect x="592" y="14" width="10" height="10" fill="#C99A4A"/><text x="606" y="23" font-size="10" fill="#333">Null effect</text></svg><p class="chart-interpretation"><strong>What this means: </strong>Positive / negative / null effect-direction evidence counts per outcome. This visual encodes effect_direction, not whether evidence supports a claim.</p></div><div id="chart-outcome-en" class="chart-mount" aria-label="Outcome Evidence Overview"></div><figure class="academic-figure" data-visual="outcome-evidence-balance"><svg viewBox="0 0 1518 300" xmlns="http://www.w3.org/2000/svg" role="img" aria-label="Effect direction by outcome type (publication figure)"><title>Effect direction by outcome type (publication figure)</title><desc>Counts of positive / negative / null effects per outcome type with an integer count axis, theme-independent. Source: EduEvidence result.json.</desc><rect width="1518" height="300" fill="#FFFFFF"/><line x1="70" y1="250" x2="650" y2="250" stroke="#333" stroke-width="1"/><text x="106.2" y="266" text-anchor="middle" font-size="10" fill="#333">Knowledge gain</text><rect x="88.1" y="150.0" width="18.1" height="100.0" fill="#38BDF8"/><text x="97.2" y="147.0" text-anchor="middle" font-size="9" fill="#333">1</text><rect x="106.2" y="250.0" width="18.1" height="1.0" fill="#10B981"/><rect x="124.4" y="250.0" width="18.1" height="1.0" fill="#F59E0B"/><text x="178.8" y="266" text-anchor="middle" font-size="10" fill="#333">Retention</text><rect x="160.6" y="250.0" width="18.1" height="1.0" fill="#38BDF8"/><rect x="178.8" y="250.0" width="18.1" height="1.0" fill="#10B981"/><rect x="196.9" y="150.0" width="18.1" height="100.0" fill="#F59E0B"/><text x="205.9" y="147.0" text-anchor="middle" font-size="9" fill="#333">1</text><text x="251.2" y="266" text-anchor="middle" font-size="10" fill="#333">Independent problem solving</text><rect x="233.1" y="250.0" width="18.1" height="1.0" fill="#38BDF8"/><rect x="251.2" y="150.0" width="18.1" height="100.0" fill="#10B981"/><text x="260.3" y="147.0" text-anchor="middle" font-size="9" fill="#333">1</text><rect x="269.4" y="50.0" width="18.1" height="200.0" fill="#F59E0B"/><text x="278.4" y="47.0" text-anchor="middle" font-size="9" fill="#333">2</text><text x="323.8" y="266" text-anchor="middle" font-size="10" fill="#333">Completion time</text><rect x="305.6" y="50.0" width="18.1" height="200.0" fill="#38BDF8"/><text x="314.7" y="47.0" text-anchor="middle" font-size="9" fill="#333">2</text><rect x="323.8" y="250.0" width="18.1" height="1.0" fill="#10B981"/><rect x="341.9" y="250.0" width="18.1" height="1.0" fill="#F59E0B"/><text x="396.2" y="266" text-anchor="middle" font-size="10" fill="#333">Code quality</text><rect x="378.1" y="250.0" width="18.1" height="1.0" fill="#38BDF8"/><rect x="396.2" y="250.0" width="18.1" height="1.0" fill="#10B981"/><rect x="414.4" y="150.0" width="18.1" height="100.0" fill="#F59E0B"/><text x="423.4" y="147.0" text-anchor="middle" font-size="9" fill="#333">1</text><text x="468.8" y="266" text-anchor="middle" font-size="10" fill="#333">Assignment score</text><rect x="450.6" y="50.0" width="18.1" height="200.0" fill="#38BDF8"/><text x="459.7" y="47.0" text-anchor="middle" font-size="9" fill="#333">2</text><rect x="468.8" y="250.0" width="18.1" height="1.0" fill="#10B981"/><rect x="486.9" y="250.0" width="18.1" height="1.0" fill="#F59E0B"/><text x="541.2" y="266" text-anchor="middle" font-size="10" fill="#333">Metacognition</text><rect x="523.1" y="150.0" width="18.1" height="100.0" fill="#38BDF8"/><text x="532.2" y="147.0" text-anchor="middle" font-size="9" fill="#333">1</text><rect x="541.2" y="250.0" width="18.1" height="1.0" fill="#10B981"/><rect x="559.4" y="250.0" width="18.1" height="1.0" fill="#F59E0B"/><text x="613.8" y="266" text-anchor="middle" font-size="10" fill="#333">Over-reliance</text><rect x="595.6" y="250.0" width="18.1" height="1.0" fill="#38BDF8"/><rect x="613.8" y="150.0" width="18.1" height="100.0" fill="#10B981"/><text x="622.8" y="147.0" text-anchor="middle" font-size="9" fill="#333">1</text><rect x="631.9" y="250.0" width="18.1" height="1.0" fill="#F59E0B"/><line x1="65" y1="250.0" x2="70" y2="250.0" stroke="#999"/><text x="62" y="253.0" text-anchor="end" font-size="9" fill="#666">0</text><line x1="65" y1="150.0" x2="70" y2="150.0" stroke="#999"/><text x="62" y="153.0" text-anchor="end" font-size="9" fill="#666">1</text><line x1="65" y1="50.0" x2="70" y2="50.0" stroke="#999"/><text x="62" y="53.0" text-anchor="end" font-size="9" fill="#666">2</text><text x="360.0" y="30" text-anchor="middle" font-size="14" font-weight="700" fill="#111">Effect direction by outcome type</text><rect x="70" y="8" width="10" height="10" fill="#38BDF8"/><text x="84" y="17" font-size="10" fill="#333">Positive effect</text><rect x="267" y="8" width="10" height="10" fill="#10B981"/><text x="281" y="17" font-size="10" fill="#333">Negative effect</text><rect x="464" y="8" width="10" height="10" fill="#F59E0B"/><text x="478" y="17" font-size="10" fill="#333">Null effect</text><text x="20" y="290" font-size="11" fill="#333333" font-style="italic">Fig. 1. Counts of positive / negative / null effects per outcome type (based on effect_direction; publication figure, theme-independent). Source: EduEvidence result.json.</text></svg><figcaption>Fig. 1. Positive / negative / null effect counts per outcome (based on effect_direction, not claim support).</figcaption></figure><div class='matrix-controls'><div class='matrix-tools'><input id='matrix-search-full-en' type='search' placeholder='Search evidence…' aria-label='Filter / search evidence'><select id='matrix-direction-full-en' aria-label='Filter by effect direction'><option value=''>All effects</option><option value='positive'>Positive effect</option><option value='negative'>Negative effect</option><option value='null'>Null effect</option></select><select id='matrix-outcome-full-en' aria-label='Filter by outcome type'><option value=''>All outcomes</option><option value='assignment_score'>Assignment score</option><option value='code_quality'>Code quality</option><option value='completion_time'>Completion time</option><option value='independent_problem_solving'>Independent problem solving</option><option value='knowledge_gain'>Knowledge gain</option><option value='metacognition'>Metacognition</option><option value='over_reliance'>Over-reliance</option><option value='retention'>Retention</option></select></div></div><div class='table-wrap matrix-wrap'><table id='evidence-matrix-full-en' class='data-table evidence-matrix'><thead><tr><th>ID</th><th>Outcome</th><th>Effect</th><th>Quality</th><th>Claim</th><th>Source</th></tr></thead><tbody><tr data-effect="positive" data-direction="positive" data-outcome="completion_time" data-search="e-001 study-kazemitabaar-2023 smpl-kazemitabaar-2023-n69 studying the effect of ai code generators on supporting novice learners in introductory programming ai coding assistants significantly increase task completion speed and completion rate during training. 69 novices ages 10-17 with no prior text-based programming experience access_to_openai_codex_ai_coding_assistant_during_training baseline_group_without_ai_coding_assistant positive s-2023-kazemitabaar"><td><code>E-001</code></td><td><strong>Completion time</strong></td><td><span class="dir pos">Positive effect</span><span class="relation-note">Claim relation: Support</span></td><td><div class="quality-cell"><span class="num">9</span><span class="quality-meter" aria-hidden="true"><i style="width:90%"></i></span></div></td><td class="claim-cell"><p>AI coding assistants significantly increase task completion speed and completion rate during training.</p><details class="matrix-row-detail"><summary>View full evidence</summary><dl class="evidence-detail-grid"><div class="evidence-detail-row"><dt>Study ID</dt><dd>STUDY-KAZEMITABAAR-2023</dd></div><div class="evidence-detail-row"><dt>Sample ID</dt><dd>SMPL-KAZEMITABAAR-2023-N69</dd></div><div class="evidence-detail-row"><dt>Study title</dt><dd>Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming</dd></div><div class="evidence-detail-row"><dt>Source title</dt><dd>Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming</dd></div><div class="evidence-detail-row"><dt>Year</dt><dd>2023</dd></div><div class="evidence-detail-row"><dt>Study design</dt><dd>Randomized controlled trial</dd></div><div class="evidence-detail-row"><dt>Education level</dt><dd>K-12 Ages 10 17</dd></div><div class="evidence-detail-row"><dt>Population</dt><dd>69 novices ages 10-17 with no prior text-based programming experience</dd></div><div class="evidence-detail-row"><dt>Sample size</dt><dd>69</dd></div><div class="evidence-detail-row"><dt>Intervention</dt><dd>Access To Openai Codex AI Coding Assistant During Training</dd></div><div class="evidence-detail-row"><dt>Comparison</dt><dd>Baseline Group Without AI Coding Assistant</dd></div><div class="evidence-detail-row"><dt>Outcome measure</dt><dd>Code Authoring Task Progress And Time</dd></div><div class="evidence-detail-row"><dt>Effect / result</dt><dd>1.15x completion rate, 0.57x time, 1.8x correctness</dd></div><div class="evidence-detail-row"><dt>Effect direction</dt><dd>Positive effect</dd></div><div class="evidence-detail-row"><dt>Relation to claim</dt><dd>Support</dd></div><div class="evidence-detail-row"><dt>Duration</dt><dd>3 Weeks Training</dd></div><div class="evidence-detail-row"><dt>Method</dt><dd>controlled experiment with random assignment, immediate post-test and 1-week retention test</dd></div><div class="evidence-detail-row"><dt>Strengths</dt><dd>randomized_controlled_design；immediate_post_test_and_retention_test；code_modification_task_guard</dd></div><div class="evidence-detail-row"><dt>Limitations</dt><dd>non_university_population_ages_10_17；small_sample_69；self-paced environment differs from classroom</dd></div><div class="evidence-detail-row"><dt>Confounders</dt><dd>Prior Programming Competency Interaction</dd></div><div class="evidence-detail-row"><dt>Quality dimensions</dt><dd>D1 Study design = 2 · D2 Sample quality = 2 · D3 Measurement validity = 2 · D4 Temporal strength = 2 · D5 Directness = 1</dd></div><div class="evidence-detail-row"><dt>Quality score</dt><dd>9.0</dd></div><div class="evidence-detail-row"><dt>Evidence level</dt><dd>strong</dd></div><div class="evidence-detail-row"><dt>Applicability</dt><dd>learner_match=partial_novice_programmers_but_younger · subject_match=introductory_programming · tool_match=codex_like_generative_ai · scope=task_performance_during_training</dd></div><div class="evidence-detail-row"><dt>Confidence</dt><dd>0.7</dd></div><div class="evidence-detail-row"><dt>Evidence status</dt><dd>Supported</dd></div><div class="evidence-detail-row"><dt>Full claim</dt><dd>AI coding assistants significantly increase task completion speed and completion rate during training.</dd></div><div class="evidence-detail-row"><dt>Source location</dt><dd>https://dl.acm.org/doi/10.1145/3544548.3580919</dd></div><div class="evidence-detail-row"><dt>Verifiable link</dt><dd><a href="https://dl.acm.org/doi/10.1145/3544548.3580919">https://dl.acm.org/doi/10.1145/3544548.3580919</a></dd></div></dl></details></td><td><a class="source-link" href="https://dl.acm.org/doi/10.1145/3544548.3580919" title="Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming"><code>S-2023-kazemitabaar</code></a></td></tr><tr data-effect="null" data-direction="null" data-outcome="independent_problem_solving" data-search="e-002 study-kazemitabaar-2023 smpl-kazemitabaar-2023-n69 studying the effect of ai code generators on supporting novice learners in introductory programming access to ai code generation did not decrease performance on manual code-modification tasks. 69 novices ages 10-17 with no prior text-based programming experience access_to_openai_codex_ai_coding_assistant_during_training baseline_group_without_ai_coding_assistant null s-2023-kazemitabaar"><td><code>E-002</code></td><td><strong>Independent problem solving</strong></td><td><span class="dir neu">Null effect</span><span class="relation-note">Claim relation: Neutral</span></td><td><div class="quality-cell"><span class="num">7</span><span class="quality-meter" aria-hidden="true"><i style="width:70%"></i></span></div></td><td class="claim-cell"><p>Access to AI code generation did not decrease performance on manual code-modification tasks.</p><details class="matrix-row-detail"><summary>View full evidence</summary><dl class="evidence-detail-grid"><div class="evidence-detail-row"><dt>Study ID</dt><dd>STUDY-KAZEMITABAAR-2023</dd></div><div class="evidence-detail-row"><dt>Sample ID</dt><dd>SMPL-KAZEMITABAAR-2023-N69</dd></div><div class="evidence-detail-row"><dt>Study title</dt><dd>Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming</dd></div><div class="evidence-detail-row"><dt>Source title</dt><dd>Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming</dd></div><div class="evidence-detail-row"><dt>Year</dt><dd>2023</dd></div><div class="evidence-detail-row"><dt>Study design</dt><dd>Randomized controlled trial</dd></div><div class="evidence-detail-row"><dt>Education level</dt><dd>K-12 Ages 10 17</dd></div><div class="evidence-detail-row"><dt>Population</dt><dd>69 novices ages 10-17 with no prior text-based programming experience</dd></div><div class="evidence-detail-row"><dt>Sample size</dt><dd>69</dd></div><div class="evidence-detail-row"><dt>Intervention</dt><dd>Access To Openai Codex AI Coding Assistant During Training</dd></div><div class="evidence-detail-row"><dt>Comparison</dt><dd>Baseline Group Without AI Coding Assistant</dd></div><div class="evidence-detail-row"><dt>Outcome measure</dt><dd>manual code-modification tasks during training</dd></div><div class="evidence-detail-row"><dt>Effect / result</dt><dd>no significant difference between groups</dd></div><div class="evidence-detail-row"><dt>Effect direction</dt><dd>Null effect</dd></div><div class="evidence-detail-row"><dt>Relation to claim</dt><dd>Neutral</dd></div><div class="evidence-detail-row"><dt>Duration</dt><dd>3 Weeks Training</dd></div><div class="evidence-detail-row"><dt>Method</dt><dd>controlled experiment, code-modification task followed each code-authoring task</dd></div><div class="evidence-detail-row"><dt>Strengths</dt><dd>direct_test_of_transfer-adjacent_skill；same_session_measurement</dd></div><div class="evidence-detail-row"><dt>Limitations</dt><dd>code modification is not full independent problem solving；non_university population</dd></div><div class="evidence-detail-row"><dt>Confounders</dt><dd>Practice Effect</dd></div><div class="evidence-detail-row"><dt>Quality dimensions</dt><dd>D1 Study design = 2 · D2 Sample quality = 2 · D3 Measurement validity = 1 · D4 Temporal strength = 1 · D5 Directness = 1</dd></div><div class="evidence-detail-row"><dt>Quality score</dt><dd>7.0</dd></div><div class="evidence-detail-row"><dt>Evidence level</dt><dd>moderate</dd></div><div class="evidence-detail-row"><dt>Applicability</dt><dd>learner_match=partial · subject_match=introductory_programming · tool_match=codex · scope=short-term manual code modification</dd></div><div class="evidence-detail-row"><dt>Confidence</dt><dd>0.5</dd></div><div class="evidence-detail-row"><dt>Evidence status</dt><dd>Supported</dd></div><div class="evidence-detail-row"><dt>Full claim</dt><dd>Access to AI code generation did not decrease performance on manual code-modification tasks.</dd></div><div class="evidence-detail-row"><dt>Source location</dt><dd>https://dl.acm.org/doi/10.1145/3544548.3580919</dd></div><div class="evidence-detail-row"><dt>Verifiable link</dt><dd><a href="https://dl.acm.org/doi/10.1145/3544548.3580919">https://dl.acm.org/doi/10.1145/3544548.3580919</a></dd></div></dl></details></td><td><a class="source-link" href="https://dl.acm.org/doi/10.1145/3544548.3580919" title="Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming"><code>S-2023-kazemitabaar</code></a></td></tr><tr data-effect="null" data-direction="null" data-outcome="retention" data-search="e-003 study-kazemitabaar-2023 smpl-kazemitabaar-2023-n69 studying the effect of ai code generators on supporting novice learners in introductory programming one week after training, retention differences between codex and baseline groups did not reach statistical significance. 69 novices ages 10-17 with no prior text-based programming experience access_to_openai_codex_ai_coding_assistant_during_training baseline_group_without_ai_coding_assistant null s-2023-kazemitabaar"><td><code>E-003</code></td><td><strong>Retention</strong></td><td><span class="dir neu">Null effect</span><span class="relation-note">Claim relation: Neutral</span></td><td><div class="quality-cell"><span class="num">9</span><span class="quality-meter" aria-hidden="true"><i style="width:90%"></i></span></div></td><td class="claim-cell"><p>One week after training, retention differences between Codex and baseline groups did not reach statistical significance.</p><details class="matrix-row-detail"><summary>View full evidence</summary><dl class="evidence-detail-grid"><div class="evidence-detail-row"><dt>Study ID</dt><dd>STUDY-KAZEMITABAAR-2023</dd></div><div class="evidence-detail-row"><dt>Sample ID</dt><dd>SMPL-KAZEMITABAAR-2023-N69</dd></div><div class="evidence-detail-row"><dt>Study title</dt><dd>Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming</dd></div><div class="evidence-detail-row"><dt>Source title</dt><dd>Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming</dd></div><div class="evidence-detail-row"><dt>Year</dt><dd>2023</dd></div><div class="evidence-detail-row"><dt>Study design</dt><dd>Randomized controlled trial</dd></div><div class="evidence-detail-row"><dt>Education level</dt><dd>K-12 Ages 10 17</dd></div><div class="evidence-detail-row"><dt>Population</dt><dd>69 novices ages 10-17 with no prior text-based programming experience</dd></div><div class="evidence-detail-row"><dt>Sample size</dt><dd>69</dd></div><div class="evidence-detail-row"><dt>Intervention</dt><dd>Access To Openai Codex AI Coding Assistant During Training</dd></div><div class="evidence-detail-row"><dt>Comparison</dt><dd>Baseline Group Without AI Coding Assistant</dd></div><div class="evidence-detail-row"><dt>Outcome measure</dt><dd>retention post-test one week after training</dd></div><div class="evidence-detail-row"><dt>Effect / result</dt><dd>slightly better for Codex group but not significant</dd></div><div class="evidence-detail-row"><dt>Effect direction</dt><dd>Null effect</dd></div><div class="evidence-detail-row"><dt>Relation to claim</dt><dd>Neutral</dd></div><div class="evidence-detail-row"><dt>Duration</dt><dd>3 Weeks Training Plus 1 Week Retention</dd></div><div class="evidence-detail-row"><dt>Method</dt><dd>controlled experiment with delayed retention test</dd></div><div class="evidence-detail-row"><dt>Strengths</dt><dd>delayed_test_included</dd></div><div class="evidence-detail-row"><dt>Limitations</dt><dd>1-week retention window is short；small sample；non-university population</dd></div><div class="evidence-detail-row"><dt>Confounders</dt><dd>Prior Competency</dd></div><div class="evidence-detail-row"><dt>Quality dimensions</dt><dd>D1 Study design = 2 · D2 Sample quality = 2 · D3 Measurement validity = 2 · D4 Temporal strength = 2 · D5 Directness = 1</dd></div><div class="evidence-detail-row"><dt>Quality score</dt><dd>9.0</dd></div><div class="evidence-detail-row"><dt>Evidence level</dt><dd>strong</dd></div><div class="evidence-detail-row"><dt>Applicability</dt><dd>learner_match=partial · subject_match=introductory_programming · tool_match=codex · scope=retention over one week</dd></div><div class="evidence-detail-row"><dt>Confidence</dt><dd>0.5</dd></div><div class="evidence-detail-row"><dt>Evidence status</dt><dd>Supported</dd></div><div class="evidence-detail-row"><dt>Full claim</dt><dd>One week after training, retention differences between Codex and baseline groups did not reach statistical significance.</dd></div><div class="evidence-detail-row"><dt>Source location</dt><dd>https://dl.acm.org/doi/10.1145/3544548.3580919</dd></div><div class="evidence-detail-row"><dt>Verifiable link</dt><dd><a href="https://dl.acm.org/doi/10.1145/3544548.3580919">https://dl.acm.org/doi/10.1145/3544548.3580919</a></dd></div></dl></details></td><td><a class="source-link" href="https://dl.acm.org/doi/10.1145/3544548.3580919" title="Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming"><code>S-2023-kazemitabaar</code></a></td></tr><tr data-effect="negative" data-direction="negative" data-outcome="independent_problem_solving" data-search="e-004 study-bastani-2025 smpl-bastani-2025-n950 generative ai without guardrails can harm learning: evidence from high school mathematics students with unguarded gpt-4 access performed 17% worse on the independent exam than the control group, despite higher practice performance. nearly a thousand high school math students in turkey gpt4_based_tutor_gpt_base_unguarded no_generative_ai_control negative s-2025-bastani"><td><code>E-004</code></td><td><strong>Independent problem solving</strong></td><td><span class="dir neg">Negative effect</span><span class="relation-note">Claim relation: Contradict</span></td><td><div class="quality-cell"><span class="num">8</span><span class="quality-meter" aria-hidden="true"><i style="width:80%"></i></span></div></td><td class="claim-cell"><p>Students with unguarded GPT-4 access performed 17% worse on the independent exam than the control group, despite higher practice performance.</p><details class="matrix-row-detail"><summary>View full evidence</summary><dl class="evidence-detail-grid"><div class="evidence-detail-row"><dt>Study ID</dt><dd>STUDY-BASTANI-2025</dd></div><div class="evidence-detail-row"><dt>Sample ID</dt><dd>SMPL-BASTANI-2025-N950</dd></div><div class="evidence-detail-row"><dt>Study title</dt><dd>Generative AI without guardrails can harm learning: Evidence from high school mathematics</dd></div><div class="evidence-detail-row"><dt>Source title</dt><dd>Generative AI without guardrails can harm learning: Evidence from high school mathematics</dd></div><div class="evidence-detail-row"><dt>Year</dt><dd>2025</dd></div><div class="evidence-detail-row"><dt>Study design</dt><dd>Randomized controlled trial</dd></div><div class="evidence-detail-row"><dt>Education level</dt><dd>High school</dd></div><div class="evidence-detail-row"><dt>Population</dt><dd>nearly a thousand high school math students in Turkey</dd></div><div class="evidence-detail-row"><dt>Sample size</dt><dd>950</dd></div><div class="evidence-detail-row"><dt>Intervention</dt><dd>Gpt4 Based Tutor Gpt Base Unguarded</dd></div><div class="evidence-detail-row"><dt>Comparison</dt><dd>No Generative AI Control</dd></div><div class="evidence-detail-row"><dt>Outcome measure</dt><dd>exam without access to AI resources after practice phase</dd></div><div class="evidence-detail-row"><dt>Effect / result</dt><dd>negative_17_percent_on_independent_exam</dd></div><div class="evidence-detail-row"><dt>Effect direction</dt><dd>Negative effect</dd></div><div class="evidence-detail-row"><dt>Relation to claim</dt><dd>Contradict</dd></div><div class="evidence-detail-row"><dt>Duration</dt><dd>In Class Study Sessions</dd></div><div class="evidence-detail-row"><dt>Method</dt><dd>large-scale randomized controlled trial, practice phase then closed-book exam</dd></div><div class="evidence-detail-row"><dt>Strengths</dt><dd>large_scale_rct；independent_exam_without_ai；arm_wise_design</dd></div><div class="evidence-detail-row"><dt>Limitations</dt><dd>high_school_mathematics_not_university_programming；single_country</dd></div><div class="evidence-detail-row"><dt>Confounders</dt><dd>Tool Design Difference</dd></div><div class="evidence-detail-row"><dt>Quality dimensions</dt><dd>D1 Study design = 2 · D2 Sample quality = 2 · D3 Measurement validity = 2 · D4 Temporal strength = 1 · D5 Directness = 1</dd></div><div class="evidence-detail-row"><dt>Quality score</dt><dd>8.0</dd></div><div class="evidence-detail-row"><dt>Evidence level</dt><dd>strong</dd></div><div class="evidence-detail-row"><dt>Applicability</dt><dd>learner_match=partial_same_age_band_different_subject · subject_match=no_mathematics_vs_programming · tool_match=gpt4_chat_interface · scope=unguarded_general_chat_interface</dd></div><div class="evidence-detail-row"><dt>Confidence</dt><dd>0.75</dd></div><div class="evidence-detail-row"><dt>Evidence status</dt><dd>Supported</dd></div><div class="evidence-detail-row"><dt>Full claim</dt><dd>Students with unguarded GPT-4 access performed 17% worse on the independent exam than the control group, despite higher practice performance.</dd></div><div class="evidence-detail-row"><dt>Source location</dt><dd>https://www.pnas.org/doi/10.1073/pnas.2422633122</dd></div><div class="evidence-detail-row"><dt>Verifiable link</dt><dd><a href="https://www.pnas.org/doi/10.1073/pnas.2422633122">https://www.pnas.org/doi/10.1073/pnas.2422633122</a></dd></div></dl></details></td><td><a class="source-link" href="https://www.pnas.org/doi/10.1073/pnas.2422633122" title="Generative AI without guardrails can harm learning: Evidence from high school mathematics"><code>S-2025-bastani</code></a></td></tr><tr data-effect="null" data-direction="null" data-outcome="independent_problem_solving" data-search="e-005 study-bastani-2025 smpl-bastani-2025-n950 generative ai without guardrails can harm learning: evidence from high school mathematics guardrail design of the ai tutor (hints instead of answers, teacher-informed prompts) largely eliminated the negative learning effect. nearly a thousand high school math students in turkey gpt4_tutor_with_teacher_designed_guardrails no_generative_ai_control null s-2025-bastani"><td><code>E-005</code></td><td><strong>Independent problem solving</strong></td><td><span class="dir neu">Null effect</span><span class="relation-note">Claim relation: Support</span></td><td><div class="quality-cell"><span class="num">8</span><span class="quality-meter" aria-hidden="true"><i style="width:80%"></i></span></div></td><td class="claim-cell"><p>Guardrail design of the AI tutor (hints instead of answers, teacher-informed prompts) largely eliminated the negative learning effect.</p><details class="matrix-row-detail"><summary>View full evidence</summary><dl class="evidence-detail-grid"><div class="evidence-detail-row"><dt>Study ID</dt><dd>STUDY-BASTANI-2025</dd></div><div class="evidence-detail-row"><dt>Sample ID</dt><dd>SMPL-BASTANI-2025-N950</dd></div><div class="evidence-detail-row"><dt>Study title</dt><dd>Generative AI without guardrails can harm learning: Evidence from high school mathematics</dd></div><div class="evidence-detail-row"><dt>Source title</dt><dd>Generative AI without guardrails can harm learning: Evidence from high school mathematics</dd></div><div class="evidence-detail-row"><dt>Year</dt><dd>2025</dd></div><div class="evidence-detail-row"><dt>Study design</dt><dd>Randomized controlled trial</dd></div><div class="evidence-detail-row"><dt>Education level</dt><dd>High school</dd></div><div class="evidence-detail-row"><dt>Population</dt><dd>nearly a thousand high school math students in Turkey</dd></div><div class="evidence-detail-row"><dt>Sample size</dt><dd>950</dd></div><div class="evidence-detail-row"><dt>Intervention</dt><dd>Gpt4 Tutor With Teacher Designed Guardrails</dd></div><div class="evidence-detail-row"><dt>Comparison</dt><dd>No Generative AI Control</dd></div><div class="evidence-detail-row"><dt>Outcome measure</dt><dd>exam without access to AI resources after practice phase</dd></div><div class="evidence-detail-row"><dt>Effect / result</dt><dd>negative effect essentially eradicated, no positive effect observed</dd></div><div class="evidence-detail-row"><dt>Effect direction</dt><dd>Null effect</dd></div><div class="evidence-detail-row"><dt>Relation to claim</dt><dd>Support</dd></div><div class="evidence-detail-row"><dt>Duration</dt><dd>In Class Study Sessions</dd></div><div class="evidence-detail-row"><dt>Method</dt><dd>large-scale randomized controlled trial, three arms</dd></div><div class="evidence-detail-row"><dt>Strengths</dt><dd>direct_manipulation_of_tool_design；large_sample</dd></div><div class="evidence-detail-row"><dt>Limitations</dt><dd>no_positive_learning_gain_even_with_guardrails；subject_mismatch</dd></div><div class="evidence-detail-row"><dt>Confounders</dt><dd>Prompt Engineering Effort</dd></div><div class="evidence-detail-row"><dt>Quality dimensions</dt><dd>D1 Study design = 2 · D2 Sample quality = 2 · D3 Measurement validity = 2 · D4 Temporal strength = 1 · D5 Directness = 1</dd></div><div class="evidence-detail-row"><dt>Quality score</dt><dd>8.0</dd></div><div class="evidence-detail-row"><dt>Evidence level</dt><dd>strong</dd></div><div class="evidence-detail-row"><dt>Applicability</dt><dd>learner_match=partial · subject_match=no · tool_match=guardrailed_tutor_design · scope=guardrail_design_principle_transferable</dd></div><div class="evidence-detail-row"><dt>Confidence</dt><dd>0.75</dd></div><div class="evidence-detail-row"><dt>Evidence status</dt><dd>Supported</dd></div><div class="evidence-detail-row"><dt>Full claim</dt><dd>Guardrail design of the AI tutor (hints instead of answers, teacher-informed prompts) largely eliminated the negative learning effect.</dd></div><div class="evidence-detail-row"><dt>Source location</dt><dd>https://www.pnas.org/doi/10.1073/pnas.2422633122</dd></div><div class="evidence-detail-row"><dt>Verifiable link</dt><dd><a href="https://www.pnas.org/doi/10.1073/pnas.2422633122">https://www.pnas.org/doi/10.1073/pnas.2422633122</a></dd></div></dl></details></td><td><a class="source-link" href="https://www.pnas.org/doi/10.1073/pnas.2422633122" title="Generative AI without guardrails can harm learning: Evidence from high school mathematics"><code>S-2025-bastani</code></a></td></tr><tr data-effect="positive" data-direction="positive" data-outcome="assignment_score" data-search="e-006 study-bastani-2025 smpl-bastani-2025-n950 generative ai without guardrails can harm learning: evidence from high school mathematics access to gpt-4 during practice improves task performance (48% for gpt base, 127% for gpt tutor) — but this task performance does not transfer to independent exam performance. nearly a thousand high school math students in turkey gpt4_tutor_access_during_practice no_generative_ai_control positive s-2025-bastani"><td><code>E-006</code></td><td><strong>Assignment score</strong></td><td><span class="dir pos">Positive effect</span><span class="relation-note">Claim relation: Support</span></td><td><div class="quality-cell"><span class="num">8</span><span class="quality-meter" aria-hidden="true"><i style="width:80%"></i></span></div></td><td class="claim-cell"><p>Access to GPT-4 during practice improves task performance (48% for GPT Base, 127% for GPT Tutor) — but this task performance does not transfer to independent exam performance.</p><details class="matrix-row-detail"><summary>View full evidence</summary><dl class="evidence-detail-grid"><div class="evidence-detail-row"><dt>Study ID</dt><dd>STUDY-BASTANI-2025</dd></div><div class="evidence-detail-row"><dt>Sample ID</dt><dd>SMPL-BASTANI-2025-N950</dd></div><div class="evidence-detail-row"><dt>Study title</dt><dd>Generative AI without guardrails can harm learning: Evidence from high school mathematics</dd></div><div class="evidence-detail-row"><dt>Source title</dt><dd>Generative AI without guardrails can harm learning: Evidence from high school mathematics</dd></div><div class="evidence-detail-row"><dt>Year</dt><dd>2025</dd></div><div class="evidence-detail-row"><dt>Study design</dt><dd>Randomized controlled trial</dd></div><div class="evidence-detail-row"><dt>Education level</dt><dd>High school</dd></div><div class="evidence-detail-row"><dt>Population</dt><dd>nearly a thousand high school math students in Turkey</dd></div><div class="evidence-detail-row"><dt>Sample size</dt><dd>950</dd></div><div class="evidence-detail-row"><dt>Intervention</dt><dd>Gpt4 Tutor Access During Practice</dd></div><div class="evidence-detail-row"><dt>Comparison</dt><dd>No Generative AI Control</dd></div><div class="evidence-detail-row"><dt>Outcome measure</dt><dd>practice problem performance during study sessions</dd></div><div class="evidence-detail-row"><dt>Effect / result</dt><dd>48-127 percent improvement on practice problems</dd></div><div class="evidence-detail-row"><dt>Effect direction</dt><dd>Positive effect</dd></div><div class="evidence-detail-row"><dt>Relation to claim</dt><dd>Support</dd></div><div class="evidence-detail-row"><dt>Duration</dt><dd>In Class Study Sessions</dd></div><div class="evidence-detail-row"><dt>Method</dt><dd>randomized controlled trial with practice and closed-book exam phases</dd></div><div class="evidence-detail-row"><dt>Strengths</dt><dd>same_study_compares_task_and_learning；large_sample</dd></div><div class="evidence-detail-row"><dt>Limitations</dt><dd>subject_mismatch_mathematics</dd></div><div class="evidence-detail-row"><dt>Confounders</dt><dd>Task Familiarity</dd></div><div class="evidence-detail-row"><dt>Quality dimensions</dt><dd>D1 Study design = 2 · D2 Sample quality = 2 · D3 Measurement validity = 2 · D4 Temporal strength = 1 · D5 Directness = 1</dd></div><div class="evidence-detail-row"><dt>Quality score</dt><dd>8.0</dd></div><div class="evidence-detail-row"><dt>Evidence level</dt><dd>strong</dd></div><div class="evidence-detail-row"><dt>Applicability</dt><dd>learner_match=partial · subject_match=no · tool_match=gpt4 · scope=task_performance_vs_learning_separation</dd></div><div class="evidence-detail-row"><dt>Confidence</dt><dd>0.75</dd></div><div class="evidence-detail-row"><dt>Evidence status</dt><dd>Supported</dd></div><div class="evidence-detail-row"><dt>Full claim</dt><dd>Access to GPT-4 during practice improves task performance (48% for GPT Base, 127% for GPT Tutor) — but this task performance does not transfer to independent exam performance.</dd></div><div class="evidence-detail-row"><dt>Source location</dt><dd>https://www.pnas.org/doi/10.1073/pnas.2422633122</dd></div><div class="evidence-detail-row"><dt>Verifiable link</dt><dd><a href="https://www.pnas.org/doi/10.1073/pnas.2422633122">https://www.pnas.org/doi/10.1073/pnas.2422633122</a></dd></div></dl></details></td><td><a class="source-link" href="https://www.pnas.org/doi/10.1073/pnas.2422633122" title="Generative AI without guardrails can harm learning: Evidence from high school mathematics"><code>S-2025-bastani</code></a></td></tr><tr data-effect="positive" data-direction="positive" data-outcome="knowledge_gain" data-search="e-007 study-marzuki-2024 smpl-marzuki-2024-n72 impact of chatgpt on esl students&#x27; academic writing skills chatgpt as a formative feedback tool produced a significant positive impact on students&#x27; academic writing skills with positive student perceptions. undergraduate esl students at an indian university chatgpt_as_formative_feedback_tool traditional_instruction_control positive s-2024-marzuki"><td><code>E-007</code></td><td><strong>Knowledge gain</strong></td><td><span class="dir pos">Positive effect</span><span class="relation-note">Claim relation: Support</span></td><td><div class="quality-cell"><span class="num">6</span><span class="quality-meter" aria-hidden="true"><i style="width:60%"></i></span></div></td><td class="claim-cell"><p>ChatGPT as a formative feedback tool produced a significant positive impact on students&#x27; academic writing skills with positive student perceptions.</p><details class="matrix-row-detail"><summary>View full evidence</summary><dl class="evidence-detail-grid"><div class="evidence-detail-row"><dt>Study ID</dt><dd>STUDY-MARZUKI-2024</dd></div><div class="evidence-detail-row"><dt>Sample ID</dt><dd>SMPL-MARZUKI-2024-N72</dd></div><div class="evidence-detail-row"><dt>Study title</dt><dd>Impact of ChatGPT on ESL students&#x27; academic writing skills</dd></div><div class="evidence-detail-row"><dt>Source title</dt><dd>Impact of ChatGPT on ESL students&#x27; academic writing skills</dd></div><div class="evidence-detail-row"><dt>Year</dt><dd>2024</dd></div><div class="evidence-detail-row"><dt>Study design</dt><dd>Mixed methods</dd></div><div class="evidence-detail-row"><dt>Education level</dt><dd>Undergraduate</dd></div><div class="evidence-detail-row"><dt>Population</dt><dd>undergraduate ESL students at an Indian university</dd></div><div class="evidence-detail-row"><dt>Sample size</dt><dd>72</dd></div><div class="evidence-detail-row"><dt>Intervention</dt><dd>Chatgpt As Formative Feedback Tool</dd></div><div class="evidence-detail-row"><dt>Comparison</dt><dd>Traditional Instruction Control</dd></div><div class="evidence-detail-row"><dt>Outcome measure</dt><dd>writing tests with pre-post-delayed design</dd></div><div class="evidence-detail-row"><dt>Effect / result</dt><dd>significant positive impact on writing skills</dd></div><div class="evidence-detail-row"><dt>Effect direction</dt><dd>Positive effect</dd></div><div class="evidence-detail-row"><dt>Relation to claim</dt><dd>Support</dd></div><div class="evidence-detail-row"><dt>Duration</dt><dd>6 Hours Intervention</dd></div><div class="evidence-detail-row"><dt>Method</dt><dd>mixed methods intervention study, pre/post/delayed tests and focus groups</dd></div><div class="evidence-detail-row"><dt>Strengths</dt><dd>delayed_post_test；mixed_methods_triangulation</dd></div><div class="evidence-detail-row"><dt>Limitations</dt><dd>short_intervention_6_hours；single_institution；elite_private_university</dd></div><div class="evidence-detail-row"><dt>Confounders</dt><dd>Self Selection Consent</dd></div><div class="evidence-detail-row"><dt>Quality dimensions</dt><dd>D1 Study design = 1 · D2 Sample quality = 1 · D3 Measurement validity = 2 · D4 Temporal strength = 2 · D5 Directness = 0</dd></div><div class="evidence-detail-row"><dt>Quality score</dt><dd>6.0</dd></div><div class="evidence-detail-row"><dt>Evidence level</dt><dd>moderate</dd></div><div class="evidence-detail-row"><dt>Applicability</dt><dd>learner_match=yes_undergraduate · subject_match=no_writing_not_programming · tool_match=chatgpt · scope=formative_feedback_writing</dd></div><div class="evidence-detail-row"><dt>Confidence</dt><dd>0.55</dd></div><div class="evidence-detail-row"><dt>Evidence status</dt><dd>Supported</dd></div><div class="evidence-detail-row"><dt>Full claim</dt><dd>ChatGPT as a formative feedback tool produced a significant positive impact on students&#x27; academic writing skills with positive student perceptions.</dd></div><div class="evidence-detail-row"><dt>Source location</dt><dd>https://link.springer.com/article/10.1186/s40561-024-00295-9</dd></div><div class="evidence-detail-row"><dt>Verifiable link</dt><dd><a href="https://link.springer.com/article/10.1186/s40561-024-00295-9">https://link.springer.com/article/10.1186/s40561-024-00295-9</a></dd></div></dl></details></td><td><a class="source-link" href="https://link.springer.com/article/10.1186/s40561-024-00295-9" title="Impact of ChatGPT on ESL students&#x27; academic writing skills"><code>S-2024-marzuki</code></a></td></tr><tr data-effect="positive" data-direction="positive" data-outcome="completion_time" data-search="e-008 study-peng-2023 smpl-peng-2023-n95 the impact of ai on developer productivity: evidence from github copilot professional developers with copilot access completed a standardized coding task about 55% faster than the control group (rct, n=95). 95 recruited professional developers completing a standardized coding task on a freelance platform access_to_github_copilot_during_task control_group_without_copilot positive s-2023-peng"><td><code>E-008</code></td><td><strong>Completion time</strong></td><td><span class="dir pos">Positive effect</span><span class="relation-note">Claim relation: Support</span></td><td><div class="quality-cell"><span class="num">8</span><span class="quality-meter" aria-hidden="true"><i style="width:80%"></i></span></div></td><td class="claim-cell"><p>Professional developers with Copilot access completed a standardized coding task about 55% faster than the control group (RCT, n=95).</p><details class="matrix-row-detail"><summary>View full evidence</summary><dl class="evidence-detail-grid"><div class="evidence-detail-row"><dt>Study ID</dt><dd>STUDY-PENG-2023</dd></div><div class="evidence-detail-row"><dt>Sample ID</dt><dd>SMPL-PENG-2023-N95</dd></div><div class="evidence-detail-row"><dt>Study title</dt><dd>The Impact of AI on Developer Productivity: Evidence from GitHub Copilot</dd></div><div class="evidence-detail-row"><dt>Source title</dt><dd>The Impact of AI on Developer Productivity: Evidence from GitHub Copilot</dd></div><div class="evidence-detail-row"><dt>Year</dt><dd>2023</dd></div><div class="evidence-detail-row"><dt>Study design</dt><dd>Randomized controlled trial</dd></div><div class="evidence-detail-row"><dt>Education level</dt><dd>Professional Developers Not Students</dd></div><div class="evidence-detail-row"><dt>Population</dt><dd>95 recruited professional developers completing a standardized coding task on a freelance platform</dd></div><div class="evidence-detail-row"><dt>Sample size</dt><dd>95</dd></div><div class="evidence-detail-row"><dt>Intervention</dt><dd>Access To Github Copilot During Task</dd></div><div class="evidence-detail-row"><dt>Comparison</dt><dd>Control Group Without Copilot</dd></div><div class="evidence-detail-row"><dt>Outcome measure</dt><dd>Time To Complete Http Server Implementation</dd></div><div class="evidence-detail-row"><dt>Effect / result</dt><dd>~55.8% faster task completion in Copilot group</dd></div><div class="evidence-detail-row"><dt>Effect direction</dt><dd>Positive effect</dd></div><div class="evidence-detail-row"><dt>Relation to claim</dt><dd>Support</dd></div><div class="evidence-detail-row"><dt>Duration</dt><dd>Single Task Session</dd></div><div class="evidence-detail-row"><dt>Method</dt><dd>online randomized controlled experiment with objective completion-time metric</dd></div><div class="evidence-detail-row"><dt>Strengths</dt><dd>randomized_controlled_design；objective_completion_time_metric</dd></div><div class="evidence-detail-row"><dt>Limitations</dt><dd>professional_population_not_students；single_task_ecology；preprint_not_peer_reviewed</dd></div><div class="evidence-detail-row"><dt>Confounders</dt><dd>Task Familiarity；Platform Recruitment Self Selection</dd></div><div class="evidence-detail-row"><dt>Quality dimensions</dt><dd>D1 Study design = 2 · D2 Sample quality = 2 · D3 Measurement validity = 2 · D4 Temporal strength = 1 · D5 Directness = 1</dd></div><div class="evidence-detail-row"><dt>Quality score</dt><dd>8.0</dd></div><div class="evidence-detail-row"><dt>Evidence level</dt><dd>moderate</dd></div><div class="evidence-detail-row"><dt>Applicability</dt><dd>learner_match=mismatch_professional_developers · subject_match=adjacent_web_development_task · tool_match=copilot_like_generative_ai · scope=task_performance_only_no_learning_outcome</dd></div><div class="evidence-detail-row"><dt>Confidence</dt><dd>0.6</dd></div><div class="evidence-detail-row"><dt>Evidence status</dt><dd>Supported</dd></div><div class="evidence-detail-row"><dt>Full claim</dt><dd>Professional developers with Copilot access completed a standardized coding task about 55% faster than the control group (RCT, n=95).</dd></div><div class="evidence-detail-row"><dt>Source location</dt><dd>https://doi.org/10.48550/arXiv.2302.06590</dd></div><div class="evidence-detail-row"><dt>Verifiable link</dt><dd><a href="https://doi.org/10.48550/arXiv.2302.06590">https://doi.org/10.48550/arXiv.2302.06590</a></dd></div></dl></details></td><td><a class="source-link" href="https://doi.org/10.48550/arXiv.2302.06590" title="The Impact of AI on Developer Productivity: Evidence from GitHub Copilot"><code>S-2023-peng</code></a></td></tr><tr data-effect="null" data-direction="null" data-outcome="code_quality" data-search="e-009 study-yetistiren-2023 smpl-yetistiren-2023-bench github copilot ai pair programmer: asset or liability? systematic benchmark evaluation reports mixed quality results for copilot-generated code relative to human code: correctness competitive on parts of the benchmark while security-relevant defects are documented. copilot-generated and human-written programs drawn from published benchmark datasets copilot_generated_programs human_written_programs_on_same_benchmarks null s-2023-yetistiren"><td><code>E-009</code></td><td><strong>Code quality</strong></td><td><span class="dir neu">Null effect</span><span class="relation-note">Claim relation: Neutral</span></td><td><div class="quality-cell"><span class="num">7</span><span class="quality-meter" aria-hidden="true"><i style="width:70%"></i></span></div></td><td class="claim-cell"><p>Systematic benchmark evaluation reports mixed quality results for Copilot-generated code relative to human code: correctness competitive on parts of the benchmark while security-relevant defects are documented.</p><details class="matrix-row-detail"><summary>View full evidence</summary><dl class="evidence-detail-grid"><div class="evidence-detail-row"><dt>Study ID</dt><dd>STUDY-YETISTIREN-2023</dd></div><div class="evidence-detail-row"><dt>Sample ID</dt><dd>SMPL-YETISTIREN-2023-BENCH</dd></div><div class="evidence-detail-row"><dt>Study title</dt><dd>GitHub Copilot AI Pair Programmer: Asset or Liability?</dd></div><div class="evidence-detail-row"><dt>Source title</dt><dd>GitHub Copilot AI Pair Programmer: Asset or Liability?</dd></div><div class="evidence-detail-row"><dt>Year</dt><dd>2023</dd></div><div class="evidence-detail-row"><dt>Study design</dt><dd>Observational</dd></div><div class="evidence-detail-row"><dt>Education level</dt><dd>Not Applicable Code Artifacts</dd></div><div class="evidence-detail-row"><dt>Population</dt><dd>Copilot-generated and human-written programs drawn from published benchmark datasets</dd></div><div class="evidence-detail-row"><dt>Intervention</dt><dd>Copilot Generated Programs</dd></div><div class="evidence-detail-row"><dt>Comparison</dt><dd>Human Written Programs On Same Benchmarks</dd></div><div class="evidence-detail-row"><dt>Outcome measure</dt><dd>Correctness Security Maintainability Metrics On Benchmarks</dd></div><div class="evidence-detail-row"><dt>Effect / result</dt><dd>mixed quality profile; no single-direction summary</dd></div><div class="evidence-detail-row"><dt>Effect direction</dt><dd>Null effect</dd></div><div class="evidence-detail-row"><dt>Relation to claim</dt><dd>Neutral</dd></div><div class="evidence-detail-row"><dt>Duration</dt><dd>Not Applicable Artifact Study</dd></div><div class="evidence-detail-row"><dt>Method</dt><dd>systematic empirical evaluation of generated code against human baselines on public benchmarks</dd></div><div class="evidence-detail-row"><dt>Strengths</dt><dd>multi_dimensional_quality_metrics；reproducible_benchmark_protocol</dd></div><div class="evidence-detail-row"><dt>Limitations</dt><dd>artifact_benchmark_not_classroom；no_learning_outcome；tool_version_from_2023</dd></div><div class="evidence-detail-row"><dt>Confounders</dt><dd>Benchmark Task Distribution</dd></div><div class="evidence-detail-row"><dt>Quality dimensions</dt><dd>D1 Study design = 1 · D2 Sample quality = 2 · D3 Measurement validity = 2 · D4 Temporal strength = 1 · D5 Directness = 1</dd></div><div class="evidence-detail-row"><dt>Quality score</dt><dd>7.0</dd></div><div class="evidence-detail-row"><dt>Evidence level</dt><dd>moderate</dd></div><div class="evidence-detail-row"><dt>Applicability</dt><dd>learner_match=mismatch_no_learners_in_study · subject_match=introductory_adjacent_code_tasks · tool_match=copilot_like_generative_ai · scope=output_quality_only</dd></div><div class="evidence-detail-row"><dt>Confidence</dt><dd>0.55</dd></div><div class="evidence-detail-row"><dt>Evidence status</dt><dd>Supported</dd></div><div class="evidence-detail-row"><dt>Full claim</dt><dd>Systematic benchmark evaluation reports mixed quality results for Copilot-generated code relative to human code: correctness competitive on parts of the benchmark while security-relevant defects are documented.</dd></div><div class="evidence-detail-row"><dt>Source location</dt><dd>https://doi.org/10.1016/j.jss.2023.111734</dd></div><div class="evidence-detail-row"><dt>Verifiable link</dt><dd><a href="https://doi.org/10.1016/j.jss.2023.111734">https://doi.org/10.1016/j.jss.2023.111734</a></dd></div></dl></details></td><td><a class="source-link" href="https://doi.org/10.1016/j.jss.2023.111734" title="GitHub Copilot AI Pair Programmer: Asset or Liability?"><code>S-2023-yetistiren</code></a></td></tr><tr data-effect="positive" data-direction="positive" data-outcome="assignment_score" data-search="e-010 study-finnieansley-2022 smpl-finnieansley-2022-qsets using github copilot to solve introductory programming problems codex produced passing-level solutions for roughly half to three-quarters of cs1 exam-style questions depending on the dataset, indicating substantial task-capability headroom available to novices. cs1 exam-style question sets answered by codex and compared against published student score distributions codex_answer_generation_on_cs1_questions published_student_cohort_score_distributions positive s-2022-finnie-ansley"><td><code>E-010</code></td><td><strong>Assignment score</strong></td><td><span class="dir pos">Positive effect</span><span class="relation-note">Claim relation: Support</span></td><td><div class="quality-cell"><span class="num">7</span><span class="quality-meter" aria-hidden="true"><i style="width:70%"></i></span></div></td><td class="claim-cell"><p>Codex produced passing-level solutions for roughly half to three-quarters of CS1 exam-style questions depending on the dataset, indicating substantial task-capability headroom available to novices.</p><details class="matrix-row-detail"><summary>View full evidence</summary><dl class="evidence-detail-grid"><div class="evidence-detail-row"><dt>Study ID</dt><dd>STUDY-FINNIEANSLEY-2022</dd></div><div class="evidence-detail-row"><dt>Sample ID</dt><dd>SMPL-FINNIEANSLEY-2022-QSETS</dd></div><div class="evidence-detail-row"><dt>Study title</dt><dd>Using GitHub Copilot to Solve Introductory Programming Problems</dd></div><div class="evidence-detail-row"><dt>Source title</dt><dd>Using GitHub Copilot to Solve Introductory Programming Problems</dd></div><div class="evidence-detail-row"><dt>Year</dt><dd>2022</dd></div><div class="evidence-detail-row"><dt>Study design</dt><dd>Observational</dd></div><div class="evidence-detail-row"><dt>Education level</dt><dd>University Year 1 Question Sets</dd></div><div class="evidence-detail-row"><dt>Population</dt><dd>CS1 exam-style question sets answered by Codex and compared against published student score distributions</dd></div><div class="evidence-detail-row"><dt>Intervention</dt><dd>Codex Answer Generation On CS1 Questions</dd></div><div class="evidence-detail-row"><dt>Comparison</dt><dd>Published Student Cohort Score Distributions</dd></div><div class="evidence-detail-row"><dt>Outcome measure</dt><dd>Pass Rate On CS1 Exam Style Questions</dd></div><div class="evidence-detail-row"><dt>Effect / result</dt><dd>passing solutions on ~50-75% of questions across datasets</dd></div><div class="evidence-detail-row"><dt>Effect direction</dt><dd>Positive effect</dd></div><div class="evidence-detail-row"><dt>Relation to claim</dt><dd>Support</dd></div><div class="evidence-detail-row"><dt>Duration</dt><dd>Not Applicable Capability Probe</dd></div><div class="evidence-detail-row"><dt>Method</dt><dd>capability benchmark against published student distributions; reproducible question sets</dd></div><div class="evidence-detail-row"><dt>Strengths</dt><dd>public_reproducible_question_sets；directly_relevant_task_domain</dd></div><div class="evidence-detail-row"><dt>Limitations</dt><dd>tool_solves_task_does_not_equate_student_learning；codex_2021_model_version_outdated</dd></div><div class="evidence-detail-row"><dt>Confounders</dt><dd>Question Leakage Into Training Data Possible</dd></div><div class="evidence-detail-row"><dt>Quality dimensions</dt><dd>D1 Study design = 1 · D2 Sample quality = 2 · D3 Measurement validity = 2 · D4 Temporal strength = 1 · D5 Directness = 1</dd></div><div class="evidence-detail-row"><dt>Quality score</dt><dd>7.0</dd></div><div class="evidence-detail-row"><dt>Evidence level</dt><dd>moderate</dd></div><div class="evidence-detail-row"><dt>Applicability</dt><dd>learner_match=partial_measures_tool_not_students · subject_match=introductory_programming · tool_match=copilot_like_generative_ai · scope=tool_capability_headroom</dd></div><div class="evidence-detail-row"><dt>Confidence</dt><dd>0.55</dd></div><div class="evidence-detail-row"><dt>Evidence status</dt><dd>Supported</dd></div><div class="evidence-detail-row"><dt>Full claim</dt><dd>Codex produced passing-level solutions for roughly half to three-quarters of CS1 exam-style questions depending on the dataset, indicating substantial task-capability headroom available to novices.</dd></div><div class="evidence-detail-row"><dt>Source location</dt><dd>https://dl.acm.org/doi/10.1145/3545945.3569830</dd></div><div class="evidence-detail-row"><dt>Verifiable link</dt><dd><a href="https://dl.acm.org/doi/10.1145/3545945.3569830">https://dl.acm.org/doi/10.1145/3545945.3569830</a></dd></div></dl></details></td><td><a class="source-link" href="https://dl.acm.org/doi/10.1145/3545945.3569830" title="Using GitHub Copilot to Solve Introductory Programming Problems"><code>S-2022-finnie-ansley</code></a></td></tr><tr data-effect="positive" data-direction="positive" data-outcome="metacognition" data-search="e-011 study-explcomp-2023 smpl-explcomp-2023-ratings comparing code explanations created by students and large language models controlled comparisons find llm-generated code explanations comparable to (in places better than) student-authored explanations, suggesting viability as explanatory scaffold material rather than as a replacement for student explanation practice. student-produced versus llm-produced explanations of short programs under controlled comparison llm_generated_code_explanations student_generated_explanations_of_same_programs positive s-2023-explanations-compare"><td><code>E-011</code></td><td><strong>Metacognition</strong></td><td><span class="dir pos">Positive effect</span><span class="relation-note">Claim relation: Support</span></td><td><div class="quality-cell"><span class="num">7</span><span class="quality-meter" aria-hidden="true"><i style="width:70%"></i></span></div></td><td class="claim-cell"><p>Controlled comparisons find LLM-generated code explanations comparable to (in places better than) student-authored explanations, suggesting viability as explanatory scaffold material rather than as a replacemen…</p><details class="matrix-row-detail"><summary>View full evidence</summary><dl class="evidence-detail-grid"><div class="evidence-detail-row"><dt>Study ID</dt><dd>STUDY-EXPLCOMP-2023</dd></div><div class="evidence-detail-row"><dt>Sample ID</dt><dd>SMPL-EXPLCOMP-2023-RATINGS</dd></div><div class="evidence-detail-row"><dt>Study title</dt><dd>Comparing Code Explanations Created by Students and Large Language Models</dd></div><div class="evidence-detail-row"><dt>Source title</dt><dd>Comparing Code Explanations Created by Students and Large Language Models</dd></div><div class="evidence-detail-row"><dt>Year</dt><dd>2023</dd></div><div class="evidence-detail-row"><dt>Study design</dt><dd>Observational</dd></div><div class="evidence-detail-row"><dt>Education level</dt><dd>University Introductory</dd></div><div class="evidence-detail-row"><dt>Population</dt><dd>Student-produced versus LLM-produced explanations of short programs under controlled comparison</dd></div><div class="evidence-detail-row"><dt>Intervention</dt><dd>LLM Generated Code Explanations</dd></div><div class="evidence-detail-row"><dt>Comparison</dt><dd>Student Generated Explanations Of Same Programs</dd></div><div class="evidence-detail-row"><dt>Outcome measure</dt><dd>Rated Explanation Quality And Comprehensibility</dd></div><div class="evidence-detail-row"><dt>Effect / result</dt><dd>comparable-or-better rated quality vs student explanations</dd></div><div class="evidence-detail-row"><dt>Effect direction</dt><dd>Positive effect</dd></div><div class="evidence-detail-row"><dt>Relation to claim</dt><dd>Support</dd></div><div class="evidence-detail-row"><dt>Duration</dt><dd>Single Session Ratings</dd></div><div class="evidence-detail-row"><dt>Method</dt><dd>controlled comparison with blind rating of explanation pairs</dd></div><div class="evidence-detail-row"><dt>Strengths</dt><dd>controlled_pairwise_comparison；learning_process_relevant_construct</dd></div><div class="evidence-detail-row"><dt>Limitations</dt><dd>short_term_ratings_not_learning_gains；small_program_snippets_ecology</dd></div><div class="evidence-detail-row"><dt>Confounders</dt><dd>Rating Criteria Subjectivity</dd></div><div class="evidence-detail-row"><dt>Quality dimensions</dt><dd>D1 Study design = 1 · D2 Sample quality = 2 · D3 Measurement validity = 2 · D4 Temporal strength = 1 · D5 Directness = 1</dd></div><div class="evidence-detail-row"><dt>Quality score</dt><dd>7.0</dd></div><div class="evidence-detail-row"><dt>Evidence level</dt><dd>moderate</dd></div><div class="evidence-detail-row"><dt>Applicability</dt><dd>learner_match=partial_scaffold_material_only · subject_match=introductory_programming · tool_match=llm_explanations · scope=scaffold_quality_not_effectiveness</dd></div><div class="evidence-detail-row"><dt>Confidence</dt><dd>0.55</dd></div><div class="evidence-detail-row"><dt>Evidence status</dt><dd>Supported</dd></div><div class="evidence-detail-row"><dt>Full claim</dt><dd>Controlled comparisons find LLM-generated code explanations comparable to (in places better than) student-authored explanations, suggesting viability as explanatory scaffold material rather than as a replacement for student explanation practice.</dd></div><div class="evidence-detail-row"><dt>Source location</dt><dd>https://dl.acm.org/doi/10.1145/3587102.3588785</dd></div><div class="evidence-detail-row"><dt>Verifiable link</dt><dd><a href="https://dl.acm.org/doi/10.1145/3587102.3588785">https://dl.acm.org/doi/10.1145/3587102.3588785</a></dd></div></dl></details></td><td><a class="source-link" href="https://dl.acm.org/doi/10.1145/3587102.3588785" title="Comparing Code Explanations Created by Students and Large Language Models"><code>S-2023-explanations-compare</code></a></td></tr><tr data-effect="negative" data-direction="negative" data-outcome="over_reliance" data-search="e-012 study-vaithilingam-2022 smpl-vaithilingam-2022-n24 expectation vs. experience: evaluating the usability of code generation tools despite faster first-task completion, participants struggled to understand and debug ai-generated solutions and reported low ownership of the final program - documenting metacognitive and dependence risks that pure speed metrics miss. 24 participants in a within-subjects usability study of copilot-style tools copilot_assisted_program_writing within_subject_baseline_without_tool negative s-2022-vaithilingam"><td><code>E-012</code></td><td><strong>Over-reliance</strong></td><td><span class="dir neg">Negative effect</span><span class="relation-note">Claim relation: Contradict</span></td><td><div class="quality-cell"><span class="num">7</span><span class="quality-meter" aria-hidden="true"><i style="width:70%"></i></span></div></td><td class="claim-cell"><p>Despite faster first-task completion, participants struggled to understand and debug AI-generated solutions and reported low ownership of the final program - documenting metacognitive and dependence risks that…</p><details class="matrix-row-detail"><summary>View full evidence</summary><dl class="evidence-detail-grid"><div class="evidence-detail-row"><dt>Study ID</dt><dd>STUDY-VAITHILINGAM-2022</dd></div><div class="evidence-detail-row"><dt>Sample ID</dt><dd>SMPL-VAITHILINGAM-2022-N24</dd></div><div class="evidence-detail-row"><dt>Study title</dt><dd>Expectation vs. Experience: Evaluating the Usability of Code Generation Tools</dd></div><div class="evidence-detail-row"><dt>Source title</dt><dd>Expectation vs. Experience: Evaluating the Usability of Code Generation Tools</dd></div><div class="evidence-detail-row"><dt>Year</dt><dd>2022</dd></div><div class="evidence-detail-row"><dt>Study design</dt><dd>Qualitative</dd></div><div class="evidence-detail-row"><dt>Education level</dt><dd>Mixed Cs Students And Professionals</dd></div><div class="evidence-detail-row"><dt>Population</dt><dd>24 participants in a within-subjects usability study of Copilot-style tools</dd></div><div class="evidence-detail-row"><dt>Sample size</dt><dd>24</dd></div><div class="evidence-detail-row"><dt>Intervention</dt><dd>Copilot Assisted Program Writing</dd></div><div class="evidence-detail-row"><dt>Comparison</dt><dd>Within Subject Baseline Without Tool</dd></div><div class="evidence-detail-row"><dt>Outcome measure</dt><dd>Understanding Ownership And Debugging Reports</dd></div><div class="evidence-detail-row"><dt>Effect / result</dt><dd>documented comprehension/ownership difficulties despite speed gain</dd></div><div class="evidence-detail-row"><dt>Effect direction</dt><dd>Negative effect</dd></div><div class="evidence-detail-row"><dt>Relation to claim</dt><dd>Contradict</dd></div><div class="evidence-detail-row"><dt>Duration</dt><dd>Single Session</dd></div><div class="evidence-detail-row"><dt>Method</dt><dd>within-subject usability study with tasks, observation and interviews</dd></div><div class="evidence-detail-row"><dt>Strengths</dt><dd>rich_qualitative_process_data；constructs_missed_by_speed_metrics</dd></div><div class="evidence-detail-row"><dt>Limitations</dt><dd>small_n_24；single_session；self_reported_understanding</dd></div><div class="evidence-detail-row"><dt>Confounders</dt><dd>Participant AI Familiarity</dd></div><div class="evidence-detail-row"><dt>Quality dimensions</dt><dd>D1 Study design = 1 · D2 Sample quality = 2 · D3 Measurement validity = 2 · D4 Temporal strength = 1 · D5 Directness = 1</dd></div><div class="evidence-detail-row"><dt>Quality score</dt><dd>7.0</dd></div><div class="evidence-detail-row"><dt>Evidence level</dt><dd>moderate</dd></div><div class="evidence-detail-row"><dt>Applicability</dt><dd>learner_match=partial_includes_cs_students · subject_match=programming_adjacent · tool_match=copilot_like_generative_ai · scope=risk_identification</dd></div><div class="evidence-detail-row"><dt>Confidence</dt><dd>0.55</dd></div><div class="evidence-detail-row"><dt>Evidence status</dt><dd>Contradicted</dd></div><div class="evidence-detail-row"><dt>Full claim</dt><dd>Despite faster first-task completion, participants struggled to understand and debug AI-generated solutions and reported low ownership of the final program - documenting metacognitive and dependence risks that pure speed metrics miss.</dd></div><div class="evidence-detail-row"><dt>Source location</dt><dd>https://dl.acm.org/doi/10.1145/3491101.3519665</dd></div><div class="evidence-detail-row"><dt>Verifiable link</dt><dd><a href="https://dl.acm.org/doi/10.1145/3491101.3519665">https://dl.acm.org/doi/10.1145/3491101.3519665</a></dd></div></dl></details></td><td><a class="source-link" href="https://dl.acm.org/doi/10.1145/3491101.3519665" title="Expectation vs. Experience: Evaluating the Usability of Code Generation Tools"><code>S-2022-vaithilingam</code></a></td></tr></tbody></table></div></div></section><section id="full-03-quality-en" class="full-chapter" data-full-chapter><header class="full-chapter-header"><h2>03 Evidence Quality, Counterevidence &amp; Method Audit</h2><p class="full-chapter-lead">Examine why evidence is credible, where it conflicts, and which conclusions require downgrading.</p></header><div class="full-chapter-body"><section class="method-review-group"><header class="method-review-title"><h3>Audit target: overall</h3><span class="method-verdict">Concern</span></header><div class="method-audit-grid"><article class="method-audit-item method-met"><div class="method-audit-head"><strong>Control Group</strong><span class="method-status">Met</span></div><p>All three studies include a no-AI control group.</p></article><article class="method-audit-item method-met"><div class="method-audit-head"><strong>Randomization</strong><span class="method-status">Met</span></div><p>Kazemitabaar 2023 and Bastani 2025 use randomized assignment.</p></article><article class="method-audit-item method-met"><div class="method-audit-head"><strong>Pre Test</strong><span class="method-status">Met</span></div><p>Kazemitabaar 2023 has a pre-study evaluation; Bastani 2025 measures baseline covariates.</p></article><article class="method-audit-item method-met"><div class="method-audit-head"><strong>Post Test</strong><span class="method-status">Met</span></div><p>Immediate post-tests present in all studies.</p></article><article class="method-audit-item method-partial"><div class="method-audit-head"><strong>Retention Test</strong><span class="method-status">Partial</span></div><p>Kazemitabaar 2023 has 1-week retention; Bastani 2025 has no delayed test; Marzuki 2024 has delayed test.</p></article><article class="method-audit-item method-partial"><div class="method-audit-head"><strong>Transfer Test</strong><span class="method-status">Partial</span></div><p>Kazemitabaar 2023 code-modification task is transfer-adjacent; no full no-AI transfer task.</p></article><article class="method-audit-item method-met"><div class="method-audit-head"><strong>Sample Bias</strong><span class="method-status">Met</span></div><p>Bastani 2025 nearly 1000 students; Kazemitabaar 2023 small (69) young sample.</p></article><article class="method-audit-item method-partial"><div class="method-audit-head"><strong>Self Selection</strong><span class="method-status">Partial</span></div><p>Marzuki 2024 consent-based participation risks self-selection.</p></article><article class="method-audit-item method-partial"><div class="method-audit-head"><strong>Measurement Validity</strong><span class="method-status">Partial</span></div><p>Practice/task performance is not equated to learning; independent exams present in Bastani 2025 only.</p></article><article class="method-audit-item method-partial"><div class="method-audit-head"><strong>Confounders</strong><span class="method-status">Partial</span></div><p>Prior programming competency interacts with AI benefit in Kazemitabaar 2023.</p></article><article class="method-audit-item method-not_applicable"><div class="method-audit-head"><strong>Instructor Effect</strong><span class="method-status">N/A</span></div><p>Kazemitabaar 2023 is self-paced; classroom studies may carry instructor effects.</p></article><article class="method-audit-item method-partial"><div class="method-audit-head"><strong>Novelty Effect</strong><span class="method-status">Partial</span></div><p>Short interventions likely inflate engagement; none of the studies controlled for novelty.</p></article><article class="method-audit-item method-not_applicable"><div class="method-audit-head"><strong>Tool Version Effect</strong><span class="method-status">N/A</span></div><p>Single tool versions studied; rapid tool change limits durability.</p></article><article class="method-audit-item method-partial"><div class="method-audit-head"><strong>Ai Usage Policy</strong><span class="method-status">Partial</span></div><p>Bastani 2025 explicitly contrasts unguarded vs guardrailed usage policies.</p></article><article class="method-audit-item method-partial"><div class="method-audit-head"><strong>Dropout</strong><span class="method-status">Partial</span></div><p>Marzuki 2024 reports attrition; others not detailed.</p></article></div><div class="method-guard-wrap"><strong>Task vs learning guard: </strong><p class="method-guard">Bastani 2025 demonstrates the danger of equating the two: +48-127% practice performance coexisted with -17% independent exam performance.</p></div></section><article class="conflict-card"><strong>Tribunal note: </strong><div class="conflict-text"><p>Disagreement comes from outcome separation (task vs learning), tool design (guarded vs unguarded), and population (K-12 / professionals vs university). Task-performance evidence is consistently positive across randomized and benchmark studies;…</p><details class="detail-expander"><summary>Expand full explanation</summary><div class="detail-body"><p>Disagreement comes from outcome separation (task vs learning), tool design (guarded vs unguarded), and population (K-12 / professionals vs university). Task-performance evidence is consistently positive across randomized and benchmark studies; the only study measuring independent performance after AI removal shows harm without guardrails; usability and artifact studies add dependence and quality caveats rather than resolving the learning question.</p></div></details></div></article><div class="evidence-tribunal" data-visual="evidence-tribunal-grid"><p class="tribunal-summary"><strong>Decision: </strong>Pilot · <strong>Confidence: </strong>Moderate</p><div class="tribunal-grid"><article class="tribunal-card supported"><header><span aria-hidden="true">✓</span><h3>Can claim</h3><span class="tribunal-count">7</span></header><ul><li><p>AI coding assistants reliably increase task performance during training (completion speed, correctness) — E-001, E-006.</p><div class="tribunal-evidence-refs"><code>E-001</code><code>E-006</code></div></li><li><p>Unguarded generative AI access can harm independent problem solving when access is removed — E-004.</p><div class="tribunal-evidence-refs"><code>E-004</code></div></li><li><p>Guardrail design (hints instead of answers) substantially mitigates the negative learning effect — E-005.</p><div class="tribunal-evidence-refs"><code>E-005</code></div></li><li><p>Task performance gains do not automatically imply learning gains — E-004 vs E-006 (within-study contrast).</p><div class="tribunal-evidence-refs"><code>E-004</code><code>E-006</code></div></li><li><p>Tool capability is substantial: Codex solves roughly half to three-quarters of CS1 exam-style questions — E-010.</p><div class="tribunal-evidence-refs"><code>E-010</code></div></li><li><p>Professional-developer RCT shows ~55% faster task completion with Copilot; directness limited by professional population — E-008.</p><div class="tribunal-evidence-refs"><code>E-008</code></div></li><li><p>LLM code explanations rate comparable to student-authored explanations, viable as scaffold material — E-011.</p><div class="tribunal-evidence-refs"><code>E-011</code></div></li></ul></article><article class="tribunal-card uncertain"><header><span aria-hidden="true">?</span><h3>Cannot yet claim</h3><span class="tribunal-count">4</span></header><ul><li><p>Whether AI coding assistants improve or preserve actual programming learning in university novices — no direct university-level RCT in reviewed set [无直接证据]</p></li><li><p>Whether one-week neutral retention (Kazemitabaar 2023) extends to a semester — E-003.</p><div class="tribunal-evidence-refs"><code>E-003</code></div></li><li><p>Whether benchmark quality findings (E-009) and explanation-quality ratings (E-011) translate into classroom learning gains.</p><div class="tribunal-evidence-refs"><code>E-009</code><code>E-011</code></div></li><li><p>How comprehension/ownership difficulties documented in usability studies (E-012) behave over a full semester with guardrails.</p><div class="tribunal-evidence-refs"><code>E-012</code></div></li></ul></article><article class="tribunal-card contradicted"><header><span aria-hidden="true">×</span><h3>Contradicted claims</h3><span class="tribunal-count">2</span></header><ul><li><p>The claim &#x27;AI tools always improve learning&#x27; is contradicted by E-004 (unguarded access, -17% independent exam).</p><div class="tribunal-evidence-refs"><code>E-004</code></div></li><li><p>The claim &#x27;speed gains equal learning gains&#x27; is contradicted by the task-vs-learning separation across E-001/E-006/E-008 vs E-004.</p><div class="tribunal-evidence-refs"><code>E-001</code><code>E-006</code><code>E-008</code><code>E-004</code></div></li></ul></article><article class="tribunal-card missing"><header><span aria-hidden="true">…</span><h3>Missing evidence</h3><span class="tribunal-count">4</span></header><ul><li><p>RCT of AI coding assistants in university programming courses with retention and no-AI transfer tests.</p></li><li><p>Studies varying AI usage policy within the same course.</p></li><li><p>Longitudinal data on AI dependency beyond one course.</p></li><li><p>Peer-reviewed replication of the professional speed RCT (Peng et al. remains a preprint).</p></li></ul></article></div><details class="supporting-visual"><summary>EvidenceFlow Protocol</summary><svg viewBox="0 0 720 260" xmlns="http://www.w3.org/2000/svg" role="img" aria-label="EvidenceFlow Protocol"><title>EvidenceFlow Protocol</title><desc>Research flow from framing, retrieval, fetch/verify, extraction, challenge, method audit and adjudication to applicability and intervention evaluation.</desc><defs><marker id="arr-en-full-workflow" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto"><path d="M0,0 L6,3 L0,6 Z" fill="#8A867E"/></marker></defs><rect width="720" height="260" fill="#FCFAF6" rx="12"/><text x="24" y="34" font-size="16" font-weight="700" fill="#3A3833">EvidenceFlow Protocol</text><rect x="24" y="100" width="64" height="52" rx="8" fill="#B8694A"/><text x="56.0" y="126.0" text-anchor="middle" fill="#FFFFFF" font-size="10" font-weight="600"><tspan x="56.0" y="126.0">Frame</tspan></text><line x1="88" y1="126" x2="100" y2="126" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-en-full-workflow)"/><rect x="100" y="100" width="64" height="52" rx="8" fill="#B8694A"/><text x="132.0" y="126.0" text-anchor="middle" fill="#FFFFFF" font-size="10" font-weight="600"><tspan x="132.0" y="126.0">Retrieve</tspan></text><line x1="164" y1="126" x2="176" y2="126" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-en-full-workflow)"/><rect x="176" y="100" width="64" height="52" rx="8" fill="#B8694A"/><text x="208.0" y="126.0" text-anchor="middle" fill="#FFFFFF" font-size="10" font-weight="600"><tspan x="208.0" y="126.0">Fetch</tspan><tspan x="208.0" y="140.0">Verify</tspan></text><line x1="240" y1="126" x2="252" y2="126" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-en-full-workflow)"/><rect x="252" y="100" width="64" height="52" rx="8" fill="#B8694A"/><text x="284.0" y="126.0" text-anchor="middle" fill="#FFFFFF" font-size="10" font-weight="600"><tspan x="284.0" y="126.0">Extract</tspan></text><line x1="316" y1="126" x2="328" y2="126" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-en-full-workflow)"/><rect x="328" y="100" width="64" height="52" rx="8" fill="#5E8A6A"/><text x="360.0" y="126.0" text-anchor="middle" fill="#FFFFFF" font-size="10" font-weight="600"><tspan x="360.0" y="126.0">Challenge</tspan></text><line x1="392" y1="126" x2="404" y2="126" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-en-full-workflow)"/><rect x="404" y="100" width="64" height="52" rx="8" fill="#5E8A6A"/><text x="436.0" y="126.0" text-anchor="middle" fill="#FFFFFF" font-size="10" font-weight="600"><tspan x="436.0" y="126.0">Audit</tspan><tspan x="436.0" y="140.0">Method</tspan></text><line x1="468" y1="126" x2="480" y2="126" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-en-full-workflow)"/><rect x="480" y="100" width="64" height="52" rx="8" fill="#C99A4A"/><text x="512.0" y="126.0" text-anchor="middle" fill="#FFFFFF" font-size="10" font-weight="600"><tspan x="512.0" y="126.0">Adjudicate</tspan></text><line x1="544" y1="126" x2="556" y2="126" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-en-full-workflow)"/><rect x="556" y="100" width="64" height="52" rx="8" fill="#C99A4A"/><text x="588.0" y="126.0" text-anchor="middle" fill="#FFFFFF" font-size="10" font-weight="600"><tspan x="588.0" y="126.0">Applicability</tspan></text><line x1="620" y1="126" x2="632" y2="126" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-en-full-workflow)"/><rect x="632" y="100" width="64" height="52" rx="8" fill="#C99A4A"/><text x="664.0" y="126.0" text-anchor="middle" fill="#FFFFFF" font-size="10" font-weight="600"><tspan x="664.0" y="126.0">Intervene</tspan><tspan x="664.0" y="140.0">Evaluate</tspan></text></svg></details><details class="supporting-visual"><summary>Tribunal infographic</summary><svg viewBox="0 0 720 260" xmlns="http://www.w3.org/2000/svg" role="img" aria-label="Evidence Tribunal infographic"><title>Evidence Tribunal infographic</title><desc>Evidence IDs for claims that can and cannot be claimed, plus the recommended action badge; full claim text is in the tribunal cards below.</desc><defs><marker id="arr-en-full-tribunal" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto"><path d="M0,0 L6,3 L0,6 Z" fill="#8A867E"/></marker></defs><rect width="720" height="260" fill="#FCFAF6" rx="12"/><text x="24" y="34" font-size="16" font-weight="700" fill="#3A3833">Evidence Tribunal</text><text x="24" y="180" font-size="13" font-weight="700" fill="#3A3833">Source of conflict</text><text x="24" y="202" font-size="11" fill="#8A867E">See tribunal cards below</text><rect x="24" y="216" width="180" height="28" rx="14" fill="#C99A4A"/><text x="114" y="235" text-anchor="middle" font-size="13" font-weight="700" fill="#fff">Pilot</text><text x="250" y="80" font-size="13" font-weight="700" fill="#5E8A6A">Can claim (7)</text><circle cx="254" cy="100" r="3" fill="#5E8A6A"/><text x="266" y="105" font-size="11" fill="#3A3833">E-001</text><circle cx="254" cy="122" r="3" fill="#5E8A6A"/><text x="266" y="127" font-size="11" fill="#3A3833">E-006</text><circle cx="254" cy="144" r="3" fill="#5E8A6A"/><text x="266" y="149" font-size="11" fill="#3A3833">E-004</text><circle cx="254" cy="166" r="3" fill="#5E8A6A"/><text x="266" y="171" font-size="11" fill="#3A3833">E-005</text><text x="490" y="80" font-size="13" font-weight="700" fill="#A85B53">Cannot claim (2)</text><circle cx="494" cy="100" r="3" fill="#A85B53"/><text x="506" y="105" font-size="11" fill="#3A3833">E-004</text><circle cx="494" cy="122" r="3" fill="#A85B53"/><text x="506" y="127" font-size="11" fill="#3A3833">E-001</text><circle cx="494" cy="144" r="3" fill="#A85B53"/><text x="506" y="149" font-size="11" fill="#3A3833">E-006</text><circle cx="494" cy="166" r="3" fill="#A85B53"/><text x="506" y="171" font-size="11" fill="#3A3833">E-008</text></svg></details></div><div class="trace-chain" data-visual="trace-chain"><article class="trace-chain-card"><div class="trace-claim-node"><strong>C-001</strong><p class="trace-claim-text">AI coding assistants significantly increase task completion speed and completion rate during training.</p></div><div class="trace-arrow">↓</div><div class="trace-evidence-list"><div class="trace-evidence-node"><code>E-001</code><span>Completion time</span><span class="dir pos">Positive effect</span><span class="trace-relation">claim relation: Support</span><span>Q 9.0</span><span class="trace-arrow">→</span><a href="https://dl.acm.org/doi/10.1145/3544548.3580919"><code>S-2023-kazemitabaar</code></a></div></div></article><article class="trace-chain-card"><div class="trace-claim-node"><strong>C-002</strong><p class="trace-claim-text">Access to AI code generation did not decrease performance on manual code-modification tasks.</p></div><div class="trace-arrow">↓</div><div class="trace-evidence-list"><div class="trace-evidence-node"><code>E-002</code><span>Independent problem solving</span><span class="dir neu">Null effect</span><span class="trace-relation">claim relation: Neutral</span><span>Q 7.0</span><span class="trace-arrow">→</span><a href="https://dl.acm.org/doi/10.1145/3544548.3580919"><code>S-2023-kazemitabaar</code></a></div></div></article><article class="trace-chain-card"><div class="trace-claim-node"><strong>C-003</strong><p class="trace-claim-text">One week after training, retention differences between Codex and baseline groups did not reach statistical significance.</p></div><div class="trace-arrow">↓</div><div class="trace-evidence-list"><div class="trace-evidence-node"><code>E-003</code><span>Retention</span><span class="dir neu">Null effect</span><span class="trace-relation">claim relation: Neutral</span><span>Q 9.0</span><span class="trace-arrow">→</span><a href="https://dl.acm.org/doi/10.1145/3544548.3580919"><code>S-2023-kazemitabaar</code></a></div></div></article><article class="trace-chain-card"><div class="trace-claim-node"><strong>C-004</strong><p class="trace-claim-text">Students with unguarded GPT-4 access performed 17% worse on the independent exam than the control group, despite higher practice performance.</p></div><div class="trace-arrow">↓</div><div class="trace-evidence-list"><div class="trace-evidence-node"><code>E-004</code><span>Independent problem solving</span><span class="dir neg">Negative effect</span><span class="trace-relation">claim relation: Contradict</span><span>Q 8.0</span><span class="trace-arrow">→</span><a href="https://www.pnas.org/doi/10.1073/pnas.2422633122"><code>S-2025-bastani</code></a></div></div></article><article class="trace-chain-card"><div class="trace-claim-node"><strong>C-005</strong><p class="trace-claim-text">Guardrail design of the AI tutor (hints instead of answers, teacher-informed prompts) largely eliminated the negative learning effect.</p></div><div class="trace-arrow">↓</div><div class="trace-evidence-list"><div class="trace-evidence-node"><code>E-005</code><span>Independent problem solving</span><span class="dir neu">Null effect</span><span class="trace-relation">claim relation: Support</span><span>Q 8.0</span><span class="trace-arrow">→</span><a href="https://www.pnas.org/doi/10.1073/pnas.2422633122"><code>S-2025-bastani</code></a></div></div></article><article class="trace-chain-card"><div class="trace-claim-node"><strong>C-006</strong><p class="trace-claim-text">Access to GPT-4 during practice improves task performance (48% for GPT Base, 127% for GPT Tutor) — but this task performance does not transfer to independent exam performance.</p></div><div class="trace-arrow">↓</div><div class="trace-evidence-list"><div class="trace-evidence-node"><code>E-006</code><span>Assignment score</span><span class="dir pos">Positive effect</span><span class="trace-relation">claim relation: Support</span><span>Q 8.0</span><span class="trace-arrow">→</span><a href="https://www.pnas.org/doi/10.1073/pnas.2422633122"><code>S-2025-bastani</code></a></div></div></article><article class="trace-chain-card"><div class="trace-claim-node"><strong>C-007</strong><p class="trace-claim-text">ChatGPT as a formative feedback tool produced a significant positive impact on students&#x27; academic writing skills with positive student perceptions.</p></div><div class="trace-arrow">↓</div><div class="trace-evidence-list"><div class="trace-evidence-node"><code>E-007</code><span>Knowledge gain</span><span class="dir pos">Positive effect</span><span class="trace-relation">claim relation: Support</span><span>Q 6.0</span><span class="trace-arrow">→</span><a href="https://link.springer.com/article/10.1186/s40561-024-00295-9"><code>S-2024-marzuki</code></a></div></div></article><article class="trace-chain-card"><div class="trace-claim-node"><strong>C-008</strong><p class="trace-claim-text">Professional developers with Copilot access completed a standardized coding task about 55% faster than the control group (RCT, n=95).</p></div><div class="trace-arrow">↓</div><div class="trace-evidence-list"><div class="trace-evidence-node"><code>E-008</code><span>Completion time</span><span class="dir pos">Positive effect</span><span class="trace-relation">claim relation: Support</span><span>Q 8.0</span><span class="trace-arrow">→</span><a href="https://doi.org/10.48550/arXiv.2302.06590"><code>S-2023-peng</code></a></div></div></article><article class="trace-chain-card"><div class="trace-claim-node"><strong>C-009</strong><p class="trace-claim-text">Systematic benchmark evaluation reports mixed quality results for Copilot-generated code relative to human code: correctness competitive on parts of the benchmark while security-relevant defects are documented.</p></div><div class="trace-arrow">↓</div><div class="trace-evidence-list"><div class="trace-evidence-node"><code>E-009</code><span>Code quality</span><span class="dir neu">Null effect</span><span class="trace-relation">claim relation: Neutral</span><span>Q 7.0</span><span class="trace-arrow">→</span><a href="https://doi.org/10.1016/j.jss.2023.111734"><code>S-2023-yetistiren</code></a></div></div></article><article class="trace-chain-card"><div class="trace-claim-node"><strong>C-010</strong><p class="trace-claim-text">Codex produced passing-level solutions for roughly half to three-quarters of CS1 exam-style questions depending on the dataset, indicating substantial task-capability headroom available to novices.</p></div><div class="trace-arrow">↓</div><div class="trace-evidence-list"><div class="trace-evidence-node"><code>E-010</code><span>Assignment score</span><span class="dir pos">Positive effect</span><span class="trace-relation">claim relation: Support</span><span>Q 7.0</span><span class="trace-arrow">→</span><a href="https://dl.acm.org/doi/10.1145/3545945.3569830"><code>S-2022-finnie-ansley</code></a></div></div></article><article class="trace-chain-card"><div class="trace-claim-node"><strong>C-011</strong><div class="trace-claim-text"><p>Controlled comparisons find LLM-generated code explanations comparable to (in places better than) student-authored explanations, suggesting viability as explanatory scaffold material rather than as a replacement for student explanation prac…</p><details class="detail-expander"><summary>Expand full explanation</summary><div class="detail-body"><p>Controlled comparisons find LLM-generated code explanations comparable to (in places better than) student-authored explanations, suggesting viability as explanatory scaffold material rather than as a replacement for student explanation practice.</p></div></details></div></div><div class="trace-arrow">↓</div><div class="trace-evidence-list"><div class="trace-evidence-node"><code>E-011</code><span>Metacognition</span><span class="dir pos">Positive effect</span><span class="trace-relation">claim relation: Support</span><span>Q 7.0</span><span class="trace-arrow">→</span><a href="https://dl.acm.org/doi/10.1145/3587102.3588785"><code>S-2023-explanations-compare</code></a></div></div></article><article class="trace-chain-card"><div class="trace-claim-node"><strong>C-012</strong><p class="trace-claim-text">Despite faster first-task completion, participants struggled to understand and debug AI-generated solutions and reported low ownership of the final program - documenting metacognitive and dependence risks that pure speed metrics miss.</p></div><div class="trace-arrow">↓</div><div class="trace-evidence-list"><div class="trace-evidence-node"><code>E-012</code><span>Over-reliance</span><span class="dir neg">Negative effect</span><span class="trace-relation">claim relation: Contradict</span><span>Q 7.0</span><span class="trace-arrow">→</span><a href="https://dl.acm.org/doi/10.1145/3491101.3519665"><code>S-2022-vaithilingam</code></a></div></div></article></div><div id="chart-trace-en" class="chart-mount" aria-label="Claim-Evidence Trace"></div><p class="chart-interpretation"><strong>What this means: </strong>Every important claim must resolve to Evidence IDs and original sources.</p></div></section><section id="full-04-action-en" class="full-chapter" data-full-chapter><header class="full-chapter-header"><h2>04 Applicability &amp; Teaching Action</h2><p class="full-chapter-lead">Connect applicability, guardrails and teaching actions to specific evidence.</p></header><div class="full-chapter-body"><p><strong>Target population: </strong>university first-year computer science students learning C programming for the first time</p>
2509
+ </div><div class="scope-grid"><article class="scope-card"><h3>Research question</h3><p class="scope-text">我们准备在大学一年级 C 语言课程中允许学生使用生成式 AI 编程助手。它到底会不会提高学习效果？应该怎样引入？</p></article><article class="scope-card"><h3>Target learners</h3><p class="scope-text">Education level: First-year undergraduate; Major: Computer science; Prior knowledge: First programming course, no prior text-based programming; Learner characteristics: Mixed-ability large class (60 students)</p></article><article class="scope-card"><h3>Course context</h3><p class="scope-text">Subject: C programming; Course type: Lecture + lab; Duration: 16 weeks (one semester)</p></article><article class="scope-card"><h3>AI intervention</h3><p class="scope-text">Method: Lecture with lab exercises; AI tool: Generative AI coding assistant; Allowed usage: Under design (pending evidence review); Frequency: Weekly lab sessions; Duration: One semester</p></article><article class="scope-card"><h3>Comparison</h3><p class="scope-text">No AI Coding Assistant Control</p></article><article class="scope-card"><h3>Outcome constructs</h3><p class="scope-text">Primary outcomes: Independent problem solving, Code quality; Secondary outcomes: Completion time, Retention, Knowledge gain; Risk outcomes: AI dependency, Over-reliance, Reduced transfer</p></article><article class="scope-card"><h3>Research scope</h3><p class="scope-text">Time range: 2021 2026; Geography: Worldwide; Study designs: Randomized controlled trial, Quasi-experimental, Observational</p></article><article class="scope-card"><h3>Decision success condition</h3><p class="scope-text">independent problem solving and code quality improve (or do not decline) while AI dependency risk stays controlled; evidence base supports a bounded pilot.</p></article></div></div></section><section id="full-02-evidence-en" class="full-chapter" data-full-chapter><header class="full-chapter-header"><h2>02 Key Evidence &amp; Outcome Separation</h2><p class="full-chapter-lead">Place task performance, actual learning, retention and risk on one evidence map without conflating them.</p></header><div class="full-chapter-body"><div class="retrieval-grid"><article><h3>Inclusion criteria</h3><ul><li>Studies Of Generative AI Coding Tools In Learning To Program</li><li>Outcomes Measuring Learning Not Only Task Speed</li><li>University Or Novice Programming Populations</li></ul></article><article><h3>Exclusion criteria</h3><ul><li>Practitioner Anecdotes Without Data</li><li>Industry Professional Populations Only</li></ul></article></div><div class="retrieval-coverage"><h3>Source coverage</h3><p><code>S-2023-kazemitabaar</code> <code>S-2025-bastani</code> <code>S-2024-marzuki</code> <code>S-2023-peng</code> <code>S-2023-yetistiren</code> <code>S-2022-finnie-ansley</code> <code>S-2023-explanations-compare</code> <code>S-2022-vaithilingam</code></p><p class="retrieval-note">This report shows only retrieval metadata present in result; it does not fabricate PRISMA/funnel counts when none exist.</p></div><div class="outcome-separation" data-visual="outcome-separation"><div class="visual-heading"><h3>Outcome Separation · Task performance ≠ learning</h3><p>Outcomes are adjudicated separately so faster training performance is not silently treated as evidence of learning.</p><p class="semantic-note">Effect direction comes from evidence.effect_direction; supporting a claim does not imply a positive outcome.</p></div><div class="outcome-groups"><article class="outcome-group outcome-task"><h3>Task / proximal performance</h3><ul><li><strong>Completion time</strong><span class="outcome-states"><span class="dir pos">Positive effect 2</span></span></li><li><strong>Code quality</strong><span class="outcome-states"><span class="dir neu">Null effect 1</span></span></li><li><strong>Assignment score</strong><span class="outcome-states"><span class="dir pos">Positive effect 2</span></span></li></ul></article><article class="outcome-group outcome-learning"><h3>Learning / retention / transfer</h3><ul><li><strong>Knowledge gain</strong><span class="outcome-states"><span class="dir pos">Positive effect 1</span></span></li><li><strong>Retention</strong><span class="outcome-states"><span class="dir neu">Null effect 1</span></span></li><li><strong>Independent problem solving</strong><span class="outcome-states"><span class="dir neg">Negative effect 1</span><span class="dir neu">Null effect 2</span></span></li></ul></article><article class="outcome-group outcome-risk"><h3>Risk / dependency</h3><ul><li><strong>Over-reliance</strong><span class="outcome-states"><span class="dir neg">Negative effect 1</span></span></li></ul></article><article class="outcome-group outcome-other"><h3>Other outcomes</h3><ul><li><strong>Metacognition</strong><span class="outcome-states"><span class="dir pos">Positive effect 1</span></span></li></ul></article></div></div><div class='table-wrap outcome-table'><table class='data-table'><thead><tr><th>Outcome</th><th>Positive effect</th><th>Negative effect</th><th>Null effect</th><th>Evidence</th></tr></thead><tbody><tr><td><strong>Knowledge gain</strong><span class='raw-tag' title='raw id'>knowledge_gain</span></td><td class='num'>1</td><td class='num'>0</td><td class='num'>0</td><td><code>E-007</code> </td></tr><tr><td><strong>Retention</strong><span class='raw-tag' title='raw id'>retention</span></td><td class='num'>0</td><td class='num'>0</td><td class='num'>1</td><td><code>E-003</code> </td></tr><tr><td><strong>Independent problem solving</strong><span class='raw-tag' title='raw id'>independent_problem_solving</span></td><td class='num'>0</td><td class='num'>1</td><td class='num'>2</td><td><code>E-002</code> <code>E-004</code> <code>E-005</code> </td></tr><tr><td><strong>Completion time</strong><span class='raw-tag' title='raw id'>completion_time</span></td><td class='num'>2</td><td class='num'>0</td><td class='num'>0</td><td><code>E-001</code> <code>E-008</code> </td></tr><tr><td><strong>Code quality</strong><span class='raw-tag' title='raw id'>code_quality</span></td><td class='num'>0</td><td class='num'>0</td><td class='num'>1</td><td><code>E-009</code> </td></tr><tr><td><strong>Assignment score</strong><span class='raw-tag' title='raw id'>assignment_score</span></td><td class='num'>2</td><td class='num'>0</td><td class='num'>0</td><td><code>E-006</code> <code>E-010</code> </td></tr><tr><td><strong>Metacognition</strong><span class='raw-tag' title='raw id'>metacognition</span></td><td class='num'>1</td><td class='num'>0</td><td class='num'>0</td><td><code>E-011</code> </td></tr><tr><td><strong>Over-reliance</strong><span class='raw-tag' title='raw id'>over_reliance</span></td><td class='num'>0</td><td class='num'>1</td><td class='num'>0</td><td><code>E-012</code> </td></tr></tbody></table></div><div class="visual-surface" data-visual="outcome-evidence-balance"><svg viewBox="0 0 720 300" xmlns="http://www.w3.org/2000/svg" role="img" aria-label="Outcome evidence effect balance"><title>Outcome evidence effect balance</title><desc>Positive / negative / null effect counts per outcome (based on effect_direction, not claim support).</desc><rect x="0" y="0" width="720" height="300" fill="#FFFFFF"/><line x1="439.0" y1="46" x2="439.0" y2="182" stroke="#999" stroke-width="1" stroke-dasharray="3,3"/><text x="190" y="58.0" text-anchor="end" font-size="11" fill="#333">Knowledge gain</text><rect x="439.0" y="49.8" width="120.5" height="9.4" fill="#5E8A6A"/><text x="499.2" y="58.0" text-anchor="middle" font-size="9" fill="#fff">1</text><text x="190" y="75.0" text-anchor="end" font-size="11" fill="#333">Retention</text><text x="190" y="92.0" text-anchor="end" font-size="11" fill="#333">Independent problem solving</text><rect x="318.5" y="83.8" width="120.5" height="9.4" fill="#A85B53"/><text x="378.8" y="92.0" text-anchor="middle" font-size="9" fill="#fff">-1</text><text x="190" y="109.0" text-anchor="end" font-size="11" fill="#333">Completion time</text><rect x="439.0" y="100.8" width="241.0" height="9.4" fill="#5E8A6A"/><text x="559.5" y="109.0" text-anchor="middle" font-size="9" fill="#fff">2</text><text x="190" y="126.0" text-anchor="end" font-size="11" fill="#333">Code quality</text><text x="190" y="143.0" text-anchor="end" font-size="11" fill="#333">Assignment score</text><rect x="439.0" y="134.8" width="241.0" height="9.4" fill="#5E8A6A"/><text x="559.5" y="143.0" text-anchor="middle" font-size="9" fill="#fff">2</text><text x="190" y="160.0" text-anchor="end" font-size="11" fill="#333">Metacognition</text><rect x="439.0" y="151.8" width="120.5" height="9.4" fill="#5E8A6A"/><text x="499.2" y="160.0" text-anchor="middle" font-size="9" fill="#fff">1</text><text x="190" y="177.0" text-anchor="end" font-size="11" fill="#333">Over-reliance</text><rect x="318.5" y="168.8" width="120.5" height="9.4" fill="#A85B53"/><text x="378.8" y="177.0" text-anchor="middle" font-size="9" fill="#fff">-1</text><rect x="439.0" y="206.1" width="8.0" height="6" fill="#C99A4A"/><rect x="439.0" y="214.9" width="8.0" height="6" fill="#C99A4A"/><rect x="439.0" y="232.4" width="8.0" height="6" fill="#C99A4A"/><rect x="198" y="14" width="10" height="10" fill="#5E8A6A"/><text x="212" y="23" font-size="10" fill="#333">Positive effect</text><rect x="395" y="14" width="10" height="10" fill="#A85B53"/><text x="409" y="23" font-size="10" fill="#333">Negative effect</text><rect x="592" y="14" width="10" height="10" fill="#C99A4A"/><text x="606" y="23" font-size="10" fill="#333">Null effect</text></svg><p class="chart-interpretation"><strong>What this means: </strong>Positive / negative / null effect-direction evidence counts per outcome. This visual encodes effect_direction, not whether evidence supports a claim.</p></div><div id="chart-outcome-en" class="chart-mount" aria-label="Outcome Evidence Overview"></div><figure class="academic-figure" data-visual="outcome-evidence-balance"><svg viewBox="0 0 1518 300" xmlns="http://www.w3.org/2000/svg" role="img" aria-label="Effect direction by outcome type (publication figure)"><title>Effect direction by outcome type (publication figure)</title><desc>Counts of positive / negative / null effects per outcome type with an integer count axis, theme-independent. Source: EduEvidence result.json.</desc><rect width="1518" height="300" fill="#FFFFFF"/><line x1="70" y1="250" x2="650" y2="250" stroke="#333" stroke-width="1"/><text x="106.2" y="266" text-anchor="middle" font-size="10" fill="#333">Knowledge gain</text><rect x="88.1" y="150.0" width="18.1" height="100.0" fill="#38BDF8"/><text x="97.2" y="147.0" text-anchor="middle" font-size="9" fill="#333">1</text><rect x="106.2" y="250.0" width="18.1" height="1.0" fill="#10B981"/><rect x="124.4" y="250.0" width="18.1" height="1.0" fill="#F59E0B"/><text x="178.8" y="266" text-anchor="middle" font-size="10" fill="#333">Retention</text><rect x="160.6" y="250.0" width="18.1" height="1.0" fill="#38BDF8"/><rect x="178.8" y="250.0" width="18.1" height="1.0" fill="#10B981"/><rect x="196.9" y="150.0" width="18.1" height="100.0" fill="#F59E0B"/><text x="205.9" y="147.0" text-anchor="middle" font-size="9" fill="#333">1</text><text x="251.2" y="266" text-anchor="middle" font-size="10" fill="#333">Independent problem solving</text><rect x="233.1" y="250.0" width="18.1" height="1.0" fill="#38BDF8"/><rect x="251.2" y="150.0" width="18.1" height="100.0" fill="#10B981"/><text x="260.3" y="147.0" text-anchor="middle" font-size="9" fill="#333">1</text><rect x="269.4" y="50.0" width="18.1" height="200.0" fill="#F59E0B"/><text x="278.4" y="47.0" text-anchor="middle" font-size="9" fill="#333">2</text><text x="323.8" y="266" text-anchor="middle" font-size="10" fill="#333">Completion time</text><rect x="305.6" y="50.0" width="18.1" height="200.0" fill="#38BDF8"/><text x="314.7" y="47.0" text-anchor="middle" font-size="9" fill="#333">2</text><rect x="323.8" y="250.0" width="18.1" height="1.0" fill="#10B981"/><rect x="341.9" y="250.0" width="18.1" height="1.0" fill="#F59E0B"/><text x="396.2" y="266" text-anchor="middle" font-size="10" fill="#333">Code quality</text><rect x="378.1" y="250.0" width="18.1" height="1.0" fill="#38BDF8"/><rect x="396.2" y="250.0" width="18.1" height="1.0" fill="#10B981"/><rect x="414.4" y="150.0" width="18.1" height="100.0" fill="#F59E0B"/><text x="423.4" y="147.0" text-anchor="middle" font-size="9" fill="#333">1</text><text x="468.8" y="266" text-anchor="middle" font-size="10" fill="#333">Assignment score</text><rect x="450.6" y="50.0" width="18.1" height="200.0" fill="#38BDF8"/><text x="459.7" y="47.0" text-anchor="middle" font-size="9" fill="#333">2</text><rect x="468.8" y="250.0" width="18.1" height="1.0" fill="#10B981"/><rect x="486.9" y="250.0" width="18.1" height="1.0" fill="#F59E0B"/><text x="541.2" y="266" text-anchor="middle" font-size="10" fill="#333">Metacognition</text><rect x="523.1" y="150.0" width="18.1" height="100.0" fill="#38BDF8"/><text x="532.2" y="147.0" text-anchor="middle" font-size="9" fill="#333">1</text><rect x="541.2" y="250.0" width="18.1" height="1.0" fill="#10B981"/><rect x="559.4" y="250.0" width="18.1" height="1.0" fill="#F59E0B"/><text x="613.8" y="266" text-anchor="middle" font-size="10" fill="#333">Over-reliance</text><rect x="595.6" y="250.0" width="18.1" height="1.0" fill="#38BDF8"/><rect x="613.8" y="150.0" width="18.1" height="100.0" fill="#10B981"/><text x="622.8" y="147.0" text-anchor="middle" font-size="9" fill="#333">1</text><rect x="631.9" y="250.0" width="18.1" height="1.0" fill="#F59E0B"/><line x1="65" y1="250.0" x2="70" y2="250.0" stroke="#999"/><text x="62" y="253.0" text-anchor="end" font-size="9" fill="#666">0</text><line x1="65" y1="150.0" x2="70" y2="150.0" stroke="#999"/><text x="62" y="153.0" text-anchor="end" font-size="9" fill="#666">1</text><line x1="65" y1="50.0" x2="70" y2="50.0" stroke="#999"/><text x="62" y="53.0" text-anchor="end" font-size="9" fill="#666">2</text><text x="360.0" y="30" text-anchor="middle" font-size="14" font-weight="700" fill="#111">Effect direction by outcome type</text><rect x="70" y="8" width="10" height="10" fill="#38BDF8"/><text x="84" y="17" font-size="10" fill="#333">Positive effect</text><rect x="267" y="8" width="10" height="10" fill="#10B981"/><text x="281" y="17" font-size="10" fill="#333">Negative effect</text><rect x="464" y="8" width="10" height="10" fill="#F59E0B"/><text x="478" y="17" font-size="10" fill="#333">Null effect</text><text x="20" y="290" font-size="11" fill="#333333" font-style="italic">Fig. 1. Counts of positive / negative / null effects per outcome type (based on effect_direction; publication figure, theme-independent). Source: EduEvidence result.json.</text></svg><figcaption>Fig. 1. Positive / negative / null effect counts per outcome (based on effect_direction, not claim support).</figcaption></figure><div class='matrix-controls'><div class='matrix-tools'><input id='matrix-search-full-en' type='search' placeholder='Search evidence…' aria-label='Filter / search evidence'><select id='matrix-direction-full-en' aria-label='Filter by effect direction'><option value=''>All effects</option><option value='positive'>Positive effect</option><option value='negative'>Negative effect</option><option value='null'>Null effect</option></select><select id='matrix-outcome-full-en' aria-label='Filter by outcome type'><option value=''>All outcomes</option><option value='assignment_score'>Assignment score</option><option value='code_quality'>Code quality</option><option value='completion_time'>Completion time</option><option value='independent_problem_solving'>Independent problem solving</option><option value='knowledge_gain'>Knowledge gain</option><option value='metacognition'>Metacognition</option><option value='over_reliance'>Over-reliance</option><option value='retention'>Retention</option></select></div></div><div class='table-wrap matrix-wrap'><table id='evidence-matrix-full-en' class='data-table evidence-matrix'><thead><tr><th>ID</th><th>Outcome</th><th>Effect</th><th>Quality</th><th>Claim</th><th>Source</th></tr></thead><tbody><tr data-effect="positive" data-direction="positive" data-outcome="completion_time" data-search="e-001 study-kazemitabaar-2023 smpl-kazemitabaar-2023-n69 studying the effect of ai code generators on supporting novice learners in introductory programming ai coding assistants significantly increase task completion speed and completion rate during training. 69 novices ages 10-17 with no prior text-based programming experience access_to_openai_codex_ai_coding_assistant_during_training baseline_group_without_ai_coding_assistant positive s-2023-kazemitabaar"><td><code>E-001</code></td><td><strong>Completion time</strong></td><td><span class="dir pos">Positive effect</span><span class="relation-note">Claim relation: Support</span></td><td><div class="quality-cell"><span class="num">9</span><span class="quality-meter" aria-hidden="true"><i style="width:90%"></i></span></div></td><td class="claim-cell"><p>AI coding assistants significantly increase task completion speed and completion rate during training.</p><details class="matrix-row-detail"><summary>View full evidence</summary><dl class="evidence-detail-grid"><div class="evidence-detail-row"><dt>Study ID</dt><dd>STUDY-KAZEMITABAAR-2023</dd></div><div class="evidence-detail-row"><dt>Sample ID</dt><dd>SMPL-KAZEMITABAAR-2023-N69</dd></div><div class="evidence-detail-row"><dt>Study title</dt><dd>Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming</dd></div><div class="evidence-detail-row"><dt>Source title</dt><dd>Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming</dd></div><div class="evidence-detail-row"><dt>Year</dt><dd>2023</dd></div><div class="evidence-detail-row"><dt>Study design</dt><dd>Randomized controlled trial</dd></div><div class="evidence-detail-row"><dt>Education level</dt><dd>K-12 Ages 10 17</dd></div><div class="evidence-detail-row"><dt>Population</dt><dd>69 novices ages 10-17 with no prior text-based programming experience</dd></div><div class="evidence-detail-row"><dt>Sample size</dt><dd>69</dd></div><div class="evidence-detail-row"><dt>Intervention</dt><dd>Access To Openai Codex AI Coding Assistant During Training</dd></div><div class="evidence-detail-row"><dt>Comparison</dt><dd>Baseline Group Without AI Coding Assistant</dd></div><div class="evidence-detail-row"><dt>Outcome measure</dt><dd>Code Authoring Task Progress And Time</dd></div><div class="evidence-detail-row"><dt>Effect / result</dt><dd>1.15x completion rate, 0.57x time, 1.8x correctness</dd></div><div class="evidence-detail-row"><dt>Effect direction</dt><dd>Positive effect</dd></div><div class="evidence-detail-row"><dt>Relation to claim</dt><dd>Support</dd></div><div class="evidence-detail-row"><dt>Duration</dt><dd>3 Weeks Training</dd></div><div class="evidence-detail-row"><dt>Method</dt><dd>controlled experiment with random assignment, immediate post-test and 1-week retention test</dd></div><div class="evidence-detail-row"><dt>Strengths</dt><dd>randomized_controlled_design；immediate_post_test_and_retention_test；code_modification_task_guard</dd></div><div class="evidence-detail-row"><dt>Limitations</dt><dd>non_university_population_ages_10_17；small_sample_69；self-paced environment differs from classroom</dd></div><div class="evidence-detail-row"><dt>Confounders</dt><dd>Prior Programming Competency Interaction</dd></div><div class="evidence-detail-row"><dt>Quality dimensions</dt><dd>D1 Study design = 2 · D2 Sample quality = 2 · D3 Measurement validity = 2 · D4 Temporal strength = 2 · D5 Directness = 1</dd></div><div class="evidence-detail-row"><dt>Quality score</dt><dd>9.0</dd></div><div class="evidence-detail-row"><dt>Evidence level</dt><dd>strong</dd></div><div class="evidence-detail-row"><dt>Applicability</dt><dd>learner_match=partial_novice_programmers_but_younger · subject_match=introductory_programming · tool_match=codex_like_generative_ai · scope=task_performance_during_training</dd></div><div class="evidence-detail-row"><dt>Confidence</dt><dd>0.7</dd></div><div class="evidence-detail-row"><dt>Evidence status</dt><dd>Supported</dd></div><div class="evidence-detail-row"><dt>Full claim</dt><dd>AI coding assistants significantly increase task completion speed and completion rate during training.</dd></div><div class="evidence-detail-row"><dt>Source location</dt><dd>https://dl.acm.org/doi/10.1145/3544548.3580919</dd></div><div class="evidence-detail-row"><dt>Verifiable URL</dt><dd><a href="https://dl.acm.org/doi/10.1145/3544548.3580919">https://dl.acm.org/doi/10.1145/3544548.3580919</a></dd></div></dl></details></td><td><a class="source-link" href="https://dl.acm.org/doi/10.1145/3544548.3580919" title="Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming"><code>S-2023-kazemitabaar</code></a></td></tr><tr data-effect="null" data-direction="null" data-outcome="independent_problem_solving" data-search="e-002 study-kazemitabaar-2023 smpl-kazemitabaar-2023-n69 studying the effect of ai code generators on supporting novice learners in introductory programming access to ai code generation did not decrease performance on manual code-modification tasks. 69 novices ages 10-17 with no prior text-based programming experience access_to_openai_codex_ai_coding_assistant_during_training baseline_group_without_ai_coding_assistant null s-2023-kazemitabaar"><td><code>E-002</code></td><td><strong>Independent problem solving</strong></td><td><span class="dir neu">Null effect</span><span class="relation-note">Claim relation: Neutral</span></td><td><div class="quality-cell"><span class="num">7</span><span class="quality-meter" aria-hidden="true"><i style="width:70%"></i></span></div></td><td class="claim-cell"><p>Access to AI code generation did not decrease performance on manual code-modification tasks.</p><details class="matrix-row-detail"><summary>View full evidence</summary><dl class="evidence-detail-grid"><div class="evidence-detail-row"><dt>Study ID</dt><dd>STUDY-KAZEMITABAAR-2023</dd></div><div class="evidence-detail-row"><dt>Sample ID</dt><dd>SMPL-KAZEMITABAAR-2023-N69</dd></div><div class="evidence-detail-row"><dt>Study title</dt><dd>Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming</dd></div><div class="evidence-detail-row"><dt>Source title</dt><dd>Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming</dd></div><div class="evidence-detail-row"><dt>Year</dt><dd>2023</dd></div><div class="evidence-detail-row"><dt>Study design</dt><dd>Randomized controlled trial</dd></div><div class="evidence-detail-row"><dt>Education level</dt><dd>K-12 Ages 10 17</dd></div><div class="evidence-detail-row"><dt>Population</dt><dd>69 novices ages 10-17 with no prior text-based programming experience</dd></div><div class="evidence-detail-row"><dt>Sample size</dt><dd>69</dd></div><div class="evidence-detail-row"><dt>Intervention</dt><dd>Access To Openai Codex AI Coding Assistant During Training</dd></div><div class="evidence-detail-row"><dt>Comparison</dt><dd>Baseline Group Without AI Coding Assistant</dd></div><div class="evidence-detail-row"><dt>Outcome measure</dt><dd>manual code-modification tasks during training</dd></div><div class="evidence-detail-row"><dt>Effect / result</dt><dd>no significant difference between groups</dd></div><div class="evidence-detail-row"><dt>Effect direction</dt><dd>Null effect</dd></div><div class="evidence-detail-row"><dt>Relation to claim</dt><dd>Neutral</dd></div><div class="evidence-detail-row"><dt>Duration</dt><dd>3 Weeks Training</dd></div><div class="evidence-detail-row"><dt>Method</dt><dd>controlled experiment, code-modification task followed each code-authoring task</dd></div><div class="evidence-detail-row"><dt>Strengths</dt><dd>direct_test_of_transfer-adjacent_skill；same_session_measurement</dd></div><div class="evidence-detail-row"><dt>Limitations</dt><dd>code modification is not full independent problem solving；non_university population</dd></div><div class="evidence-detail-row"><dt>Confounders</dt><dd>Practice Effect</dd></div><div class="evidence-detail-row"><dt>Quality dimensions</dt><dd>D1 Study design = 2 · D2 Sample quality = 2 · D3 Measurement validity = 1 · D4 Temporal strength = 1 · D5 Directness = 1</dd></div><div class="evidence-detail-row"><dt>Quality score</dt><dd>7.0</dd></div><div class="evidence-detail-row"><dt>Evidence level</dt><dd>moderate</dd></div><div class="evidence-detail-row"><dt>Applicability</dt><dd>learner_match=partial · subject_match=introductory_programming · tool_match=codex · scope=short-term manual code modification</dd></div><div class="evidence-detail-row"><dt>Confidence</dt><dd>0.5</dd></div><div class="evidence-detail-row"><dt>Evidence status</dt><dd>Supported</dd></div><div class="evidence-detail-row"><dt>Full claim</dt><dd>Access to AI code generation did not decrease performance on manual code-modification tasks.</dd></div><div class="evidence-detail-row"><dt>Source location</dt><dd>https://dl.acm.org/doi/10.1145/3544548.3580919</dd></div><div class="evidence-detail-row"><dt>Verifiable URL</dt><dd><a href="https://dl.acm.org/doi/10.1145/3544548.3580919">https://dl.acm.org/doi/10.1145/3544548.3580919</a></dd></div></dl></details></td><td><a class="source-link" href="https://dl.acm.org/doi/10.1145/3544548.3580919" title="Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming"><code>S-2023-kazemitabaar</code></a></td></tr><tr data-effect="null" data-direction="null" data-outcome="retention" data-search="e-003 study-kazemitabaar-2023 smpl-kazemitabaar-2023-n69 studying the effect of ai code generators on supporting novice learners in introductory programming one week after training, retention differences between codex and baseline groups did not reach statistical significance. 69 novices ages 10-17 with no prior text-based programming experience access_to_openai_codex_ai_coding_assistant_during_training baseline_group_without_ai_coding_assistant null s-2023-kazemitabaar"><td><code>E-003</code></td><td><strong>Retention</strong></td><td><span class="dir neu">Null effect</span><span class="relation-note">Claim relation: Neutral</span></td><td><div class="quality-cell"><span class="num">9</span><span class="quality-meter" aria-hidden="true"><i style="width:90%"></i></span></div></td><td class="claim-cell"><p>One week after training, retention differences between Codex and baseline groups did not reach statistical significance.</p><details class="matrix-row-detail"><summary>View full evidence</summary><dl class="evidence-detail-grid"><div class="evidence-detail-row"><dt>Study ID</dt><dd>STUDY-KAZEMITABAAR-2023</dd></div><div class="evidence-detail-row"><dt>Sample ID</dt><dd>SMPL-KAZEMITABAAR-2023-N69</dd></div><div class="evidence-detail-row"><dt>Study title</dt><dd>Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming</dd></div><div class="evidence-detail-row"><dt>Source title</dt><dd>Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming</dd></div><div class="evidence-detail-row"><dt>Year</dt><dd>2023</dd></div><div class="evidence-detail-row"><dt>Study design</dt><dd>Randomized controlled trial</dd></div><div class="evidence-detail-row"><dt>Education level</dt><dd>K-12 Ages 10 17</dd></div><div class="evidence-detail-row"><dt>Population</dt><dd>69 novices ages 10-17 with no prior text-based programming experience</dd></div><div class="evidence-detail-row"><dt>Sample size</dt><dd>69</dd></div><div class="evidence-detail-row"><dt>Intervention</dt><dd>Access To Openai Codex AI Coding Assistant During Training</dd></div><div class="evidence-detail-row"><dt>Comparison</dt><dd>Baseline Group Without AI Coding Assistant</dd></div><div class="evidence-detail-row"><dt>Outcome measure</dt><dd>retention post-test one week after training</dd></div><div class="evidence-detail-row"><dt>Effect / result</dt><dd>slightly better for Codex group but not significant</dd></div><div class="evidence-detail-row"><dt>Effect direction</dt><dd>Null effect</dd></div><div class="evidence-detail-row"><dt>Relation to claim</dt><dd>Neutral</dd></div><div class="evidence-detail-row"><dt>Duration</dt><dd>3 Weeks Training Plus 1 Week Retention</dd></div><div class="evidence-detail-row"><dt>Method</dt><dd>controlled experiment with delayed retention test</dd></div><div class="evidence-detail-row"><dt>Strengths</dt><dd>delayed_test_included</dd></div><div class="evidence-detail-row"><dt>Limitations</dt><dd>1-week retention window is short；small sample；non-university population</dd></div><div class="evidence-detail-row"><dt>Confounders</dt><dd>Prior Competency</dd></div><div class="evidence-detail-row"><dt>Quality dimensions</dt><dd>D1 Study design = 2 · D2 Sample quality = 2 · D3 Measurement validity = 2 · D4 Temporal strength = 2 · D5 Directness = 1</dd></div><div class="evidence-detail-row"><dt>Quality score</dt><dd>9.0</dd></div><div class="evidence-detail-row"><dt>Evidence level</dt><dd>strong</dd></div><div class="evidence-detail-row"><dt>Applicability</dt><dd>learner_match=partial · subject_match=introductory_programming · tool_match=codex · scope=retention over one week</dd></div><div class="evidence-detail-row"><dt>Confidence</dt><dd>0.5</dd></div><div class="evidence-detail-row"><dt>Evidence status</dt><dd>Supported</dd></div><div class="evidence-detail-row"><dt>Full claim</dt><dd>One week after training, retention differences between Codex and baseline groups did not reach statistical significance.</dd></div><div class="evidence-detail-row"><dt>Source location</dt><dd>https://dl.acm.org/doi/10.1145/3544548.3580919</dd></div><div class="evidence-detail-row"><dt>Verifiable URL</dt><dd><a href="https://dl.acm.org/doi/10.1145/3544548.3580919">https://dl.acm.org/doi/10.1145/3544548.3580919</a></dd></div></dl></details></td><td><a class="source-link" href="https://dl.acm.org/doi/10.1145/3544548.3580919" title="Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming"><code>S-2023-kazemitabaar</code></a></td></tr><tr data-effect="negative" data-direction="negative" data-outcome="independent_problem_solving" data-search="e-004 study-bastani-2025 smpl-bastani-2025-n950 generative ai without guardrails can harm learning: evidence from high school mathematics students with unguarded gpt-4 access performed 17% worse on the independent exam than the control group, despite higher practice performance. nearly a thousand high school math students in turkey gpt4_based_tutor_gpt_base_unguarded no_generative_ai_control negative s-2025-bastani"><td><code>E-004</code></td><td><strong>Independent problem solving</strong></td><td><span class="dir neg">Negative effect</span><span class="relation-note">Claim relation: Contradict</span></td><td><div class="quality-cell"><span class="num">8</span><span class="quality-meter" aria-hidden="true"><i style="width:80%"></i></span></div></td><td class="claim-cell"><p>Students with unguarded GPT-4 access performed 17% worse on the independent exam than the control group, despite higher practice performance.</p><details class="matrix-row-detail"><summary>View full evidence</summary><dl class="evidence-detail-grid"><div class="evidence-detail-row"><dt>Study ID</dt><dd>STUDY-BASTANI-2025</dd></div><div class="evidence-detail-row"><dt>Sample ID</dt><dd>SMPL-BASTANI-2025-N950</dd></div><div class="evidence-detail-row"><dt>Study title</dt><dd>Generative AI without guardrails can harm learning: Evidence from high school mathematics</dd></div><div class="evidence-detail-row"><dt>Source title</dt><dd>Generative AI without guardrails can harm learning: Evidence from high school mathematics</dd></div><div class="evidence-detail-row"><dt>Year</dt><dd>2025</dd></div><div class="evidence-detail-row"><dt>Study design</dt><dd>Randomized controlled trial</dd></div><div class="evidence-detail-row"><dt>Education level</dt><dd>High school</dd></div><div class="evidence-detail-row"><dt>Population</dt><dd>nearly a thousand high school math students in Turkey</dd></div><div class="evidence-detail-row"><dt>Sample size</dt><dd>950</dd></div><div class="evidence-detail-row"><dt>Intervention</dt><dd>Gpt4 Based Tutor Gpt Base Unguarded</dd></div><div class="evidence-detail-row"><dt>Comparison</dt><dd>No Generative AI Control</dd></div><div class="evidence-detail-row"><dt>Outcome measure</dt><dd>exam without access to AI resources after practice phase</dd></div><div class="evidence-detail-row"><dt>Effect / result</dt><dd>negative_17_percent_on_independent_exam</dd></div><div class="evidence-detail-row"><dt>Effect direction</dt><dd>Negative effect</dd></div><div class="evidence-detail-row"><dt>Relation to claim</dt><dd>Contradict</dd></div><div class="evidence-detail-row"><dt>Duration</dt><dd>In Class Study Sessions</dd></div><div class="evidence-detail-row"><dt>Method</dt><dd>large-scale randomized controlled trial, practice phase then closed-book exam</dd></div><div class="evidence-detail-row"><dt>Strengths</dt><dd>large_scale_rct；independent_exam_without_ai；arm_wise_design</dd></div><div class="evidence-detail-row"><dt>Limitations</dt><dd>high_school_mathematics_not_university_programming；single_country</dd></div><div class="evidence-detail-row"><dt>Confounders</dt><dd>Tool Design Difference</dd></div><div class="evidence-detail-row"><dt>Quality dimensions</dt><dd>D1 Study design = 2 · D2 Sample quality = 2 · D3 Measurement validity = 2 · D4 Temporal strength = 1 · D5 Directness = 1</dd></div><div class="evidence-detail-row"><dt>Quality score</dt><dd>8.0</dd></div><div class="evidence-detail-row"><dt>Evidence level</dt><dd>strong</dd></div><div class="evidence-detail-row"><dt>Applicability</dt><dd>learner_match=partial_same_age_band_different_subject · subject_match=no_mathematics_vs_programming · tool_match=gpt4_chat_interface · scope=unguarded_general_chat_interface</dd></div><div class="evidence-detail-row"><dt>Confidence</dt><dd>0.75</dd></div><div class="evidence-detail-row"><dt>Evidence status</dt><dd>Supported</dd></div><div class="evidence-detail-row"><dt>Full claim</dt><dd>Students with unguarded GPT-4 access performed 17% worse on the independent exam than the control group, despite higher practice performance.</dd></div><div class="evidence-detail-row"><dt>Source location</dt><dd>https://www.pnas.org/doi/10.1073/pnas.2422633122</dd></div><div class="evidence-detail-row"><dt>Verifiable URL</dt><dd><a href="https://www.pnas.org/doi/10.1073/pnas.2422633122">https://www.pnas.org/doi/10.1073/pnas.2422633122</a></dd></div></dl></details></td><td><a class="source-link" href="https://www.pnas.org/doi/10.1073/pnas.2422633122" title="Generative AI without guardrails can harm learning: Evidence from high school mathematics"><code>S-2025-bastani</code></a></td></tr><tr data-effect="null" data-direction="null" data-outcome="independent_problem_solving" data-search="e-005 study-bastani-2025 smpl-bastani-2025-n950 generative ai without guardrails can harm learning: evidence from high school mathematics guardrail design of the ai tutor (hints instead of answers, teacher-informed prompts) largely eliminated the negative learning effect. nearly a thousand high school math students in turkey gpt4_tutor_with_teacher_designed_guardrails no_generative_ai_control null s-2025-bastani"><td><code>E-005</code></td><td><strong>Independent problem solving</strong></td><td><span class="dir neu">Null effect</span><span class="relation-note">Claim relation: Support</span></td><td><div class="quality-cell"><span class="num">8</span><span class="quality-meter" aria-hidden="true"><i style="width:80%"></i></span></div></td><td class="claim-cell"><p>Guardrail design of the AI tutor (hints instead of answers, teacher-informed prompts) largely eliminated the negative learning effect.</p><details class="matrix-row-detail"><summary>View full evidence</summary><dl class="evidence-detail-grid"><div class="evidence-detail-row"><dt>Study ID</dt><dd>STUDY-BASTANI-2025</dd></div><div class="evidence-detail-row"><dt>Sample ID</dt><dd>SMPL-BASTANI-2025-N950</dd></div><div class="evidence-detail-row"><dt>Study title</dt><dd>Generative AI without guardrails can harm learning: Evidence from high school mathematics</dd></div><div class="evidence-detail-row"><dt>Source title</dt><dd>Generative AI without guardrails can harm learning: Evidence from high school mathematics</dd></div><div class="evidence-detail-row"><dt>Year</dt><dd>2025</dd></div><div class="evidence-detail-row"><dt>Study design</dt><dd>Randomized controlled trial</dd></div><div class="evidence-detail-row"><dt>Education level</dt><dd>High school</dd></div><div class="evidence-detail-row"><dt>Population</dt><dd>nearly a thousand high school math students in Turkey</dd></div><div class="evidence-detail-row"><dt>Sample size</dt><dd>950</dd></div><div class="evidence-detail-row"><dt>Intervention</dt><dd>Gpt4 Tutor With Teacher Designed Guardrails</dd></div><div class="evidence-detail-row"><dt>Comparison</dt><dd>No Generative AI Control</dd></div><div class="evidence-detail-row"><dt>Outcome measure</dt><dd>exam without access to AI resources after practice phase</dd></div><div class="evidence-detail-row"><dt>Effect / result</dt><dd>negative effect essentially eradicated, no positive effect observed</dd></div><div class="evidence-detail-row"><dt>Effect direction</dt><dd>Null effect</dd></div><div class="evidence-detail-row"><dt>Relation to claim</dt><dd>Support</dd></div><div class="evidence-detail-row"><dt>Duration</dt><dd>In Class Study Sessions</dd></div><div class="evidence-detail-row"><dt>Method</dt><dd>large-scale randomized controlled trial, three arms</dd></div><div class="evidence-detail-row"><dt>Strengths</dt><dd>direct_manipulation_of_tool_design；large_sample</dd></div><div class="evidence-detail-row"><dt>Limitations</dt><dd>no_positive_learning_gain_even_with_guardrails；subject_mismatch</dd></div><div class="evidence-detail-row"><dt>Confounders</dt><dd>Prompt Engineering Effort</dd></div><div class="evidence-detail-row"><dt>Quality dimensions</dt><dd>D1 Study design = 2 · D2 Sample quality = 2 · D3 Measurement validity = 2 · D4 Temporal strength = 1 · D5 Directness = 1</dd></div><div class="evidence-detail-row"><dt>Quality score</dt><dd>8.0</dd></div><div class="evidence-detail-row"><dt>Evidence level</dt><dd>strong</dd></div><div class="evidence-detail-row"><dt>Applicability</dt><dd>learner_match=partial · subject_match=no · tool_match=guardrailed_tutor_design · scope=guardrail_design_principle_transferable</dd></div><div class="evidence-detail-row"><dt>Confidence</dt><dd>0.75</dd></div><div class="evidence-detail-row"><dt>Evidence status</dt><dd>Supported</dd></div><div class="evidence-detail-row"><dt>Full claim</dt><dd>Guardrail design of the AI tutor (hints instead of answers, teacher-informed prompts) largely eliminated the negative learning effect.</dd></div><div class="evidence-detail-row"><dt>Source location</dt><dd>https://www.pnas.org/doi/10.1073/pnas.2422633122</dd></div><div class="evidence-detail-row"><dt>Verifiable URL</dt><dd><a href="https://www.pnas.org/doi/10.1073/pnas.2422633122">https://www.pnas.org/doi/10.1073/pnas.2422633122</a></dd></div></dl></details></td><td><a class="source-link" href="https://www.pnas.org/doi/10.1073/pnas.2422633122" title="Generative AI without guardrails can harm learning: Evidence from high school mathematics"><code>S-2025-bastani</code></a></td></tr><tr data-effect="positive" data-direction="positive" data-outcome="assignment_score" data-search="e-006 study-bastani-2025 smpl-bastani-2025-n950 generative ai without guardrails can harm learning: evidence from high school mathematics access to gpt-4 during practice improves task performance (48% for gpt base, 127% for gpt tutor) — but this task performance does not transfer to independent exam performance. nearly a thousand high school math students in turkey gpt4_tutor_access_during_practice no_generative_ai_control positive s-2025-bastani"><td><code>E-006</code></td><td><strong>Assignment score</strong></td><td><span class="dir pos">Positive effect</span><span class="relation-note">Claim relation: Support</span></td><td><div class="quality-cell"><span class="num">8</span><span class="quality-meter" aria-hidden="true"><i style="width:80%"></i></span></div></td><td class="claim-cell"><p>Access to GPT-4 during practice improves task performance (48% for GPT Base, 127% for GPT Tutor) — but this task performance does not transfer to independent exam performance.</p><details class="matrix-row-detail"><summary>View full evidence</summary><dl class="evidence-detail-grid"><div class="evidence-detail-row"><dt>Study ID</dt><dd>STUDY-BASTANI-2025</dd></div><div class="evidence-detail-row"><dt>Sample ID</dt><dd>SMPL-BASTANI-2025-N950</dd></div><div class="evidence-detail-row"><dt>Study title</dt><dd>Generative AI without guardrails can harm learning: Evidence from high school mathematics</dd></div><div class="evidence-detail-row"><dt>Source title</dt><dd>Generative AI without guardrails can harm learning: Evidence from high school mathematics</dd></div><div class="evidence-detail-row"><dt>Year</dt><dd>2025</dd></div><div class="evidence-detail-row"><dt>Study design</dt><dd>Randomized controlled trial</dd></div><div class="evidence-detail-row"><dt>Education level</dt><dd>High school</dd></div><div class="evidence-detail-row"><dt>Population</dt><dd>nearly a thousand high school math students in Turkey</dd></div><div class="evidence-detail-row"><dt>Sample size</dt><dd>950</dd></div><div class="evidence-detail-row"><dt>Intervention</dt><dd>Gpt4 Tutor Access During Practice</dd></div><div class="evidence-detail-row"><dt>Comparison</dt><dd>No Generative AI Control</dd></div><div class="evidence-detail-row"><dt>Outcome measure</dt><dd>practice problem performance during study sessions</dd></div><div class="evidence-detail-row"><dt>Effect / result</dt><dd>48-127 percent improvement on practice problems</dd></div><div class="evidence-detail-row"><dt>Effect direction</dt><dd>Positive effect</dd></div><div class="evidence-detail-row"><dt>Relation to claim</dt><dd>Support</dd></div><div class="evidence-detail-row"><dt>Duration</dt><dd>In Class Study Sessions</dd></div><div class="evidence-detail-row"><dt>Method</dt><dd>randomized controlled trial with practice and closed-book exam phases</dd></div><div class="evidence-detail-row"><dt>Strengths</dt><dd>same_study_compares_task_and_learning；large_sample</dd></div><div class="evidence-detail-row"><dt>Limitations</dt><dd>subject_mismatch_mathematics</dd></div><div class="evidence-detail-row"><dt>Confounders</dt><dd>Task Familiarity</dd></div><div class="evidence-detail-row"><dt>Quality dimensions</dt><dd>D1 Study design = 2 · D2 Sample quality = 2 · D3 Measurement validity = 2 · D4 Temporal strength = 1 · D5 Directness = 1</dd></div><div class="evidence-detail-row"><dt>Quality score</dt><dd>8.0</dd></div><div class="evidence-detail-row"><dt>Evidence level</dt><dd>strong</dd></div><div class="evidence-detail-row"><dt>Applicability</dt><dd>learner_match=partial · subject_match=no · tool_match=gpt4 · scope=task_performance_vs_learning_separation</dd></div><div class="evidence-detail-row"><dt>Confidence</dt><dd>0.75</dd></div><div class="evidence-detail-row"><dt>Evidence status</dt><dd>Supported</dd></div><div class="evidence-detail-row"><dt>Full claim</dt><dd>Access to GPT-4 during practice improves task performance (48% for GPT Base, 127% for GPT Tutor) — but this task performance does not transfer to independent exam performance.</dd></div><div class="evidence-detail-row"><dt>Source location</dt><dd>https://www.pnas.org/doi/10.1073/pnas.2422633122</dd></div><div class="evidence-detail-row"><dt>Verifiable URL</dt><dd><a href="https://www.pnas.org/doi/10.1073/pnas.2422633122">https://www.pnas.org/doi/10.1073/pnas.2422633122</a></dd></div></dl></details></td><td><a class="source-link" href="https://www.pnas.org/doi/10.1073/pnas.2422633122" title="Generative AI without guardrails can harm learning: Evidence from high school mathematics"><code>S-2025-bastani</code></a></td></tr><tr data-effect="positive" data-direction="positive" data-outcome="knowledge_gain" data-search="e-007 study-marzuki-2024 smpl-marzuki-2024-n72 impact of chatgpt on esl students&#x27; academic writing skills chatgpt as a formative feedback tool produced a significant positive impact on students&#x27; academic writing skills with positive student perceptions. undergraduate esl students at an indian university chatgpt_as_formative_feedback_tool traditional_instruction_control positive s-2024-marzuki"><td><code>E-007</code></td><td><strong>Knowledge gain</strong></td><td><span class="dir pos">Positive effect</span><span class="relation-note">Claim relation: Support</span></td><td><div class="quality-cell"><span class="num">6</span><span class="quality-meter" aria-hidden="true"><i style="width:60%"></i></span></div></td><td class="claim-cell"><p>ChatGPT as a formative feedback tool produced a significant positive impact on students&#x27; academic writing skills with positive student perceptions.</p><details class="matrix-row-detail"><summary>View full evidence</summary><dl class="evidence-detail-grid"><div class="evidence-detail-row"><dt>Study ID</dt><dd>STUDY-MARZUKI-2024</dd></div><div class="evidence-detail-row"><dt>Sample ID</dt><dd>SMPL-MARZUKI-2024-N72</dd></div><div class="evidence-detail-row"><dt>Study title</dt><dd>Impact of ChatGPT on ESL students&#x27; academic writing skills</dd></div><div class="evidence-detail-row"><dt>Source title</dt><dd>Impact of ChatGPT on ESL students&#x27; academic writing skills</dd></div><div class="evidence-detail-row"><dt>Year</dt><dd>2024</dd></div><div class="evidence-detail-row"><dt>Study design</dt><dd>Mixed methods</dd></div><div class="evidence-detail-row"><dt>Education level</dt><dd>Undergraduate</dd></div><div class="evidence-detail-row"><dt>Population</dt><dd>undergraduate ESL students at an Indian university</dd></div><div class="evidence-detail-row"><dt>Sample size</dt><dd>72</dd></div><div class="evidence-detail-row"><dt>Intervention</dt><dd>Chatgpt As Formative Feedback Tool</dd></div><div class="evidence-detail-row"><dt>Comparison</dt><dd>Traditional Instruction Control</dd></div><div class="evidence-detail-row"><dt>Outcome measure</dt><dd>writing tests with pre-post-delayed design</dd></div><div class="evidence-detail-row"><dt>Effect / result</dt><dd>significant positive impact on writing skills</dd></div><div class="evidence-detail-row"><dt>Effect direction</dt><dd>Positive effect</dd></div><div class="evidence-detail-row"><dt>Relation to claim</dt><dd>Support</dd></div><div class="evidence-detail-row"><dt>Duration</dt><dd>6 Hours Intervention</dd></div><div class="evidence-detail-row"><dt>Method</dt><dd>mixed methods intervention study, pre/post/delayed tests and focus groups</dd></div><div class="evidence-detail-row"><dt>Strengths</dt><dd>delayed_post_test；mixed_methods_triangulation</dd></div><div class="evidence-detail-row"><dt>Limitations</dt><dd>short_intervention_6_hours；single_institution；elite_private_university</dd></div><div class="evidence-detail-row"><dt>Confounders</dt><dd>Self Selection Consent</dd></div><div class="evidence-detail-row"><dt>Quality dimensions</dt><dd>D1 Study design = 1 · D2 Sample quality = 1 · D3 Measurement validity = 2 · D4 Temporal strength = 2 · D5 Directness = 0</dd></div><div class="evidence-detail-row"><dt>Quality score</dt><dd>6.0</dd></div><div class="evidence-detail-row"><dt>Evidence level</dt><dd>moderate</dd></div><div class="evidence-detail-row"><dt>Applicability</dt><dd>learner_match=yes_undergraduate · subject_match=no_writing_not_programming · tool_match=chatgpt · scope=formative_feedback_writing</dd></div><div class="evidence-detail-row"><dt>Confidence</dt><dd>0.55</dd></div><div class="evidence-detail-row"><dt>Evidence status</dt><dd>Supported</dd></div><div class="evidence-detail-row"><dt>Full claim</dt><dd>ChatGPT as a formative feedback tool produced a significant positive impact on students&#x27; academic writing skills with positive student perceptions.</dd></div><div class="evidence-detail-row"><dt>Source location</dt><dd>https://link.springer.com/article/10.1186/s40561-024-00295-9</dd></div><div class="evidence-detail-row"><dt>Verifiable URL</dt><dd><a href="https://link.springer.com/article/10.1186/s40561-024-00295-9">https://link.springer.com/article/10.1186/s40561-024-00295-9</a></dd></div></dl></details></td><td><a class="source-link" href="https://link.springer.com/article/10.1186/s40561-024-00295-9" title="Impact of ChatGPT on ESL students&#x27; academic writing skills"><code>S-2024-marzuki</code></a></td></tr><tr data-effect="positive" data-direction="positive" data-outcome="completion_time" data-search="e-008 study-peng-2023 smpl-peng-2023-n95 the impact of ai on developer productivity: evidence from github copilot professional developers with copilot access completed a standardized coding task about 55% faster than the control group (rct, n=95). 95 recruited professional developers completing a standardized coding task on a freelance platform access_to_github_copilot_during_task control_group_without_copilot positive s-2023-peng"><td><code>E-008</code></td><td><strong>Completion time</strong></td><td><span class="dir pos">Positive effect</span><span class="relation-note">Claim relation: Support</span></td><td><div class="quality-cell"><span class="num">8</span><span class="quality-meter" aria-hidden="true"><i style="width:80%"></i></span></div></td><td class="claim-cell"><p>Professional developers with Copilot access completed a standardized coding task about 55% faster than the control group (RCT, n=95).</p><details class="matrix-row-detail"><summary>View full evidence</summary><dl class="evidence-detail-grid"><div class="evidence-detail-row"><dt>Study ID</dt><dd>STUDY-PENG-2023</dd></div><div class="evidence-detail-row"><dt>Sample ID</dt><dd>SMPL-PENG-2023-N95</dd></div><div class="evidence-detail-row"><dt>Study title</dt><dd>The Impact of AI on Developer Productivity: Evidence from GitHub Copilot</dd></div><div class="evidence-detail-row"><dt>Source title</dt><dd>The Impact of AI on Developer Productivity: Evidence from GitHub Copilot</dd></div><div class="evidence-detail-row"><dt>Year</dt><dd>2023</dd></div><div class="evidence-detail-row"><dt>Study design</dt><dd>Randomized controlled trial</dd></div><div class="evidence-detail-row"><dt>Education level</dt><dd>Professional Developers Not Students</dd></div><div class="evidence-detail-row"><dt>Population</dt><dd>95 recruited professional developers completing a standardized coding task on a freelance platform</dd></div><div class="evidence-detail-row"><dt>Sample size</dt><dd>95</dd></div><div class="evidence-detail-row"><dt>Intervention</dt><dd>Access To Github Copilot During Task</dd></div><div class="evidence-detail-row"><dt>Comparison</dt><dd>Control Group Without Copilot</dd></div><div class="evidence-detail-row"><dt>Outcome measure</dt><dd>Time To Complete Http Server Implementation</dd></div><div class="evidence-detail-row"><dt>Effect / result</dt><dd>~55.8% faster task completion in Copilot group</dd></div><div class="evidence-detail-row"><dt>Effect direction</dt><dd>Positive effect</dd></div><div class="evidence-detail-row"><dt>Relation to claim</dt><dd>Support</dd></div><div class="evidence-detail-row"><dt>Duration</dt><dd>Single Task Session</dd></div><div class="evidence-detail-row"><dt>Method</dt><dd>online randomized controlled experiment with objective completion-time metric</dd></div><div class="evidence-detail-row"><dt>Strengths</dt><dd>randomized_controlled_design；objective_completion_time_metric</dd></div><div class="evidence-detail-row"><dt>Limitations</dt><dd>professional_population_not_students；single_task_ecology；preprint_not_peer_reviewed</dd></div><div class="evidence-detail-row"><dt>Confounders</dt><dd>Task Familiarity；Platform Recruitment Self Selection</dd></div><div class="evidence-detail-row"><dt>Quality dimensions</dt><dd>D1 Study design = 2 · D2 Sample quality = 2 · D3 Measurement validity = 2 · D4 Temporal strength = 1 · D5 Directness = 1</dd></div><div class="evidence-detail-row"><dt>Quality score</dt><dd>8.0</dd></div><div class="evidence-detail-row"><dt>Evidence level</dt><dd>moderate</dd></div><div class="evidence-detail-row"><dt>Applicability</dt><dd>learner_match=mismatch_professional_developers · subject_match=adjacent_web_development_task · tool_match=copilot_like_generative_ai · scope=task_performance_only_no_learning_outcome</dd></div><div class="evidence-detail-row"><dt>Confidence</dt><dd>0.6</dd></div><div class="evidence-detail-row"><dt>Evidence status</dt><dd>Supported</dd></div><div class="evidence-detail-row"><dt>Full claim</dt><dd>Professional developers with Copilot access completed a standardized coding task about 55% faster than the control group (RCT, n=95).</dd></div><div class="evidence-detail-row"><dt>Source location</dt><dd>https://doi.org/10.48550/arXiv.2302.06590</dd></div><div class="evidence-detail-row"><dt>Verifiable URL</dt><dd><a href="https://doi.org/10.48550/arXiv.2302.06590">https://doi.org/10.48550/arXiv.2302.06590</a></dd></div></dl></details></td><td><a class="source-link" href="https://doi.org/10.48550/arXiv.2302.06590" title="The Impact of AI on Developer Productivity: Evidence from GitHub Copilot"><code>S-2023-peng</code></a></td></tr><tr data-effect="null" data-direction="null" data-outcome="code_quality" data-search="e-009 study-yetistiren-2023 smpl-yetistiren-2023-bench github copilot ai pair programmer: asset or liability? systematic benchmark evaluation reports mixed quality results for copilot-generated code relative to human code: correctness competitive on parts of the benchmark while security-relevant defects are documented. copilot-generated and human-written programs drawn from published benchmark datasets copilot_generated_programs human_written_programs_on_same_benchmarks null s-2023-yetistiren"><td><code>E-009</code></td><td><strong>Code quality</strong></td><td><span class="dir neu">Null effect</span><span class="relation-note">Claim relation: Neutral</span></td><td><div class="quality-cell"><span class="num">7</span><span class="quality-meter" aria-hidden="true"><i style="width:70%"></i></span></div></td><td class="claim-cell"><p>Systematic benchmark evaluation reports mixed quality results for Copilot-generated code relative to human code: correctness competitive on parts of the benchmark while security-relevant defects are documented.</p><details class="matrix-row-detail"><summary>View full evidence</summary><dl class="evidence-detail-grid"><div class="evidence-detail-row"><dt>Study ID</dt><dd>STUDY-YETISTIREN-2023</dd></div><div class="evidence-detail-row"><dt>Sample ID</dt><dd>SMPL-YETISTIREN-2023-BENCH</dd></div><div class="evidence-detail-row"><dt>Study title</dt><dd>GitHub Copilot AI Pair Programmer: Asset or Liability?</dd></div><div class="evidence-detail-row"><dt>Source title</dt><dd>GitHub Copilot AI Pair Programmer: Asset or Liability?</dd></div><div class="evidence-detail-row"><dt>Year</dt><dd>2023</dd></div><div class="evidence-detail-row"><dt>Study design</dt><dd>Observational</dd></div><div class="evidence-detail-row"><dt>Education level</dt><dd>Not Applicable Code Artifacts</dd></div><div class="evidence-detail-row"><dt>Population</dt><dd>Copilot-generated and human-written programs drawn from published benchmark datasets</dd></div><div class="evidence-detail-row"><dt>Intervention</dt><dd>Copilot Generated Programs</dd></div><div class="evidence-detail-row"><dt>Comparison</dt><dd>Human Written Programs On Same Benchmarks</dd></div><div class="evidence-detail-row"><dt>Outcome measure</dt><dd>Correctness Security Maintainability Metrics On Benchmarks</dd></div><div class="evidence-detail-row"><dt>Effect / result</dt><dd>mixed quality profile; no single-direction summary</dd></div><div class="evidence-detail-row"><dt>Effect direction</dt><dd>Null effect</dd></div><div class="evidence-detail-row"><dt>Relation to claim</dt><dd>Neutral</dd></div><div class="evidence-detail-row"><dt>Duration</dt><dd>Not Applicable Artifact Study</dd></div><div class="evidence-detail-row"><dt>Method</dt><dd>systematic empirical evaluation of generated code against human baselines on public benchmarks</dd></div><div class="evidence-detail-row"><dt>Strengths</dt><dd>multi_dimensional_quality_metrics；reproducible_benchmark_protocol</dd></div><div class="evidence-detail-row"><dt>Limitations</dt><dd>artifact_benchmark_not_classroom；no_learning_outcome；tool_version_from_2023</dd></div><div class="evidence-detail-row"><dt>Confounders</dt><dd>Benchmark Task Distribution</dd></div><div class="evidence-detail-row"><dt>Quality dimensions</dt><dd>D1 Study design = 1 · D2 Sample quality = 2 · D3 Measurement validity = 2 · D4 Temporal strength = 1 · D5 Directness = 1</dd></div><div class="evidence-detail-row"><dt>Quality score</dt><dd>7.0</dd></div><div class="evidence-detail-row"><dt>Evidence level</dt><dd>moderate</dd></div><div class="evidence-detail-row"><dt>Applicability</dt><dd>learner_match=mismatch_no_learners_in_study · subject_match=introductory_adjacent_code_tasks · tool_match=copilot_like_generative_ai · scope=output_quality_only</dd></div><div class="evidence-detail-row"><dt>Confidence</dt><dd>0.55</dd></div><div class="evidence-detail-row"><dt>Evidence status</dt><dd>Supported</dd></div><div class="evidence-detail-row"><dt>Full claim</dt><dd>Systematic benchmark evaluation reports mixed quality results for Copilot-generated code relative to human code: correctness competitive on parts of the benchmark while security-relevant defects are documented.</dd></div><div class="evidence-detail-row"><dt>Source location</dt><dd>https://doi.org/10.1016/j.jss.2023.111734</dd></div><div class="evidence-detail-row"><dt>Verifiable URL</dt><dd><a href="https://doi.org/10.1016/j.jss.2023.111734">https://doi.org/10.1016/j.jss.2023.111734</a></dd></div></dl></details></td><td><a class="source-link" href="https://doi.org/10.1016/j.jss.2023.111734" title="GitHub Copilot AI Pair Programmer: Asset or Liability?"><code>S-2023-yetistiren</code></a></td></tr><tr data-effect="positive" data-direction="positive" data-outcome="assignment_score" data-search="e-010 study-finnieansley-2022 smpl-finnieansley-2022-qsets using github copilot to solve introductory programming problems codex produced passing-level solutions for roughly half to three-quarters of cs1 exam-style questions depending on the dataset, indicating substantial task-capability headroom available to novices. cs1 exam-style question sets answered by codex and compared against published student score distributions codex_answer_generation_on_cs1_questions published_student_cohort_score_distributions positive s-2022-finnie-ansley"><td><code>E-010</code></td><td><strong>Assignment score</strong></td><td><span class="dir pos">Positive effect</span><span class="relation-note">Claim relation: Support</span></td><td><div class="quality-cell"><span class="num">7</span><span class="quality-meter" aria-hidden="true"><i style="width:70%"></i></span></div></td><td class="claim-cell"><p>Codex produced passing-level solutions for roughly half to three-quarters of CS1 exam-style questions depending on the dataset, indicating substantial task-capability headroom available to novices.</p><details class="matrix-row-detail"><summary>View full evidence</summary><dl class="evidence-detail-grid"><div class="evidence-detail-row"><dt>Study ID</dt><dd>STUDY-FINNIEANSLEY-2022</dd></div><div class="evidence-detail-row"><dt>Sample ID</dt><dd>SMPL-FINNIEANSLEY-2022-QSETS</dd></div><div class="evidence-detail-row"><dt>Study title</dt><dd>Using GitHub Copilot to Solve Introductory Programming Problems</dd></div><div class="evidence-detail-row"><dt>Source title</dt><dd>Using GitHub Copilot to Solve Introductory Programming Problems</dd></div><div class="evidence-detail-row"><dt>Year</dt><dd>2022</dd></div><div class="evidence-detail-row"><dt>Study design</dt><dd>Observational</dd></div><div class="evidence-detail-row"><dt>Education level</dt><dd>University Year 1 Question Sets</dd></div><div class="evidence-detail-row"><dt>Population</dt><dd>CS1 exam-style question sets answered by Codex and compared against published student score distributions</dd></div><div class="evidence-detail-row"><dt>Intervention</dt><dd>Codex Answer Generation On CS1 Questions</dd></div><div class="evidence-detail-row"><dt>Comparison</dt><dd>Published Student Cohort Score Distributions</dd></div><div class="evidence-detail-row"><dt>Outcome measure</dt><dd>Pass Rate On CS1 Exam Style Questions</dd></div><div class="evidence-detail-row"><dt>Effect / result</dt><dd>passing solutions on ~50-75% of questions across datasets</dd></div><div class="evidence-detail-row"><dt>Effect direction</dt><dd>Positive effect</dd></div><div class="evidence-detail-row"><dt>Relation to claim</dt><dd>Support</dd></div><div class="evidence-detail-row"><dt>Duration</dt><dd>Not Applicable Capability Probe</dd></div><div class="evidence-detail-row"><dt>Method</dt><dd>capability benchmark against published student distributions; reproducible question sets</dd></div><div class="evidence-detail-row"><dt>Strengths</dt><dd>public_reproducible_question_sets；directly_relevant_task_domain</dd></div><div class="evidence-detail-row"><dt>Limitations</dt><dd>tool_solves_task_does_not_equate_student_learning；codex_2021_model_version_outdated</dd></div><div class="evidence-detail-row"><dt>Confounders</dt><dd>Question Leakage Into Training Data Possible</dd></div><div class="evidence-detail-row"><dt>Quality dimensions</dt><dd>D1 Study design = 1 · D2 Sample quality = 2 · D3 Measurement validity = 2 · D4 Temporal strength = 1 · D5 Directness = 1</dd></div><div class="evidence-detail-row"><dt>Quality score</dt><dd>7.0</dd></div><div class="evidence-detail-row"><dt>Evidence level</dt><dd>moderate</dd></div><div class="evidence-detail-row"><dt>Applicability</dt><dd>learner_match=partial_measures_tool_not_students · subject_match=introductory_programming · tool_match=copilot_like_generative_ai · scope=tool_capability_headroom</dd></div><div class="evidence-detail-row"><dt>Confidence</dt><dd>0.55</dd></div><div class="evidence-detail-row"><dt>Evidence status</dt><dd>Supported</dd></div><div class="evidence-detail-row"><dt>Full claim</dt><dd>Codex produced passing-level solutions for roughly half to three-quarters of CS1 exam-style questions depending on the dataset, indicating substantial task-capability headroom available to novices.</dd></div><div class="evidence-detail-row"><dt>Source location</dt><dd>https://dl.acm.org/doi/10.1145/3545945.3569830</dd></div><div class="evidence-detail-row"><dt>Verifiable URL</dt><dd><a href="https://dl.acm.org/doi/10.1145/3545945.3569830">https://dl.acm.org/doi/10.1145/3545945.3569830</a></dd></div></dl></details></td><td><a class="source-link" href="https://dl.acm.org/doi/10.1145/3545945.3569830" title="Using GitHub Copilot to Solve Introductory Programming Problems"><code>S-2022-finnie-ansley</code></a></td></tr><tr data-effect="positive" data-direction="positive" data-outcome="metacognition" data-search="e-011 study-explcomp-2023 smpl-explcomp-2023-ratings comparing code explanations created by students and large language models controlled comparisons find llm-generated code explanations comparable to (in places better than) student-authored explanations, suggesting viability as explanatory scaffold material rather than as a replacement for student explanation practice. student-produced versus llm-produced explanations of short programs under controlled comparison llm_generated_code_explanations student_generated_explanations_of_same_programs positive s-2023-explanations-compare"><td><code>E-011</code></td><td><strong>Metacognition</strong></td><td><span class="dir pos">Positive effect</span><span class="relation-note">Claim relation: Support</span></td><td><div class="quality-cell"><span class="num">7</span><span class="quality-meter" aria-hidden="true"><i style="width:70%"></i></span></div></td><td class="claim-cell"><p>Controlled comparisons find LLM-generated code explanations comparable to (in places better than) student-authored explanations, suggesting viability as explanatory scaffold material rather than as a replacemen…</p><details class="matrix-row-detail"><summary>View full evidence</summary><dl class="evidence-detail-grid"><div class="evidence-detail-row"><dt>Study ID</dt><dd>STUDY-EXPLCOMP-2023</dd></div><div class="evidence-detail-row"><dt>Sample ID</dt><dd>SMPL-EXPLCOMP-2023-RATINGS</dd></div><div class="evidence-detail-row"><dt>Study title</dt><dd>Comparing Code Explanations Created by Students and Large Language Models</dd></div><div class="evidence-detail-row"><dt>Source title</dt><dd>Comparing Code Explanations Created by Students and Large Language Models</dd></div><div class="evidence-detail-row"><dt>Year</dt><dd>2023</dd></div><div class="evidence-detail-row"><dt>Study design</dt><dd>Observational</dd></div><div class="evidence-detail-row"><dt>Education level</dt><dd>University Introductory</dd></div><div class="evidence-detail-row"><dt>Population</dt><dd>Student-produced versus LLM-produced explanations of short programs under controlled comparison</dd></div><div class="evidence-detail-row"><dt>Intervention</dt><dd>LLM Generated Code Explanations</dd></div><div class="evidence-detail-row"><dt>Comparison</dt><dd>Student Generated Explanations Of Same Programs</dd></div><div class="evidence-detail-row"><dt>Outcome measure</dt><dd>Rated Explanation Quality And Comprehensibility</dd></div><div class="evidence-detail-row"><dt>Effect / result</dt><dd>comparable-or-better rated quality vs student explanations</dd></div><div class="evidence-detail-row"><dt>Effect direction</dt><dd>Positive effect</dd></div><div class="evidence-detail-row"><dt>Relation to claim</dt><dd>Support</dd></div><div class="evidence-detail-row"><dt>Duration</dt><dd>Single Session Ratings</dd></div><div class="evidence-detail-row"><dt>Method</dt><dd>controlled comparison with blind rating of explanation pairs</dd></div><div class="evidence-detail-row"><dt>Strengths</dt><dd>controlled_pairwise_comparison；learning_process_relevant_construct</dd></div><div class="evidence-detail-row"><dt>Limitations</dt><dd>short_term_ratings_not_learning_gains；small_program_snippets_ecology</dd></div><div class="evidence-detail-row"><dt>Confounders</dt><dd>Rating Criteria Subjectivity</dd></div><div class="evidence-detail-row"><dt>Quality dimensions</dt><dd>D1 Study design = 1 · D2 Sample quality = 2 · D3 Measurement validity = 2 · D4 Temporal strength = 1 · D5 Directness = 1</dd></div><div class="evidence-detail-row"><dt>Quality score</dt><dd>7.0</dd></div><div class="evidence-detail-row"><dt>Evidence level</dt><dd>moderate</dd></div><div class="evidence-detail-row"><dt>Applicability</dt><dd>learner_match=partial_scaffold_material_only · subject_match=introductory_programming · tool_match=llm_explanations · scope=scaffold_quality_not_effectiveness</dd></div><div class="evidence-detail-row"><dt>Confidence</dt><dd>0.55</dd></div><div class="evidence-detail-row"><dt>Evidence status</dt><dd>Supported</dd></div><div class="evidence-detail-row"><dt>Full claim</dt><dd>Controlled comparisons find LLM-generated code explanations comparable to (in places better than) student-authored explanations, suggesting viability as explanatory scaffold material rather than as a replacement for student explanation practice.</dd></div><div class="evidence-detail-row"><dt>Source location</dt><dd>https://dl.acm.org/doi/10.1145/3587102.3588785</dd></div><div class="evidence-detail-row"><dt>Verifiable URL</dt><dd><a href="https://dl.acm.org/doi/10.1145/3587102.3588785">https://dl.acm.org/doi/10.1145/3587102.3588785</a></dd></div></dl></details></td><td><a class="source-link" href="https://dl.acm.org/doi/10.1145/3587102.3588785" title="Comparing Code Explanations Created by Students and Large Language Models"><code>S-2023-explanations-compare</code></a></td></tr><tr data-effect="negative" data-direction="negative" data-outcome="over_reliance" data-search="e-012 study-vaithilingam-2022 smpl-vaithilingam-2022-n24 expectation vs. experience: evaluating the usability of code generation tools despite faster first-task completion, participants struggled to understand and debug ai-generated solutions and reported low ownership of the final program - documenting metacognitive and dependence risks that pure speed metrics miss. 24 participants in a within-subjects usability study of copilot-style tools copilot_assisted_program_writing within_subject_baseline_without_tool negative s-2022-vaithilingam"><td><code>E-012</code></td><td><strong>Over-reliance</strong></td><td><span class="dir neg">Negative effect</span><span class="relation-note">Claim relation: Contradict</span></td><td><div class="quality-cell"><span class="num">7</span><span class="quality-meter" aria-hidden="true"><i style="width:70%"></i></span></div></td><td class="claim-cell"><p>Despite faster first-task completion, participants struggled to understand and debug AI-generated solutions and reported low ownership of the final program - documenting metacognitive and dependence risks that…</p><details class="matrix-row-detail"><summary>View full evidence</summary><dl class="evidence-detail-grid"><div class="evidence-detail-row"><dt>Study ID</dt><dd>STUDY-VAITHILINGAM-2022</dd></div><div class="evidence-detail-row"><dt>Sample ID</dt><dd>SMPL-VAITHILINGAM-2022-N24</dd></div><div class="evidence-detail-row"><dt>Study title</dt><dd>Expectation vs. Experience: Evaluating the Usability of Code Generation Tools</dd></div><div class="evidence-detail-row"><dt>Source title</dt><dd>Expectation vs. Experience: Evaluating the Usability of Code Generation Tools</dd></div><div class="evidence-detail-row"><dt>Year</dt><dd>2022</dd></div><div class="evidence-detail-row"><dt>Study design</dt><dd>Qualitative</dd></div><div class="evidence-detail-row"><dt>Education level</dt><dd>Mixed Cs Students And Professionals</dd></div><div class="evidence-detail-row"><dt>Population</dt><dd>24 participants in a within-subjects usability study of Copilot-style tools</dd></div><div class="evidence-detail-row"><dt>Sample size</dt><dd>24</dd></div><div class="evidence-detail-row"><dt>Intervention</dt><dd>Copilot Assisted Program Writing</dd></div><div class="evidence-detail-row"><dt>Comparison</dt><dd>Within Subject Baseline Without Tool</dd></div><div class="evidence-detail-row"><dt>Outcome measure</dt><dd>Understanding Ownership And Debugging Reports</dd></div><div class="evidence-detail-row"><dt>Effect / result</dt><dd>documented comprehension/ownership difficulties despite speed gain</dd></div><div class="evidence-detail-row"><dt>Effect direction</dt><dd>Negative effect</dd></div><div class="evidence-detail-row"><dt>Relation to claim</dt><dd>Contradict</dd></div><div class="evidence-detail-row"><dt>Duration</dt><dd>Single Session</dd></div><div class="evidence-detail-row"><dt>Method</dt><dd>within-subject usability study with tasks, observation and interviews</dd></div><div class="evidence-detail-row"><dt>Strengths</dt><dd>rich_qualitative_process_data；constructs_missed_by_speed_metrics</dd></div><div class="evidence-detail-row"><dt>Limitations</dt><dd>small_n_24；single_session；self_reported_understanding</dd></div><div class="evidence-detail-row"><dt>Confounders</dt><dd>Participant AI Familiarity</dd></div><div class="evidence-detail-row"><dt>Quality dimensions</dt><dd>D1 Study design = 1 · D2 Sample quality = 2 · D3 Measurement validity = 2 · D4 Temporal strength = 1 · D5 Directness = 1</dd></div><div class="evidence-detail-row"><dt>Quality score</dt><dd>7.0</dd></div><div class="evidence-detail-row"><dt>Evidence level</dt><dd>moderate</dd></div><div class="evidence-detail-row"><dt>Applicability</dt><dd>learner_match=partial_includes_cs_students · subject_match=programming_adjacent · tool_match=copilot_like_generative_ai · scope=risk_identification</dd></div><div class="evidence-detail-row"><dt>Confidence</dt><dd>0.55</dd></div><div class="evidence-detail-row"><dt>Evidence status</dt><dd>Contradicted</dd></div><div class="evidence-detail-row"><dt>Full claim</dt><dd>Despite faster first-task completion, participants struggled to understand and debug AI-generated solutions and reported low ownership of the final program - documenting metacognitive and dependence risks that pure speed metrics miss.</dd></div><div class="evidence-detail-row"><dt>Source location</dt><dd>https://dl.acm.org/doi/10.1145/3491101.3519665</dd></div><div class="evidence-detail-row"><dt>Verifiable URL</dt><dd><a href="https://dl.acm.org/doi/10.1145/3491101.3519665">https://dl.acm.org/doi/10.1145/3491101.3519665</a></dd></div></dl></details></td><td><a class="source-link" href="https://dl.acm.org/doi/10.1145/3491101.3519665" title="Expectation vs. Experience: Evaluating the Usability of Code Generation Tools"><code>S-2022-vaithilingam</code></a></td></tr></tbody></table></div></div></section><section id="full-03-quality-en" class="full-chapter" data-full-chapter><header class="full-chapter-header"><h2>03 Evidence Quality, Counterevidence &amp; Method Audit</h2><p class="full-chapter-lead">Examine why evidence is credible, where it conflicts, and which conclusions require downgrading.</p></header><div class="full-chapter-body"><section class="method-review-group"><header class="method-review-title"><h3>Audit target: overall</h3><span class="method-verdict">Concern</span></header><div class="method-audit-grid"><article class="method-audit-item method-met"><div class="method-audit-head"><strong>Control group</strong><span class="method-status">Met</span></div><p>All three studies include a no-AI control group.</p></article><article class="method-audit-item method-met"><div class="method-audit-head"><strong>Randomization</strong><span class="method-status">Met</span></div><p>Kazemitabaar 2023 and Bastani 2025 use randomized assignment.</p></article><article class="method-audit-item method-met"><div class="method-audit-head"><strong>Pre-test</strong><span class="method-status">Met</span></div><p>Kazemitabaar 2023 has a pre-study evaluation; Bastani 2025 measures baseline covariates.</p></article><article class="method-audit-item method-met"><div class="method-audit-head"><strong>Post-test</strong><span class="method-status">Met</span></div><p>Immediate post-tests present in all studies.</p></article><article class="method-audit-item method-partial"><div class="method-audit-head"><strong>Retention test</strong><span class="method-status">Partial</span></div><p>Kazemitabaar 2023 has 1-week retention; Bastani 2025 has no delayed test; Marzuki 2024 has delayed test.</p></article><article class="method-audit-item method-partial"><div class="method-audit-head"><strong>Transfer test</strong><span class="method-status">Partial</span></div><p>Kazemitabaar 2023 code-modification task is transfer-adjacent; no full no-AI transfer task.</p></article><article class="method-audit-item method-met"><div class="method-audit-head"><strong>Sample bias</strong><span class="method-status">Met</span></div><p>Bastani 2025 nearly 1000 students; Kazemitabaar 2023 small (69) young sample.</p></article><article class="method-audit-item method-partial"><div class="method-audit-head"><strong>Self-selection bias</strong><span class="method-status">Partial</span></div><p>Marzuki 2024 consent-based participation risks self-selection.</p></article><article class="method-audit-item method-partial"><div class="method-audit-head"><strong>Measurement validity</strong><span class="method-status">Partial</span></div><p>Practice/task performance is not equated to learning; independent exams present in Bastani 2025 only.</p></article><article class="method-audit-item method-partial"><div class="method-audit-head"><strong>Confounders</strong><span class="method-status">Partial</span></div><p>Prior programming competency interacts with AI benefit in Kazemitabaar 2023.</p></article><article class="method-audit-item method-not_applicable"><div class="method-audit-head"><strong>Instructor effect</strong><span class="method-status">N/A</span></div><p>Kazemitabaar 2023 is self-paced; classroom studies may carry instructor effects.</p></article><article class="method-audit-item method-partial"><div class="method-audit-head"><strong>Novelty effect</strong><span class="method-status">Partial</span></div><p>Short interventions likely inflate engagement; none of the studies controlled for novelty.</p></article><article class="method-audit-item method-not_applicable"><div class="method-audit-head"><strong>Tool-version effect</strong><span class="method-status">N/A</span></div><p>Single tool versions studied; rapid tool change limits durability.</p></article><article class="method-audit-item method-partial"><div class="method-audit-head"><strong>AI usage policy</strong><span class="method-status">Partial</span></div><p>Bastani 2025 explicitly contrasts unguarded vs guardrailed usage policies.</p></article><article class="method-audit-item method-partial"><div class="method-audit-head"><strong>Attrition</strong><span class="method-status">Partial</span></div><p>Marzuki 2024 reports attrition; others not detailed.</p></article></div><div class="method-guard-wrap"><strong>Task vs learning guard: </strong><p class="method-guard">Bastani 2025 demonstrates the danger of equating the two: +48-127% practice performance coexisted with -17% independent exam performance.</p></div></section><article class="conflict-card"><strong>Tribunal note: </strong><div class="conflict-text"><p>Disagreement comes from outcome separation (task vs learning), tool design (guarded vs unguarded), and population (K-12 / professionals vs university). Task-performance evidence is consistently positive across randomized and benchmark studies;…</p><details class="detail-expander"><summary>Expand full explanation</summary><div class="detail-body"><p>Disagreement comes from outcome separation (task vs learning), tool design (guarded vs unguarded), and population (K-12 / professionals vs university). Task-performance evidence is consistently positive across randomized and benchmark studies; the only study measuring independent performance after AI removal shows harm without guardrails; usability and artifact studies add dependence and quality caveats rather than resolving the learning question.</p></div></details></div></article><div class="evidence-tribunal" data-visual="evidence-tribunal-grid"><p class="tribunal-summary"><strong>Decision: </strong>Pilot · <strong>Confidence: </strong>Moderate</p><div class="tribunal-grid"><article class="tribunal-card supported"><header><span aria-hidden="true">✓</span><h3>Can claim</h3><span class="tribunal-count">7</span></header><ul><li><p>AI coding assistants reliably increase task performance during training (completion speed, correctness) — E-001, E-006.</p><div class="tribunal-evidence-refs"><code>E-001</code><code>E-006</code></div></li><li><p>Unguarded generative AI access can harm independent problem solving when access is removed — E-004.</p><div class="tribunal-evidence-refs"><code>E-004</code></div></li><li><p>Guardrail design (hints instead of answers) substantially mitigates the negative learning effect — E-005.</p><div class="tribunal-evidence-refs"><code>E-005</code></div></li><li><p>Task performance gains do not automatically imply learning gains — E-004 vs E-006 (within-study contrast).</p><div class="tribunal-evidence-refs"><code>E-004</code><code>E-006</code></div></li><li><p>Tool capability is substantial: Codex solves roughly half to three-quarters of CS1 exam-style questions — E-010.</p><div class="tribunal-evidence-refs"><code>E-010</code></div></li><li><p>Professional-developer RCT shows ~55% faster task completion with Copilot; directness limited by professional population — E-008.</p><div class="tribunal-evidence-refs"><code>E-008</code></div></li><li><p>LLM code explanations rate comparable to student-authored explanations, viable as scaffold material — E-011.</p><div class="tribunal-evidence-refs"><code>E-011</code></div></li></ul></article><article class="tribunal-card uncertain"><header><span aria-hidden="true">?</span><h3>Cannot yet claim</h3><span class="tribunal-count">4</span></header><ul><li><p>Whether AI coding assistants improve or preserve actual programming learning in university novices — no direct university-level RCT in reviewed set [无直接证据]</p></li><li><p>Whether one-week neutral retention (Kazemitabaar 2023) extends to a semester — E-003.</p><div class="tribunal-evidence-refs"><code>E-003</code></div></li><li><p>Whether benchmark quality findings (E-009) and explanation-quality ratings (E-011) translate into classroom learning gains.</p><div class="tribunal-evidence-refs"><code>E-009</code><code>E-011</code></div></li><li><p>How comprehension/ownership difficulties documented in usability studies (E-012) behave over a full semester with guardrails.</p><div class="tribunal-evidence-refs"><code>E-012</code></div></li></ul></article><article class="tribunal-card contradicted"><header><span aria-hidden="true">×</span><h3>Contradicted claims</h3><span class="tribunal-count">2</span></header><ul><li><p>The claim &#x27;AI tools always improve learning&#x27; is contradicted by E-004 (unguarded access, -17% independent exam).</p><div class="tribunal-evidence-refs"><code>E-004</code></div></li><li><p>The claim &#x27;speed gains equal learning gains&#x27; is contradicted by the task-vs-learning separation across E-001/E-006/E-008 vs E-004.</p><div class="tribunal-evidence-refs"><code>E-001</code><code>E-006</code><code>E-008</code><code>E-004</code></div></li></ul></article><article class="tribunal-card missing"><header><span aria-hidden="true">…</span><h3>Missing evidence</h3><span class="tribunal-count">4</span></header><ul><li><p>RCT of AI coding assistants in university programming courses with retention and no-AI transfer tests.</p></li><li><p>Studies varying AI usage policy within the same course.</p></li><li><p>Longitudinal data on AI dependency beyond one course.</p></li><li><p>Peer-reviewed replication of the professional speed RCT (Peng et al. remains a preprint).</p></li></ul></article></div><details class="supporting-visual"><summary>EvidenceFlow Protocol</summary><svg viewBox="0 0 720 260" xmlns="http://www.w3.org/2000/svg" role="img" aria-label="EvidenceFlow Protocol"><title>EvidenceFlow Protocol</title><desc>Research flow from framing, retrieval, fetch/verify, extraction, challenge, method audit and adjudication to applicability and intervention evaluation.</desc><defs><marker id="arr-en-full-workflow" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto"><path d="M0,0 L6,3 L0,6 Z" fill="#8A867E"/></marker></defs><rect width="720" height="260" fill="#FCFAF6" rx="12"/><text x="24" y="34" font-size="16" font-weight="700" fill="#3A3833">EvidenceFlow Protocol</text><rect x="24" y="100" width="64" height="52" rx="8" fill="#B8694A"/><text x="56.0" y="126.0" text-anchor="middle" fill="#FFFFFF" font-size="10" font-weight="600"><tspan x="56.0" y="126.0">Frame</tspan></text><line x1="88" y1="126" x2="100" y2="126" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-en-full-workflow)"/><rect x="100" y="100" width="64" height="52" rx="8" fill="#B8694A"/><text x="132.0" y="126.0" text-anchor="middle" fill="#FFFFFF" font-size="10" font-weight="600"><tspan x="132.0" y="126.0">Retrieve</tspan></text><line x1="164" y1="126" x2="176" y2="126" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-en-full-workflow)"/><rect x="176" y="100" width="64" height="52" rx="8" fill="#B8694A"/><text x="208.0" y="126.0" text-anchor="middle" fill="#FFFFFF" font-size="10" font-weight="600"><tspan x="208.0" y="126.0">Fetch</tspan><tspan x="208.0" y="140.0">Verify</tspan></text><line x1="240" y1="126" x2="252" y2="126" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-en-full-workflow)"/><rect x="252" y="100" width="64" height="52" rx="8" fill="#B8694A"/><text x="284.0" y="126.0" text-anchor="middle" fill="#FFFFFF" font-size="10" font-weight="600"><tspan x="284.0" y="126.0">Extract</tspan></text><line x1="316" y1="126" x2="328" y2="126" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-en-full-workflow)"/><rect x="328" y="100" width="64" height="52" rx="8" fill="#5E8A6A"/><text x="360.0" y="126.0" text-anchor="middle" fill="#FFFFFF" font-size="10" font-weight="600"><tspan x="360.0" y="126.0">Challenge</tspan></text><line x1="392" y1="126" x2="404" y2="126" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-en-full-workflow)"/><rect x="404" y="100" width="64" height="52" rx="8" fill="#5E8A6A"/><text x="436.0" y="126.0" text-anchor="middle" fill="#FFFFFF" font-size="10" font-weight="600"><tspan x="436.0" y="126.0">Audit</tspan><tspan x="436.0" y="140.0">Method</tspan></text><line x1="468" y1="126" x2="480" y2="126" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-en-full-workflow)"/><rect x="480" y="100" width="64" height="52" rx="8" fill="#C99A4A"/><text x="512.0" y="126.0" text-anchor="middle" fill="#FFFFFF" font-size="10" font-weight="600"><tspan x="512.0" y="126.0">Adjudicate</tspan></text><line x1="544" y1="126" x2="556" y2="126" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-en-full-workflow)"/><rect x="556" y="100" width="64" height="52" rx="8" fill="#C99A4A"/><text x="588.0" y="126.0" text-anchor="middle" fill="#FFFFFF" font-size="10" font-weight="600"><tspan x="588.0" y="126.0">Applicability</tspan></text><line x1="620" y1="126" x2="632" y2="126" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-en-full-workflow)"/><rect x="632" y="100" width="64" height="52" rx="8" fill="#C99A4A"/><text x="664.0" y="126.0" text-anchor="middle" fill="#FFFFFF" font-size="10" font-weight="600"><tspan x="664.0" y="126.0">Intervene</tspan><tspan x="664.0" y="140.0">Evaluate</tspan></text></svg></details><details class="supporting-visual"><summary>Tribunal infographic</summary><svg viewBox="0 0 720 260" xmlns="http://www.w3.org/2000/svg" role="img" aria-label="Evidence Tribunal infographic"><title>Evidence Tribunal infographic</title><desc>Evidence IDs for claims that can and cannot be claimed, plus the recommended action badge; full claim text is in the tribunal cards below.</desc><defs><marker id="arr-en-full-tribunal" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto"><path d="M0,0 L6,3 L0,6 Z" fill="#8A867E"/></marker></defs><rect width="720" height="260" fill="#FCFAF6" rx="12"/><text x="24" y="34" font-size="16" font-weight="700" fill="#3A3833">Evidence Tribunal</text><text x="24" y="180" font-size="13" font-weight="700" fill="#3A3833">Source of conflict</text><text x="24" y="202" font-size="11" fill="#8A867E">See tribunal cards below</text><rect x="24" y="216" width="180" height="28" rx="14" fill="#C99A4A"/><text x="114" y="235" text-anchor="middle" font-size="13" font-weight="700" fill="#fff">Pilot</text><text x="250" y="80" font-size="13" font-weight="700" fill="#5E8A6A">Can claim (7)</text><circle cx="254" cy="100" r="3" fill="#5E8A6A"/><text x="266" y="105" font-size="11" fill="#3A3833">E-001</text><circle cx="254" cy="122" r="3" fill="#5E8A6A"/><text x="266" y="127" font-size="11" fill="#3A3833">E-006</text><circle cx="254" cy="144" r="3" fill="#5E8A6A"/><text x="266" y="149" font-size="11" fill="#3A3833">E-004</text><circle cx="254" cy="166" r="3" fill="#5E8A6A"/><text x="266" y="171" font-size="11" fill="#3A3833">E-005</text><text x="490" y="80" font-size="13" font-weight="700" fill="#A85B53">Cannot claim (2)</text><circle cx="494" cy="100" r="3" fill="#A85B53"/><text x="506" y="105" font-size="11" fill="#3A3833">E-004</text><circle cx="494" cy="122" r="3" fill="#A85B53"/><text x="506" y="127" font-size="11" fill="#3A3833">E-001</text><circle cx="494" cy="144" r="3" fill="#A85B53"/><text x="506" y="149" font-size="11" fill="#3A3833">E-006</text><circle cx="494" cy="166" r="3" fill="#A85B53"/><text x="506" y="171" font-size="11" fill="#3A3833">E-008</text></svg></details></div><div class="trace-chain" data-visual="trace-chain"><article class="trace-chain-card"><div class="trace-claim-node"><strong>C-001</strong><p class="trace-claim-text">AI coding assistants significantly increase task completion speed and completion rate during training.</p></div><div class="trace-arrow">↓</div><div class="trace-evidence-list"><div class="trace-evidence-node"><code>E-001</code><span>Completion time</span><span class="dir pos">Positive effect</span><span class="trace-relation">claim relation: Support</span><span>Q 9.0</span><span class="trace-arrow">→</span><a href="https://dl.acm.org/doi/10.1145/3544548.3580919"><code>S-2023-kazemitabaar</code></a></div></div></article><article class="trace-chain-card"><div class="trace-claim-node"><strong>C-002</strong><p class="trace-claim-text">Access to AI code generation did not decrease performance on manual code-modification tasks.</p></div><div class="trace-arrow">↓</div><div class="trace-evidence-list"><div class="trace-evidence-node"><code>E-002</code><span>Independent problem solving</span><span class="dir neu">Null effect</span><span class="trace-relation">claim relation: Neutral</span><span>Q 7.0</span><span class="trace-arrow">→</span><a href="https://dl.acm.org/doi/10.1145/3544548.3580919"><code>S-2023-kazemitabaar</code></a></div></div></article><article class="trace-chain-card"><div class="trace-claim-node"><strong>C-003</strong><p class="trace-claim-text">One week after training, retention differences between Codex and baseline groups did not reach statistical significance.</p></div><div class="trace-arrow">↓</div><div class="trace-evidence-list"><div class="trace-evidence-node"><code>E-003</code><span>Retention</span><span class="dir neu">Null effect</span><span class="trace-relation">claim relation: Neutral</span><span>Q 9.0</span><span class="trace-arrow">→</span><a href="https://dl.acm.org/doi/10.1145/3544548.3580919"><code>S-2023-kazemitabaar</code></a></div></div></article><article class="trace-chain-card"><div class="trace-claim-node"><strong>C-004</strong><p class="trace-claim-text">Students with unguarded GPT-4 access performed 17% worse on the independent exam than the control group, despite higher practice performance.</p></div><div class="trace-arrow">↓</div><div class="trace-evidence-list"><div class="trace-evidence-node"><code>E-004</code><span>Independent problem solving</span><span class="dir neg">Negative effect</span><span class="trace-relation">claim relation: Contradict</span><span>Q 8.0</span><span class="trace-arrow">→</span><a href="https://www.pnas.org/doi/10.1073/pnas.2422633122"><code>S-2025-bastani</code></a></div></div></article><article class="trace-chain-card"><div class="trace-claim-node"><strong>C-005</strong><p class="trace-claim-text">Guardrail design of the AI tutor (hints instead of answers, teacher-informed prompts) largely eliminated the negative learning effect.</p></div><div class="trace-arrow">↓</div><div class="trace-evidence-list"><div class="trace-evidence-node"><code>E-005</code><span>Independent problem solving</span><span class="dir neu">Null effect</span><span class="trace-relation">claim relation: Support</span><span>Q 8.0</span><span class="trace-arrow">→</span><a href="https://www.pnas.org/doi/10.1073/pnas.2422633122"><code>S-2025-bastani</code></a></div></div></article><article class="trace-chain-card"><div class="trace-claim-node"><strong>C-006</strong><p class="trace-claim-text">Access to GPT-4 during practice improves task performance (48% for GPT Base, 127% for GPT Tutor) — but this task performance does not transfer to independent exam performance.</p></div><div class="trace-arrow">↓</div><div class="trace-evidence-list"><div class="trace-evidence-node"><code>E-006</code><span>Assignment score</span><span class="dir pos">Positive effect</span><span class="trace-relation">claim relation: Support</span><span>Q 8.0</span><span class="trace-arrow">→</span><a href="https://www.pnas.org/doi/10.1073/pnas.2422633122"><code>S-2025-bastani</code></a></div></div></article><article class="trace-chain-card"><div class="trace-claim-node"><strong>C-007</strong><p class="trace-claim-text">ChatGPT as a formative feedback tool produced a significant positive impact on students&#x27; academic writing skills with positive student perceptions.</p></div><div class="trace-arrow">↓</div><div class="trace-evidence-list"><div class="trace-evidence-node"><code>E-007</code><span>Knowledge gain</span><span class="dir pos">Positive effect</span><span class="trace-relation">claim relation: Support</span><span>Q 6.0</span><span class="trace-arrow">→</span><a href="https://link.springer.com/article/10.1186/s40561-024-00295-9"><code>S-2024-marzuki</code></a></div></div></article><article class="trace-chain-card"><div class="trace-claim-node"><strong>C-008</strong><p class="trace-claim-text">Professional developers with Copilot access completed a standardized coding task about 55% faster than the control group (RCT, n=95).</p></div><div class="trace-arrow">↓</div><div class="trace-evidence-list"><div class="trace-evidence-node"><code>E-008</code><span>Completion time</span><span class="dir pos">Positive effect</span><span class="trace-relation">claim relation: Support</span><span>Q 8.0</span><span class="trace-arrow">→</span><a href="https://doi.org/10.48550/arXiv.2302.06590"><code>S-2023-peng</code></a></div></div></article><article class="trace-chain-card"><div class="trace-claim-node"><strong>C-009</strong><p class="trace-claim-text">Systematic benchmark evaluation reports mixed quality results for Copilot-generated code relative to human code: correctness competitive on parts of the benchmark while security-relevant defects are documented.</p></div><div class="trace-arrow">↓</div><div class="trace-evidence-list"><div class="trace-evidence-node"><code>E-009</code><span>Code quality</span><span class="dir neu">Null effect</span><span class="trace-relation">claim relation: Neutral</span><span>Q 7.0</span><span class="trace-arrow">→</span><a href="https://doi.org/10.1016/j.jss.2023.111734"><code>S-2023-yetistiren</code></a></div></div></article><article class="trace-chain-card"><div class="trace-claim-node"><strong>C-010</strong><p class="trace-claim-text">Codex produced passing-level solutions for roughly half to three-quarters of CS1 exam-style questions depending on the dataset, indicating substantial task-capability headroom available to novices.</p></div><div class="trace-arrow">↓</div><div class="trace-evidence-list"><div class="trace-evidence-node"><code>E-010</code><span>Assignment score</span><span class="dir pos">Positive effect</span><span class="trace-relation">claim relation: Support</span><span>Q 7.0</span><span class="trace-arrow">→</span><a href="https://dl.acm.org/doi/10.1145/3545945.3569830"><code>S-2022-finnie-ansley</code></a></div></div></article><article class="trace-chain-card"><div class="trace-claim-node"><strong>C-011</strong><div class="trace-claim-text"><p>Controlled comparisons find LLM-generated code explanations comparable to (in places better than) student-authored explanations, suggesting viability as explanatory scaffold material rather than as a replacement for student explanation prac…</p><details class="detail-expander"><summary>Expand full explanation</summary><div class="detail-body"><p>Controlled comparisons find LLM-generated code explanations comparable to (in places better than) student-authored explanations, suggesting viability as explanatory scaffold material rather than as a replacement for student explanation practice.</p></div></details></div></div><div class="trace-arrow">↓</div><div class="trace-evidence-list"><div class="trace-evidence-node"><code>E-011</code><span>Metacognition</span><span class="dir pos">Positive effect</span><span class="trace-relation">claim relation: Support</span><span>Q 7.0</span><span class="trace-arrow">→</span><a href="https://dl.acm.org/doi/10.1145/3587102.3588785"><code>S-2023-explanations-compare</code></a></div></div></article><article class="trace-chain-card"><div class="trace-claim-node"><strong>C-012</strong><p class="trace-claim-text">Despite faster first-task completion, participants struggled to understand and debug AI-generated solutions and reported low ownership of the final program - documenting metacognitive and dependence risks that pure speed metrics miss.</p></div><div class="trace-arrow">↓</div><div class="trace-evidence-list"><div class="trace-evidence-node"><code>E-012</code><span>Over-reliance</span><span class="dir neg">Negative effect</span><span class="trace-relation">claim relation: Contradict</span><span>Q 7.0</span><span class="trace-arrow">→</span><a href="https://dl.acm.org/doi/10.1145/3491101.3519665"><code>S-2022-vaithilingam</code></a></div></div></article></div><div id="chart-trace-en" class="chart-mount" aria-label="Claim-Evidence Trace"></div><p class="chart-interpretation"><strong>What this means: </strong>Every important claim must resolve to Evidence IDs and original sources.</p></div></section><section id="full-04-action-en" class="full-chapter" data-full-chapter><header class="full-chapter-header"><h2>04 Applicability &amp; Teaching Action</h2><p class="full-chapter-lead">Connect applicability, guardrails and teaching actions to specific evidence.</p></header><div class="full-chapter-body"><p><strong>Target population: </strong>university first-year computer science students learning C programming for the first time</p>
2510
2510
  <p><strong>Target context: </strong>16-week lecture-lab course, 60 students, TA support, offline</p>
2511
2511
  <p><strong>Suitable for: </strong>pilot in first-year C course with guardrailed usage policy</p>
2512
2512
  <p><strong>Not suitable for: </strong>unrestricted AI adoption without usage policy</p>
@@ -2530,7 +2530,7 @@ select:focus-visible,
2530
2530
  </div>
2531
2531
  <h3>Stop conditions</h3><ul><li>transfer-test scores drop significantly below baseline cohort expectations</li><li>widespread integrity violations in reasoning traces</li><li>AI dependency signals exceed threshold in risk metrics</li><li>TA/teacher workload becomes unsustainable</li></ul>
2532
2532
  <h3>Intervention timeline infographic</h3>
2533
- <svg viewBox="0 0 720 260" xmlns="http://www.w3.org/2000/svg" role="img" aria-label="Teaching intervention timeline"><title>Teaching intervention timeline</title><desc>Short phase names and activity counts; full AI usage rules are in the phase blocks.</desc><defs><marker id="arr-en-full-intervention" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto"><path d="M0,0 L6,3 L0,6 Z" fill="#8A867E"/></marker></defs><rect width="720" height="260" fill="#FCFAF6" rx="12"/><text x="24" y="34" font-size="16" font-weight="700" fill="#3A3833">Intervention Timeline</text><rect x="24" y="100" width="160" height="96" rx="8" fill="#B8694A"/><text x="104.0" y="143.0" text-anchor="middle" fill="#FFFFFF" font-size="12" font-weight="600"><tspan x="104.0" y="143.0">Phase 1</tspan></text><text x="104.0" y="159.0" text-anchor="middle" fill="#FFFFFF" font-size="11" opacity="0.85">2 activities</text><line x1="184" y1="148" x2="200" y2="148" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-en-full-intervention)"/><rect x="200" y="100" width="160" height="96" rx="8" fill="#B8694A"/><text x="280.0" y="143.0" text-anchor="middle" fill="#FFFFFF" font-size="12" font-weight="600"><tspan x="280.0" y="143.0">Phase 2</tspan></text><text x="280.0" y="159.0" text-anchor="middle" fill="#FFFFFF" font-size="11" opacity="0.85">1 activities</text><line x1="360" y1="148" x2="376" y2="148" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-en-full-intervention)"/><rect x="376" y="100" width="160" height="96" rx="8" fill="#B8694A"/><text x="456.0" y="143.0" text-anchor="middle" fill="#FFFFFF" font-size="12" font-weight="600"><tspan x="456.0" y="143.0">Phase 3</tspan></text><text x="456.0" y="159.0" text-anchor="middle" fill="#FFFFFF" font-size="11" opacity="0.85">1 activities</text><line x1="536" y1="148" x2="552" y2="148" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-en-full-intervention)"/><rect x="552" y="100" width="160" height="96" rx="8" fill="#B8694A"/><text x="632.0" y="143.0" text-anchor="middle" fill="#FFFFFF" font-size="12" font-weight="600"><tspan x="632.0" y="143.0">Phase 4</tspan></text><text x="632.0" y="159.0" text-anchor="middle" fill="#FFFFFF" font-size="11" opacity="0.85">1 activities</text></svg></div></section><section id="full-05-evaluation-en" class="full-chapter" data-full-chapter><header class="full-chapter-header"><h2>05 Pilot, Evaluation &amp; Stop Conditions</h2><p class="full-chapter-lead">Validate the pilot with independent learning outcomes and pre-specified stop conditions.</p></header><div class="full-chapter-body"><p><strong>Research question: </strong>In a first-year C programming course, does a guardrailed AI coding assistant (explain-only → structured collaboration) improve independent problem solving without increasing AI dependency, compared to no-AI instruction?</p>
2533
+ <svg viewBox="0 0 720 260" xmlns="http://www.w3.org/2000/svg" role="img" aria-label="Teaching intervention timeline"><title>Teaching intervention timeline</title><desc>Short phase names and activity counts; full AI usage rules are in the phase blocks.</desc><defs><marker id="arr-en-full-intervention" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto"><path d="M0,0 L6,3 L0,6 Z" fill="#8A867E"/></marker></defs><rect width="720" height="260" fill="#FCFAF6" rx="12"/><text x="24" y="34" font-size="16" font-weight="700" fill="#3A3833">Teaching intervention timeline</text><rect x="24" y="100" width="160" height="96" rx="8" fill="#B8694A"/><text x="104.0" y="143.0" text-anchor="middle" fill="#FFFFFF" font-size="12" font-weight="600"><tspan x="104.0" y="143.0">Phase 1</tspan></text><text x="104.0" y="159.0" text-anchor="middle" fill="#FFFFFF" font-size="11" opacity="0.85">2 activities</text><line x1="184" y1="148" x2="200" y2="148" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-en-full-intervention)"/><rect x="200" y="100" width="160" height="96" rx="8" fill="#B8694A"/><text x="280.0" y="143.0" text-anchor="middle" fill="#FFFFFF" font-size="12" font-weight="600"><tspan x="280.0" y="143.0">Phase 2</tspan></text><text x="280.0" y="159.0" text-anchor="middle" fill="#FFFFFF" font-size="11" opacity="0.85">1 activities</text><line x1="360" y1="148" x2="376" y2="148" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-en-full-intervention)"/><rect x="376" y="100" width="160" height="96" rx="8" fill="#B8694A"/><text x="456.0" y="143.0" text-anchor="middle" fill="#FFFFFF" font-size="12" font-weight="600"><tspan x="456.0" y="143.0">Phase 3</tspan></text><text x="456.0" y="159.0" text-anchor="middle" fill="#FFFFFF" font-size="11" opacity="0.85">1 activities</text><line x1="536" y1="148" x2="552" y2="148" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-en-full-intervention)"/><rect x="552" y="100" width="160" height="96" rx="8" fill="#B8694A"/><text x="632.0" y="143.0" text-anchor="middle" fill="#FFFFFF" font-size="12" font-weight="600"><tspan x="632.0" y="143.0">Phase 4</tspan></text><text x="632.0" y="159.0" text-anchor="middle" fill="#FFFFFF" font-size="11" opacity="0.85">1 activities</text></svg></div></section><section id="full-05-evaluation-en" class="full-chapter" data-full-chapter><header class="full-chapter-header"><h2>05 Pilot, Evaluation &amp; Stop Conditions</h2><p class="full-chapter-lead">Validate the pilot with independent learning outcomes and pre-specified stop conditions.</p></header><div class="full-chapter-body"><p><strong>Research question: </strong>In a first-year C programming course, does a guardrailed AI coding assistant (explain-only → structured collaboration) improve independent problem solving without increasing AI dependency, compared to no-AI instruction?</p>
2534
2534
  <p><strong>Baseline: </strong>week-1 no-AI programming quiz (independent problem solving, completion time)</p>
2535
2535
  <p><strong>Post test: </strong>week-8 no-AI programming quiz (independent problem solving, code quality)</p>
2536
2536
  <p><strong>Retention: </strong>final exam (week 16) — delayed measure 8 weeks after pilot</p>
@@ -2541,7 +2541,7 @@ select:focus-visible,
2541
2541
  <p><strong>Success threshold: </strong>treatment group shows non-inferior independent problem solving (delta within 5%) AND superior or equal retention AND AI dependency index below threshold; if independent problem solving declines &gt;10%, the pilot is judged unsuccessful regardless of task-performance gains.</p>
2542
2542
  <p><strong>Analysis plan: </strong>pre-registered comparison of treatment vs comparison sections on baseline-adjusted learning metrics (ANCOVA); task-performance metrics reported separately from learning metrics; subgroup analysis by prior programming competency; stop-condition monitoring at weeks 3, 5, 7.</p>
2543
2543
  <h3>Evaluation design infographic</h3>
2544
- <svg viewBox="0 0 720 260" xmlns="http://www.w3.org/2000/svg" role="img" aria-label="Evaluation design flow"><title>Evaluation design flow</title><desc>Evaluation flow across baseline, post test, retention and transfer; full metrics and analysis plan are in the evaluation section.</desc><defs><marker id="arr-en-full-evaluation" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto"><path d="M0,0 L6,3 L0,6 Z" fill="#8A867E"/></marker></defs><rect width="720" height="260" fill="#FCFAF6" rx="12"/><text x="24" y="34" font-size="16" font-weight="700" fill="#3A3833">Evaluation Design Flow</text><rect x="30" y="110" width="150" height="56" rx="8" fill="#5E8A6A"/><text x="105.0" y="138.0" text-anchor="middle" fill="#FFFFFF" font-size="12" font-weight="600"><tspan x="105.0" y="138.0">Baseline</tspan></text><text x="105.0" y="148" text-anchor="middle" font-size="9" fill="#fff" opacity="0.95">pre-test</text><line x1="180" y1="138" x2="192" y2="138" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-en-full-evaluation)"/><rect x="192" y="110" width="150" height="56" rx="8" fill="#5E8A6A"/><text x="267.0" y="138.0" text-anchor="middle" fill="#FFFFFF" font-size="12" font-weight="600"><tspan x="267.0" y="138.0">Post test</tspan></text><text x="267.0" y="148" text-anchor="middle" font-size="9" fill="#fff" opacity="0.95">post-test</text><line x1="342" y1="138" x2="354" y2="138" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-en-full-evaluation)"/><rect x="354" y="110" width="150" height="56" rx="8" fill="#5E8A6A"/><text x="429.0" y="138.0" text-anchor="middle" fill="#FFFFFF" font-size="12" font-weight="600"><tspan x="429.0" y="138.0">Retention</tspan></text><text x="429.0" y="148" text-anchor="middle" font-size="9" fill="#fff" opacity="0.95">retention test</text><line x1="504" y1="138" x2="516" y2="138" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-en-full-evaluation)"/><rect x="516" y="110" width="150" height="56" rx="8" fill="#5E8A6A"/><text x="591.0" y="138.0" text-anchor="middle" fill="#FFFFFF" font-size="12" font-weight="600"><tspan x="591.0" y="138.0">Transfer</tspan></text><text x="591.0" y="148" text-anchor="middle" font-size="9" fill="#fff" opacity="0.95">transfer test (no AI)</text></svg><div class="visual-suppressed"><strong>Benchmark visual suppressed</strong><p>result.json carries no benchmark.baselines, so this visual is omitted; see the standalone benchmark report.</p></div></div></section><section id="full-06-sources-en" class="full-chapter" data-full-chapter><header class="full-chapter-header"><h2>06 Sources, Traceability &amp; Appendix</h2><p class="full-chapter-lead">Preserve original sources, URLs, evidence IDs and retrieval metadata for auditability.</p></header><div class="full-chapter-body"><h3>Source list</h3><div class='table-wrap'><table class='data-table source-table'><thead><tr><th>ID</th><th>Title</th><th>Year</th><th>Authority</th><th>Verifiable location</th></tr></thead><tbody><tr><td><code>S-2023-kazemitabaar</code></td><td class='cell-main source-title-cell'>Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming <span class="cite-badge cite-ok">DOI ✓</span><details class="source-expander detail-expander"><summary>View source &amp; provenance</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>Original title</dt><dd>Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming</dd></div><div class="source-detail-row"><dt>Year</dt><dd>2023</dd></div><div class="source-detail-row"><dt>Authority</dt><dd>Tier 1 DOI-verified paper</dd></div><div class="source-detail-row"><dt>Source location</dt><dd>https://dl.acm.org/doi/10.1145/3544548.3580919</dd></div><div class="source-detail-row"><dt>Verifiable link</dt><dd><a href="https://dl.acm.org/doi/10.1145/3544548.3580919">https://dl.acm.org/doi/10.1145/3544548.3580919</a></dd></div></dl></details></td><td>2023</td><td>Tier 1 DOI-verified paper</td><td class='cell-main'><a href='https://dl.acm.org/doi/10.1145/3544548.3580919'>https://dl.acm.org/doi/10.1145/3544548.3580919</a></td></tr><tr><td><code>S-2025-bastani</code></td><td class='cell-main source-title-cell'>Generative AI without guardrails can harm learning: Evidence from high school mathematics <span class="cite-badge cite-ok">DOI ✓</span><details class="source-expander detail-expander"><summary>View source &amp; provenance</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>Original title</dt><dd>Generative AI without guardrails can harm learning: Evidence from high school mathematics</dd></div><div class="source-detail-row"><dt>Year</dt><dd>2025</dd></div><div class="source-detail-row"><dt>Authority</dt><dd>Tier 1 DOI-verified paper</dd></div><div class="source-detail-row"><dt>Source location</dt><dd>https://www.pnas.org/doi/10.1073/pnas.2422633122</dd></div><div class="source-detail-row"><dt>Verifiable link</dt><dd><a href="https://www.pnas.org/doi/10.1073/pnas.2422633122">https://www.pnas.org/doi/10.1073/pnas.2422633122</a></dd></div></dl></details></td><td>2025</td><td>Tier 1 DOI-verified paper</td><td class='cell-main'><a href='https://www.pnas.org/doi/10.1073/pnas.2422633122'>https://www.pnas.org/doi/10.1073/pnas.2422633122</a></td></tr><tr><td><code>S-2024-marzuki</code></td><td class='cell-main source-title-cell'>Impact of ChatGPT on ESL students&#x27; academic writing skills <span class="cite-badge cite-ok">DOI ✓</span><details class="source-expander detail-expander"><summary>View source &amp; provenance</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>Original title</dt><dd>Impact of ChatGPT on ESL students&#x27; academic writing skills</dd></div><div class="source-detail-row"><dt>Year</dt><dd>2024</dd></div><div class="source-detail-row"><dt>Authority</dt><dd>Tier 1 DOI-verified paper</dd></div><div class="source-detail-row"><dt>Source location</dt><dd>https://link.springer.com/article/10.1186/s40561-024-00295-9</dd></div><div class="source-detail-row"><dt>Verifiable link</dt><dd><a href="https://link.springer.com/article/10.1186/s40561-024-00295-9">https://link.springer.com/article/10.1186/s40561-024-00295-9</a></dd></div></dl></details></td><td>2024</td><td>Tier 1 DOI-verified paper</td><td class='cell-main'><a href='https://link.springer.com/article/10.1186/s40561-024-00295-9'>https://link.springer.com/article/10.1186/s40561-024-00295-9</a></td></tr><tr><td><code>S-2023-peng</code></td><td class='cell-main source-title-cell'>The Impact of AI on Developer Productivity: Evidence from GitHub Copilot <span class="cite-badge cite-ok">DOI ✓</span><details class="source-expander detail-expander"><summary>View source &amp; provenance</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>Original title</dt><dd>The Impact of AI on Developer Productivity: Evidence from GitHub Copilot</dd></div><div class="source-detail-row"><dt>Year</dt><dd>2023</dd></div><div class="source-detail-row"><dt>Authority</dt><dd>Tier2 Academic Database</dd></div><div class="source-detail-row"><dt>Source location</dt><dd>https://arxiv.org/abs/2302.06590</dd></div><div class="source-detail-row"><dt>Verifiable link</dt><dd><a href="https://doi.org/10.48550/arXiv.2302.06590">https://doi.org/10.48550/arXiv.2302.06590</a></dd></div></dl></details></td><td>2023</td><td>Tier2 Academic Database</td><td class='cell-main'><a href='https://doi.org/10.48550/arXiv.2302.06590'>https://doi.org/10.48550/arXiv.2302.06590</a></td></tr><tr><td><code>S-2023-yetistiren</code></td><td class='cell-main source-title-cell'>GitHub Copilot AI Pair Programmer: Asset or Liability? <span class="cite-badge cite-ok">DOI ✓</span><details class="source-expander detail-expander"><summary>View source &amp; provenance</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>Original title</dt><dd>GitHub Copilot AI Pair Programmer: Asset or Liability?</dd></div><div class="source-detail-row"><dt>Year</dt><dd>2023</dd></div><div class="source-detail-row"><dt>Authority</dt><dd>Tier 1 DOI-verified paper</dd></div><div class="source-detail-row"><dt>Source location</dt><dd>https://doi.org/10.1016/j.jss.2023.111734</dd></div><div class="source-detail-row"><dt>Verifiable link</dt><dd><a href="https://doi.org/10.1016/j.jss.2023.111734">https://doi.org/10.1016/j.jss.2023.111734</a></dd></div></dl></details></td><td>2023</td><td>Tier 1 DOI-verified paper</td><td class='cell-main'><a href='https://doi.org/10.1016/j.jss.2023.111734'>https://doi.org/10.1016/j.jss.2023.111734</a></td></tr><tr><td><code>S-2022-finnie-ansley</code></td><td class='cell-main source-title-cell'>Using GitHub Copilot to Solve Introductory Programming Problems <span class="cite-badge cite-ok">DOI ✓</span><details class="source-expander detail-expander"><summary>View source &amp; provenance</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>Original title</dt><dd>Using GitHub Copilot to Solve Introductory Programming Problems</dd></div><div class="source-detail-row"><dt>Year</dt><dd>2022</dd></div><div class="source-detail-row"><dt>Authority</dt><dd>Tier 1 DOI-verified paper</dd></div><div class="source-detail-row"><dt>Source location</dt><dd>https://dl.acm.org/doi/10.1145/3545945.3569830</dd></div><div class="source-detail-row"><dt>Verifiable link</dt><dd><a href="https://dl.acm.org/doi/10.1145/3545945.3569830">https://dl.acm.org/doi/10.1145/3545945.3569830</a></dd></div></dl></details></td><td>2022</td><td>Tier 1 DOI-verified paper</td><td class='cell-main'><a href='https://dl.acm.org/doi/10.1145/3545945.3569830'>https://dl.acm.org/doi/10.1145/3545945.3569830</a></td></tr><tr><td><code>S-2023-explanations-compare</code></td><td class='cell-main source-title-cell'>Comparing Code Explanations Created by Students and Large Language Models <span class="cite-badge cite-ok">DOI ✓</span><details class="source-expander detail-expander"><summary>View source &amp; provenance</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>Original title</dt><dd>Comparing Code Explanations Created by Students and Large Language Models</dd></div><div class="source-detail-row"><dt>Year</dt><dd>2023</dd></div><div class="source-detail-row"><dt>Authority</dt><dd>Tier 1 DOI-verified paper</dd></div><div class="source-detail-row"><dt>Source location</dt><dd>https://dl.acm.org/doi/10.1145/3587102.3588785</dd></div><div class="source-detail-row"><dt>Verifiable link</dt><dd><a href="https://dl.acm.org/doi/10.1145/3587102.3588785">https://dl.acm.org/doi/10.1145/3587102.3588785</a></dd></div></dl></details></td><td>2023</td><td>Tier 1 DOI-verified paper</td><td class='cell-main'><a href='https://dl.acm.org/doi/10.1145/3587102.3588785'>https://dl.acm.org/doi/10.1145/3587102.3588785</a></td></tr><tr><td><code>S-2022-vaithilingam</code></td><td class='cell-main source-title-cell'>Expectation vs. Experience: Evaluating the Usability of Code Generation Tools <span class="cite-badge cite-ok">DOI ✓</span><details class="source-expander detail-expander"><summary>View source &amp; provenance</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>Original title</dt><dd>Expectation vs. Experience: Evaluating the Usability of Code Generation Tools</dd></div><div class="source-detail-row"><dt>Year</dt><dd>2022</dd></div><div class="source-detail-row"><dt>Authority</dt><dd>Tier 1 DOI-verified paper</dd></div><div class="source-detail-row"><dt>Source location</dt><dd>https://dl.acm.org/doi/10.1145/3491101.3519665</dd></div><div class="source-detail-row"><dt>Verifiable link</dt><dd><a href="https://dl.acm.org/doi/10.1145/3491101.3519665">https://dl.acm.org/doi/10.1145/3491101.3519665</a></dd></div></dl></details></td><td>2022</td><td>Tier 1 DOI-verified paper</td><td class='cell-main'><a href='https://dl.acm.org/doi/10.1145/3491101.3519665'>https://dl.acm.org/doi/10.1145/3491101.3519665</a></td></tr></tbody></table></div><h3>Fetch provenance</h3><p class="provenance-summary">Search provider: n/a</p><p class='provenance-empty'>No per-source fetch records (sources provided directly by the research pipeline).</p></div></section></main></div>
2544
+ <svg viewBox="0 0 720 260" xmlns="http://www.w3.org/2000/svg" role="img" aria-label="Evaluation design flow"><title>Evaluation design flow</title><desc>Evaluation flow across baseline, post test, retention and transfer; full metrics and analysis plan are in the evaluation section.</desc><defs><marker id="arr-en-full-evaluation" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto"><path d="M0,0 L6,3 L0,6 Z" fill="#8A867E"/></marker></defs><rect width="720" height="260" fill="#FCFAF6" rx="12"/><text x="24" y="34" font-size="16" font-weight="700" fill="#3A3833">Evaluation design flow</text><rect x="30" y="110" width="150" height="56" rx="8" fill="#5E8A6A"/><text x="105.0" y="138.0" text-anchor="middle" fill="#FFFFFF" font-size="12" font-weight="600"><tspan x="105.0" y="138.0">Baseline</tspan></text><text x="105.0" y="148" text-anchor="middle" font-size="9" fill="#fff" opacity="0.95">pre-test</text><line x1="180" y1="138" x2="192" y2="138" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-en-full-evaluation)"/><rect x="192" y="110" width="150" height="56" rx="8" fill="#5E8A6A"/><text x="267.0" y="138.0" text-anchor="middle" fill="#FFFFFF" font-size="12" font-weight="600"><tspan x="267.0" y="138.0">Post test</tspan></text><text x="267.0" y="148" text-anchor="middle" font-size="9" fill="#fff" opacity="0.95">post-test</text><line x1="342" y1="138" x2="354" y2="138" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-en-full-evaluation)"/><rect x="354" y="110" width="150" height="56" rx="8" fill="#5E8A6A"/><text x="429.0" y="138.0" text-anchor="middle" fill="#FFFFFF" font-size="12" font-weight="600"><tspan x="429.0" y="138.0">Retention</tspan></text><text x="429.0" y="148" text-anchor="middle" font-size="9" fill="#fff" opacity="0.95">Retention</text><line x1="504" y1="138" x2="516" y2="138" stroke="#8A867E" stroke-width="2" marker-end="url(#arr-en-full-evaluation)"/><rect x="516" y="110" width="150" height="56" rx="8" fill="#5E8A6A"/><text x="591.0" y="138.0" text-anchor="middle" fill="#FFFFFF" font-size="12" font-weight="600"><tspan x="591.0" y="138.0">Transfer</tspan></text><text x="591.0" y="148" text-anchor="middle" font-size="9" fill="#fff" opacity="0.95">Transfer</text></svg><div class="visual-suppressed"><strong>Benchmark visual suppressed</strong><p>result.json carries no benchmark.baselines, so this visual is omitted; see the standalone benchmark report.</p></div></div></section><section id="full-06-sources-en" class="full-chapter" data-full-chapter><header class="full-chapter-header"><h2>06 Sources, Traceability &amp; Appendix</h2><p class="full-chapter-lead">Preserve original sources, URLs, evidence IDs and retrieval metadata for auditability.</p></header><div class="full-chapter-body"><h3>Source list</h3><div class='table-wrap'><table class='data-table source-table'><thead><tr><th>ID</th><th>Title</th><th>Year</th><th>Authority</th><th>Verifiable location</th></tr></thead><tbody><tr><td><code>S-2023-kazemitabaar</code></td><td class='cell-main source-title-cell'>Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming <span class="cite-badge cite-ok">DOI verified</span><details class="source-expander detail-expander"><summary>View source &amp; provenance</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>Original title</dt><dd>Studying the effect of AI Code Generators on Supporting Novice Learners in Introductory Programming</dd></div><div class="source-detail-row"><dt>Year</dt><dd>2023</dd></div><div class="source-detail-row"><dt>Authority</dt><dd>Tier 1 DOI-verified paper</dd></div><div class="source-detail-row"><dt>Source location</dt><dd>https://dl.acm.org/doi/10.1145/3544548.3580919</dd></div><div class="source-detail-row"><dt>Verifiable link</dt><dd><a href="https://dl.acm.org/doi/10.1145/3544548.3580919">https://dl.acm.org/doi/10.1145/3544548.3580919</a></dd></div></dl></details></td><td>2023</td><td>Tier 1 DOI-verified paper</td><td class='cell-main'><a href='https://dl.acm.org/doi/10.1145/3544548.3580919'>https://dl.acm.org/doi/10.1145/3544548.3580919</a></td></tr><tr><td><code>S-2025-bastani</code></td><td class='cell-main source-title-cell'>Generative AI without guardrails can harm learning: Evidence from high school mathematics <span class="cite-badge cite-ok">DOI verified</span><details class="source-expander detail-expander"><summary>View source &amp; provenance</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>Original title</dt><dd>Generative AI without guardrails can harm learning: Evidence from high school mathematics</dd></div><div class="source-detail-row"><dt>Year</dt><dd>2025</dd></div><div class="source-detail-row"><dt>Authority</dt><dd>Tier 1 DOI-verified paper</dd></div><div class="source-detail-row"><dt>Source location</dt><dd>https://www.pnas.org/doi/10.1073/pnas.2422633122</dd></div><div class="source-detail-row"><dt>Verifiable link</dt><dd><a href="https://www.pnas.org/doi/10.1073/pnas.2422633122">https://www.pnas.org/doi/10.1073/pnas.2422633122</a></dd></div></dl></details></td><td>2025</td><td>Tier 1 DOI-verified paper</td><td class='cell-main'><a href='https://www.pnas.org/doi/10.1073/pnas.2422633122'>https://www.pnas.org/doi/10.1073/pnas.2422633122</a></td></tr><tr><td><code>S-2024-marzuki</code></td><td class='cell-main source-title-cell'>Impact of ChatGPT on ESL students&#x27; academic writing skills <span class="cite-badge cite-ok">DOI verified</span><details class="source-expander detail-expander"><summary>View source &amp; provenance</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>Original title</dt><dd>Impact of ChatGPT on ESL students&#x27; academic writing skills</dd></div><div class="source-detail-row"><dt>Year</dt><dd>2024</dd></div><div class="source-detail-row"><dt>Authority</dt><dd>Tier 1 DOI-verified paper</dd></div><div class="source-detail-row"><dt>Source location</dt><dd>https://link.springer.com/article/10.1186/s40561-024-00295-9</dd></div><div class="source-detail-row"><dt>Verifiable link</dt><dd><a href="https://link.springer.com/article/10.1186/s40561-024-00295-9">https://link.springer.com/article/10.1186/s40561-024-00295-9</a></dd></div></dl></details></td><td>2024</td><td>Tier 1 DOI-verified paper</td><td class='cell-main'><a href='https://link.springer.com/article/10.1186/s40561-024-00295-9'>https://link.springer.com/article/10.1186/s40561-024-00295-9</a></td></tr><tr><td><code>S-2023-peng</code></td><td class='cell-main source-title-cell'>The Impact of AI on Developer Productivity: Evidence from GitHub Copilot <span class="cite-badge cite-ok">DOI verified</span><details class="source-expander detail-expander"><summary>View source &amp; provenance</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>Original title</dt><dd>The Impact of AI on Developer Productivity: Evidence from GitHub Copilot</dd></div><div class="source-detail-row"><dt>Year</dt><dd>2023</dd></div><div class="source-detail-row"><dt>Authority</dt><dd>Tier2 Academic Database</dd></div><div class="source-detail-row"><dt>Source location</dt><dd>https://arxiv.org/abs/2302.06590</dd></div><div class="source-detail-row"><dt>Verifiable link</dt><dd><a href="https://doi.org/10.48550/arXiv.2302.06590">https://doi.org/10.48550/arXiv.2302.06590</a></dd></div></dl></details></td><td>2023</td><td>Tier2 Academic Database</td><td class='cell-main'><a href='https://doi.org/10.48550/arXiv.2302.06590'>https://doi.org/10.48550/arXiv.2302.06590</a></td></tr><tr><td><code>S-2023-yetistiren</code></td><td class='cell-main source-title-cell'>GitHub Copilot AI Pair Programmer: Asset or Liability? <span class="cite-badge cite-ok">DOI verified</span><details class="source-expander detail-expander"><summary>View source &amp; provenance</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>Original title</dt><dd>GitHub Copilot AI Pair Programmer: Asset or Liability?</dd></div><div class="source-detail-row"><dt>Year</dt><dd>2023</dd></div><div class="source-detail-row"><dt>Authority</dt><dd>Tier 1 DOI-verified paper</dd></div><div class="source-detail-row"><dt>Source location</dt><dd>https://doi.org/10.1016/j.jss.2023.111734</dd></div><div class="source-detail-row"><dt>Verifiable link</dt><dd><a href="https://doi.org/10.1016/j.jss.2023.111734">https://doi.org/10.1016/j.jss.2023.111734</a></dd></div></dl></details></td><td>2023</td><td>Tier 1 DOI-verified paper</td><td class='cell-main'><a href='https://doi.org/10.1016/j.jss.2023.111734'>https://doi.org/10.1016/j.jss.2023.111734</a></td></tr><tr><td><code>S-2022-finnie-ansley</code></td><td class='cell-main source-title-cell'>Using GitHub Copilot to Solve Introductory Programming Problems <span class="cite-badge cite-ok">DOI verified</span><details class="source-expander detail-expander"><summary>View source &amp; provenance</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>Original title</dt><dd>Using GitHub Copilot to Solve Introductory Programming Problems</dd></div><div class="source-detail-row"><dt>Year</dt><dd>2022</dd></div><div class="source-detail-row"><dt>Authority</dt><dd>Tier 1 DOI-verified paper</dd></div><div class="source-detail-row"><dt>Source location</dt><dd>https://dl.acm.org/doi/10.1145/3545945.3569830</dd></div><div class="source-detail-row"><dt>Verifiable link</dt><dd><a href="https://dl.acm.org/doi/10.1145/3545945.3569830">https://dl.acm.org/doi/10.1145/3545945.3569830</a></dd></div></dl></details></td><td>2022</td><td>Tier 1 DOI-verified paper</td><td class='cell-main'><a href='https://dl.acm.org/doi/10.1145/3545945.3569830'>https://dl.acm.org/doi/10.1145/3545945.3569830</a></td></tr><tr><td><code>S-2023-explanations-compare</code></td><td class='cell-main source-title-cell'>Comparing Code Explanations Created by Students and Large Language Models <span class="cite-badge cite-ok">DOI verified</span><details class="source-expander detail-expander"><summary>View source &amp; provenance</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>Original title</dt><dd>Comparing Code Explanations Created by Students and Large Language Models</dd></div><div class="source-detail-row"><dt>Year</dt><dd>2023</dd></div><div class="source-detail-row"><dt>Authority</dt><dd>Tier 1 DOI-verified paper</dd></div><div class="source-detail-row"><dt>Source location</dt><dd>https://dl.acm.org/doi/10.1145/3587102.3588785</dd></div><div class="source-detail-row"><dt>Verifiable link</dt><dd><a href="https://dl.acm.org/doi/10.1145/3587102.3588785">https://dl.acm.org/doi/10.1145/3587102.3588785</a></dd></div></dl></details></td><td>2023</td><td>Tier 1 DOI-verified paper</td><td class='cell-main'><a href='https://dl.acm.org/doi/10.1145/3587102.3588785'>https://dl.acm.org/doi/10.1145/3587102.3588785</a></td></tr><tr><td><code>S-2022-vaithilingam</code></td><td class='cell-main source-title-cell'>Expectation vs. Experience: Evaluating the Usability of Code Generation Tools <span class="cite-badge cite-ok">DOI verified</span><details class="source-expander detail-expander"><summary>View source &amp; provenance</summary><dl class="source-detail-grid"><div class="source-detail-row"><dt>Original title</dt><dd>Expectation vs. Experience: Evaluating the Usability of Code Generation Tools</dd></div><div class="source-detail-row"><dt>Year</dt><dd>2022</dd></div><div class="source-detail-row"><dt>Authority</dt><dd>Tier 1 DOI-verified paper</dd></div><div class="source-detail-row"><dt>Source location</dt><dd>https://dl.acm.org/doi/10.1145/3491101.3519665</dd></div><div class="source-detail-row"><dt>Verifiable link</dt><dd><a href="https://dl.acm.org/doi/10.1145/3491101.3519665">https://dl.acm.org/doi/10.1145/3491101.3519665</a></dd></div></dl></details></td><td>2022</td><td>Tier 1 DOI-verified paper</td><td class='cell-main'><a href='https://dl.acm.org/doi/10.1145/3491101.3519665'>https://dl.acm.org/doi/10.1145/3491101.3519665</a></td></tr></tbody></table></div><h3>Fetch provenance</h3><p class="provenance-summary">Search provider: n/a</p><p class='provenance-empty'>No per-source fetch records (sources provided directly by the research pipeline).</p></div></section></main></div>
2545
2545
  </div>
2546
2546
  <footer class="report-footer"><p>EduEvidence Evidence Report · Schema PASS · Claim Binding PASS · Numeric Consistency PASS · Bilingual Structure PASS · Human Language PASS · False Precision PASS · Lieflat Data Bound PASS · Axis Distortion NOT_CHECKED · Colorblind Safe NOT_CHECKED · single-file offline · source: result.json</p></footer>
2547
2547
  </div>