eduevidence 5.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +506 -0
- package/README.zh-CN.md +434 -0
- package/SKILL.md +195 -0
- package/bin/eduevidence.js +127 -0
- package/domains/education/manifest.json +183 -0
- package/domains/education/outcome_taxonomy.json +127 -0
- package/domains/manifest.json +26 -0
- package/domains/policy/frame.schema.json +234 -0
- package/domains/policy/manifest.json +10 -0
- package/domains/policy/methodology_checklist.json +109 -0
- package/domains/policy/outcome_taxonomy.json +53 -0
- package/domains/policy/references/causal-identification.md +45 -0
- package/domains/policy/references/cost-evidence.md +44 -0
- package/domains/policy/references/equity.md +42 -0
- package/domains/policy/references/evidence-hierarchy.md +41 -0
- package/domains/policy/references/implementation-evidence.md +47 -0
- package/eduevidence_cli.py +26 -0
- package/engine/__init__.py +11 -0
- package/engine/__pycache__/__init__.cpython-312.pyc +0 -0
- package/engine/__pycache__/analysis.cpython-312.pyc +0 -0
- package/engine/__pycache__/bias.cpython-312.pyc +0 -0
- package/engine/__pycache__/briefs.cpython-312.pyc +0 -0
- package/engine/__pycache__/capabilities.cpython-312.pyc +0 -0
- package/engine/__pycache__/citation_check.cpython-312.pyc +0 -0
- package/engine/__pycache__/contracts.cpython-312.pyc +0 -0
- package/engine/__pycache__/datasets.cpython-312.pyc +0 -0
- package/engine/__pycache__/events.cpython-312.pyc +0 -0
- package/engine/__pycache__/evidence_graph.cpython-312.pyc +0 -0
- package/engine/__pycache__/evidence_review.cpython-312.pyc +0 -0
- package/engine/__pycache__/evidencecore.cpython-312.pyc +0 -0
- package/engine/__pycache__/gap_lens.cpython-312.pyc +0 -0
- package/engine/__pycache__/gaps.cpython-312.pyc +0 -0
- package/engine/__pycache__/graph_store.cpython-312.pyc +0 -0
- package/engine/__pycache__/graph_validate.cpython-312.pyc +0 -0
- package/engine/__pycache__/ids.cpython-312.pyc +0 -0
- package/engine/__pycache__/library.cpython-312.pyc +0 -0
- package/engine/__pycache__/library_builtin.cpython-312.pyc +0 -0
- package/engine/__pycache__/living.cpython-312.pyc +0 -0
- package/engine/__pycache__/log.cpython-312.pyc +0 -0
- package/engine/__pycache__/meta_analysis.cpython-312.pyc +0 -0
- package/engine/__pycache__/meta_synthesis.cpython-312.pyc +0 -0
- package/engine/__pycache__/migration.cpython-312.pyc +0 -0
- package/engine/__pycache__/mode_router.cpython-312.pyc +0 -0
- package/engine/__pycache__/paths.cpython-312.pyc +0 -0
- package/engine/__pycache__/pilot.cpython-312.pyc +0 -0
- package/engine/__pycache__/planner.cpython-312.pyc +0 -0
- package/engine/__pycache__/project.cpython-312.pyc +0 -0
- package/engine/__pycache__/projections.cpython-312.pyc +0 -0
- package/engine/__pycache__/robustness.cpython-312.pyc +0 -0
- package/engine/__pycache__/run.cpython-312.pyc +0 -0
- package/engine/__pycache__/semantics.cpython-312.pyc +0 -0
- package/engine/__pycache__/study_design.cpython-312.pyc +0 -0
- package/engine/__pycache__/synthesis.cpython-312.pyc +0 -0
- package/engine/__pycache__/tribunal.cpython-312.pyc +0 -0
- package/engine/__pycache__/update.cpython-312.pyc +0 -0
- package/engine/__pycache__/versions.cpython-312.pyc +0 -0
- package/engine/analysis.py +308 -0
- package/engine/bias.py +178 -0
- package/engine/briefs.py +106 -0
- package/engine/capabilities.py +99 -0
- package/engine/citation_check.py +192 -0
- package/engine/contracts.py +117 -0
- package/engine/datasets.py +165 -0
- package/engine/events.py +67 -0
- package/engine/evidence_graph.py +571 -0
- package/engine/evidence_review.py +88 -0
- package/engine/evidencecore.py +182 -0
- package/engine/gap_lens.py +132 -0
- package/engine/gaps.py +169 -0
- package/engine/graph_store.py +335 -0
- package/engine/graph_validate.py +87 -0
- package/engine/ids.py +77 -0
- package/engine/library.py +268 -0
- package/engine/library_builtin.py +301 -0
- package/engine/living.py +671 -0
- package/engine/log.py +39 -0
- package/engine/meta_analysis.py +333 -0
- package/engine/meta_synthesis.py +111 -0
- package/engine/migration.py +397 -0
- package/engine/mode_router.py +72 -0
- package/engine/paths.py +15 -0
- package/engine/pilot.py +368 -0
- package/engine/planner.py +126 -0
- package/engine/project.py +118 -0
- package/engine/projections.py +240 -0
- package/engine/robustness.py +109 -0
- package/engine/run.py +85 -0
- package/engine/semantics.py +135 -0
- package/engine/study_design.py +87 -0
- package/engine/synthesis.py +187 -0
- package/engine/tribunal.py +408 -0
- package/engine/update.py +113 -0
- package/engine/versions.py +12 -0
- package/install.sh +510 -0
- package/integrations/__init__.py +1 -0
- package/integrations/__pycache__/__init__.cpython-312.pyc +0 -0
- package/integrations/__pycache__/agent_mcp.cpython-312.pyc +0 -0
- package/integrations/__pycache__/smart_web_fetch.cpython-312.pyc +0 -0
- package/integrations/agent_mcp.py +856 -0
- package/integrations/smart_web_fetch.py +59 -0
- package/package.json +50 -0
- package/pyproject.toml +55 -0
- package/references/applicability-policy.md +88 -0
- package/references/education-framing.md +132 -0
- package/references/effect_size_formulas.md +35 -0
- package/references/evaluation-design.md +111 -0
- package/references/evidence-quality.md +79 -0
- package/references/grade_framework.md +29 -0
- package/references/intervention-design.md +98 -0
- package/references/methodology-audit.md +103 -0
- package/references/outcome-taxonomy.md +106 -0
- package/references/retrieval-protocol.md +142 -0
- package/references/skeptic-protocol.md +93 -0
- package/references/social_science_pitfalls.md +48 -0
- package/references/source-validity.md +140 -0
- package/references/tribunal-policy.md +112 -0
- package/references/wwc_standards.md +29 -0
- package/retrieval/__init__.py +1 -0
- package/retrieval/__pycache__/__init__.cpython-312.pyc +0 -0
- package/retrieval/__pycache__/corpus_store.cpython-312.pyc +0 -0
- package/retrieval/__pycache__/dedupe.cpython-312.pyc +0 -0
- package/retrieval/__pycache__/failures.cpython-312.pyc +0 -0
- package/retrieval/__pycache__/fetch.cpython-312.pyc +0 -0
- package/retrieval/__pycache__/search.cpython-312.pyc +0 -0
- package/retrieval/__pycache__/source.cpython-312.pyc +0 -0
- package/retrieval/__pycache__/validate.cpython-312.pyc +0 -0
- package/retrieval/corpus_store.py +181 -0
- package/retrieval/dedupe.py +127 -0
- package/retrieval/failures.py +90 -0
- package/retrieval/fetch.py +435 -0
- package/retrieval/search.py +493 -0
- package/retrieval/source.py +160 -0
- package/retrieval/validate.py +257 -0
- package/schemas/agent-mcp-approval.schema.json +57 -0
- package/schemas/chart-spec.schema.json +88 -0
- package/schemas/cross-model-review.schema.json +28 -0
- package/schemas/education-frame.schema.json +106 -0
- package/schemas/evaluation.schema.json +35 -0
- package/schemas/evidence.schema.json +81 -0
- package/schemas/fetch-result.schema.json +119 -0
- package/schemas/intervention.schema.json +46 -0
- package/schemas/methodology.schema.json +102 -0
- package/schemas/report-result.schema.json +381 -0
- package/schemas/report-spec.schema.json +130 -0
- package/schemas/source.schema.json +311 -0
- package/schemas/v2/analysis-plan.schema.json +28 -0
- package/schemas/v2/analysis-run.schema.json +33 -0
- package/schemas/v2/claim.schema.json +26 -0
- package/schemas/v2/dataset-asset.schema.json +40 -0
- package/schemas/v2/decision-snapshot.schema.json +53 -0
- package/schemas/v2/evidence-link.schema.json +38 -0
- package/schemas/v2/finding.schema.json +47 -0
- package/schemas/v2/graph-revision.schema.json +30 -0
- package/schemas/v2/knowledge-gap.schema.json +40 -0
- package/schemas/v2/methodology-audit.schema.json +30 -0
- package/schemas/v2/outcome.schema.json +18 -0
- package/schemas/v2/project.schema.json +31 -0
- package/schemas/v2/research-intent.schema.json +24 -0
- package/schemas/v2/run.schema.json +43 -0
- package/schemas/v2/source.schema.json +24 -0
- package/schemas/v2/study-design.schema.json +67 -0
- package/schemas/v2/study.schema.json +37 -0
- package/schemas/v3/pilot-outcome.schema.json +132 -0
- package/schemas/v3/run-manifest.schema.json +193 -0
- package/schemas/v3/synthesis.schema.json +86 -0
- package/schemas/v4/drift-report.schema.json +66 -0
- package/schemas/v4/evidence-library.schema.json +42 -0
- package/schemas/v4/living-subscription.schema.json +55 -0
- package/schemas/v4/meta-analysis.schema.json +152 -0
- package/schemas/verdict.schema.json +56 -0
- package/scripts/__init__.py +0 -0
- package/scripts/__pycache__/__init__.cpython-312.pyc +0 -0
- package/scripts/__pycache__/benchmark.cpython-312.pyc +0 -0
- package/scripts/__pycache__/benchmark_evaluator.cpython-312.pyc +0 -0
- package/scripts/__pycache__/benchmark_judge.cpython-312.pyc +0 -0
- package/scripts/__pycache__/benchmark_routing.cpython-312.pyc +0 -0
- package/scripts/__pycache__/benchmark_v2.cpython-312.pyc +0 -0
- package/scripts/__pycache__/benchmark_v3.cpython-312.pyc +0 -0
- package/scripts/__pycache__/build_result.cpython-312.pyc +0 -0
- package/scripts/__pycache__/claim_audit.cpython-312.pyc +0 -0
- package/scripts/__pycache__/complexity_gate.cpython-312.pyc +0 -0
- package/scripts/__pycache__/compute_confidence.cpython-312.pyc +0 -0
- package/scripts/__pycache__/dashboard_server.cpython-312.pyc +0 -0
- package/scripts/__pycache__/did_regression.cpython-312.pyc +0 -0
- package/scripts/__pycache__/effect_calculator.cpython-312.pyc +0 -0
- package/scripts/__pycache__/evidence_matrix.cpython-312.pyc +0 -0
- package/scripts/__pycache__/evidence_score.cpython-312.pyc +0 -0
- package/scripts/__pycache__/evidence_semantics.cpython-312.pyc +0 -0
- package/scripts/__pycache__/fetch_benchmark.cpython-312.pyc +0 -0
- package/scripts/__pycache__/lint_report_layout.cpython-312.pyc +0 -0
- package/scripts/__pycache__/orchestrator.cpython-312.pyc +0 -0
- package/scripts/__pycache__/pre_verdict_gate.cpython-312.pyc +0 -0
- package/scripts/__pycache__/recompute_demo_quality.cpython-312.pyc +0 -0
- package/scripts/__pycache__/render_report.cpython-312.pyc +0 -0
- package/scripts/__pycache__/render_report_html.cpython-312.pyc +0 -0
- package/scripts/__pycache__/run_workspace.cpython-312.pyc +0 -0
- package/scripts/__pycache__/skill_lint.cpython-312.pyc +0 -0
- package/scripts/__pycache__/startup_probe.cpython-312.pyc +0 -0
- package/scripts/__pycache__/sync_killer_demo_report.cpython-312.pyc +0 -0
- package/scripts/__pycache__/test_adversarial_empirical.cpython-312-pytest-9.0.2.pyc +0 -0
- package/scripts/__pycache__/test_adversarial_empirical.cpython-312-pytest-9.1.1.pyc +0 -0
- package/scripts/__pycache__/validate_schema.cpython-312.pyc +0 -0
- package/scripts/audit_dois.py +292 -0
- package/scripts/bake_pack.sh +37 -0
- package/scripts/benchmark.py +183 -0
- package/scripts/benchmark_evaluator.py +371 -0
- package/scripts/benchmark_judge.py +535 -0
- package/scripts/benchmark_routing.py +120 -0
- package/scripts/benchmark_v2.py +304 -0
- package/scripts/benchmark_v3.py +552 -0
- package/scripts/build_esl_artifacts.py +1921 -0
- package/scripts/build_evidence_library.py +307 -0
- package/scripts/build_killer_demo.py +295 -0
- package/scripts/build_result.py +311 -0
- package/scripts/check_version_consistency.py +96 -0
- package/scripts/citation_check.py +123 -0
- package/scripts/claim_audit.py +157 -0
- package/scripts/complexity_gate.py +180 -0
- package/scripts/compute_confidence.py +176 -0
- package/scripts/dashboard_server.py +536 -0
- package/scripts/did_regression.py +315 -0
- package/scripts/effect_calculator.py +99 -0
- package/scripts/enrich_projects_human_and_lieflat.py +315 -0
- package/scripts/evidence_matrix.py +129 -0
- package/scripts/evidence_score.py +234 -0
- package/scripts/evidence_semantics.py +87 -0
- package/scripts/fetch_benchmark.py +177 -0
- package/scripts/generate_metrics.py +99 -0
- package/scripts/generate_new_projects.py +686 -0
- package/scripts/generate_promo.py +192 -0
- package/scripts/lint_report_layout.py +182 -0
- package/scripts/orchestrator.py +1456 -0
- package/scripts/pre_verdict_gate.py +513 -0
- package/scripts/quickstart.py +121 -0
- package/scripts/rebake_all_5themes.py +88 -0
- package/scripts/recompute_demo_quality.py +205 -0
- package/scripts/render_report.py +270 -0
- package/scripts/render_report_html.py +356 -0
- package/scripts/retraction_watch.py +110 -0
- package/scripts/run_workspace.py +337 -0
- package/scripts/serve_web.py +54 -0
- package/scripts/skill_lint.py +150 -0
- package/scripts/startup_probe.py +265 -0
- package/scripts/sync_killer_demo_report.py +270 -0
- package/scripts/test_adversarial_empirical.py +541 -0
- package/scripts/validate_schema.py +256 -0
- package/skill/agents/education-planner.md +80 -0
- package/skill/agents/evaluation-designer.md +74 -0
- package/skill/agents/evidence-analyst.md +106 -0
- package/skill/agents/evidence-judge.md +111 -0
- package/skill/agents/evidence-retriever.md +80 -0
- package/skill/agents/intervention-designer.md +82 -0
- package/skill/agents/method-reviewer.md +104 -0
- package/skill/agents/skeptic.md +89 -0
- package/skill/sub-skills/aihot-trend-analysis/SKILL.md +31 -0
- package/skill/sub-skills/contradiction-analysis/SKILL.md +17 -0
- package/skill/sub-skills/data-analysis/SKILL.md +23 -0
- package/skill/sub-skills/ethics-review/SKILL.md +25 -0
- package/skill/sub-skills/evidence-extraction/SKILL.md +19 -0
- package/skill/sub-skills/evidence-review/SKILL.md +18 -0
- package/skill/sub-skills/gap-analysis/SKILL.md +25 -0
- package/skill/sub-skills/literature-review/SKILL.md +21 -0
- package/skill/sub-skills/methodology-audit/SKILL.md +20 -0
- package/skill/sub-skills/report-generation/SKILL.md +51 -0
- package/skill/sub-skills/research-planning/SKILL.md +21 -0
- package/skill/sub-skills/study-design/SKILL.md +16 -0
- package/skill/task-briefs/adjudicate.md +17 -0
- package/skill/task-briefs/audit.md +15 -0
- package/skill/task-briefs/challenge.md +15 -0
- package/skill/task-briefs/evaluate.md +13 -0
- package/skill/task-briefs/extract.md +16 -0
- package/skill/task-briefs/frame.md +17 -0
- package/skill/task-briefs/intervene.md +14 -0
- package/skill/task-briefs/present.md +16 -0
- package/skill/task-briefs/retrieve.md +15 -0
- package/visualization/eduevidence-report/assets/base.css +337 -0
- package/visualization/eduevidence-report/motion/motion.css +157 -0
- package/visualization/eduevidence-report/motion/motion.js +107 -0
- package/visualization/eduevidence-report/references/bilingual-style.md +7 -0
- package/visualization/eduevidence-report/references/component-catalog.md +145 -0
- package/visualization/eduevidence-report/references/evidence-expansion.md +65 -0
- package/visualization/eduevidence-report/references/full-report-outline.md +86 -0
- package/visualization/eduevidence-report/references/layout-constraints.md +63 -0
- package/visualization/eduevidence-report/references/lieflat-composition.md +79 -0
- package/visualization/eduevidence-report/references/motion-system.md +31 -0
- package/visualization/eduevidence-report/schemas/adapter-envelope.schema.json +22 -0
- package/visualization/eduevidence-report/schemas/visual-layout.schema.json +87 -0
- package/visualization/eduevidence-report/scripts/__pycache__/adapter_contract.cpython-312.pyc +0 -0
- package/visualization/eduevidence-report/scripts/__pycache__/build_artifact_manifest.cpython-312.pyc +0 -0
- package/visualization/eduevidence-report/scripts/__pycache__/build_charts.cpython-312.pyc +0 -0
- package/visualization/eduevidence-report/scripts/__pycache__/build_figures.cpython-312.pyc +0 -0
- package/visualization/eduevidence-report/scripts/__pycache__/build_infographics.cpython-312.pyc +0 -0
- package/visualization/eduevidence-report/scripts/__pycache__/build_report.cpython-312.pyc +0 -0
- package/visualization/eduevidence-report/scripts/__pycache__/charts_data.cpython-312.pyc +0 -0
- package/visualization/eduevidence-report/scripts/__pycache__/lieflat_engine.cpython-312.pyc +0 -0
- package/visualization/eduevidence-report/scripts/__pycache__/zh_labels.cpython-312.pyc +0 -0
- package/visualization/eduevidence-report/scripts/adapter_contract.py +72 -0
- package/visualization/eduevidence-report/scripts/build_artifact_manifest.py +70 -0
- package/visualization/eduevidence-report/scripts/build_charts.py +283 -0
- package/visualization/eduevidence-report/scripts/build_figures.py +515 -0
- package/visualization/eduevidence-report/scripts/build_infographics.py +268 -0
- package/visualization/eduevidence-report/scripts/build_report.py +3211 -0
- package/visualization/eduevidence-report/scripts/charts_data.py +617 -0
- package/visualization/eduevidence-report/scripts/check_mobile_layout.js +220 -0
- package/visualization/eduevidence-report/scripts/lieflat_engine.py +852 -0
- package/visualization/eduevidence-report/scripts/zh_labels.py +245 -0
- package/visualization/eduevidence-report/themes/academic.css +94 -0
- package/visualization/eduevidence-report/themes/claude.css +96 -0
- package/visualization/eduevidence-report/themes/datalab-dark.css +147 -0
- package/visualization/eduevidence-report/themes/datalab.css +151 -0
- package/visualization/eduevidence-report/themes/presentation.css +140 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 EduEvidence Contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,506 @@
|
|
|
1
|
+
# EduEvidence
|
|
2
|
+
|
|
3
|
+
> **🌐 English | [中文](README.zh-CN.md)**
|
|
4
|
+
|
|
5
|
+
## EduEvidence Research Engine — Evidence-Based Education Decision Skill
|
|
6
|
+
|
|
7
|
+
> **From Education Questions to Evidence-Based Decisions.**
|
|
8
|
+
|
|
9
|
+
EduEvidence is delivered as an **AI Agent Skill**; inside the Skill operates
|
|
10
|
+
the **EduEvidence Research Engine** — a persistent, auditable engine that
|
|
11
|
+
turns education questions into evidence-grounded decisions.
|
|
12
|
+
|
|
13
|
+
- **Two Research Modes** — **Evidence Review** (secondary-evidence research)
|
|
14
|
+
and **Full Research Cycle** (Evidence Review → Knowledge Gap → study design
|
|
15
|
+
→ your data → analysis → updated decision).
|
|
16
|
+
- **Project Workspace + Evidence Graph** — long-lived Projects with versioned,
|
|
17
|
+
immutable graph revisions; `result.json`/HTML/Markdown are projections, not
|
|
18
|
+
fact stores.
|
|
19
|
+
- **Shared Research Library** — verified external facts reused via snapshot
|
|
20
|
+
imports; interpretations stay Project-local.
|
|
21
|
+
- **Frozen scientific rule** — *No new study design without evidence
|
|
22
|
+
grounding*: designs must reference explicit, evidence-grounded Knowledge
|
|
23
|
+
Gap IDs.
|
|
24
|
+
- ⚖️ It does not generate answers for teachers — it shows what the evidence supports, what it cannot support, who it applies to, and how to pilot and verify it.
|
|
25
|
+
- 🧪 Built on real research (examples include CHI 2023 / PNAS 2025 / ACL 2025 / Springer 2024 empirical evidence); no claims without sources.
|
|
26
|
+
- 🚦 The output is not a binary "allow/forbid" answer but a four-state decision — **ADOPT / PILOT / REJECT / INSUFFICIENT EVIDENCE** — plus an actionable teaching intervention and evaluation plan.
|
|
27
|
+
- 🧩 The engine is an internal capability architecture — not a standalone
|
|
28
|
+
server/app; Native Core runs on Python stdlib only and never requires
|
|
29
|
+
Agent MCP or a daemon.
|
|
30
|
+
|
|
31
|
+

|
|
32
|
+
|
|
33
|
+
---
|
|
34
|
+
|
|
35
|
+
## Quick Install
|
|
36
|
+
|
|
37
|
+
**npm (recommended for Skill install)**
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
npm install -g eduevidence
|
|
41
|
+
eduevidence skill # interactive: pick your agent (claude / cursor / codex / …)
|
|
42
|
+
eduevidence skill --list-hosts # or preview all supported agents first
|
|
43
|
+
# non-interactive: eduevidence skill --host cursor
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
**git clone (full dev setup + examples)**
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
git clone https://github.com/37chengshan/eduevidence.git
|
|
50
|
+
cd eduevidence
|
|
51
|
+
bash install.sh # one-click: venv + deps + self-check + tests
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Open the example report right away:
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
open examples/ai-coding-assistant/EduEvidence_Report.html
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
> Requires Python 3.10+; the core has zero third-party dependencies. `pip install matplotlib` is optional for academic-figure PNG/PDF export.
|
|
61
|
+
> After install, the script prompts you to star the repo (prompt only — it never runs any GitHub command on your behalf).
|
|
62
|
+
|
|
63
|
+
---
|
|
64
|
+
|
|
65
|
+
## Install as a Skill (for AI Agent users)
|
|
66
|
+
|
|
67
|
+
> EduEvidence itself is an **AI Agent Skill** (`SKILL.md` + `skill/agents/` + `references/` + `schemas/` + `scripts/` + `retrieval/` + `integrations/` + `visualization/`).
|
|
68
|
+
> Once installed, your host agent (Claude Code / OMP / Codex / OpenCode / Kimi / ZCode / OpenClaw / Harness / Grok / Copilot / Cline …) can auto-load this Skill when it receives teaching-decision questions.
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
npm install -g eduevidence
|
|
72
|
+
eduevidence skill # interactive host picker (default)
|
|
73
|
+
eduevidence skill --list-hosts
|
|
74
|
+
eduevidence skill --dry-run
|
|
75
|
+
eduevidence skill --host cursor # optional: skip the menu
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Or from a git clone:
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
bash install.sh --skill # interactive: choose which agent to install to
|
|
82
|
+
bash install.sh --list-hosts # list supported agents and skill locations
|
|
83
|
+
bash install.sh --skill --host claude
|
|
84
|
+
bash install.sh --skill --dry-run # preview only, write nothing
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
You can also run it remotely without cloning:
|
|
88
|
+
|
|
89
|
+
> ⚠️ Supply-chain note (E7): `curl | bash` executes a remote script. Prefer
|
|
90
|
+
> cloning and reviewing first, or pin the URL to a specific commit.
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
bash -c "$(curl -fsSL https://raw.githubusercontent.com/37chengshan/eduevidence/main/install.sh)"
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Before writing, the script automatically backs up any existing skill directory (`cp -r` to `.bak-<timestamp>`); `--dry-run` only previews the changes. Side effects: creates a Python venv, writes `~/.eduevidence/env` (`AGENT_MCP_INSTALLED=1`).
|
|
97
|
+
|
|
98
|
+
### Supported agents and install locations
|
|
99
|
+
|
|
100
|
+
| Agent | Detection path | Skill install location |
|
|
101
|
+
|---|---|---|
|
|
102
|
+
| Cursor | `~/.cursor` | `~/.cursor/skills/eduevidence/` |
|
|
103
|
+
| Claude Code | `~/.claude` | `~/.claude/skills/eduevidence/` (project `.claude/skills/` when no user-level config) |
|
|
104
|
+
| Codex | `~/.codex` or `codex` command | `~/.agents/skills/` (falls back to `~/.codex/skills/`, `~/.codex/prompts/`) |
|
|
105
|
+
| OMP | `~/.omp` | `~/.omp/agent/skills/eduevidence/` |
|
|
106
|
+
| OpenCode | `~/.config/opencode` | `~/.config/opencode/skills/eduevidence/` |
|
|
107
|
+
| Kimi Code | `$KIMI_CODE_HOME` or `~/.kimi-code` | `~/.kimi-code/skills/eduevidence/` |
|
|
108
|
+
| ZCode | `~/.zcode` | `~/.zcode/skills/eduevidence/` |
|
|
109
|
+
| OpenClaw | `~/.openclaw` | `~/.openclaw/skills/eduevidence/` |
|
|
110
|
+
| Harness | `~/.harness` | `~/.harness/skills/eduevidence/` |
|
|
111
|
+
| Grok | `~/.grok` | `~/.grok/skills/eduevidence/` |
|
|
112
|
+
| GitHub Copilot CLI | `~/.copilot` | `~/.copilot/skills/eduevidence/` |
|
|
113
|
+
| Cline | `~/.cline` or `~/.config/cline` | `~/.cline/skills/eduevidence/` |
|
|
114
|
+
|
|
115
|
+
In the interactive menu: pick `all` to install to every agent, `custom` to type a directory manually, or `local` for local-only install (venv + pytest + self-check).
|
|
116
|
+
|
|
117
|
+
### Method 3: Universal prompt (agents not listed)
|
|
118
|
+
|
|
119
|
+
Your agent is not in the list? Paste the following prompt **verbatim** into any AI that supports skills / custom instructions:
|
|
120
|
+
|
|
121
|
+
```text
|
|
122
|
+
Follow the install guide at https://github.com/37chengshan/eduevidence/blob/main/docs/install-guide.md
|
|
123
|
+
to install EduEvidence as a skill for me: read the doc first, then per Section 2's
|
|
124
|
+
landing table copy SKILL.md, skill/, references/, schemas/, scripts/, retrieval/,
|
|
125
|
+
integrations/, visualization/ into my skill directory (or import via my loading
|
|
126
|
+
mechanism), then complete the Section 3 verification (SKILL.md readable + scripts
|
|
127
|
+
runnable + sample report renderable).
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
## What Problem We Solve
|
|
131
|
+
|
|
132
|
+
A typical AI answers an education question like this:
|
|
133
|
+
|
|
134
|
+
```text
|
|
135
|
+
Question → Search a few sources → Summarize opinions → Give advice
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
EduEvidence does this instead:
|
|
139
|
+
|
|
140
|
+
```text
|
|
141
|
+
Education question
|
|
142
|
+
→ Education Research Framing (learner / intervention / comparison / outcomes / context)
|
|
143
|
+
→ Literature & evidence retrieval (supporting evidence + independent counter-evidence)
|
|
144
|
+
→ Claim-Level Evidence Extraction
|
|
145
|
+
→ Skeptic challenge protocol + Method Reviewer audit
|
|
146
|
+
→ Evidence Tribunal
|
|
147
|
+
→ Applicability Analysis
|
|
148
|
+
→ Decision: ADOPT / PILOT / REJECT / INSUFFICIENT EVIDENCE
|
|
149
|
+
→ Teaching Intervention (minimum viable pilot)
|
|
150
|
+
→ Evaluation Plan
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
It answers six questions:
|
|
154
|
+
|
|
155
|
+
1. What does the current evidence actually support?
|
|
156
|
+
2. What can the current evidence not support?
|
|
157
|
+
3. Why do different studies reach different results?
|
|
158
|
+
4. Which students, which courses, under which conditions does it apply to?
|
|
159
|
+
5. If an institution adopts it, how to roll it out with low risk?
|
|
160
|
+
6. How to verify whether it actually works after implementation?
|
|
161
|
+
|
|
162
|
+
## 30-second Demo
|
|
163
|
+
|
|
164
|
+
> Main demo: **Should first-year C programming students be allowed to use generative AI coding assistants?**
|
|
165
|
+
|
|
166
|
+
| Time | Stage |
|
|
167
|
+
|---|---|
|
|
168
|
+
| 0–20s | Ask the education question |
|
|
169
|
+
| 20–45s | Education Research Frame |
|
|
170
|
+
| 45–75s | Evidence Retrieval |
|
|
171
|
+
| 75–110s | Evidence Matrix |
|
|
172
|
+
| 110–135s | Methodology + Skeptic |
|
|
173
|
+
| 135–155s | Evidence Tribunal |
|
|
174
|
+
| 155–170s | Teaching Intervention + Evaluation |
|
|
175
|
+
| 170–180s | Benchmark |
|
|
176
|
+
|
|
177
|
+
Full example pack: [`examples/ai-coding-assistant/`](examples/ai-coding-assistant/).
|
|
178
|
+
|
|
179
|
+
## Why Education Evidence Is Hard
|
|
180
|
+
|
|
181
|
+
Education evidence has natural pitfalls. EduEvidence's core contribution is standardizing the countermeasures:
|
|
182
|
+
|
|
183
|
+
- **Outcome Separation**: `faster task completion ≠ actually learning to program`; `short-term score gains ≠ long-term retention`; `completing tasks with AI ≠ transferring skills without AI`.
|
|
184
|
+
- **Counter-Evidence Search**: it does not just verify the user's initial assumption — it independently searches for null / negative / contradictory evidence, AI dependency, novelty effects, self-selection bias, and more.
|
|
185
|
+
- **Evidence Tribunal**: instead of listing pros and cons, it judges which studies are more credible, whether conflicts come from samples / measurement / course / tool / design, and what can be concluded so far.
|
|
186
|
+
- **Evidence-to-Action Bridge**: it does not stop at "research shows…" — it connects to applicability, the teaching decision, pilot intervention, and evaluation design.
|
|
187
|
+
|
|
188
|
+
## How EduEvidence Works
|
|
189
|
+
|
|
190
|
+
```text
|
|
191
|
+
┌─────────────────────────────────────┐
|
|
192
|
+
│ EduEvidence │
|
|
193
|
+
│ education knowledge + decision + │
|
|
194
|
+
│ intervention + evaluation │
|
|
195
|
+
└────────────────┬────────────────────┘
|
|
196
|
+
│
|
|
197
|
+
┌────────────────▼────────────────────┐
|
|
198
|
+
│ EvidenceFlow Protocol │
|
|
199
|
+
│ Frame / Retrieve / Extract / │
|
|
200
|
+
│ Challenge / Audit / Adjudicate │
|
|
201
|
+
└────────────────┬────────────────────┘
|
|
202
|
+
│
|
|
203
|
+
┌────────┴────────┐
|
|
204
|
+
▼ ▼
|
|
205
|
+
Platform Native Agent MCP
|
|
206
|
+
Execution Mode Enhanced Mode
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
The 9-step workflow:
|
|
210
|
+
|
|
211
|
+
```text
|
|
212
|
+
1. Frame Build the EducationResearchFrame
|
|
213
|
+
2. Retrieve Retrieve literature & evidence (support + independent counter-evidence)
|
|
214
|
+
3. Extract Extract claim-level evidence (bound to outcomes)
|
|
215
|
+
4. Challenge Skeptic protocol (fixed 9 checks)
|
|
216
|
+
5. Audit Method Reviewer audit (15-item checklist)
|
|
217
|
+
6. Adjudicate Evidence Tribunal (Evidence Matrix + Verdict)
|
|
218
|
+
7. Applicability Applicability analysis
|
|
219
|
+
8. Intervene Teaching Intervention design (minimum viable pilot)
|
|
220
|
+
9. Evaluate Evaluation Plan design
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
Every step is validated against JSON Schemas (`schemas/`), deterministic logic lives in `scripts/`, and the education methodology is documented independently in `references/`.
|
|
224
|
+
|
|
225
|
+
## Outcome Separation
|
|
226
|
+
|
|
227
|
+
EduEvidence enforces 20 outcome types (`references/outcome-taxonomy.md`):
|
|
228
|
+
|
|
229
|
+
```text
|
|
230
|
+
Learning: Knowledge Gain / Concept Understanding / Retention / Transfer / Independent Problem Solving
|
|
231
|
+
Task: Completion Time / Accuracy / Code Quality / Assignment Score
|
|
232
|
+
Process: Engagement / Motivation / Cognitive Load / Help-Seeking / Metacognition
|
|
233
|
+
Risk: AI Dependency / Over-reliance / Reduced Effort / Reduced Transfer / Academic Integrity Risk / False Confidence
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
The demo's highlight: in Kazemitabaar et al. (CHI 2023), the AI code assistant raised task completion by 1.15× and correctness by 1.8×, but the one-week retention test showed no significant difference — **task performance ≠ learning**.
|
|
237
|
+
|
|
238
|
+
## Evidence Tribunal
|
|
239
|
+
|
|
240
|
+
`references/tribunal-policy.md` defines the adjudication rules: input = Frame + Evidence Matrix + Skeptic Findings + Method Reviews; output = EducationVerdict (`schemas/verdict.schema.json`), including:
|
|
241
|
+
|
|
242
|
+
- supported / uncertain / contradicted claims
|
|
243
|
+
- conflict-source analysis (sample / measurement / course / tool / design)
|
|
244
|
+
- Can Claim / Cannot Claim boundaries
|
|
245
|
+
- four-state decision + Confidence (rule-based, not model-generated freely)
|
|
246
|
+
|
|
247
|
+

|
|
248
|
+
|
|
249
|
+
## From Evidence to Action
|
|
250
|
+
|
|
251
|
+
Evidence must connect to the real classroom (`references/applicability-policy.md`, `intervention-design.md`, `evaluation-design.md`):
|
|
252
|
+
|
|
253
|
+
- **Applicability**: For whom? For which course? For which outcome? Under what conditions? With what AI usage policy?
|
|
254
|
+
- **Intervention**: always a "minimum viable pilot", never direct full deployment; includes AI usage rules, teacher/student roles, reflection requirements, and stop conditions.
|
|
255
|
+
- **Evaluation**: every PILOT/ADOPT recommendation must come with an evaluation plan; distinguishes baseline / post-test / retention / transfer, and task-performance vs learning metrics.
|
|
256
|
+
|
|
257
|
+
## Benchmark
|
|
258
|
+
|
|
259
|
+
30 education research questions in v1 (`benchmarks/questions.jsonl`), S×10 / M×10 / L×10; 15 in the core domain "AI-assisted university teaching", 10 with human gold annotations (`benchmarks/annotations/`).
|
|
260
|
+
|
|
261
|
+
Baseline design:
|
|
262
|
+
|
|
263
|
+
```text
|
|
264
|
+
B0 Direct LLM
|
|
265
|
+
B1 Search + LLM
|
|
266
|
+
B2 Standard Research Agent
|
|
267
|
+
B3 EduEvidence Single-Agent ← demonstrates the value of the education methodology (B2 vs B3)
|
|
268
|
+
B4 EduEvidence + Agent MCP ← demonstrates the value of multi-agent enhancement (B3 vs B4)
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
Key metrics: Citation Support Precision / Unsupported Claim Rate / Contradiction Discovery Rate / Outcome Separation Accuracy / Scope Calibration / Intervention Evidence Alignment. See `docs/benchmark.md`.
|
|
272
|
+
|
|
273
|
+
> ⚠️ `benchmarks/results/` is **harness validation (deterministic simulation, marked SIMULATED)** — it proves the evaluation framework runs, not real model performance. The **first round of Layer B empirical runs has been launched** (B2 vs B3, 10 questions × 3 repeats, `omp` driver with `deepseek-v4-flash` — see `benchmarks/empirical/run-empirical-01`, report at `benchmarks/empirical/v3-report.md`). Metrics are gold-based heuristics (`method: heuristic`); results remain limited by the model and question set, so no definitive effectiveness claim is made until the runs are reviewed.
|
|
274
|
+
|
|
275
|
+
## Example: AI Coding Assistant
|
|
276
|
+
|
|
277
|
+
> **Should first-year C programming students be allowed to use generative AI coding assistants?**
|
|
278
|
+
|
|
279
|
+
`examples/ai-coding-assistant/` shows the full path from question to decision:
|
|
280
|
+
|
|
281
|
+
- **Evidence** (7 items, all bound to real sources): task-performance gains (Kazemitabaar 2023), unguarded access harming independent exam performance by −17% (Bastani 2025, PNAS), guardrails eliminating the negative effect (Bastani 2025), formative-feedback writing evidence (Marzuki 2024).
|
|
282
|
+
- **Decision**: **PILOT** — task-performance evidence is strong, but direct learning-effect evidence for university programming courses is missing, and the unguarded-access risk is documented.
|
|
283
|
+
- **Intervention**: 4-phase pilot (Independent Foundation → Explain Don't Solve → Structured Collaboration → Transfer Check).
|
|
284
|
+
- **Evaluation**: no-AI baseline / post-test / final-exam retention / no-AI transfer task + AI-dependency risk metrics.
|
|
285
|
+
|
|
286
|
+
Two more examples — AI writing assistant (`examples/ai-writing-assistant/`) and a calculus AI tutor (`examples/ai-tutor/`) — show the skill is not hard-coded to one question.
|
|
287
|
+
|
|
288
|
+
**Example provenance (read before citing)**: every example pack carries a `data_origin`
|
|
289
|
+
badge in its report header and in `result.json.meta`:
|
|
290
|
+
|
|
291
|
+
- `examples/ai-coding-assistant-evidence/` — **flagship, real literature**: 8 sources with
|
|
292
|
+
registry-verified DOIs (Crossref/DataCite audit: `benchmarks/doi-audit/report.md`),
|
|
293
|
+
engine-computed confidence; data_origin=`manual_curated`.
|
|
294
|
+
- `examples/ai-coding-assistant/`, `ai-tutor/`, `ai-writing-assistant/` — real-source
|
|
295
|
+
walkthrough packs.
|
|
296
|
+
- `examples/esl-academic-writing-ai/`, `highschool-math-ai-tutor/` — **synthetic
|
|
297
|
+
demonstrations** (data_origin=`synthetic`). Their numbers and citations are illustrative,
|
|
298
|
+
not real studies; reports carry a loud SYNTHETIC badge. Do not cite them as evidence.
|
|
299
|
+
|
|
300
|
+
### Start your own research in ~30 minutes
|
|
301
|
+
|
|
302
|
+
```bash
|
|
303
|
+
python3 scripts/quickstart.py "你的教育研究问题" # creates runs/<id> + NEXT_STEPS.md
|
|
304
|
+
# hand the LLM stages to your AI agent per NEXT_STEPS.md, then finish with:
|
|
305
|
+
python3 scripts/orchestrator.py adjudicate --project runs/<id>
|
|
306
|
+
bash scripts/bake_pack.sh <pack_dir> # 5-theme bilingual report
|
|
307
|
+
python3 scripts/citation_check.py --pack <pack_dir> --write-back # DOI ✓ badges
|
|
308
|
+
```
|
|
309
|
+
|
|
310
|
+
## Visualization: Bilingual HTML Report + Infographics + Academic Figures
|
|
311
|
+
|
|
312
|
+
After research completes, `result.json` is rendered into three visualization outputs by deterministic Python adapters. The adapters themselves use the standard library; the optional Web Studio chart enhancement has a separate browser dependency.
|
|
313
|
+
|
|
314
|
+
```text
|
|
315
|
+
result.json + result.zh.json (Chinese parallel data)
|
|
316
|
+
├─ build_charts.py → chart_specs.json (ECharts option data; no ECharts runtime bundled)
|
|
317
|
+
├─ build_infographics.py → infographics.json (hand-authored SVGs)
|
|
318
|
+
├─ build_figures.py → figures/ (publication figures: figure_data.json + SVG/PNG/PDF)
|
|
319
|
+
└─ build_report.py → EduEvidence_Report.html (single-file bilingual report + report_spec.json)
|
|
320
|
+
```
|
|
321
|
+
|
|
322
|
+
**EduEvidence_Report.html (main deliverable)**:
|
|
323
|
+
|
|
324
|
+
- **Bilingual switch**: Chinese by default, one click to EN; data remains isomorphic.
|
|
325
|
+
- **Executive summary narrative**: question → evidence → action, with traceable source sections.
|
|
326
|
+
- **Two-page layout**: Visual Brief + Full Report (AI-planned 5–7 dynamic chapters, not a fixed template).
|
|
327
|
+
- **Five styles (chosen at generation time, no in-HTML switcher)**: claude / academic / datalab / datalab-dark / presentation.
|
|
328
|
+
- **Static-first**: decision, matrix, tribunal, intervention and sources remain readable without JavaScript; ECharts is an optional enhancement.
|
|
329
|
+
- **Integrity gate**: chart numbers are checked against result.json item by item; publishing is blocked with `REPORT_INVALID` on mismatch.
|
|
330
|
+
|
|
331
|
+
**Local Web Studio** (`python3 scripts/dashboard_server.py --port 8765`) has exactly three read-only views: Dashboard, Report Browser and Data Visualization. It loads ECharts 5.4.3 from jsDelivr for interactive charts; the submission package does not bundle that runtime, so Web interactivity requires network access. The baked report's static HTML/SVG remains the offline artifact.
|
|
332
|
+
|
|
333
|
+
> Open the example directly: `examples/ai-coding-assistant/EduEvidence_Report.html`
|
|
334
|
+
|
|
335
|
+
## Architecture
|
|
336
|
+
|
|
337
|
+
The repository is a complete **Skill package**: `SKILL.md` is the entry point; everything else is layered as *skill core → quality assurance → demos*. See [`docs/architecture.md`](docs/architecture.md):
|
|
338
|
+
|
|
339
|
+
```text
|
|
340
|
+
EduEvidence/ (= one Skill package)
|
|
341
|
+
│
|
|
342
|
+
├─ SKILL.md ← Skill entry: When to Use / Inputs / Workflow / Output Contract
|
|
343
|
+
│
|
|
344
|
+
├─ Skill core (required to run)
|
|
345
|
+
│ ├─ engine/ V2 Research Engine (Project Workspace / immutable Evidence
|
|
346
|
+
│ │ Graph / Library / synthesis / tribunal / study design /
|
|
347
|
+
│ │ datasets / analysis / projections / migration)
|
|
348
|
+
│ ├─ skill/agents/ role protocols (capability execution profiles)
|
|
349
|
+
│ ├─ references/ 11 education methodology documents (evidence quality / skeptic /
|
|
350
|
+
│ │ tribunal policy / intervention design…)
|
|
351
|
+
│ ├─ schemas/ V1 + v2/v3 contracts (13 top-level + 17 v2 + v3 pilot/synthesis/run-manifest)
|
|
352
|
+
│ ├─ domains/ v4 domain registry (manifest.json) + per-domain packages
|
|
353
|
+
│ │ (education: registration-only; policy: frame schema /
|
|
354
|
+
│ │ outcome taxonomy / methodology checklist / references)
|
|
355
|
+
│ ├─ scripts/ deterministic logic scripts (scoring / matrix / audit /
|
|
356
|
+
│ │ confidence / orchestrator / startup probe / V2 CLI)
|
|
357
|
+
│ ├─ retrieval/ Search & fetch layer (fetch / validate / dedupe / failures)
|
|
358
|
+
│ ├─ integrations/ Agent MCP enhancement layer + Smart Web Fetch integration
|
|
359
|
+
│ └─ visualization/ Presentation layer (ECharts / infographics / academic figures /
|
|
360
|
+
│ bilingual HTML composer + V2 project surfaces)
|
|
361
|
+
│
|
|
362
|
+
├─ Quality assurance
|
|
363
|
+
│ ├─ tests/ pytest test matrix (V1–V4, 752 test functions / 73 files — docs/metrics.json)
|
|
364
|
+
│ └─ benchmarks/ V1 questions + benchmarks/v2/ (graph/contract metrics)
|
|
365
|
+
│
|
|
366
|
+
└─ Demos & distribution
|
|
367
|
+
├─ examples/ Research & Decision Packs + full-research-cycle-fixture (synthetic)
|
|
368
|
+
├─ docs/ architecture / methodology / benchmark / demo / reproducibility
|
|
369
|
+
├─ install.sh one-click install (local / multi-agent Skill) + self-check
|
|
370
|
+
├─ pyproject.toml packaging metadata (wheel ships CLI + engine; stdlib-only core)
|
|
371
|
+
└─ README(.en).md bilingual docs
|
|
372
|
+
```
|
|
373
|
+
|
|
374
|
+
> Skill-package principle: the **minimal runtime set is `SKILL.md + engine/ + skill/ + references/ + schemas/ + scripts/`**; `retrieval/`, `integrations/`, `visualization/` are the execution/presentation layers that make the Skill actually runnable; `tests/`, `benchmarks/`, `examples/`, `docs/` provide credibility and onboarding — none of them affect the Skill body itself.
|
|
375
|
+
|
|
376
|
+
### v4 Domain Registry (EvidenceCore)
|
|
377
|
+
|
|
378
|
+
The first step of the **EvidenceCore** abstraction: a `domains/` registry
|
|
379
|
+
(`domains/manifest.json`) plus per-domain packages, consumed by
|
|
380
|
+
`engine/evidencecore.py` (`list_domains()` / `load_domain()` /
|
|
381
|
+
`validate_frame()`).
|
|
382
|
+
|
|
383
|
+
- **education** is a **registration-only domain**: it points at the existing
|
|
384
|
+
contracts — `schemas/education-frame.schema.json` (frame),
|
|
385
|
+
the 20-token outcome taxonomy from `schemas/evidence.schema.json` (four
|
|
386
|
+
categories per `engine/pilot.py` `_OUTCOME_CATEGORY`), the 15-item
|
|
387
|
+
methodology checklist from `skill/agents/method-reviewer.md`,
|
|
388
|
+
`benchmarks/annotations` (golds) and `references/`. It **adds no new logic
|
|
389
|
+
path** — no new schema, no new validator, no new methodology.
|
|
390
|
+
- **policy** is the first self-contained domain: its own frame schema
|
|
391
|
+
(`decision_object` / `intervention` / `population` / `stakeholders` /
|
|
392
|
+
`outcomes` / `context` / `scope` — deliberately **not** education's
|
|
393
|
+
`learner` / `course`), a 5-token policy outcome taxonomy, a 12-item policy
|
|
394
|
+
evidence-quality checklist and 5 methodology notes in
|
|
395
|
+
`domains/policy/references/`.
|
|
396
|
+
- **Domain select is handled by the main agent via CLI** — the engine layer
|
|
397
|
+
never picks a domain itself; `load_domain(domain_id)` only validates that
|
|
398
|
+
the requested domain's contracts exist.
|
|
399
|
+
|
|
400
|
+
### SCP / Platform Native Mode
|
|
401
|
+
|
|
402
|
+
EduEvidence runs fully standalone without Agent MCP (no external service required):
|
|
403
|
+
|
|
404
|
+
- No local daemon
|
|
405
|
+
- No dependency on any single CLI
|
|
406
|
+
- No Agent MCP dependency
|
|
407
|
+
- SKILL.md is self-contained; the core workflow runs end to end
|
|
408
|
+
- All schemas / methods / output contracts exist independently
|
|
409
|
+
|
|
410
|
+
### Agent MCP Enhanced Mode
|
|
411
|
+
|
|
412
|
+
Agent MCP is a **performance & reliability enhancement layer, not a prerequisite** (Complexity Gate in `docs/methodology.md`):
|
|
413
|
+
|
|
414
|
+
- S-level tasks: single agent, 0 spawns
|
|
415
|
+
- M-level tasks: Primary Analysis + Independent Check
|
|
416
|
+
- L-level tasks: 8-role workflow (Planner / Retriever / Analyst / Skeptic / Method Reviewer / Judge / Intervention Designer / Evaluation Designer)
|
|
417
|
+
|
|
418
|
+
> Number of roles ≠ number of agents that must be launched. Platform Native Mode runs the role protocols sequentially in a single agent.
|
|
419
|
+
|
|
420
|
+
> 🔒 Agent MCP principle: **Scan first. Recommend second. Ask the user. Execute only after explicit confirmation.** No spawn without user approval; reject → fall back to Native.
|
|
421
|
+
|
|
422
|
+

|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
## Usage
|
|
426
|
+
|
|
427
|
+
```bash
|
|
428
|
+
# 1. Validate data against the schema contracts
|
|
429
|
+
python3 scripts/validate_schema.py --schema schemas/evidence.schema.json \
|
|
430
|
+
--data examples/ai-coding-assistant/evidence.jsonl
|
|
431
|
+
|
|
432
|
+
# 2. Compute evidence quality scores and Confidence
|
|
433
|
+
python3 scripts/evidence_score.py examples/ai-coding-assistant/evidence.jsonl
|
|
434
|
+
|
|
435
|
+
# 3. Generate the Evidence Matrix (one of the core views)
|
|
436
|
+
python3 scripts/evidence_matrix.py examples/ai-coding-assistant/evidence.jsonl
|
|
437
|
+
|
|
438
|
+
# 4. Run the Citation Audit (claim-evidence traceability)
|
|
439
|
+
python3 scripts/claim_audit.py --claims claims.jsonl --evidence evidence.jsonl
|
|
440
|
+
|
|
441
|
+
# 5. Render the Research & Decision Pack (Markdown)
|
|
442
|
+
python3 scripts/render_report.py \
|
|
443
|
+
--frame examples/ai-coding-assistant/frame.json \
|
|
444
|
+
--evidence examples/ai-coding-assistant/evidence.jsonl \
|
|
445
|
+
--methodology examples/ai-coding-assistant/methodology.json \
|
|
446
|
+
--verdict examples/ai-coding-assistant/verdict.json \
|
|
447
|
+
--intervention examples/ai-coding-assistant/intervention.json \
|
|
448
|
+
--evaluation examples/ai-coding-assistant/evaluation.json \
|
|
449
|
+
--out REPORT.md
|
|
450
|
+
|
|
451
|
+
# 6. Render the single-file bilingual HTML report (main deliverable)
|
|
452
|
+
python3 visualization/eduevidence-report/scripts/build_report.py \
|
|
453
|
+
--result examples/ai-coding-assistant/result.json \
|
|
454
|
+
--out examples/ai-coding-assistant/EduEvidence_Report.html
|
|
455
|
+
|
|
456
|
+
# 7. Validate the benchmark question set
|
|
457
|
+
python3 scripts/benchmark.py --questions benchmarks/questions.jsonl
|
|
458
|
+
|
|
459
|
+
# 8. Run the tests
|
|
460
|
+
pytest
|
|
461
|
+
```
|
|
462
|
+
|
|
463
|
+
> In real use, the Skill is executed by an agent that reads SKILL.md and runs the 9-step workflow; `scripts/` guarantees deterministic validation of structured data, `visualization/` guarantees deterministic rendering, and `examples/` are complete runnable packs.
|
|
464
|
+
|
|
465
|
+
## Methodology
|
|
466
|
+
|
|
467
|
+
- Education evidence quality framework: five dimensions, 0–2 each (D1 study design / D2 sample quality / D3 measurement validity / D4 temporal strength / D5 directness), total 0–10 (`references/evidence-quality.md`).
|
|
468
|
+
- 15-item methodology audit; top-priority rule: **task performance must not be equated with learning** (`references/methodology-audit.md`).
|
|
469
|
+
- Rule-based Confidence: `Evidence Quality + Consistency + Directness + Evidence Count − Conflict Penalty − Unsupported Penalty` → High / Moderate / Low / Insufficient (`scripts/evidence_score.py`).
|
|
470
|
+
- Failure handling: INSUFFICIENT_SOURCES / UNSUPPORTED_CLAIM / CONFLICT_UNRESOLVED / SCOPE_MISMATCH / METHODOLOGY_TOO_WEAK / NEEDS_USER_CONTEXT / TOOL_FAILURE — on failure, high-confidence recommendations are never forced.
|
|
471
|
+
|
|
472
|
+
## Limitations
|
|
473
|
+
|
|
474
|
+
- The benchmark is based on retrievable evidence from real literature; actual model runs must be collected per the B0–B4 baselines in `docs/benchmark.md`.
|
|
475
|
+
- Search and extraction depend on available retrieval resources; `TOOL_FAILURE` never fabricates sources.
|
|
476
|
+
- EduEvidence assists teaching decisions — **it does not replace the final decision of teachers or institutions**; it never auto-decides on high-stakes assessment, student discipline, individual psychological judgments, or major student educational opportunities.
|
|
477
|
+
|
|
478
|
+
## Roadmap
|
|
479
|
+
|
|
480
|
+
**Completed — harness / simulation only (marked as SIMULATED, not empirical):**
|
|
481
|
+
|
|
482
|
+
- [x] Benchmark v2 harness / simulation — `benchmarks/results/` is a deterministic
|
|
483
|
+
simulation that proves the evaluation framework runs; it is **not** real model
|
|
484
|
+
performance (see the ⚠️ note in [Benchmark](#benchmark)).
|
|
485
|
+
- [x] Skill core & pipeline: 9-step protocol (Research Core 6 + Decision Extension 3),
|
|
486
|
+
13 top-level JSON Schemas, deterministic scripts, 8-role protocols (original Phases 0–6).
|
|
487
|
+
- [x] Evidence-to-action: applicability / four-state decision / intervention / evaluation.
|
|
488
|
+
- [x] Product UI: single-file bilingual HTML report + infographics + academic figures
|
|
489
|
+
(original Phase 8).
|
|
490
|
+
|
|
491
|
+
**Planned — not yet done, and not claimed as done:**
|
|
492
|
+
|
|
493
|
+
- [ ] **Empirical Benchmark** — first round **launched** (B2 vs B3, 10 questions × 3
|
|
494
|
+
repeats, `omp` / `deepseek-v4-flash` → `benchmarks/empirical/v3-report.md`);
|
|
495
|
+
full 30-question coverage, **B3 vs B4**, variance reporting, and
|
|
496
|
+
gold-annotation / independent-judge scoring still planned (see `docs/benchmark.md`).
|
|
497
|
+
- [ ] **HTML accessibility** — sync `<html lang>` on language switch, `aria-pressed` on
|
|
498
|
+
theme/lang buttons, labeled table filter controls, bilingual `title`/`desc` on SVG
|
|
499
|
+
figures, and a safe URL scheme allowlist for source links.
|
|
500
|
+
- [ ] **Vertical decision loop** — close **PILOT → real data → re-adjudication**:
|
|
501
|
+
feed pilot outcomes back into the project evidence graph and produce an updated
|
|
502
|
+
decision (Full Research Cycle end-to-end with real user data).
|
|
503
|
+
|
|
504
|
+
## License
|
|
505
|
+
|
|
506
|
+
MIT — see [LICENSE](LICENSE).
|