eduevidence 5.2.0 → 6.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +105 -0
- package/README.md +142 -75
- package/README.zh-CN.md +73 -30
- package/SKILL.md +397 -131
- package/agents/openai.yaml +4 -0
- package/assets/readme/controlled-execution.svg +34 -0
- package/assets/readme/landing-tour.gif +0 -0
- package/assets/readme/logo.png +0 -0
- package/assets/readme/research-workflow.svg +56 -0
- package/assets/readme/studio-graph.png +0 -0
- package/assets/readme/studio-overview.png +0 -0
- package/assets/readme/studio-reports.png +0 -0
- package/assets/readme/studio-tour.gif +0 -0
- package/autoevolve/config.yaml +17 -0
- package/autoevolve/program.md +25 -0
- package/autoevolve/protected.manifest.yaml +34 -0
- package/benchmarks/adversarial/cases.jsonl +7 -0
- package/benchmarks/evidence-library.json +5268 -0
- package/benchmarks/partitions.json +8 -0
- package/bin/eduevidence.js +2 -1
- package/docs/architecture.md +496 -0
- package/docs/autoresearch-evolution-plan.md +2903 -0
- package/docs/autoresearch-implementation-status.md +101 -0
- package/docs/demo-storyboard.md +20 -0
- package/docs/demo-workplace-ai.md +92 -0
- package/docs/demo.md +32 -0
- package/docs/install-guide.md +150 -0
- package/docs/orchestration-role-model.md +1254 -0
- package/docs/release-closeout/README.md +17 -0
- package/docs/release-closeout/frontend-acceptance.md +23 -0
- package/docs/release-closeout/issues.md +19 -0
- package/docs/release-closeout/verification.md +28 -0
- package/docs/release-contract.md +108 -0
- package/docs/research-studio-guide.zh-CN.md +166 -0
- package/docs/sciverse-api.md +125 -0
- package/eduevidence_cli.py +29 -13
- package/engine/_resources.py +13 -0
- package/engine/autoevolve/__init__.py +3 -0
- package/engine/autoevolve/agent_view.py +167 -0
- package/engine/autoevolve/core.py +357 -0
- package/engine/autoevolve/events.py +11 -0
- package/engine/autoevolve/git_workspace.py +77 -0
- package/engine/autoevolve/projection.py +23 -0
- package/engine/autoevolve/runner.py +413 -0
- package/engine/autoevolve/trust.py +146 -0
- package/engine/autoresearch/__init__.py +6 -0
- package/engine/autoresearch/commit.py +132 -0
- package/engine/autoresearch/contracts.py +126 -0
- package/engine/autoresearch/controller.py +207 -0
- package/engine/autoresearch/events.py +12 -0
- package/engine/autoresearch/gap_priority.py +168 -0
- package/engine/autoresearch/projection.py +30 -0
- package/engine/autoresearch/research_memory.py +59 -0
- package/engine/autoresearch/saturation.py +91 -0
- package/engine/briefs.py +2 -1
- package/engine/capabilities.py +1 -0
- package/engine/contracts.py +3 -1
- package/engine/decision_policy.py +96 -0
- package/engine/evidence_graph.py +14 -10
- package/engine/evidencecore.py +7 -5
- package/engine/gaps.py +132 -73
- package/engine/ids.py +2 -0
- package/engine/judge_pack.py +65 -0
- package/engine/library.py +6 -2
- package/engine/library_builtin.py +3 -1
- package/engine/living.py +36 -5
- package/engine/meta_synthesis.py +3 -1
- package/engine/migration.py +88 -3
- package/engine/orchestration.py +460 -0
- package/engine/paths.py +2 -0
- package/engine/pilot.py +36 -33
- package/engine/project.py +2 -2
- package/engine/research_service.py +113 -0
- package/engine/studio_read_model.py +400 -0
- package/engine/taxonomy.py +211 -0
- package/engine/tribunal.py +44 -33
- package/engine/update.py +1 -0
- package/engine/versions.py +1 -1
- package/engine/worker_result.py +109 -0
- package/engine/workflows.py +70 -0
- package/examples/ai-coding-assistant-evidence/EduEvidence_Report.html +2934 -0
- package/examples/ai-coding-assistant-evidence/artifact_manifest.json +15 -0
- package/examples/ai-coding-assistant-evidence/citation_check.json +79 -0
- package/examples/ai-coding-assistant-evidence/claims.jsonl +12 -0
- package/examples/ai-coding-assistant-evidence/evaluation.json +35 -0
- package/examples/ai-coding-assistant-evidence/evidence.jsonl +12 -0
- package/examples/ai-coding-assistant-evidence/final_verdict.json +107 -0
- package/examples/ai-coding-assistant-evidence/frame.json +48 -0
- package/examples/ai-coding-assistant-evidence/gate_report.json +101 -0
- package/examples/ai-coding-assistant-evidence/intervention.json +51 -0
- package/examples/ai-coding-assistant-evidence/methodology.json +36 -0
- package/examples/ai-coding-assistant-evidence/raw_verdict.json +86 -0
- package/examples/ai-coding-assistant-evidence/report_spec.json +230 -0
- package/examples/ai-coding-assistant-evidence/reports-5themes/EduEvidence_Report_academic.html +2934 -0
- package/examples/ai-coding-assistant-evidence/reports-5themes/EduEvidence_Report_claude.html +2934 -0
- package/examples/ai-coding-assistant-evidence/reports-5themes/EduEvidence_Report_datalab-dark.html +2934 -0
- package/examples/ai-coding-assistant-evidence/reports-5themes/EduEvidence_Report_datalab.html +2934 -0
- package/examples/ai-coding-assistant-evidence/reports-5themes/EduEvidence_Report_presentation.html +2934 -0
- package/examples/ai-coding-assistant-evidence/reports-5themes/report_academic.html +2934 -0
- package/examples/ai-coding-assistant-evidence/reports-5themes/report_claude.html +2934 -0
- package/examples/ai-coding-assistant-evidence/reports-5themes/report_datalab-dark.html +2934 -0
- package/examples/ai-coding-assistant-evidence/reports-5themes/report_datalab.html +2934 -0
- package/examples/ai-coding-assistant-evidence/reports-5themes/report_presentation.html +2934 -0
- package/examples/ai-coding-assistant-evidence/result.json +1457 -0
- package/examples/ai-coding-assistant-evidence/result.zh.json +1457 -0
- package/examples/ai-coding-assistant-evidence/skeptic.json +72 -0
- package/examples/ai-coding-assistant-evidence/sources.jsonl +8 -0
- package/examples/ai-coding-assistant-evidence/verdict.json +107 -0
- package/examples/spaced-retrieval-practice/applicability.json +14 -0
- package/examples/spaced-retrieval-practice/artifact_manifest.json +15 -0
- package/examples/spaced-retrieval-practice/claims.jsonl +3 -0
- package/examples/spaced-retrieval-practice/evidence.jsonl +6 -0
- package/examples/spaced-retrieval-practice/final_verdict.json +93 -0
- package/examples/spaced-retrieval-practice/frame.json +58 -0
- package/examples/spaced-retrieval-practice/gate_report.json +101 -0
- package/examples/spaced-retrieval-practice/methodology.json +78 -0
- package/examples/spaced-retrieval-practice/report_spec.json +212 -0
- package/examples/spaced-retrieval-practice/reports-5themes/EduEvidence_Report_academic.html +2728 -0
- package/examples/spaced-retrieval-practice/reports-5themes/EduEvidence_Report_claude.html +2728 -0
- package/examples/spaced-retrieval-practice/reports-5themes/EduEvidence_Report_datalab-dark.html +2728 -0
- package/examples/spaced-retrieval-practice/reports-5themes/EduEvidence_Report_datalab.html +2728 -0
- package/examples/spaced-retrieval-practice/reports-5themes/EduEvidence_Report_presentation.html +2728 -0
- package/examples/spaced-retrieval-practice/reports-5themes/report_academic.html +2728 -0
- package/examples/spaced-retrieval-practice/reports-5themes/report_claude.html +2728 -0
- package/examples/spaced-retrieval-practice/reports-5themes/report_datalab-dark.html +2728 -0
- package/examples/spaced-retrieval-practice/reports-5themes/report_datalab.html +2728 -0
- package/examples/spaced-retrieval-practice/reports-5themes/report_presentation.html +2728 -0
- package/examples/spaced-retrieval-practice/result.json +942 -0
- package/examples/spaced-retrieval-practice/result.zh.json +942 -0
- package/examples/spaced-retrieval-practice/skeptic.json +70 -0
- package/examples/spaced-retrieval-practice/sources.jsonl +7 -0
- package/examples/spaced-retrieval-practice/verdict.json +93 -0
- package/examples/workplace-ai-assistant/artifact_manifest.json +15 -0
- package/examples/workplace-ai-assistant/claims.jsonl +4 -0
- package/examples/workplace-ai-assistant/evaluation.json +19 -0
- package/examples/workplace-ai-assistant/evidence.jsonl +4 -0
- package/examples/workplace-ai-assistant/evidence_graph.json +444 -0
- package/examples/workplace-ai-assistant/final_verdict.json +78 -0
- package/examples/workplace-ai-assistant/frame.json +41 -0
- package/examples/workplace-ai-assistant/gate_report.json +101 -0
- package/examples/workplace-ai-assistant/intervention.json +27 -0
- package/examples/workplace-ai-assistant/legacy-link-check.json +16 -0
- package/examples/workplace-ai-assistant/methodology.json +60 -0
- package/examples/workplace-ai-assistant/report_spec.json +224 -0
- package/examples/workplace-ai-assistant/reports-5themes/EduEvidence_Report_academic.html +2814 -0
- package/examples/workplace-ai-assistant/reports-5themes/EduEvidence_Report_claude.html +2814 -0
- package/examples/workplace-ai-assistant/reports-5themes/EduEvidence_Report_datalab-dark.html +2814 -0
- package/examples/workplace-ai-assistant/reports-5themes/EduEvidence_Report_datalab.html +2814 -0
- package/examples/workplace-ai-assistant/reports-5themes/EduEvidence_Report_presentation.html +2814 -0
- package/examples/workplace-ai-assistant/reports-5themes/report_academic.html +2814 -0
- package/examples/workplace-ai-assistant/reports-5themes/report_claude.html +2814 -0
- package/examples/workplace-ai-assistant/reports-5themes/report_datalab-dark.html +2814 -0
- package/examples/workplace-ai-assistant/reports-5themes/report_datalab.html +2814 -0
- package/examples/workplace-ai-assistant/reports-5themes/report_presentation.html +2814 -0
- package/examples/workplace-ai-assistant/result.json +615 -0
- package/examples/workplace-ai-assistant/result.zh.json +615 -0
- package/examples/workplace-ai-assistant/search_log.json +19 -0
- package/examples/workplace-ai-assistant/skeptic.json +72 -0
- package/examples/workplace-ai-assistant/sources.jsonl +3 -0
- package/examples/workplace-ai-assistant/validation_result.json +9 -0
- package/examples/workplace-ai-assistant/verdict.json +78 -0
- package/install.sh +7 -7
- package/integrations/agent_mcp.py +2 -2
- package/integrations/orchestration_dispatch.py +146 -0
- package/package.json +46 -3
- package/pyproject.toml +14 -22
- package/references/autoresearch.md +30 -0
- package/references/evaluation-policy.md +24 -0
- package/references/orchestration.md +22 -0
- package/references/report-copy-style.md +67 -0
- package/references/retrieval-compliance.md +75 -0
- package/references/retrieval-protocol.md +20 -0
- package/references/scientific-invariants.md +19 -0
- package/retrieval/audit.py +178 -0
- package/retrieval/fetch.py +96 -0
- package/retrieval/sciverse.py +398 -0
- package/retrieval/search.py +47 -7
- package/schemas/applicability.schema.json +94 -0
- package/schemas/chart-spec.schema.json +10 -3
- package/schemas/evidence.schema.json +316 -43
- package/schemas/fetch-result.schema.json +2 -1
- package/schemas/intervention.schema.json +106 -21
- package/schemas/report-result.schema.json +12 -4
- package/schemas/report-spec.schema.json +98 -100
- package/schemas/skeptic.schema.json +86 -0
- package/schemas/source.schema.json +21 -2
- package/schemas/v2/finding.schema.json +5 -1
- package/schemas/v2/methodology-audit.schema.json +5 -1
- package/schemas/v2/outcome.schema.json +28 -5
- package/schemas/v2/project.schema.json +2 -2
- package/schemas/v2/run.schema.json +1 -1
- package/schemas/v2/study.schema.json +5 -1
- package/schemas/vNext/autoevolve-session.schema.json +34 -0
- package/schemas/vNext/eval-snapshot.schema.json +77 -0
- package/schemas/vNext/execution-plan.schema.json +50 -0
- package/schemas/vNext/gap-priority.schema.json +54 -0
- package/schemas/vNext/negative-search-record.schema.json +68 -0
- package/schemas/vNext/research-iteration.schema.json +87 -0
- package/schemas/vNext/research-strategy.schema.json +62 -0
- package/schemas/vNext/skill-experiment.schema.json +90 -0
- package/schemas/vNext/task-spec.schema.json +156 -0
- package/schemas/vNext/worker-result.schema.json +60 -0
- package/schemas/verdict.schema.json +164 -28
- package/scripts/benchmark_judge.py +2 -2
- package/scripts/benchmark_v3.py +26 -43
- package/scripts/build_esl_artifacts.py +4 -4
- package/scripts/build_evidence_library.py +2 -2
- package/scripts/build_gh_pages.py +98 -0
- package/scripts/build_readme_diagrams.py +72 -0
- package/scripts/build_report_variants.py +101 -0
- package/scripts/build_result.py +74 -9
- package/scripts/check_autoresearch_invariants.py +95 -0
- package/scripts/check_package_parity.py +85 -0
- package/scripts/check_protocol_alignment.py +375 -0
- package/scripts/check_versioned_schemas.py +254 -0
- package/scripts/claim_audit.py +13 -8
- package/scripts/compute_confidence.py +10 -0
- package/scripts/daily_evolve.py +30 -0
- package/scripts/dashboard_server.py +130 -101
- package/scripts/did_regression.py +17 -32
- package/scripts/enrich_projects_human_and_lieflat.py +1 -1
- package/scripts/evidence_score.py +5 -2
- package/scripts/generate_metrics.py +4 -3
- package/scripts/generate_new_projects.py +5 -5
- package/scripts/orchestrator.py +286 -36
- package/scripts/pre_verdict_gate.py +224 -26
- package/scripts/quickstart.py +18 -2
- package/scripts/rebake_all_5themes.py +1 -2
- package/scripts/research_auto_cli.py +475 -0
- package/scripts/run_workspace.py +24 -8
- package/scripts/search_provenance.py +64 -0
- package/scripts/serve_web.py +9 -10
- package/scripts/skill_lint.py +1 -1
- package/scripts/skill_payload.py +81 -0
- package/scripts/test_adversarial_empirical.py +26 -19
- package/scripts/validate_schema.py +46 -2
- package/scripts/vnext_cli.py +133 -0
- package/setup.py +12 -0
- package/skill/agents/evaluation-designer.md +20 -4
- package/skill/agents/evidence-analyst.md +19 -3
- package/skill/agents/evidence-judge.md +50 -2
- package/skill/agents/evidence-retriever.md +20 -3
- package/skill/agents/intervention-designer.md +20 -4
- package/skill/agents/method-reviewer.md +18 -2
- package/skill/agents/{education-planner.md → research-planner.md} +19 -3
- package/skill/agents/skeptic.md +18 -2
- package/skill/roles/registry.yaml +45 -0
- package/skill/sub-skills/aihot-trend-analysis/SKILL.md +28 -9
- package/skill/sub-skills/contradiction-analysis/SKILL.md +31 -11
- package/skill/sub-skills/data-analysis/SKILL.md +34 -15
- package/skill/sub-skills/ethics-review/SKILL.md +33 -10
- package/skill/sub-skills/evidence-extraction/SKILL.md +29 -11
- package/skill/sub-skills/evidence-review/SKILL.md +31 -12
- package/skill/sub-skills/gap-analysis/SKILL.md +31 -9
- package/skill/sub-skills/literature-review/SKILL.md +35 -14
- package/skill/sub-skills/methodology-audit/SKILL.md +29 -12
- package/skill/sub-skills/report-generation/SKILL.md +40 -6
- package/skill/sub-skills/research-planning/SKILL.md +41 -14
- package/skill/sub-skills/study-design/SKILL.md +30 -9
- package/skill/task-briefs/adjudicate.md +32 -7
- package/skill/task-briefs/applicability.md +38 -0
- package/skill/task-briefs/audit.md +32 -7
- package/skill/task-briefs/challenge.md +34 -5
- package/skill/task-briefs/evaluate.md +30 -5
- package/skill/task-briefs/extract.md +31 -8
- package/skill/task-briefs/frame.md +39 -10
- package/skill/task-briefs/intervene.md +32 -6
- package/skill/task-briefs/present.md +32 -8
- package/skill/task-briefs/projection.md +37 -0
- package/skill/task-briefs/retrieve.md +36 -6
- package/skill/workflows/decision-and-pilot.md +85 -0
- package/skill/workflows/evaluate-and-update.md +93 -0
- package/skill/workflows/evidence-review.md +117 -0
- package/visualization/eduevidence-report/assets/base.css +2 -2
- package/visualization/eduevidence-report/assets/reader.css +752 -0
- package/visualization/eduevidence-report/assets/reader.js +132 -0
- package/visualization/eduevidence-report/references/chart-selection-catalog.md +109 -0
- package/visualization/eduevidence-report/references/lieflat-composition.md +3 -1
- package/visualization/eduevidence-report/scripts/build_figures.py +25 -3
- package/visualization/eduevidence-report/scripts/build_infographics.py +5 -1
- package/visualization/eduevidence-report/scripts/build_report.py +561 -121
- package/visualization/eduevidence-report/scripts/charts_data.py +2 -0
- package/visualization/eduevidence-report/scripts/lieflat_engine.py +371 -136
- package/visualization/eduevidence-report/scripts/zh_labels.py +80 -1
- package/visualization/eduevidence-report/themes/academic.css +1 -1
- package/visualization/eduevidence-report/themes/claude.css +1 -1
- package/visualization/eduevidence-report/themes/datalab-dark.css +2 -2
- package/visualization/eduevidence-report/themes/datalab.css +2 -2
- package/visualization/eduevidence-report/themes/presentation.css +2 -2
- package/web/README.md +18 -0
- package/web/architecture.html +14885 -0
- package/web/index.html +53 -0
- package/web/studio/THIRD_PARTY_LICENSES.txt +146 -0
- package/web/studio/assets/index-B8tkF44Q.css +1 -0
- package/web/studio/assets/index-CQ6Keoyc.js +230 -0
- package/web/studio/config.json +1 -0
- package/web/studio/index.html +14 -0
- package/engine/__pycache__/__init__.cpython-312.pyc +0 -0
- package/engine/__pycache__/analysis.cpython-312.pyc +0 -0
- package/engine/__pycache__/bias.cpython-312.pyc +0 -0
- package/engine/__pycache__/briefs.cpython-312.pyc +0 -0
- package/engine/__pycache__/capabilities.cpython-312.pyc +0 -0
- package/engine/__pycache__/citation_check.cpython-312.pyc +0 -0
- package/engine/__pycache__/contracts.cpython-312.pyc +0 -0
- package/engine/__pycache__/datasets.cpython-312.pyc +0 -0
- package/engine/__pycache__/events.cpython-312.pyc +0 -0
- package/engine/__pycache__/evidence_graph.cpython-312.pyc +0 -0
- package/engine/__pycache__/evidence_review.cpython-312.pyc +0 -0
- package/engine/__pycache__/evidencecore.cpython-312.pyc +0 -0
- package/engine/__pycache__/gap_lens.cpython-312.pyc +0 -0
- package/engine/__pycache__/gaps.cpython-312.pyc +0 -0
- package/engine/__pycache__/graph_store.cpython-312.pyc +0 -0
- package/engine/__pycache__/graph_validate.cpython-312.pyc +0 -0
- package/engine/__pycache__/ids.cpython-312.pyc +0 -0
- package/engine/__pycache__/library.cpython-312.pyc +0 -0
- package/engine/__pycache__/library_builtin.cpython-312.pyc +0 -0
- package/engine/__pycache__/living.cpython-312.pyc +0 -0
- package/engine/__pycache__/log.cpython-312.pyc +0 -0
- package/engine/__pycache__/meta_analysis.cpython-312.pyc +0 -0
- package/engine/__pycache__/meta_synthesis.cpython-312.pyc +0 -0
- package/engine/__pycache__/migration.cpython-312.pyc +0 -0
- package/engine/__pycache__/mode_router.cpython-312.pyc +0 -0
- package/engine/__pycache__/paths.cpython-312.pyc +0 -0
- package/engine/__pycache__/pilot.cpython-312.pyc +0 -0
- package/engine/__pycache__/planner.cpython-312.pyc +0 -0
- package/engine/__pycache__/project.cpython-312.pyc +0 -0
- package/engine/__pycache__/projections.cpython-312.pyc +0 -0
- package/engine/__pycache__/robustness.cpython-312.pyc +0 -0
- package/engine/__pycache__/run.cpython-312.pyc +0 -0
- package/engine/__pycache__/semantics.cpython-312.pyc +0 -0
- package/engine/__pycache__/study_design.cpython-312.pyc +0 -0
- package/engine/__pycache__/synthesis.cpython-312.pyc +0 -0
- package/engine/__pycache__/tribunal.cpython-312.pyc +0 -0
- package/engine/__pycache__/update.cpython-312.pyc +0 -0
- package/engine/__pycache__/versions.cpython-312.pyc +0 -0
- package/integrations/__pycache__/__init__.cpython-312.pyc +0 -0
- package/integrations/__pycache__/agent_mcp.cpython-312.pyc +0 -0
- package/integrations/__pycache__/smart_web_fetch.cpython-312.pyc +0 -0
- package/retrieval/__pycache__/__init__.cpython-312.pyc +0 -0
- package/retrieval/__pycache__/corpus_store.cpython-312.pyc +0 -0
- package/retrieval/__pycache__/dedupe.cpython-312.pyc +0 -0
- package/retrieval/__pycache__/failures.cpython-312.pyc +0 -0
- package/retrieval/__pycache__/fetch.cpython-312.pyc +0 -0
- package/retrieval/__pycache__/search.cpython-312.pyc +0 -0
- package/retrieval/__pycache__/source.cpython-312.pyc +0 -0
- package/retrieval/__pycache__/validate.cpython-312.pyc +0 -0
- package/scripts/__pycache__/__init__.cpython-312.pyc +0 -0
- package/scripts/__pycache__/benchmark.cpython-312.pyc +0 -0
- package/scripts/__pycache__/benchmark_evaluator.cpython-312.pyc +0 -0
- package/scripts/__pycache__/benchmark_judge.cpython-312.pyc +0 -0
- package/scripts/__pycache__/benchmark_routing.cpython-312.pyc +0 -0
- package/scripts/__pycache__/benchmark_v2.cpython-312.pyc +0 -0
- package/scripts/__pycache__/benchmark_v3.cpython-312.pyc +0 -0
- package/scripts/__pycache__/build_result.cpython-312.pyc +0 -0
- package/scripts/__pycache__/claim_audit.cpython-312.pyc +0 -0
- package/scripts/__pycache__/complexity_gate.cpython-312.pyc +0 -0
- package/scripts/__pycache__/compute_confidence.cpython-312.pyc +0 -0
- package/scripts/__pycache__/dashboard_server.cpython-312.pyc +0 -0
- package/scripts/__pycache__/did_regression.cpython-312.pyc +0 -0
- package/scripts/__pycache__/effect_calculator.cpython-312.pyc +0 -0
- package/scripts/__pycache__/evidence_matrix.cpython-312.pyc +0 -0
- package/scripts/__pycache__/evidence_score.cpython-312.pyc +0 -0
- package/scripts/__pycache__/evidence_semantics.cpython-312.pyc +0 -0
- package/scripts/__pycache__/fetch_benchmark.cpython-312.pyc +0 -0
- package/scripts/__pycache__/lint_report_layout.cpython-312.pyc +0 -0
- package/scripts/__pycache__/orchestrator.cpython-312.pyc +0 -0
- package/scripts/__pycache__/pre_verdict_gate.cpython-312.pyc +0 -0
- package/scripts/__pycache__/recompute_demo_quality.cpython-312.pyc +0 -0
- package/scripts/__pycache__/render_report.cpython-312.pyc +0 -0
- package/scripts/__pycache__/render_report_html.cpython-312.pyc +0 -0
- package/scripts/__pycache__/run_workspace.cpython-312.pyc +0 -0
- package/scripts/__pycache__/skill_lint.cpython-312.pyc +0 -0
- package/scripts/__pycache__/startup_probe.cpython-312.pyc +0 -0
- package/scripts/__pycache__/sync_killer_demo_report.cpython-312.pyc +0 -0
- package/scripts/__pycache__/test_adversarial_empirical.cpython-312-pytest-9.0.2.pyc +0 -0
- package/scripts/__pycache__/test_adversarial_empirical.cpython-312-pytest-9.1.1.pyc +0 -0
- package/scripts/__pycache__/validate_schema.cpython-312.pyc +0 -0
- package/visualization/eduevidence-report/scripts/__pycache__/adapter_contract.cpython-312.pyc +0 -0
- package/visualization/eduevidence-report/scripts/__pycache__/build_artifact_manifest.cpython-312.pyc +0 -0
- package/visualization/eduevidence-report/scripts/__pycache__/build_charts.cpython-312.pyc +0 -0
- package/visualization/eduevidence-report/scripts/__pycache__/build_figures.cpython-312.pyc +0 -0
- package/visualization/eduevidence-report/scripts/__pycache__/build_infographics.cpython-312.pyc +0 -0
- package/visualization/eduevidence-report/scripts/__pycache__/build_report.cpython-312.pyc +0 -0
- package/visualization/eduevidence-report/scripts/__pycache__/charts_data.cpython-312.pyc +0 -0
- package/visualization/eduevidence-report/scripts/__pycache__/lieflat_engine.cpython-312.pyc +0 -0
- package/visualization/eduevidence-report/scripts/__pycache__/zh_labels.cpython-312.pyc +0 -0
|
@@ -5,52 +5,188 @@
|
|
|
5
5
|
"description": "Output of the Evidence Tribunal: what the evidence supports, cannot support, and the recommended action. Decision is one of ADOPT / PILOT / REJECT / INSUFFICIENT EVIDENCE (expressed here as adopt|pilot|reject|insufficient_evidence). confidence 由 scripts/compute_confidence.py 确定性计算并覆盖模型值;confidence_score 是规则化指数(0-1),不是概率。扩展字段一律放在 extensions 内。",
|
|
6
6
|
"type": "object",
|
|
7
7
|
"additionalProperties": false,
|
|
8
|
-
"required": [
|
|
8
|
+
"required": [
|
|
9
|
+
"decision_question",
|
|
10
|
+
"recommended_action",
|
|
11
|
+
"confidence"
|
|
12
|
+
],
|
|
9
13
|
"properties": {
|
|
10
|
-
"decision_question": {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
"
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
"
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
"
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
14
|
+
"decision_question": {
|
|
15
|
+
"type": "string"
|
|
16
|
+
},
|
|
17
|
+
"target_population": {
|
|
18
|
+
"type": "string"
|
|
19
|
+
},
|
|
20
|
+
"target_context": {
|
|
21
|
+
"type": "string"
|
|
22
|
+
},
|
|
23
|
+
"supported_claims": {
|
|
24
|
+
"type": "array",
|
|
25
|
+
"items": {
|
|
26
|
+
"type": "string"
|
|
27
|
+
}
|
|
28
|
+
},
|
|
29
|
+
"uncertain_claims": {
|
|
30
|
+
"type": "array",
|
|
31
|
+
"items": {
|
|
32
|
+
"type": "string"
|
|
33
|
+
}
|
|
34
|
+
},
|
|
35
|
+
"contradicted_claims": {
|
|
36
|
+
"type": "array",
|
|
37
|
+
"items": {
|
|
38
|
+
"type": "string"
|
|
39
|
+
}
|
|
40
|
+
},
|
|
41
|
+
"reason_for_disagreement": {
|
|
42
|
+
"type": "string"
|
|
43
|
+
},
|
|
44
|
+
"methodology_summary": {
|
|
45
|
+
"type": "string"
|
|
46
|
+
},
|
|
47
|
+
"outcome_specific_findings": {
|
|
48
|
+
"type": "object",
|
|
49
|
+
"additionalProperties": true
|
|
50
|
+
},
|
|
51
|
+
"short_term_effect": {
|
|
52
|
+
"type": [
|
|
53
|
+
"string",
|
|
54
|
+
"null"
|
|
55
|
+
]
|
|
56
|
+
},
|
|
57
|
+
"long_term_effect": {
|
|
58
|
+
"type": [
|
|
59
|
+
"string",
|
|
60
|
+
"null"
|
|
61
|
+
]
|
|
62
|
+
},
|
|
63
|
+
"transfer_effect": {
|
|
64
|
+
"type": [
|
|
65
|
+
"string",
|
|
66
|
+
"null"
|
|
67
|
+
]
|
|
68
|
+
},
|
|
69
|
+
"risk_effect": {
|
|
70
|
+
"type": [
|
|
71
|
+
"string",
|
|
72
|
+
"null"
|
|
73
|
+
]
|
|
74
|
+
},
|
|
75
|
+
"applicability": {
|
|
76
|
+
"type": "object",
|
|
77
|
+
"additionalProperties": true
|
|
78
|
+
},
|
|
79
|
+
"confidence": {
|
|
80
|
+
"type": "string",
|
|
81
|
+
"enum": [
|
|
82
|
+
"High",
|
|
83
|
+
"Moderate",
|
|
84
|
+
"Low",
|
|
85
|
+
"Insufficient"
|
|
86
|
+
]
|
|
87
|
+
},
|
|
25
88
|
"confidence_score": {
|
|
26
|
-
"type": [
|
|
89
|
+
"type": [
|
|
90
|
+
"number",
|
|
91
|
+
"null"
|
|
92
|
+
],
|
|
27
93
|
"minimum": 0,
|
|
28
94
|
"maximum": 1,
|
|
29
95
|
"description": "规则化置信度指数(0-1),由 compute_confidence.py 覆盖模型值。不是概率,禁止宣传为百分比。"
|
|
30
96
|
},
|
|
31
|
-
"confidence_policy_version": {
|
|
32
|
-
|
|
33
|
-
|
|
97
|
+
"confidence_policy_version": {
|
|
98
|
+
"type": "string",
|
|
99
|
+
"description": "确定性置信度策略版本号(如 2026-08-12.v1)。"
|
|
100
|
+
},
|
|
101
|
+
"independent_studies": {
|
|
102
|
+
"type": [
|
|
103
|
+
"integer",
|
|
104
|
+
"null"
|
|
105
|
+
],
|
|
106
|
+
"minimum": 0,
|
|
107
|
+
"description": "独立研究数(按 study_id/source_id 去重)。"
|
|
108
|
+
},
|
|
109
|
+
"independent_samples": {
|
|
110
|
+
"type": [
|
|
111
|
+
"integer",
|
|
112
|
+
"null"
|
|
113
|
+
],
|
|
114
|
+
"minimum": 0,
|
|
115
|
+
"description": "独立样本数(按 sample_id 去重)。"
|
|
116
|
+
},
|
|
34
117
|
"confidence_breakdown": {
|
|
35
118
|
"type": "object",
|
|
36
119
|
"additionalProperties": true,
|
|
37
120
|
"description": "Rule-based components: evidence_quality, consistency, directness, evidence_count, independent_studies, independent_samples, conflict_penalty, unsupported_penalty."
|
|
38
121
|
},
|
|
39
122
|
"raw_model_confidence": {
|
|
40
|
-
"type": [
|
|
123
|
+
"type": [
|
|
124
|
+
"string",
|
|
125
|
+
"null"
|
|
126
|
+
],
|
|
41
127
|
"description": "模型原始 confidence 输出(被确定性值覆盖前的值,仅供审计比对)。"
|
|
42
128
|
},
|
|
43
|
-
"raw_model_confidence_breakdown": {
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
"
|
|
48
|
-
|
|
49
|
-
|
|
129
|
+
"raw_model_confidence_breakdown": {
|
|
130
|
+
"type": "object",
|
|
131
|
+
"additionalProperties": true
|
|
132
|
+
},
|
|
133
|
+
"what_can_be_claimed": {
|
|
134
|
+
"type": "array",
|
|
135
|
+
"items": {
|
|
136
|
+
"type": "string"
|
|
137
|
+
}
|
|
138
|
+
},
|
|
139
|
+
"what_cannot_be_claimed": {
|
|
140
|
+
"type": "array",
|
|
141
|
+
"items": {
|
|
142
|
+
"type": "string"
|
|
143
|
+
}
|
|
144
|
+
},
|
|
145
|
+
"missing_evidence": {
|
|
146
|
+
"type": "array",
|
|
147
|
+
"items": {
|
|
148
|
+
"type": "string"
|
|
149
|
+
}
|
|
150
|
+
},
|
|
151
|
+
"recommended_action": {
|
|
152
|
+
"type": "string",
|
|
153
|
+
"enum": [
|
|
154
|
+
"adopt",
|
|
155
|
+
"pilot",
|
|
156
|
+
"reject",
|
|
157
|
+
"insufficient_evidence"
|
|
158
|
+
]
|
|
159
|
+
},
|
|
160
|
+
"decision_rationale": {
|
|
161
|
+
"type": "string"
|
|
162
|
+
},
|
|
163
|
+
"exceeds_evidence_boundary": {
|
|
164
|
+
"type": "array",
|
|
165
|
+
"items": {
|
|
166
|
+
"type": "string"
|
|
167
|
+
},
|
|
168
|
+
"description": "Conclusions that currently go beyond the evidence boundary."
|
|
169
|
+
},
|
|
50
170
|
"extensions": {
|
|
51
171
|
"type": "object",
|
|
52
172
|
"description": "结构化扩展字段的统一容器(P1-01)。未列入本 schema 的字段必须放在这里,禁止在顶层新增属性。",
|
|
53
173
|
"additionalProperties": true
|
|
174
|
+
},
|
|
175
|
+
"strongest_support": {
|
|
176
|
+
"type": "string",
|
|
177
|
+
"description": "The single strongest conclusion the evidence supports, as a complete reader-facing sentence (<=60 chars zh / ~15 words en). Written by the adjudicator, not assembled by the renderer."
|
|
178
|
+
},
|
|
179
|
+
"key_uncertainty": {
|
|
180
|
+
"type": "string",
|
|
181
|
+
"description": "The decision-relevant uncertainty or counter-evidence, as a complete reader-facing sentence (<=70 chars zh / ~18 words en)."
|
|
182
|
+
},
|
|
183
|
+
"main_risk": {
|
|
184
|
+
"type": "string",
|
|
185
|
+
"description": "The principal risk of acting, as a complete reader-facing sentence (<=60 chars zh / ~15 words en)."
|
|
186
|
+
},
|
|
187
|
+
"next_action": {
|
|
188
|
+
"type": "string",
|
|
189
|
+
"description": "The recommended next step, as a complete reader-facing sentence (<=80 chars zh / ~20 words en)."
|
|
54
190
|
}
|
|
55
191
|
}
|
|
56
192
|
}
|
|
@@ -45,7 +45,7 @@ from benchmark_evaluator import extract_json_block # noqa: E402
|
|
|
45
45
|
|
|
46
46
|
JUDGE_DIMS = ("citation_support", "outcome_correctness", "scope_calibration",
|
|
47
47
|
"contradiction_handling", "decision_calibration")
|
|
48
|
-
DEFAULT_JUDGE_MODEL = "
|
|
48
|
+
DEFAULT_JUDGE_MODEL = "" # 无默认:judge 模型必须显式指定或经 EDUEVIDENCE_LLM_MODEL 提供
|
|
49
49
|
DEFAULT_LIMIT = 60
|
|
50
50
|
HEURISTIC_METRICS = ("outcome_separation_accuracy", "decision_calibration",
|
|
51
51
|
"contradiction_recall", "contradiction_precision",
|
|
@@ -514,7 +514,7 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
514
514
|
p_run.add_argument("--questions", default="benchmarks/questions.jsonl")
|
|
515
515
|
p_run.add_argument("--annotations", default="benchmarks/annotations")
|
|
516
516
|
p_run.add_argument("--model", default=DEFAULT_JUDGE_MODEL)
|
|
517
|
-
p_run.add_argument("--thinking", default="
|
|
517
|
+
p_run.add_argument("--thinking", default="max", choices=["low", "high", "max"])
|
|
518
518
|
p_run.add_argument("--limit", type=int, default=DEFAULT_LIMIT,
|
|
519
519
|
help="max completed attempts to judge (default 60; <=0 = unlimited)")
|
|
520
520
|
p_run.set_defaults(func=_cmd_run)
|
package/scripts/benchmark_v3.py
CHANGED
|
@@ -31,6 +31,7 @@ from __future__ import annotations
|
|
|
31
31
|
import argparse
|
|
32
32
|
import json
|
|
33
33
|
import os
|
|
34
|
+
import subprocess
|
|
34
35
|
import sys
|
|
35
36
|
import urllib.error
|
|
36
37
|
import urllib.request
|
|
@@ -48,8 +49,6 @@ BASELINES = (
|
|
|
48
49
|
)
|
|
49
50
|
DEFAULT_BUDGET_TOKENS = 1_000_000
|
|
50
51
|
|
|
51
|
-
# ---------------------------------------------------------------- prompts
|
|
52
|
-
|
|
53
52
|
|
|
54
53
|
def _prompt_b0(q: dict) -> str:
|
|
55
54
|
return (
|
|
@@ -118,9 +117,6 @@ def build_prompt(baseline: str, q: dict) -> str:
|
|
|
118
117
|
return fn(q)
|
|
119
118
|
|
|
120
119
|
|
|
121
|
-
# ---------------------------------------------------------------- drivers
|
|
122
|
-
|
|
123
|
-
|
|
124
120
|
class ApiDriver:
|
|
125
121
|
"""OpenAI-compatible chat completions driver (no SDK dependency)."""
|
|
126
122
|
|
|
@@ -152,7 +148,7 @@ class ApiDriver:
|
|
|
152
148
|
headers={"Content-Type": "application/json",
|
|
153
149
|
"Authorization": f"Bearer {self.api_key}"},
|
|
154
150
|
method="POST")
|
|
155
|
-
with urllib.request.urlopen(req, timeout=self.timeout) as resp:
|
|
151
|
+
with urllib.request.urlopen(req, timeout=self.timeout) as resp:
|
|
156
152
|
payload = json.loads(resp.read().decode("utf-8"))
|
|
157
153
|
usage = payload.get("usage") or {}
|
|
158
154
|
text = (payload.get("choices") or [{}])[0].get("message", {}).get("content", "")
|
|
@@ -166,28 +162,21 @@ class ApiDriver:
|
|
|
166
162
|
|
|
167
163
|
|
|
168
164
|
class CliDriver:
|
|
169
|
-
"""omp CLI driver - host agent runtime (user-approved).
|
|
170
|
-
|
|
171
|
-
Calls `omp -p --no-session --model=<model> <prompt>` in a scratch dir;
|
|
172
|
-
captures stdout as the response. Token usage is estimated from text
|
|
173
|
-
length and recorded as such (manifest usage fields may stay null; the
|
|
174
|
-
run manifest environment records the exact invocation).
|
|
175
|
-
"""
|
|
165
|
+
"""omp CLI driver - host agent runtime (user-approved)."""
|
|
176
166
|
|
|
177
167
|
name = "cli"
|
|
178
168
|
|
|
179
|
-
def __init__(self, model: str | None = None, thinking: str = "
|
|
169
|
+
def __init__(self, model: str | None = None, thinking: str = "max",
|
|
180
170
|
timeout: int = 600):
|
|
181
|
-
self.model = model or os.environ.get("EDUEVIDENCE_LLM_MODEL", "
|
|
171
|
+
self.model = model or os.environ.get("EDUEVIDENCE_LLM_MODEL", "")
|
|
182
172
|
self.thinking = thinking
|
|
183
173
|
self.timeout = timeout
|
|
184
174
|
|
|
185
175
|
def available(self) -> bool:
|
|
186
176
|
import shutil
|
|
187
|
-
return shutil.which("omp") is not None
|
|
177
|
+
return bool(self.model) and shutil.which("omp") is not None
|
|
188
178
|
|
|
189
179
|
def call(self, prompt: str, *, no_tools: bool = False) -> tuple[str, dict]:
|
|
190
|
-
import subprocess
|
|
191
180
|
import tempfile
|
|
192
181
|
import time
|
|
193
182
|
|
|
@@ -200,7 +189,7 @@ class CliDriver:
|
|
|
200
189
|
t0 = time.monotonic()
|
|
201
190
|
with tempfile.TemporaryDirectory(prefix="eduevidence-bench-") as workdir:
|
|
202
191
|
proc = subprocess.run(cmd, capture_output=True, text=True,
|
|
203
|
-
|
|
192
|
+
timeout=self.timeout, cwd=workdir)
|
|
204
193
|
latency = time.monotonic() - t0
|
|
205
194
|
if proc.returncode != 0:
|
|
206
195
|
raise RuntimeError(
|
|
@@ -214,6 +203,7 @@ class CliDriver:
|
|
|
214
203
|
}
|
|
215
204
|
return text, usage
|
|
216
205
|
|
|
206
|
+
|
|
217
207
|
class SimDriver:
|
|
218
208
|
"""Deterministic simulation — harness validation ONLY. Never performance evidence."""
|
|
219
209
|
|
|
@@ -226,10 +216,8 @@ class SimDriver:
|
|
|
226
216
|
return True
|
|
227
217
|
|
|
228
218
|
def call(self, prompt: str, *, no_tools: bool = False) -> tuple[str, dict[str, Any]]:
|
|
229
|
-
from benchmark_v2 import simulate_question_result # noqa:
|
|
219
|
+
from benchmark_v2 import simulate_question_result # noqa: F401
|
|
230
220
|
|
|
231
|
-
# Deterministic pseudo-usage from prompt length; response is a stub
|
|
232
|
-
# that the evaluator must never use as model performance.
|
|
233
221
|
import random
|
|
234
222
|
rng = random.Random(len(prompt) * 7919 % 2**31)
|
|
235
223
|
usage = {
|
|
@@ -246,28 +234,26 @@ class SimDriver:
|
|
|
246
234
|
)
|
|
247
235
|
|
|
248
236
|
|
|
249
|
-
def make_driver(name: str) -> Any:
|
|
237
|
+
def make_driver(name: str, *, model: str | None = None, thinking: str = "max") -> Any:
|
|
250
238
|
if name == "api":
|
|
251
239
|
return ApiDriver()
|
|
252
240
|
if name == "cli":
|
|
253
|
-
return CliDriver()
|
|
241
|
+
return CliDriver(model=model, thinking=thinking)
|
|
254
242
|
if name == "sim":
|
|
255
243
|
return SimDriver()
|
|
256
244
|
raise ValueError(f"unknown driver: {name}")
|
|
257
245
|
|
|
258
246
|
|
|
259
|
-
# ---------------------------------------------------------------- run
|
|
260
|
-
|
|
261
|
-
|
|
262
247
|
def _now_iso() -> str:
|
|
263
248
|
return datetime.now(timezone.utc).isoformat()
|
|
264
249
|
|
|
265
250
|
|
|
266
251
|
def run_benchmark(*, questions: list[dict], baselines: list[str], repeats: int,
|
|
267
252
|
out_dir: Path, driver_name: str, budget_tokens: int | None,
|
|
268
|
-
temperature: float = 0.0, resume: bool = False
|
|
253
|
+
temperature: float = 0.0, resume: bool = False,
|
|
254
|
+
model: str | None = None, thinking: str = "max") -> dict[str, Any]:
|
|
269
255
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
270
|
-
driver = make_driver(driver_name)
|
|
256
|
+
driver = make_driver(driver_name, model=model, thinking=thinking)
|
|
271
257
|
if not driver.available():
|
|
272
258
|
raise RuntimeError(
|
|
273
259
|
f"driver '{driver_name}' unavailable (api needs EDUEVIDENCE_LLM_MODEL "
|
|
@@ -303,8 +289,6 @@ def run_benchmark(*, questions: list[dict], baselines: list[str], repeats: int,
|
|
|
303
289
|
total_tokens = 0
|
|
304
290
|
budget_stopped = False
|
|
305
291
|
import re as _re
|
|
306
|
-
# --resume: reuse previously completed attempts (their response artifacts
|
|
307
|
-
# live in out_dir); only unfinished attempts are re-run.
|
|
308
292
|
done_ids: set[str] = set()
|
|
309
293
|
resumed: dict[str, dict[str, Any]] = {}
|
|
310
294
|
if resume and out_dir.is_dir():
|
|
@@ -315,7 +299,7 @@ def run_benchmark(*, questions: list[dict], baselines: list[str], repeats: int,
|
|
|
315
299
|
except (OSError, _json.JSONDecodeError):
|
|
316
300
|
continue
|
|
317
301
|
aid = data.get("attempt_id")
|
|
318
|
-
if aid and
|
|
302
|
+
if aid and art.is_file():
|
|
319
303
|
done_ids.add(aid)
|
|
320
304
|
resumed[aid] = data
|
|
321
305
|
if done_ids:
|
|
@@ -333,9 +317,6 @@ def run_benchmark(*, questions: list[dict], baselines: list[str], repeats: int,
|
|
|
333
317
|
break
|
|
334
318
|
attempt_id = f"{question['id']}-{baseline}-a{attempt}"
|
|
335
319
|
if attempt_id in done_ids:
|
|
336
|
-
# Re-register resumed attempts in the manifest (status +
|
|
337
|
-
# usage read back from their artifact) so eval/report see
|
|
338
|
-
# the complete run.
|
|
339
320
|
art = out_dir / f"{attempt_id}.response.json"
|
|
340
321
|
data = resumed.get(attempt_id, {})
|
|
341
322
|
usage = data.get("usage") or {}
|
|
@@ -393,7 +374,7 @@ def run_benchmark(*, questions: list[dict], baselines: list[str], repeats: int,
|
|
|
393
374
|
}, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
394
375
|
entry["artifacts"] = [artifact.name]
|
|
395
376
|
except (urllib.error.URLError, OSError, ValueError, KeyError,
|
|
396
|
-
subprocess.TimeoutExpired) as exc:
|
|
377
|
+
subprocess.TimeoutExpired) as exc:
|
|
397
378
|
entry.update({"status": "failed", "error": str(exc),
|
|
398
379
|
"finished_at": _now_iso()})
|
|
399
380
|
manifest["attempts"].append(entry)
|
|
@@ -405,8 +386,6 @@ def run_benchmark(*, questions: list[dict], baselines: list[str], repeats: int,
|
|
|
405
386
|
break
|
|
406
387
|
|
|
407
388
|
if budget_stopped:
|
|
408
|
-
# P2-2: record remaining attempts as budget_stopped so the report can
|
|
409
|
-
# distinguish "stopped by budget" from "never scheduled".
|
|
410
389
|
for question in questions:
|
|
411
390
|
if any(a["question_id"] == question["id"] for a in manifest["attempts"]):
|
|
412
391
|
continue
|
|
@@ -429,7 +408,7 @@ def run_benchmark(*, questions: list[dict], baselines: list[str], repeats: int,
|
|
|
429
408
|
tmp = out_dir / "manifest.json.tmp"
|
|
430
409
|
tmp.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n",
|
|
431
410
|
encoding="utf-8")
|
|
432
|
-
tmp.replace(manifest_path)
|
|
411
|
+
tmp.replace(manifest_path)
|
|
433
412
|
_validate_manifest(manifest_path)
|
|
434
413
|
print(f"wrote {manifest_path} (attempts={len(manifest['attempts'])}, "
|
|
435
414
|
f"mode={manifest['run_mode']}, total_tokens~{total_tokens})")
|
|
@@ -444,13 +423,13 @@ def _questions_version() -> str:
|
|
|
444
423
|
proc = _sp.run(["git", "-C", str(repo), "rev-parse", "--short", "HEAD"],
|
|
445
424
|
capture_output=True, text=True, timeout=10)
|
|
446
425
|
return proc.stdout.strip() or "unknown"
|
|
447
|
-
except Exception:
|
|
426
|
+
except Exception:
|
|
448
427
|
return "unknown"
|
|
449
428
|
|
|
450
429
|
|
|
451
430
|
def _validate_manifest(path: Path) -> None:
|
|
452
431
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
453
|
-
from validate_schema import Validator, SchemaError
|
|
432
|
+
from validate_schema import Validator, SchemaError
|
|
454
433
|
|
|
455
434
|
import json as _json
|
|
456
435
|
schema = _json.loads(
|
|
@@ -479,12 +458,12 @@ def _cmd_run(args: argparse.Namespace) -> int:
|
|
|
479
458
|
baselines=baselines, repeats=args.repeats,
|
|
480
459
|
out_dir=Path(args.out), driver_name=args.driver,
|
|
481
460
|
budget_tokens=args.budget_tokens, temperature=args.temperature,
|
|
482
|
-
resume=args.resume)
|
|
461
|
+
resume=args.resume, model=args.model, thinking=args.thinking)
|
|
483
462
|
return 0
|
|
484
463
|
|
|
485
464
|
|
|
486
465
|
def _cmd_report(args: argparse.Namespace) -> int:
|
|
487
|
-
from benchmark_evaluator import report_from_run
|
|
466
|
+
from benchmark_evaluator import report_from_run
|
|
488
467
|
|
|
489
468
|
run_dir = Path(args.run)
|
|
490
469
|
manifest = json.loads((run_dir / "manifest.json").read_text(encoding="utf-8"))
|
|
@@ -508,6 +487,10 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
508
487
|
p_run.add_argument("--out", required=True)
|
|
509
488
|
p_run.add_argument("--budget-tokens", type=int, default=DEFAULT_BUDGET_TOKENS)
|
|
510
489
|
p_run.add_argument("--temperature", type=float, default=0.0)
|
|
490
|
+
p_run.add_argument("--model", default="",
|
|
491
|
+
help="OMP model for --driver cli (required; env EDUEVIDENCE_LLM_MODEL accepted; no unconfirmed default)")
|
|
492
|
+
p_run.add_argument("--thinking", default="max", choices=["low", "high", "max"],
|
|
493
|
+
help="reasoning effort for --driver cli")
|
|
511
494
|
p_run.add_argument("--resume", action="store_true",
|
|
512
495
|
help="skip attempts whose response artifacts already exist in --out")
|
|
513
496
|
p_run.set_defaults(func=_cmd_run)
|
|
@@ -536,7 +519,7 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
536
519
|
|
|
537
520
|
|
|
538
521
|
def _cmd_eval(args: argparse.Namespace) -> int:
|
|
539
|
-
from benchmark_evaluator import evaluate_run
|
|
522
|
+
from benchmark_evaluator import evaluate_run
|
|
540
523
|
|
|
541
524
|
run_dir = Path(args.run)
|
|
542
525
|
manifest = json.loads((run_dir / "manifest.json").read_text(encoding="utf-8"))
|
|
@@ -10,7 +10,7 @@ import sys
|
|
|
10
10
|
from pathlib import Path
|
|
11
11
|
|
|
12
12
|
# Add project root to sys.path
|
|
13
|
-
BASE_DIR = Path(
|
|
13
|
+
BASE_DIR = Path(__file__).resolve().parents[1]
|
|
14
14
|
sys.path.insert(0, str(BASE_DIR))
|
|
15
15
|
|
|
16
16
|
from engine.evidence_graph import (
|
|
@@ -18,7 +18,7 @@ from engine.evidence_graph import (
|
|
|
18
18
|
ClaimNode, RiskNode, GapNode, DecisionNode, GraphEdge
|
|
19
19
|
)
|
|
20
20
|
|
|
21
|
-
ESL_DIR = BASE_DIR / "examples" / "esl-academic-writing-ai"
|
|
21
|
+
ESL_DIR = BASE_DIR / "tests" / "fixtures" / "legacy-examples" / "esl-academic-writing-ai"
|
|
22
22
|
ESL_DIR.mkdir(parents=True, exist_ok=True)
|
|
23
23
|
THEMES_DIR = ESL_DIR / "reports-5themes"
|
|
24
24
|
THEMES_DIR.mkdir(parents=True, exist_ok=True)
|
|
@@ -1552,7 +1552,7 @@ result_en = {
|
|
|
1552
1552
|
"complexity": "L",
|
|
1553
1553
|
"mode": "agent_mcp_enhanced",
|
|
1554
1554
|
"agents": [
|
|
1555
|
-
"
|
|
1555
|
+
"research-planner",
|
|
1556
1556
|
"evidence-retriever",
|
|
1557
1557
|
"evidence-analyst",
|
|
1558
1558
|
"skeptic",
|
|
@@ -1651,7 +1651,7 @@ result_zh = {
|
|
|
1651
1651
|
"complexity": "L",
|
|
1652
1652
|
"mode": "agent_mcp_enhanced",
|
|
1653
1653
|
"agents": [
|
|
1654
|
-
"
|
|
1654
|
+
"research-planner",
|
|
1655
1655
|
"evidence-retriever",
|
|
1656
1656
|
"evidence-analyst",
|
|
1657
1657
|
"skeptic",
|
|
@@ -59,8 +59,8 @@ ANNOTATIONS_DIR = ROOT / "benchmarks" / "annotations"
|
|
|
59
59
|
QUESTIONS_PATH = ROOT / "benchmarks" / "questions.jsonl"
|
|
60
60
|
EXAMPLE_EVIDENCE = {
|
|
61
61
|
"ai-coding-assistant": ROOT / "examples" / "ai-coding-assistant" / "evidence.jsonl",
|
|
62
|
-
"ai-tutor": ROOT / "examples" / "ai-tutor" / "evidence.jsonl",
|
|
63
|
-
"ai-writing-assistant": ROOT / "examples" / "ai-writing-assistant" / "evidence.jsonl",
|
|
62
|
+
"ai-tutor": ROOT / "tests" / "fixtures" / "legacy-examples" / "ai-tutor" / "evidence.jsonl",
|
|
63
|
+
"ai-writing-assistant": ROOT / "tests" / "fixtures" / "legacy-examples" / "ai-writing-assistant" / "evidence.jsonl",
|
|
64
64
|
}
|
|
65
65
|
|
|
66
66
|
_WS_RE = re.compile(r"\s+")
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Build public Pages: unchanged introduction + read-only example Studio.
|
|
3
|
+
|
|
4
|
+
Only repository examples are exported. The user's EDUEVIDENCE_HOME, local
|
|
5
|
+
projects, run events and Autoevolve session data never enter this artifact.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
import json
|
|
9
|
+
import re
|
|
10
|
+
import shutil
|
|
11
|
+
import sys
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
ROOT = Path(__file__).resolve().parent.parent
|
|
15
|
+
WEB_DIR = ROOT / 'web'
|
|
16
|
+
EXAMPLES_DIR = ROOT / 'examples'
|
|
17
|
+
OUT_DIR = ROOT / 'dist_gh_pages'
|
|
18
|
+
sys.path.insert(0, str(ROOT))
|
|
19
|
+
from engine.studio_read_model import StudioReader # noqa: E402
|
|
20
|
+
from scripts.build_report_variants import bake # noqa: E402
|
|
21
|
+
from scripts.dashboard_server import scan_local_projects, build_stats, build_viz_payload # noqa: E402
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def write_json(path: Path, payload):
|
|
25
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
26
|
+
path.write_text(json.dumps(payload, ensure_ascii=False, allow_nan=False, indent=2), encoding='utf-8')
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def main():
|
|
30
|
+
if not (WEB_DIR / 'studio' / 'index.html').is_file():
|
|
31
|
+
raise SystemExit('Build the frontend first: cd studio && npm ci && npm run build')
|
|
32
|
+
bake(EXAMPLES_DIR)
|
|
33
|
+
if OUT_DIR.exists():
|
|
34
|
+
shutil.rmtree(OUT_DIR)
|
|
35
|
+
OUT_DIR.mkdir(parents=True)
|
|
36
|
+
for item in WEB_DIR.iterdir():
|
|
37
|
+
if item.name == 'api':
|
|
38
|
+
continue
|
|
39
|
+
if item.is_dir():
|
|
40
|
+
shutil.copytree(item, OUT_DIR / item.name, dirs_exist_ok=True)
|
|
41
|
+
else:
|
|
42
|
+
shutil.copy2(item, OUT_DIR / item.name)
|
|
43
|
+
|
|
44
|
+
# Legacy landing endpoints kept without modifying source web/api artifacts.
|
|
45
|
+
projects = scan_local_projects()
|
|
46
|
+
for project in projects:
|
|
47
|
+
name = project['id']
|
|
48
|
+
project['html_report_path'] = f'reports/{name}/EduEvidence_Report.html' if project.get('html_report_path') else None
|
|
49
|
+
for variant in project.get('report_variants', []):
|
|
50
|
+
variant['path'] = f"reports/{name}/{Path(variant['path']).name}"
|
|
51
|
+
write_json(OUT_DIR / 'api' / 'projects.json', {'projects': projects, 'stats': build_stats(projects)})
|
|
52
|
+
for project in projects:
|
|
53
|
+
write_json(OUT_DIR / 'api' / 'projects' / project['id'] / 'viz.json', build_viz_payload(project['id']))
|
|
54
|
+
|
|
55
|
+
reader = StudioReader(EXAMPLES_DIR, ROOT / '.static-export-no-local-state', static=True)
|
|
56
|
+
catalog = reader.catalog()
|
|
57
|
+
write_json(OUT_DIR / 'api' / 'studio' / 'catalog.json', catalog)
|
|
58
|
+
write_json(OUT_DIR / 'api' / 'studio' / 'evolution.json', {'experiments': [], 'status': 'not_exported'})
|
|
59
|
+
for project in catalog['projects']:
|
|
60
|
+
key = project['id']
|
|
61
|
+
detail = reader.detail(key)
|
|
62
|
+
write_json(OUT_DIR / 'api' / 'studio' / 'projects' / f'{key}.json', detail)
|
|
63
|
+
name = key.removeprefix('example--')
|
|
64
|
+
source = EXAMPLES_DIR / name
|
|
65
|
+
target = OUT_DIR / 'reports' / name
|
|
66
|
+
target.mkdir(parents=True, exist_ok=True)
|
|
67
|
+
for path in (source / 'reports-5themes').glob('*.html'):
|
|
68
|
+
shutil.copy2(path, target / path.name)
|
|
69
|
+
current_default = source / 'reports-5themes' / 'EduEvidence_Report_claude.html'
|
|
70
|
+
if not current_default.exists():
|
|
71
|
+
current_default = source / 'EduEvidence_Report.html'
|
|
72
|
+
if current_default.is_file():
|
|
73
|
+
shutil.copy2(current_default, target / 'EduEvidence_Report.html')
|
|
74
|
+
write_json(OUT_DIR / 'studio' / 'config.json', {'mode': 'static', 'api_base': '../api/studio', 'readonly': True})
|
|
75
|
+
# Keep old public entry links functional without changing the landing design.
|
|
76
|
+
redirect = '<!doctype html><html lang="en"><meta charset="utf-8"><meta http-equiv="refresh" content="0;url=./studio/"><title>Research Studio</title><a href="./studio/">Open Research Studio</a><script>location.replace("./studio/"+location.hash)</script></html>'
|
|
77
|
+
(OUT_DIR / 'studio.html').write_text(redirect, encoding='utf-8')
|
|
78
|
+
landing = WEB_DIR / 'landing.html'
|
|
79
|
+
if landing.is_file():
|
|
80
|
+
page = landing.read_text(encoding='utf-8')
|
|
81
|
+
page = page.replace('href="/landing.html"', 'href="index.html"').replace('href="/index.html"', 'href="studio/"')
|
|
82
|
+
theme_alias = {'claude_research':'claude', 'academic_paper':'academic', 'datalab_light':'datalab', 'datalab_dark':'datalab-dark', 'presentation_judge':'presentation'}
|
|
83
|
+
def report_link(match):
|
|
84
|
+
parts = match.group(1).replace('&', '&').split('&')
|
|
85
|
+
project_id = parts[0]
|
|
86
|
+
theme = next((s.split('=', 1)[1] for s in parts[1:] if s.startswith('theme=')), 'default')
|
|
87
|
+
theme = theme_alias.get(theme, theme)
|
|
88
|
+
filename = 'EduEvidence_Report.html' if theme == 'default' else f'EduEvidence_Report_{theme}.html'
|
|
89
|
+
return f'href="reports/{project_id}/{filename}"'
|
|
90
|
+
page = re.sub(r'href="/report\?id=([^\"]+)"', report_link, page)
|
|
91
|
+
(OUT_DIR / 'index.html').write_text(page, encoding='utf-8')
|
|
92
|
+
(OUT_DIR / 'landing.html').write_text(page, encoding='utf-8')
|
|
93
|
+
(OUT_DIR / '.nojekyll').write_text('', encoding='utf-8')
|
|
94
|
+
print(f'Pages ready: {len(catalog["projects"])} public cases; local projects excluded')
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
if __name__ == '__main__':
|
|
98
|
+
main()
|