@researai/deepscientist 1.5.16 → 1.6.0

package/AISB/catalog/aisb.b7.multimodal_fusion.yaml

@@ -0,0 +1,232 @@
+schema_version: 1
+id: aisb.b7.multimodal_fusion
+name: AISB B7 Multimodal Fusion
+version: 0.3.0
+catalog_version: 0.1.0
+one_line: MMMU-Pro image-plus-text public-dev mini package with exact-match style scoring.
+task_description: AISB B7 Multimodal Fusion is an AISB BenchStore catalog pilot entry. The package is
+  intended for public-dev workspace initialization, public evaluator checks, and paper/benchmark-track
+  submission scaffolding. Official leaderboard scores require organizer-side hidden replay.
+homepage: https://aisb.deepscientist.cc
+localized:
+  zh:
+    name: AISB B7 多模态融合
+    one_line: 带精确匹配类评分的 MMMU-Pro 图文 public-dev mini 包。
+    task_description: AISB B7 多模态融合 是 AISB BenchStore catalog pilot 条目。公开包只用于 public-dev 工作区初始化、公开 evaluator
+      检查和论文/benchmark track 提交脚手架；正式 leaderboard 分数必须由组织方 hidden replay 产生。
+capability_tags:
+- multimodal
+- image_text
+- qa
+aisb_direction: B7
+track_fit:
+- paper_track
+- benchmark_track
+task_mode: benchmark
+benchmark_mode: multimodal_answer_exact_match
+requires_execution: true
+requires_paper: true
+integrity_level: hidden_reference
+snapshot_status: partial
+support_level: preview
+discovery:
+  collection: AISB
+  collection_priority: 100
+  recommendation_weight: 500
+  featured: false
+  featured_reason: null
+display:
+  placement: grid
+  card_size: m
+  badge: AISB
+  accent_priority: normal
+official_links:
+  homepage: https://github.com/ResearAI/NLPCC-2026-Task9-AISB
+  github: https://github.com/ResearAI/NLPCC-2026-Task9-AISB
+  benchmark_overview: https://github.com/ResearAI/NLPCC-2026-Task9-AISB/tree/main/benchmarks/aisb/B7_MultimodalFusion
+release_gate:
+  release_ready: false
+  status: registry_pending
+  full_real_replay: passed
+  reason: Public-dev and scorer-private replay paths exist locally, but final registry promotion, release
+    assets, and license review remain pending.
+  next_required_step: Validate the published BenchStore release asset on a clean host, then complete
+    registry and license review.
+primary_outputs:
+- results.json
+- paper/paper.pdf
+- paper/claims.json
+- logs/experiment_log.jsonl
+- logs/iterations.jsonl
+cost_band: medium
+time_band: 1-4h
+difficulty: hard
+data_access: public
+resources:
+  minimum:
+    cpu_cores: 4
+    ram_gb: 8
+    disk_gb: 5
+    gpu_count: 0
+  recommended:
+    cpu_cores: 8
+    ram_gb: 16
+    disk_gb: 20
+    gpu_count: 0
+source:
+  benchmark_dir: benchmarks/aisb/B7_MultimodalFusion
+  bench_yaml: benchmarks/aisb/B7_MultimodalFusion/bench.yaml
+  source_repo: ResearAI/NLPCC-2026-Task9-AISB
+  source_commit: b6b7527a226ca7328697156608571f2cb7dcda3a
+download:
+  provider: github_release
+  repo: ResearAI/NLPCC-2026-Task9-AISB
+  tag: aisb-v0.0.1
+  asset_name: aisb.b7.multimodal_fusion-v0.3.0.zip
+  url: https://github.com/giao-123-sun/DeepScientist/releases/download/aisb-v0.0.1/aisb.b7.multimodal_fusion-v0.3.0.zip
+  source_url: https://github.com/ResearAI/NLPCC-2026-Task9-AISB/tree/main/benchmarks/aisb/B7_MultimodalFusion
+  archive_type: zip
+  local_dir_name: aisb.b7.multimodal_fusion
+  sha256: 1d64b497e341a5a72489d520a2ab3d099acf681f3ea6417bbdb28f900fff2e32
+  size_bytes: 160705
+  published_at: '2026-04-30T00:00:00Z'
+  asset_status: published
+dataset_download:
+  primary_method: packaged_public_dev
+  sources:
+  - kind: public_dev
+    access: public
+    url: null
+    note: Public-dev files are included in the published BenchStore package.
+  - kind: hidden_reference
+    access: private
+    url: null
+    note: Organizer-only hidden references are excluded from catalog packages.
+credential_requirements:
+  mode: none
+  notes:
+  - No credential is required for packaged public-dev evaluation.
+  - External model APIs are optional participant choices and not bundled.
+environment:
+  python: '>=3.11'
+  docker: true
+  key_packages:
+  - pytest
+  - PyYAML
+  notes:
+  - Use docker/Dockerfile when available for official replay parity.
+  - Do not copy organizer-private hidden references into participant workspaces.
+launch_profiles:
+- id: b7_public_dev_smoke
+  label: B7 public-dev smoke test
+  description: Initialize an AISB public-dev workspace, validate submission artifacts, and run the public
+    evaluator.
+  startup_contract:
+  - Use only files inside the installed benchmark workspace and the generated submission directory.
+  - Treat hidden references as organizer-only and unavailable to the agent.
+  - Produce benchmark outputs, paper artifacts, claims, and logs before packaging.
+  commands:
+  - python scripts/agent_tools.py workspace init B7 --dest .work/benchstore/B7
+  - python scripts/agent_tools.py evaluate B7 --bench-dir .work/benchstore/B7 --submission .work/benchstore/B7/submission
+  - python scripts/agent_tools.py submission validate .work/benchstore/B7/submission
+  - python scripts/agent_tools.py submission package .work/benchstore/B7/submission --output .work/benchstore/B7/submission.zip
+  - python scripts/agent_tools.py submission replay .work/benchstore/B7/submission --track B7
+  one_click_prompt: Start AISB B7 from the installed BenchStore package. Initialize the public-dev workspace,
+    inspect AGENT.md and README.md, run the baseline or your method, evaluate locally, validate/package
+    the submission, and record any release-gate risks in logs/experiment_log.jsonl.
+paper:
+  venue: AISB
+  year: 2026
+  track_fit:
+  - paper_track
+  - benchmark_track
+  requires_claims_json: true
+  url: null
+benchmark_track:
+  metric: accuracy
+  benchmark_source: MMMU-Pro mini
+  packaged_data_status: real MMMU-Pro image-plus-text mini split
+  official_score_requires_hidden_replay: true
+benchmarks:
+  primary:
+    name: MMMU-Pro tool
+    source: MMMU/MMMU_Pro (HF)
+    tasks: 2
+    hidden: 2
+    metric: exact_match_accuracy
+    sota: 82% (Gemini 3.1 Pro Preview)
+    public_score: 0.5
+  alternatives:
+  - name: MMMU
+    source: MMMU/MMMU (HF)
+    note: parent benchmark; 11.5K questions
+  - name: MathVista
+    source: huggingface.co/datasets/AI4Math/MathVista
+    note: math-visual reasoning; future candidate
+papers:
+  highlights:
+  - title: 'MMMU-Pro: A More Robust Multi-discipline Multimodal Understanding Benchmark'
+    arxiv: '2409.02813'
+    url: https://arxiv.org/abs/2409.02813
+    venue: '2024'
+    why: 10-option multimodal benchmark; SOTA 82% vs human 88.6%
+  - title: 'SenseNova-MARS: Multimodal Agent Reasoning and Search via RL'
+    arxiv: '2512.24330'
+    url: https://arxiv.org/abs/2512.24330
+    venue: Dec 2025
+    why: 74.3 MMSearch; 54.4 HR-MMSearch; beats Gemini-3-Pro
+  - title: 'P1-VL: Bridging Visual Perception and Scientific Reasoning'
+    arxiv: '2602.09443'
+    url: https://arxiv.org/abs/2602.09443
+    venue: Feb 2026
+    why: 12 gold medals in Physics Olympiad; science multimodal reasoning
+  full_references_url: https://github.com/ResearAI/NLPCC-2026-Task9-AISB/tree/main/benchmarks/aisb/B7_*/references/papers.json
+nlpcc_tracks:
+- track: T1
+  name: AI/CS Reasoning & Engineering
+  url: https://github.com/ResearAI/NLPCC-2026-Task9-AISB/tree/main/benchmarks/nlpcc/T1
+private_hidden_reference_strategy:
+  policy: organizer_only
+  public_package_includes_hidden_reference: false
+  public_catalog_discloses_private_paths: false
+  score_release: aggregate_only
+  notes:
+  - Hidden answers, labels, canaries, and private evaluator internals are never packaged.
+  - Official scores must record hidden_reference_version without disclosing hidden content.
+score_card:
+  required_fields:
+  - team_id
+  - system_name
+  - direction
+  - track
+  - benchmark_package_version
+  - evaluator_version
+  - hidden_reference_version
+  - docker_image_digest
+  - data_split_version
+  - benchmark_score
+  - paper_score
+  - cas_score
+  - final_score
+  - security_verdict
+  - license_verdict
+  - used_reference_scope
+  - official_score
+  - created_at
+  track_scoring:
+    paper_track: Final_A = 0.0 * S_benchmark + 1.0 * S_paper
+    benchmark_track: Final_B = 0.7 * S_benchmark + 0.3 * S_paper
+    cas: integrity gate, not bonus
+risk_flags:
+- hidden_reference_not_public
+- not_release_ready
+- registry_promotion_pending
+risk_notes:
+- This catalog entry is not release_ready.
+- Download URL, checksum, and size point to the published GitHub release asset.
+- Official scores require organizer-side replay against private hidden references.
+recommended_when: Use this entry for AISB public-dev integration, local agent workflow checks, and catalog
+  UI validation.
+not_recommended_when: Do not treat this entry as an official release or public leaderboard claim until
+  release gates pass.
+image_path: ../image/aisb.b7.multimodal_fusion.svg

package/AISB/catalog/aisb.b8.lifesci_drug.yaml

@@ -0,0 +1,275 @@
+schema_version: 1
+id: aisb.b8.lifesci_drug
+name: AISB B8 LifeSci Drug Discovery
+version: 0.1.1
+catalog_version: 0.1.0
+one_line: ADMET-only public-dev BenchStore pilot using five real TDC molecular-property endpoints.
+task_description: AISB B8 LifeSci Drug Discovery is an AISB BenchStore catalog pilot entry. The package
+  is intended for public-dev workspace initialization, public evaluator checks, and paper/benchmark-track
+  submission scaffolding. Official leaderboard scores require organizer-side hidden replay.
+homepage: https://aisb.deepscientist.cc
+localized:
+  zh:
+    name: AISB B8 生命科学药物发现
+    one_line: 基于五个真实 TDC 分子性质端点的 ADMET-only BenchStore pilot。
+    task_description: AISB B8 生命科学药物发现 是 AISB BenchStore catalog pilot 条目。公开包只用于 public-dev 工作区初始化、公开
+      evaluator 检查和论文/benchmark track 提交脚手架；正式 leaderboard 分数必须由组织方 hidden replay 产生。
+capability_tags:
+- life_science
+- drug_discovery
+- admet_prediction
+- molecular_property_prediction
+- tabular_ml
+aisb_direction: B8
+track_fit:
+- paper_track
+- benchmark_track
+task_mode: benchmark
+benchmark_mode: experiment_driven
+requires_execution: true
+requires_paper: true
+integrity_level: hidden_reference
+snapshot_status: partial
+support_level: advanced
+discovery:
+  collection: AISB
+  collection_priority: 100
+  recommendation_weight: 1000
+  featured: true
+  featured_reason: Prefer AISB B8 as the top BenchStore hero card for the current pilot because it is
+    the strongest low-friction public-dev package with real data and working replay evidence.
+display:
+  placement: hero
+  card_size: xl
+  badge: Recommended
+  accent_priority: highest
+official_links:
+  homepage: https://github.com/ResearAI/NLPCC-2026-Task9-AISB
+  github: https://github.com/ResearAI/NLPCC-2026-Task9-AISB
+  benchmark_overview: https://github.com/ResearAI/NLPCC-2026-Task9-AISB/tree/main/benchmarks/aisb/B8_LifeSciDrug
+release_gate:
+  release_ready: false
+  status: registry_pending
+  full_real_replay: passed
+  reason: Public-dev and scorer-private replay paths exist locally, but final registry promotion, release
+    assets, and license review remain pending.
+  next_required_step: Validate the published BenchStore release asset on a clean host, then complete
+    registry and license review.
+primary_outputs:
+- results.json
+- paper/paper.pdf
+- paper/claims.json
+- logs/experiment_log.jsonl
+- logs/iterations.jsonl
+cost_band: low
+time_band: 1-2h
+difficulty: medium
+data_access: public
+resources:
+  minimum:
+    cpu_cores: 4
+    ram_gb: 16
+    disk_gb: 20
+    gpu_count: 0
+  recommended:
+    cpu_cores: 8
+    ram_gb: 32
+    disk_gb: 40
+    gpu_count: 0
+source:
+  benchmark_dir: benchmarks/aisb/B8_LifeSciDrug
+  bench_yaml: benchmarks/aisb/B8_LifeSciDrug/bench.yaml
+  source_repo: ResearAI/NLPCC-2026-Task9-AISB
+  source_commit: b6b7527a226ca7328697156608571f2cb7dcda3a
+download:
+  provider: github_release
+  repo: ResearAI/NLPCC-2026-Task9-AISB
+  tag: aisb-v0.0.1
+  asset_name: aisb.b8.lifesci_drug-v0.1.1.zip
+  url: https://github.com/giao-123-sun/DeepScientist/releases/download/aisb-v0.0.1/aisb.b8.lifesci_drug-v0.1.1.zip
+  source_url: https://github.com/ResearAI/NLPCC-2026-Task9-AISB/tree/main/benchmarks/aisb/B8_LifeSciDrug
+  archive_type: zip
+  local_dir_name: aisb.b8.lifesci_drug
+  sha256: 7e9fde8e42baea653e20a0d67d77aa0a9c8cc0c314f8828320454556f73f851e
+  size_bytes: 154222
+  published_at: '2026-04-30T00:00:00Z'
+  asset_status: published
+dataset_download:
+  primary_method: packaged_public_dev
+  sources:
+  - kind: public_dev
+    access: public
+    url: null
+    note: Public-dev files are included in the published BenchStore package.
+  - kind: hidden_reference
+    access: private
+    url: null
+    note: Organizer-only hidden references are excluded from catalog packages.
+credential_requirements:
+  mode: none
+  notes:
+  - No credential is required for packaged public-dev evaluation.
+  - External model APIs are optional participant choices and not bundled.
+environment:
+  python: '>=3.11'
+  docker: true
+  key_packages:
+  - pytest
+  - PyYAML
+  notes:
+  - Use docker/Dockerfile when available for official replay parity.
+  - Do not copy organizer-private hidden references into participant workspaces.
+launch_profiles:
+- id: b8_public_dev_smoke
+  label: B8 public-dev smoke test
+  description: Initialize an AISB public-dev workspace, validate submission artifacts, and run the public
+    evaluator.
+  startup_contract:
+  - Use only files inside the installed benchmark workspace and the generated submission directory.
+  - Treat hidden references as organizer-only and unavailable to the agent.
+  - Produce benchmark outputs, paper artifacts, claims, and logs before packaging.
+  commands:
+  - python scripts/agent_tools.py workspace init B8 --dest .work/benchstore/B8
+  - python scripts/agent_tools.py evaluate B8 --bench-dir .work/benchstore/B8 --submission .work/benchstore/B8/submission
+  - python scripts/agent_tools.py submission validate .work/benchstore/B8/submission
+  - python scripts/agent_tools.py submission package .work/benchstore/B8/submission --output .work/benchstore/B8/submission.zip
+  - python scripts/agent_tools.py submission replay .work/benchstore/B8/submission --track B8
+  one_click_prompt: Start AISB B8 from the installed BenchStore package. Initialize the public-dev workspace,
+    inspect AGENT.md and README.md, run the baseline or your method, evaluate locally, validate/package
+    the submission, and record any release-gate risks in logs/experiment_log.jsonl.
+paper:
+  venue: AISB
+  year: 2026
+  track_fit:
+  - paper_track
+  - benchmark_track
+  requires_claims_json: true
+  url: null
+benchmark_track:
+  metric: mean_normalized_endpoint_score
+  benchmark_source: TDC ADMET endpoints scoped to molecular property prediction
+  packaged_data_status: real TDC ADMET public-dev CSVs plus label-stripped real TDC ADMET test inputs
+  official_score_requires_hidden_replay: true
+benchmarks:
+  primary:
+    name: TDC ADMET 5-endpoint
+    source: TDC (tdcommons.ai)
+    endpoints:
+    - caco2_wang (91)
+    - herg (65)
+    - ames (727)
+    - cyp3a4_veith (1232)
+    - lipophilicity_az (420)
+    total_dev_rows: 2535
+    metric: mean_normalized_endpoint_score
+    sota_per_endpoint:
+      caco2_wang: MAE 0.256 (CaliciBoost)
+      herg: AUROC 0.880 (MapLight+GNN)
+      ames: AUROC 0.871 (ZairaChem)
+      cyp3a4_veith: AUPRC 0.916 (MapLight+GNN)
+      lipophilicity_az: MAE 0.456 (MiniMol)
+    our_baseline: 0.6385
+    our_hidden: 0.5829
+  alternatives:
+  - name: TDC-2
+    source: tdcommons.ai
+    note: multimodal data + extended ADMET; future upgrade
+  - name: ADMET-AI
+    source: github.com/swansonk14/admet_ai
+    note: fast predictor; reference method only
+papers:
+  highlights:
+  - title: 'TDC: Therapeutics Data Commons'
+    arxiv: '2102.09548'
+    url: https://arxiv.org/abs/2102.09548
+    venue: '2021'
+    why: Primary benchmark data source; ADMET endpoints
+  - title: 'ADMET-AI: A Machine Learning ADMET Platform'
+    source: github.com/swansonk14/admet_ai
+    why: Fast ADMET predictor; reference method candidate
+  - title: 'STELLA: Self-Evolving LLM Agent for Biomedical Research'
+    arxiv: '2507.02004'
+    url: https://arxiv.org/abs/2507.02004
+    venue: Jul 2025
+    why: 26% HLE biomed; 63% LAB-Bench LitQA; self-evolving for life science
+  - title: Auditable Agent Platform for Automated Molecular Optimization
+    arxiv: '2508.03444'
+    url: https://arxiv.org/abs/2508.03444
+    venue: Aug 2025
+    why: 31% binding affinity improvement; multi-agent molecular optimization
+  full_references_url: https://github.com/ResearAI/NLPCC-2026-Task9-AISB/tree/main/benchmarks/aisb/B8_*/references/papers.json
+nlpcc_tracks:
+- track: T3
+  name: Scientific Discovery
+  url: https://github.com/ResearAI/NLPCC-2026-Task9-AISB/tree/main/benchmarks/nlpcc/T3
+private_hidden_reference_strategy:
+  policy: organizer_only
+  public_package_includes_hidden_reference: false
+  public_catalog_discloses_private_paths: false
+  score_release: aggregate_only
+  notes:
+  - Hidden answers, labels, canaries, and private evaluator internals are never packaged.
+  - Official scores must record hidden_reference_version without disclosing hidden content.
+score_card:
+  required_fields:
+  - team_id
+  - system_name
+  - direction
+  - track
+  - benchmark_package_version
+  - evaluator_version
+  - hidden_reference_version
+  - docker_image_digest
+  - data_split_version
+  - benchmark_score
+  - paper_score
+  - cas_score
+  - final_score
+  - security_verdict
+  - license_verdict
+  - used_reference_scope
+  - official_score
+  - created_at
+  track_scoring:
+    paper_track: Final_A = 0.0 * S_benchmark + 1.0 * S_paper
+    benchmark_track: Final_B = 0.7 * S_benchmark + 0.3 * S_paper
+    cas: integrity gate, not bonus
+risk_flags:
+- hidden_reference_not_public
+- not_release_ready
+- registry_promotion_pending
+- endpoint_license_review_pending
+risk_notes:
+- This catalog entry is not release_ready.
+- Download URL, checksum, and size point to the published GitHub release asset.
+- Official scores require organizer-side replay against private hidden references.
+recommended_when: Use this entry for AISB public-dev integration, local agent workflow checks, and catalog
+  UI validation.
+not_recommended_when: Do not treat this entry as an official release or public leaderboard claim until
+  release gates pass.
+image_path: ../image/aisb.b8.lifesci_drug.svg
+pilot:
+  status: selected
+  reason: B8 has real public-dev ADMET CSVs, label-stripped hidden inputs, CPU-safe baseline/evaluator,
+    and low install cost.
+checksums:
+  release_asset:
+    sha256: 7e9fde8e42baea653e20a0d67d77aa0a9c8cc0c314f8828320454556f73f851e
+    size_bytes: 154222
+    status: published
+  public_dev_files:
+  - path: benchmarks/aisb/B8_LifeSciDrug/data/dev/ames_dev.csv
+    rows: 727
+    sha256: 44fe16829e10ef78c0cf0456f19f355cac419bdabb930bf633f1dc714ff037c2
+  - path: benchmarks/aisb/B8_LifeSciDrug/data/dev/caco2_wang_dev.csv
+    rows: 91
+    sha256: 6c00bec1a71fef143bdfec175f71d3544e59f8d6065f8e68943333ce7d881b16
+  - path: benchmarks/aisb/B8_LifeSciDrug/data/dev/cyp3a4_veith_dev.csv
+    rows: 1232
+    sha256: 4b6abcba74d6e63078fa0e2c5adb4af48cde187cc082bb086002e64568afcfaa
+  - path: benchmarks/aisb/B8_LifeSciDrug/data/dev/herg_dev.csv
+    rows: 65
+    sha256: e3f22630a8f747a0adc972703ff44bc91e4c0f2481ac0a43eb8208ddcb5c7dd3
+  - path: benchmarks/aisb/B8_LifeSciDrug/data/dev/lipophilicity_az_dev.csv
+    rows: 420
+    sha256: 193eec001ae071e04b68d8a207d7f097f712e0e45d622bf1bbde4a962bb80321