@researai/deepscientist 1.5.16 → 1.6.0

package/AISB/catalog/aisb.b1.agentic_coding.yaml

@@ -0,0 +1,244 @@
+schema_version: 1
+id: aisb.b1.agentic_coding
+name: AISB B1 Agentic Coding
+version: 0.1.0
+catalog_version: 0.1.0
+one_line: FeatureBench-lite public-dev package for repository navigation, code edits, and test-driven
+  repair.
+task_description: AISB B1 Agentic Coding is an AISB BenchStore catalog pilot entry. The package is intended
+  for public-dev workspace initialization, public evaluator checks, and paper/benchmark-track submission
+  scaffolding. Official leaderboard scores require organizer-side hidden replay.
+homepage: https://aisb.deepscientist.cc
+localized:
+  zh:
+    name: AISB B1 智能体编程
+    one_line: 面向仓库导航、代码编辑和测试驱动修复的 FeatureBench-lite public-dev 包。
+    task_description: AISB B1 智能体编程 是 AISB BenchStore catalog pilot 条目。公开包只用于 public-dev 工作区初始化、公开 evaluator
+      检查和论文/benchmark track 提交脚手架；正式 leaderboard 分数必须由组织方 hidden replay 产生。
+capability_tags:
+- agentic_coding
+- software_engineering
+- feature_implementation
+aisb_direction: B1
+track_fit:
+- paper_track
+- benchmark_track
+task_mode: benchmark
+benchmark_mode: software_engineering_agent
+requires_execution: true
+requires_paper: true
+integrity_level: hidden_reference
+snapshot_status: partial
+support_level: preview
+discovery:
+  collection: AISB
+  collection_priority: 100
+  recommendation_weight: 700
+  featured: false
+  featured_reason: null
+display:
+  placement: grid
+  card_size: m
+  badge: AISB
+  accent_priority: normal
+official_links:
+  homepage: https://github.com/ResearAI/NLPCC-2026-Task9-AISB
+  github: https://github.com/ResearAI/NLPCC-2026-Task9-AISB
+  benchmark_overview: https://github.com/ResearAI/NLPCC-2026-Task9-AISB/tree/main/benchmarks/aisb/B1_Agentic_Coding
+release_gate:
+  release_ready: false
+  status: registry_pending
+  full_real_replay: passed
+  reason: Public-dev and scorer-private replay paths exist locally, but final registry promotion, release
+    assets, and license review remain pending.
+  next_required_step: Validate the published BenchStore release asset on a clean host, then complete
+    registry and license review.
+primary_outputs:
+- results.json
+- paper/paper.pdf
+- paper/claims.json
+- logs/experiment_log.jsonl
+- logs/iterations.jsonl
+cost_band: medium
+time_band: 1-8h
+difficulty: hard
+data_access: public
+resources:
+  minimum:
+    cpu_cores: 4
+    ram_gb: 16
+    disk_gb: 20
+    gpu_count: 0
+  recommended:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 200
+    gpu_count: 0
+source:
+  benchmark_dir: benchmarks/aisb/B1_Agentic_Coding
+  bench_yaml: benchmarks/aisb/B1_Agentic_Coding/bench.yaml
+  source_repo: ResearAI/NLPCC-2026-Task9-AISB
+  source_commit: b6b7527a226ca7328697156608571f2cb7dcda3a
+download:
+  provider: github_release
+  repo: ResearAI/NLPCC-2026-Task9-AISB
+  tag: aisb-v0.0.1
+  asset_name: aisb.b1.agentic_coding-v0.1.0.zip
+  url: https://github.com/giao-123-sun/DeepScientist/releases/download/aisb-v0.0.1/aisb.b1.agentic_coding-v0.1.0.zip
+  source_url: https://github.com/ResearAI/NLPCC-2026-Task9-AISB/tree/main/benchmarks/aisb/B1_Agentic_Coding
+  archive_type: zip
+  local_dir_name: aisb.b1.agentic_coding
+  sha256: 5abe83b517511e83ce299fa009503f31a44bc3972acdd905630d0398f0664669
+  size_bytes: 47750
+  published_at: '2026-04-30T00:00:00Z'
+  asset_status: published
+dataset_download:
+  primary_method: packaged_public_dev
+  sources:
+  - kind: public_dev
+    access: public
+    url: null
+    note: Public-dev files are included in the published BenchStore package.
+  - kind: hidden_reference
+    access: private
+    url: null
+    note: Organizer-only hidden references are excluded from catalog packages.
+credential_requirements:
+  mode: none
+  notes:
+  - No credential is required for packaged public-dev evaluation.
+  - External model APIs are optional participant choices and not bundled.
+environment:
+  python: '>=3.11'
+  docker: true
+  key_packages:
+  - pytest
+  - PyYAML
+  notes:
+  - Use docker/Dockerfile when available for official replay parity.
+  - Do not copy organizer-private hidden references into participant workspaces.
+launch_profiles:
+- id: b1_public_dev_smoke
+  label: B1 public-dev smoke test
+  description: Initialize an AISB public-dev workspace, validate submission artifacts, and run the public
+    evaluator.
+  startup_contract:
+  - Use only files inside the installed benchmark workspace and the generated submission directory.
+  - Treat hidden references as organizer-only and unavailable to the agent.
+  - Produce benchmark outputs, paper artifacts, claims, and logs before packaging.
+  commands:
+  - python scripts/agent_tools.py workspace init B1 --dest .work/benchstore/B1
+  - python scripts/agent_tools.py evaluate B1 --bench-dir .work/benchstore/B1 --submission .work/benchstore/B1/submission
+  - python scripts/agent_tools.py submission validate .work/benchstore/B1/submission
+  - python scripts/agent_tools.py submission package .work/benchstore/B1/submission --output .work/benchstore/B1/submission.zip
+  - python scripts/agent_tools.py submission replay .work/benchstore/B1/submission --track B1
+  one_click_prompt: Start AISB B1 from the installed BenchStore package. Initialize the public-dev workspace,
+    inspect AGENT.md and README.md, run the baseline or your method, evaluate locally, validate/package
+    the submission, and record any release-gate risks in logs/experiment_log.jsonl.
+paper:
+  venue: AISB
+  year: 2026
+  track_fit:
+  - paper_track
+  - benchmark_track
+  requires_claims_json: true
+  url: null
+benchmark_track:
+  metric: task_completion
+  benchmark_source: FeatureBench public subset
+  packaged_data_status: real FeatureBench-lite public-dev rows plus scorer-private official harness replay
+    rows
+  official_score_requires_hidden_replay: true
+benchmarks:
+  primary:
+    name: FeatureBench-lite
+    source: LiberCoders/FeatureBench (HF)
+    tasks: 3
+    metric: task_completion
+    sota: 20% Lite (GPT-5.1-Codex, ICLR 2026)
+    public_score: 0.0
+  alternatives:
+  - name: SWE-bench Pro
+    source: ScaleAI/SWE-bench_Pro (HF)
+    tasks: 500+
+    metric: resolved_rate
+    sota: 59.10% (GPT-5.4 xHigh)
+    note: clean benchmark; B3 primary
+  - name: SWE-bench Verified
+    source: princeton-nlp/SWE-bench_Verified
+    note: DEPRECATED — contaminated; only legacy reference
+  - name: SWE-rebench
+    source: cais/re-bench
+    metric: task_success
+    note: research engineering benchmark; B1/B3 cross
+papers:
+  highlights:
+  - title: 'FeatureBench: Benchmarking Agentic Coding for Complex Feature Development'
+    arxiv: '2602.10975'
+    url: https://arxiv.org/abs/2602.10975
+    venue: ICLR 2026
+    why: Primary benchmark paper; SOTA 12.5% Full / 20% Lite
+  - title: 'SWE-Edit: Rethinking Code Editing for Efficient SWE-Agents'
+    arxiv: '2604.26102'
+    url: https://arxiv.org/abs/2604.26102
+    venue: April 2026
+    why: +2.1% resolved rate, -17.9% inference cost
+  - title: 'AgentForge: Execution-grounded Multi-Agent Framework'
+    arxiv: '2604.13120'
+    url: https://arxiv.org/abs/2604.13120
+    venue: April 2026
+    why: 5-agent architecture; SWE-Bench Lite 40.0%
+  - title: Many SWE-bench-Passing PRs Would Not Be Merged
+    url: https://metr.org/blog/2026-03-19-many-swe-bench-prs-not-merged/
+    source: METR blog, March 2026
+    why: 'Critical evaluation: patch correctness vs mergeability'
+  full_references_url: https://github.com/ResearAI/NLPCC-2026-Task9-AISB/tree/main/benchmarks/aisb/B1_*/references/papers.json
+nlpcc_tracks:
+- track: T1
+  name: AI/CS Reasoning & Engineering
+  url: https://github.com/ResearAI/NLPCC-2026-Task9-AISB/tree/main/benchmarks/nlpcc/T1
+private_hidden_reference_strategy:
+  policy: organizer_only
+  public_package_includes_hidden_reference: false
+  public_catalog_discloses_private_paths: false
+  score_release: aggregate_only
+  notes:
+  - Hidden answers, labels, canaries, and private evaluator internals are never packaged.
+  - Official scores must record hidden_reference_version without disclosing hidden content.
+score_card:
+  required_fields:
+  - team_id
+  - system_name
+  - direction
+  - track
+  - benchmark_package_version
+  - evaluator_version
+  - hidden_reference_version
+  - docker_image_digest
+  - data_split_version
+  - benchmark_score
+  - paper_score
+  - cas_score
+  - final_score
+  - security_verdict
+  - license_verdict
+  - used_reference_scope
+  - official_score
+  - created_at
+  track_scoring:
+    paper_track: Final_A = 0.0 * S_benchmark + 1.0 * S_paper
+    benchmark_track: Final_B = 0.7 * S_benchmark + 0.3 * S_paper
+    cas: integrity gate, not bonus
+risk_flags:
+- hidden_reference_not_public
+- not_release_ready
+- registry_promotion_pending
+risk_notes:
+- This catalog entry is not release_ready.
+- Download URL, checksum, and size point to the published GitHub release asset.
+- Official scores require organizer-side replay against private hidden references.
+recommended_when: Use this entry for AISB public-dev integration, local agent workflow checks, and catalog
+  UI validation.
+not_recommended_when: Do not treat this entry as an official release or public leaderboard claim until
+  release gates pass.
+image_path: ../image/aisb.b1.agentic_coding.svg

package/AISB/catalog/aisb.b10.climate_earth.yaml

@@ -0,0 +1,235 @@
+schema_version: 1
+id: aisb.b10.climate_earth
+name: AISB B10 Climate and Earth Science
+version: 0.1.0
+catalog_version: 0.1.0
+one_line: WeatherBench2-mini ERA5 package for latitude-weighted weather forecasting validation.
+task_description: AISB B10 Climate and Earth Science is an AISB BenchStore catalog pilot entry. The package
+  is intended for public-dev workspace initialization, public evaluator checks, and paper/benchmark-track
+  submission scaffolding. Official leaderboard scores require organizer-side hidden replay.
+homepage: https://aisb.deepscientist.cc
+localized:
+  zh:
+    name: AISB B10 气候与地球科学
+    one_line: 用于纬度加权天气预报验证的 WeatherBench2-mini ERA5 包。
+    task_description: AISB B10 气候与地球科学 是 AISB BenchStore catalog pilot 条目。公开包只用于 public-dev 工作区初始化、公开
+      evaluator 检查和论文/benchmark track 提交脚手架；正式 leaderboard 分数必须由组织方 hidden replay 产生。
+capability_tags:
+- climate
+- earth_science
+- weather
+- forecasting
+aisb_direction: B10
+track_fit:
+- paper_track
+- benchmark_track
+task_mode: benchmark
+benchmark_mode: experiment_driven
+requires_execution: true
+requires_paper: true
+integrity_level: hidden_reference
+snapshot_status: partial
+support_level: preview
+discovery:
+  collection: AISB
+  collection_priority: 100
+  recommendation_weight: 700
+  featured: false
+  featured_reason: null
+display:
+  placement: grid
+  card_size: m
+  badge: AISB
+  accent_priority: normal
+official_links:
+  homepage: https://github.com/ResearAI/NLPCC-2026-Task9-AISB
+  github: https://github.com/ResearAI/NLPCC-2026-Task9-AISB
+  benchmark_overview: https://github.com/ResearAI/NLPCC-2026-Task9-AISB/tree/main/benchmarks/aisb/B10_ClimateEarth
+release_gate:
+  release_ready: false
+  status: registry_pending
+  full_real_replay: passed
+  reason: Public-dev and scorer-private replay paths exist locally, but final registry promotion, release
+    assets, and license review remain pending.
+  next_required_step: Validate the published BenchStore release asset on a clean host, then complete
+    registry and license review.
+primary_outputs:
+- results.json
+- paper/paper.pdf
+- paper/claims.json
+- logs/experiment_log.jsonl
+- logs/iterations.jsonl
+cost_band: medium
+time_band: 1-2h
+difficulty: hard
+data_access: public
+resources:
+  minimum:
+    cpu_cores: 4
+    ram_gb: 8
+    disk_gb: 10
+    gpu_count: 0
+  recommended:
+    cpu_cores: 8
+    ram_gb: 16
+    disk_gb: 20
+    gpu_count: 0
+source:
+  benchmark_dir: benchmarks/aisb/B10_ClimateEarth
+  bench_yaml: benchmarks/aisb/B10_ClimateEarth/bench.yaml
+  source_repo: ResearAI/NLPCC-2026-Task9-AISB
+  source_commit: b6b7527a226ca7328697156608571f2cb7dcda3a
+download:
+  provider: github_release
+  repo: ResearAI/NLPCC-2026-Task9-AISB
+  tag: aisb-v0.0.1
+  asset_name: aisb.b10.climate_earth-v0.1.0.zip
+  url: https://github.com/giao-123-sun/DeepScientist/releases/download/aisb-v0.0.1/aisb.b10.climate_earth-v0.1.0.zip
+  source_url: https://github.com/ResearAI/NLPCC-2026-Task9-AISB/tree/main/benchmarks/aisb/B10_ClimateEarth
+  archive_type: zip
+  local_dir_name: aisb.b10.climate_earth
+  sha256: 693a771c756519eb09e18256da519d0a1a7b3371b32cb60c81c60291428b0ece
+  size_bytes: 25280
+  published_at: '2026-04-30T00:00:00Z'
+  asset_status: published
+dataset_download:
+  primary_method: packaged_public_dev
+  sources:
+  - kind: public_dev
+    access: public
+    url: null
+    note: Public-dev files are included in the published BenchStore package.
+  - kind: hidden_reference
+    access: private
+    url: null
+    note: Organizer-only hidden references are excluded from catalog packages.
+credential_requirements:
+  mode: none
+  notes:
+  - No credential is required for packaged public-dev evaluation.
+  - External model APIs are optional participant choices and not bundled.
+environment:
+  python: '>=3.11'
+  docker: true
+  key_packages:
+  - pytest
+  - PyYAML
+  notes:
+  - Use docker/Dockerfile when available for official replay parity.
+  - Do not copy organizer-private hidden references into participant workspaces.
+launch_profiles:
+- id: b10_public_dev_smoke
+  label: B10 public-dev smoke test
+  description: Initialize an AISB public-dev workspace, validate submission artifacts, and run the public
+    evaluator.
+  startup_contract:
+  - Use only files inside the installed benchmark workspace and the generated submission directory.
+  - Treat hidden references as organizer-only and unavailable to the agent.
+  - Produce benchmark outputs, paper artifacts, claims, and logs before packaging.
+  commands:
+  - python scripts/agent_tools.py workspace init B10 --dest .work/benchstore/B10
+  - python scripts/agent_tools.py evaluate B10 --bench-dir .work/benchstore/B10 --submission .work/benchstore/B10/submission
+  - python scripts/agent_tools.py submission validate .work/benchstore/B10/submission
+  - python scripts/agent_tools.py submission package .work/benchstore/B10/submission --output .work/benchstore/B10/submission.zip
+  - python scripts/agent_tools.py submission replay .work/benchstore/B10/submission --track B10
+  one_click_prompt: Start AISB B10 from the installed BenchStore package. Initialize the public-dev workspace,
+    inspect AGENT.md and README.md, run the baseline or your method, evaluate locally, validate/package
+    the submission, and record any release-gate risks in logs/experiment_log.jsonl.
+paper:
+  venue: AISB
+  year: 2026
+  track_fit:
+  - paper_track
+  - benchmark_track
+  requires_claims_json: true
+  url: null
+benchmark_track:
+  metric: 1/(1+RMSE) latitude-weighted
+  benchmark_source: WeatherBench2-mini ERA5 64x32
+  packaged_data_status: real WeatherBench2 ERA5 64x32 grid plus label-stripped held-out inputs
+  official_score_requires_hidden_replay: true
+benchmarks:
+  primary:
+    name: WeatherBench2-mini ERA5 64x32
+    source: google-research/weatherbench2 (gs://)
+    dev_count: 64
+    hidden_count: 32
+    metric: latitude-weighted 1/(1+RMSE)
+    sota: GenCast (Nature 2024), Aurora (Microsoft 2024), NeuralGCM (Nature 2024), AIFS (ECMWF)
+    our_baseline: 0.0016
+  alternatives:
+  - name: ClimateBench v1.0 ssp245
+    source: Zenodo CC BY 4.0
+    note: legacy compatibility fixture; 48 rows; superseded by WB2
+  - name: ClimAgent/ClimaBench
+    source: arXiv:2604.16922
+    note: April 2026; AI-driven climate science discovery benchmark
+papers:
+  highlights:
+  - title: 'WeatherBench 2: A Framework for Next-Generation Data-Driven Weather Forecasting'
+    source: JAMES 2024 / google-research/weatherbench2
+    why: Primary benchmark; 62 variables, 13 pressure levels, ERA5 data
+  - title: 'GenCast: Diffusion-based Ensemble Forecasting'
+    arxiv: '2312.15796'
+    url: https://arxiv.org/abs/2312.15796
+    venue: Nature 2024
+    why: 'SOTA: beats ECMWF ENS on 99.8% of targets'
+  - title: 'ClimAgent: LLM Agents for Autonomous Climate Science Analysis'
+    arxiv: '2604.16922'
+    url: https://arxiv.org/abs/2604.16922
+    venue: April 2026
+    why: 'ClimaBench: first comprehensive climate discovery benchmark'
+  - title: ClimateBench v1.0 (legacy)
+    source: Zenodo 7064308, CC BY 4.0
+    why: Legacy compatibility fixture; superseded by WeatherBench 2
+  full_references_url: https://github.com/ResearAI/NLPCC-2026-Task9-AISB/tree/main/benchmarks/aisb/B10_*/references/papers.json
+nlpcc_tracks:
+- track: T3
+  name: Scientific Discovery
+  url: https://github.com/ResearAI/NLPCC-2026-Task9-AISB/tree/main/benchmarks/nlpcc/T3
+private_hidden_reference_strategy:
+  policy: organizer_only
+  public_package_includes_hidden_reference: false
+  public_catalog_discloses_private_paths: false
+  score_release: aggregate_only
+  notes:
+  - Hidden answers, labels, canaries, and private evaluator internals are never packaged.
+  - Official scores must record hidden_reference_version without disclosing hidden content.
+score_card:
+  required_fields:
+  - team_id
+  - system_name
+  - direction
+  - track
+  - benchmark_package_version
+  - evaluator_version
+  - hidden_reference_version
+  - docker_image_digest
+  - data_split_version
+  - benchmark_score
+  - paper_score
+  - cas_score
+  - final_score
+  - security_verdict
+  - license_verdict
+  - used_reference_scope
+  - official_score
+  - created_at
+  track_scoring:
+    paper_track: Final_A = 0.0 * S_benchmark + 1.0 * S_paper
+    benchmark_track: Final_B = 0.7 * S_benchmark + 0.3 * S_paper
+    cas: integrity gate, not bonus
+risk_flags:
+- hidden_reference_not_public
+- not_release_ready
+- registry_promotion_pending
+- attribution_review_pending
+risk_notes:
+- This catalog entry is not release_ready.
+- Download URL, checksum, and size point to the published GitHub release asset.
+- Official scores require organizer-side replay against private hidden references.
+recommended_when: Use this entry for AISB public-dev integration, local agent workflow checks, and catalog
+  UI validation.
+not_recommended_when: Do not treat this entry as an official release or public leaderboard claim until
+  release gates pass.
+image_path: ../image/aisb.b10.climate_earth.svg