@researai/deepscientist 1.5.17 → 1.6.0

package/AISB/catalog/aisb.b9.material_science.yaml

@@ -0,0 +1,237 @@
+schema_version: 1
+id: aisb.b9.material_science
+name: AISB B9 Material Science
+version: 0.1.0
+catalog_version: 0.1.0
+one_line: Matbench Discovery WBM mini package for crystal-stability prediction validation.
+task_description: AISB B9 Material Science is an AISB BenchStore catalog pilot entry. The package is intended
+  for public-dev workspace initialization, public evaluator checks, and paper/benchmark-track submission
+  scaffolding. Official leaderboard scores require organizer-side hidden replay.
+homepage: https://aisb.deepscientist.cc
+localized:
+  zh:
+    name: AISB B9 材料科学
+    one_line: 用于晶体稳定性预测验证的 Matbench Discovery WBM mini 包。
+    task_description: AISB B9 材料科学 是 AISB BenchStore catalog pilot 条目。公开包只用于 public-dev 工作区初始化、公开 evaluator
+      检查和论文/benchmark track 提交脚手架；正式 leaderboard 分数必须由组织方 hidden replay 产生。
+capability_tags:
+- material_science
+- property_prediction
+- crystal_stability
+aisb_direction: B9
+track_fit:
+- paper_track
+- benchmark_track
+task_mode: benchmark
+benchmark_mode: experiment_driven
+requires_execution: true
+requires_paper: true
+integrity_level: hidden_reference
+snapshot_status: partial
+support_level: preview
+discovery:
+  collection: AISB
+  collection_priority: 100
+  recommendation_weight: 700
+  featured: false
+  featured_reason: null
+display:
+  placement: grid
+  card_size: m
+  badge: AISB
+  accent_priority: normal
+official_links:
+  homepage: https://github.com/ResearAI/NLPCC-2026-Task9-AISB
+  github: https://github.com/ResearAI/NLPCC-2026-Task9-AISB
+  benchmark_overview: https://github.com/ResearAI/NLPCC-2026-Task9-AISB/tree/main/benchmarks/aisb/B9_MaterialScience
+release_gate:
+  release_ready: false
+  status: registry_pending
+  full_real_replay: passed
+  reason: Public-dev and scorer-private replay paths exist locally, but final registry promotion, release
+    assets, and license review remain pending.
+  next_required_step: Validate the published BenchStore release asset on a clean host, then complete
+    registry and license review.
+primary_outputs:
+- results.json
+- paper/paper.pdf
+- paper/claims.json
+- logs/experiment_log.jsonl
+- logs/iterations.jsonl
+cost_band: low
+time_band: minutes
+difficulty: medium
+data_access: public
+resources:
+  minimum:
+    cpu_cores: 2
+    ram_gb: 2
+    disk_gb: 1
+    gpu_count: 0
+  recommended:
+    cpu_cores: 4
+    ram_gb: 4
+    disk_gb: 2
+    gpu_count: 0
+source:
+  benchmark_dir: benchmarks/aisb/B9_MaterialScience
+  bench_yaml: benchmarks/aisb/B9_MaterialScience/bench.yaml
+  source_repo: ResearAI/NLPCC-2026-Task9-AISB
+  source_commit: b6b7527a226ca7328697156608571f2cb7dcda3a
+download:
+  provider: github_release
+  repo: ResearAI/NLPCC-2026-Task9-AISB
+  tag: aisb-v0.0.1
+  asset_name: aisb.b9.material_science-v0.1.0.zip
+  url: https://github.com/giao-123-sun/DeepScientist/releases/download/aisb-v0.0.1/aisb.b9.material_science-v0.1.0.zip
+  source_url: https://github.com/ResearAI/NLPCC-2026-Task9-AISB/tree/main/benchmarks/aisb/B9_MaterialScience
+  archive_type: zip
+  local_dir_name: aisb.b9.material_science
+  sha256: 58511b92525fc9320972c83cb050a33e3a80c93734c0a1c23fb8cf15bc631d32
+  size_bytes: 46754
+  published_at: '2026-04-30T00:00:00Z'
+  asset_status: published
+dataset_download:
+  primary_method: packaged_public_dev
+  sources:
+  - kind: public_dev
+    access: public
+    url: null
+    note: Public-dev files are included in the published BenchStore package.
+  - kind: hidden_reference
+    access: private
+    url: null
+    note: Organizer-only hidden references are excluded from catalog packages.
+credential_requirements:
+  mode: none
+  notes:
+  - No credential is required for packaged public-dev evaluation.
+  - External model APIs are optional participant choices and not bundled.
+environment:
+  python: '>=3.11'
+  docker: true
+  key_packages:
+  - pytest
+  - PyYAML
+  notes:
+  - Use docker/Dockerfile when available for official replay parity.
+  - Do not copy organizer-private hidden references into participant workspaces.
+launch_profiles:
+- id: b9_public_dev_smoke
+  label: B9 public-dev smoke test
+  description: Initialize an AISB public-dev workspace, validate submission artifacts, and run the public
+    evaluator.
+  startup_contract:
+  - Use only files inside the installed benchmark workspace and the generated submission directory.
+  - Treat hidden references as organizer-only and unavailable to the agent.
+  - Produce benchmark outputs, paper artifacts, claims, and logs before packaging.
+  commands:
+  - python scripts/agent_tools.py workspace init B9 --dest .work/benchstore/B9
+  - python scripts/agent_tools.py evaluate B9 --bench-dir .work/benchstore/B9 --submission .work/benchstore/B9/submission
+  - python scripts/agent_tools.py submission validate .work/benchstore/B9/submission
+  - python scripts/agent_tools.py submission package .work/benchstore/B9/submission --output .work/benchstore/B9/submission.zip
+  - python scripts/agent_tools.py submission replay .work/benchstore/B9/submission --track B9
+  one_click_prompt: Start AISB B9 from the installed BenchStore package. Initialize the public-dev workspace,
+    inspect AGENT.md and README.md, run the baseline or your method, evaluate locally, validate/package
+    the submission, and record any release-gate risks in logs/experiment_log.jsonl.
+paper:
+  venue: AISB
+  year: 2026
+  track_fit:
+  - paper_track
+  - benchmark_track
+  requires_claims_json: true
+  url: null
+benchmark_track:
+  metric: f1
+  benchmark_source: Matbench Discovery WBM mini
+  packaged_data_status: real Matbench Discovery WBM mini public-dev CSV plus label-stripped held-out inputs
+  official_score_requires_hidden_replay: true
+benchmarks:
+  primary:
+    name: Matbench Discovery WBM mini
+    source: janosh/matbench-discovery (Figshare CC BY 4.0)
+    tasks: 800
+    hidden: 200
+    metric: F1
+    sota: F1=0.931 (EquiformerV3+DeNS-OAM, 2026-04-07)
+    our_baseline: 0.8792
+    our_hidden: 0.6486
+  alternatives:
+  - name: PhononBench
+    source: arXiv:2512.21227
+    note: 108K structures; dynamic stability; avg stability 25.83%
+  - name: OC25
+    source: Open Catalyst
+    note: restricted access; not for public-dev
+papers:
+  highlights:
+  - title: 'Matbench Discovery: A Framework for ML Crystal Stability Prediction'
+    arxiv: '2308.14920'
+    url: https://arxiv.org/abs/2308.14920
+    venue: Nature MI 2025
+    why: Primary benchmark; WBM mini split used for public-dev
+  - title: 'PhononBench: Large-Scale Phonon-Based Benchmark for Crystal Generation'
+    arxiv: '2512.21227'
+    url: https://arxiv.org/abs/2512.21227
+    venue: Dec 2025
+    why: 108K structures; avg stability only 25.83%; harder than Matbench
+  - title: 'MIND: AI Co-Scientist for Materials Research'
+    arxiv: '2604.13699'
+    url: https://arxiv.org/abs/2604.13699
+    venue: April 2026
+    why: Multi-agent hypothesis-verification; SevenNet-Omni validation
+  - title: EquiformerV3+DeNS-OAM
+    source: Matbench Discovery leaderboard, 2026-04-07
+    why: 'Current SOTA: F1=0.931, MAE=0.018'
+  full_references_url: https://github.com/ResearAI/NLPCC-2026-Task9-AISB/tree/main/benchmarks/aisb/B9_*/references/papers.json
+nlpcc_tracks:
+- track: T3
+  name: Scientific Discovery
+  url: https://github.com/ResearAI/NLPCC-2026-Task9-AISB/tree/main/benchmarks/nlpcc/T3
+private_hidden_reference_strategy:
+  policy: organizer_only
+  public_package_includes_hidden_reference: false
+  public_catalog_discloses_private_paths: false
+  score_release: aggregate_only
+  notes:
+  - Hidden answers, labels, canaries, and private evaluator internals are never packaged.
+  - Official scores must record hidden_reference_version without disclosing hidden content.
+score_card:
+  required_fields:
+  - team_id
+  - system_name
+  - direction
+  - track
+  - benchmark_package_version
+  - evaluator_version
+  - hidden_reference_version
+  - docker_image_digest
+  - data_split_version
+  - benchmark_score
+  - paper_score
+  - cas_score
+  - final_score
+  - security_verdict
+  - license_verdict
+  - used_reference_scope
+  - official_score
+  - created_at
+  track_scoring:
+    paper_track: Final_A = 0.0 * S_benchmark + 1.0 * S_paper
+    benchmark_track: Final_B = 0.7 * S_benchmark + 0.3 * S_paper
+    cas: integrity gate, not bonus
+risk_flags:
+- hidden_reference_not_public
+- not_release_ready
+- registry_promotion_pending
+- attribution_review_pending
+risk_notes:
+- This catalog entry is not release_ready.
+- Download URL, checksum, and size point to the published GitHub release asset.
+- Official scores require organizer-side replay against private hidden references.
+recommended_when: Use this entry for AISB public-dev integration, local agent workflow checks, and catalog
+  UI validation.
+not_recommended_when: Do not treat this entry as an official release or public leaderboard claim until
+  release gates pass.
+image_path: ../image/aisb.b9.material_science.svg

package/AISB/catalog/aisb.t3.001_savvy.yaml

@@ -0,0 +1,159 @@
+id: aisb.t3.001_savvy
+name: 'SAVVY: Spatial Awareness via Audio-Visual LLMs through Seeing and Hearing'
+version: 0.1.0
+one_line: Training-free two-stage spatial reasoning pipeline for AV-LLMs with SAVVY-Bench
+  evaluation of 3D egocentric and allocentric audio-visual QA.
+task_description: 'SAVVY (Spatial Awareness via Audio-Visual LLMs through Seeing and
+  Hearing) is a packaged benchmark combining a training-free reasoning pipeline with
+  the SAVVY-Bench evaluation framework for dynamic 3D audio-visual question answering.
+  The pipeline operates in two stages: (1) Egocentric Spatial Tracks Estimation, which
+  leverages AV-LLMs and audio-visual methods to track object trajectories using both
+  visual and spatial audio cues; and (2) Dynamic Global Map Construction, which aggregates
+  multimodal object trajectories into a unified global 3D map for accurate egocentric
+  and allocentric spatial reasoning. SAVVY-Bench evaluates AV-LLMs on thousands of
+  QA pairs probing directional and distance relationships among static and moving
+  objects, requiring fine-grained temporal grounding, consistent 3D localization,
+  and multimodal annotation. Supported AV-LLMs include VideoLLaMA2, MiniCPM-o2.6,
+  Video-SALMONN, Gemini 2.5 Flash/Pro, Ola, EgoGPT, and Longvale. Primary staged metrics
+  track overall benchmark accuracy, spatial AVQA scores, and mean relative accuracy
+  at multiple thresholds.
+
+  '
+task_mode: evaluation_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: runnable
+support_level: advanced
+time_band: 1-2d
+cost_band: high
+difficulty: hard
+data_access: public
+primary_outputs:
+- overall_accuracy
+- spatial_avqa_score
+- mra_0.5_0.95_0.05
+- predavmap_json
+launch_profiles:
+- id: quick_eval
+  label: Quick Eval
+  description: Run the packaged SAVVY-Bench evaluation route on preprocessed benchmark
+    assets using supported AV-LLMs.
+- id: full_pipeline
+  label: Full Pipeline
+  description: Execute the complete training-free SAVVY reasoning pipeline from raw
+    multimodal input through trajectory estimation and global map construction, followed
+    by benchmark evaluation.
+dataset_download:
+  primary_method: mixed
+  sources:
+  - kind: huggingface
+    url: https://huggingface.co/datasets/uwneuroai/SAVVY-Bench
+    access: public
+    note: Primary benchmark dataset with QA pairs and annotations. Download via Hugging
+      Face Hub or Data Studio preview.
+  - kind: github_repo
+    url: https://github.com/shlizee/savvy/tree/main/SAVVY
+    access: public
+    note: SAVVY algorithm repository containing data preprocessing scripts under `data_utils/`,
+      benchmark data, and annotations.
+  - kind: github_repo
+    url: https://github.com/shlizee/savvy/tree/main/SAVVY-Bench
+    access: public
+    note: SAVVY-Bench evaluation repository with multi-model benchmarking framework
+      and evaluation scripts.
+  notes:
+  - Public Hugging Face SAVVY-Bench route covers benchmark evaluation but raw preprocessing
+    route requires Project Aria licensed data.
+  - SAVVY algorithm and SAVVY-Bench evaluation are split across separate repositories;
+    full setup may require cloning both plus external model environments.
+  - 数百GB of multimodal assets required for full pipeline; disk estimates account for
+    video, audio, trajectories, and intermediate outputs.
+credential_requirements:
+  mode: none
+  items: []
+  notes:
+  - No explicit dataset or API credentials detected in bundled snapshot.
+  - Full pipeline may require Gemini API access for snapshot descriptor component.
+  - Project Aria data access requires separate license agreement for raw preprocessing
+    route.
+resources:
+  minimum:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 250
+    gpu_count: 1
+    gpu_vram_gb: 24
+    notes: Minimum single-GPU configuration for basic evaluation; pipeline execution
+      may be memory-constrained.
+  recommended:
+    cpu_cores: 32
+    ram_gb: 128
+    disk_gb: 500
+    gpu_count: 2
+    gpu_vram_gb: 48
+    notes: Multi-GPU configuration recommended for parallel trajectory estimation
+      and reduced wall-clock time during full pipeline runs.
+environment:
+  python: '3.12'
+  cuda: '11.8'
+  pytorch: 2.8.0
+  flash_attn: null
+  key_packages:
+  - transformers==4.57.0
+  - torch==2.8.0
+  - open3d==0.19.0
+  - google-generativeai
+  - clipseg
+  - opencv-python
+  - numpy
+  - tqdm
+  notes:
+  - SAVVY install script creates Python 3.12 environment and separately clones ZoeDepth
+    and EfficientViT helper repositories.
+  - SAVVY-Bench recommends separate conda/venv environments per AV-LLM evaluator due
+    to divergent dependency stacks.
+  - External dependencies include ZoeDepth for depth estimation, EfficientViT for
+    segmentation, CLIPSeg and SAM2 for text-guided segmentation.
+risk_flags:
+- multi_env_benchmark
+- licensed_data_route
+- external_api_dependency
+- large_storage_requirement
+risk_notes:
+- SAVVY algorithm is training-free but full paper-faithful route spans separate SAVVY
+  and SAVVY-Bench repositories plus per-model evaluation environments.
+- Public HuggingFace SAVVY-Bench enables benchmark evaluation; raw preprocessing route
+  references Project Aria licensed data and hundreds of GB of multimodal assets.
+- Pipeline depends on third-party models and tools (ZoeDepth, EfficientViT, CLIPSeg,
+  SAM2) and optionally Gemini API for snapshot descriptors.
+- Disk requirements substantial (250-500GB) for video, spatial audio, trajectories,
+  3D maps, and intermediate processing outputs.
+recommended_when: Use this benchmark for training-free but systems-heavy audio-visual
+  LLM spatial reasoning evaluation requiring both pipeline execution and benchmark
+  scoring, particularly for dynamic 3D scenes with moving objects and multi-channel
+  spatial audio.
+not_recommended_when: Avoid when requiring single-environment setup, limited local
+  storage (<250GB), no external model API dependencies, or lightweight evaluation
+  without the full reasoning pipeline.
+paper:
+  title: 'SAVVY: Spatial Awareness via Audio-Visual LLMs through Seeing and Hearing'
+  venue: NeurIPS 2025 Oral
+  year: 2025
+  url: https://arxiv.org/abs/2506.05414
+  project_url: https://zijuncui02.github.io/SAVVY/
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.001_savvy.zip
+  archive_type: zip
+  local_dir_name: paper-1-SAVVY
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.001_savvy.zip
+  sha256: 2edb91c0816c40fa8734e871271e582ae21ecb66d25eb3f6437cc422534a6028
+  size_bytes: 2025393
+display:
+  palette_seed: ember-cyan-signal
+  art_style: cinematic-technical
+  accent_priority: high
+image_path: ../../../AISB/image/001_aisb.t3.001_savvy.jpg

package/AISB/catalog/aisb.t3.001_savvy.zh.yaml

@@ -0,0 +1,121 @@
+id: aisb.t3.001_savvy
+name: 'SAVVY: Spatial Awareness via Audio-Visual LLMs through Seeing and Hearing'
+version: 0.1.0
+one_line: 无需训练的两阶段空间推理流程，用于AV-LLM的SAVVY-Bench评估，涵盖3D自我中心和非自我中心视听问答
+task_description: 'SAVVY（通过视觉和听觉实现音频-视觉大语言模型的空间感知）是一个综合基准测试，将无需训练的推理流程与SAVVY-Bench评估框架相结合，用于动态3D视听问答。该流程分两个阶段运行：（1）自我中心空间轨迹估计，利用AV-LLM和视听方法，通过视觉和空间音频线索跟踪物体轨迹；（2）动态全局地图构建，将多模态物体轨迹聚合到统一的全局3D地图中，以实现精确的自我中心和非自我中心空间推理。SAVVY-Bench在数千个问答对上评估AV-LLM，探索静态和移动物体之间的方向和距离关系，需要精细的时间定位、一致的3D定位和多模态标注。支持的AV-LLM包括VideoLLaMA2、MiniCPM-o2.6、Video-SALMONN、Gemini 2.5 Flash/Pro、Ola、EgoGPT和Longvale。主要分阶段指标跟踪整体基准测试准确率、空间AVQA分数以及多个阈值下的平均相对准确率。
+
+  '
+task_mode: evaluation_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: runnable
+support_level: advanced
+time_band: 1-2d
+cost_band: high
+difficulty: hard
+data_access: public
+primary_outputs:
+- overall_accuracy
+- spatial_avqa_score
+- mra_0.5_0.95_0.05
+- predavmap_json
+launch_profiles:
+- id: quick_eval
+  label: 快速评估
+  description: 使用支持的AV-LLM在预处理后的基准测试资产上运行打包的SAVVY-Bench评估流程。
+- id: full_pipeline
+  label: 完整流程
+  description: 从原始多模态输入执行完整的无需训练的SAVVY推理流程，包括轨迹估计和全局地图构建，然后进行基准测试评估。
+dataset_download:
+  primary_method: mixed
+  sources:
+  - kind: huggingface
+    url: https://huggingface.co/datasets/uwneuroai/SAVVY-Bench
+    access: public
+    note: 主要基准测试数据集，包含问答对和标注。通过Hugging Face Hub或Data Studio预览下载。
+  - kind: github_repo
+    url: https://github.com/shlizee/savvy/tree/main/SAVVY
+    access: public
+    note: SAVVY算法仓库，包含data_utils/下的数据预处理脚本、基准测试数据和标注。
+  - kind: github_repo
+    url: https://github.com/shlizee/savvy/tree/main/SAVVY-Bench
+    access: public
+    note: SAVVY-Bench评估仓库，包含多模型基准测试框架和评估脚本。
+  notes:
+  - 公共Hugging Face SAVVY-Bench路线涵盖基准测试评估，但原始预处理路线需要Project Aria授权数据。
+  - SAVVY算法和SAVVY-Bench评估分别位于不同仓库；完整设置可能需要克隆两个仓库以及外部模型环境。
+  - 完整流程需要数百GB的多模态资产；磁盘估算包括视频、音频、轨迹和中间输出。
+credential_requirements:
+  mode: none
+  items: []
+  notes:
+  - 在打包的快照中未检测到明确的数据集或API凭据。
+  - 完整流程可能需要Gemini API访问权限用于快照描述符组件。
+  - Project Aria数据访问需要为原始预处理路线单独签署许可协议。
+resources:
+  minimum:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 250
+    gpu_count: 1
+    gpu_vram_gb: 24
+    notes: 基本评估的最低单GPU配置；流程执行可能在内存上受限。
+  recommended:
+    cpu_cores: 32
+    ram_gb: 128
+    disk_gb: 500
+    gpu_count: 2
+    gpu_vram_gb: 48
+    notes: 推荐使用多GPU配置进行并行轨迹估计，缩短完整流程运行时的挂钟时间。
+environment:
+  python: '3.12'
+  cuda: '11.8'
+  pytorch: 2.8.0
+  flash_attn: null
+  key_packages:
+  - transformers==4.57.0
+  - torch==2.8.0
+  - open3d==0.19.0
+  - google-generativeai
+  - clipseg
+  - opencv-python
+  - numpy
+  - tqdm
+  notes:
+  - SAVVY安装脚本创建Python 3.12环境，并分别克隆ZoeDepth和EfficientViT辅助仓库。
+  - SAVVY-Bench建议每个AV-LLM评估器使用独立的conda/venv环境，因为依赖栈存在分歧。
+  - 外部依赖包括用于深度估计的ZoeDepth、用于分割的EfficientViT、用于文本引导分割的CLIPSeg和SAM2。
+risk_flags:
+- multi_env_benchmark
+- licensed_data_route
+- external_api_dependency
+- large_storage_requirement
+risk_notes:
+- SAVVY算法无需训练，但忠实于论文的完整路线横跨独立的SAVVY和SAVVY-Bench仓库以及每个模型的评估环境。
+- 公共HuggingFace SAVVY-Bench支持基准测试评估；原始预处理路线引用Project Aria授权数据和数百GB的多模态资产。
+- 流程依赖于第三方模型和工具（ZoeDepth、EfficientViT、CLIPSeg、SAM2），以及可选的Gemini API用于快照描述符。
+- 磁盘需求量大（250-500GB），用于视频、空间音频、轨迹、3D地图和中间处理输出。
+recommended_when: 当需要进行无需训练但系统密集型的视听大语言模型空间推理评估时使用此基准测试，尤其适用于具有移动物体和多通道空间音频的动态3D场景。
+not_recommended_when: 避免在需要单一环境设置、本地存储有限（<250GB）、无外部模型API依赖或无需完整推理流程的轻量级评估时使用。
+paper:
+  title: 'SAVVY: Spatial Awareness via Audio-Visual LLMs through Seeing and Hearing'
+  venue: NeurIPS 2025 Oral
+  year: 2025
+  url: https://arxiv.org/abs/2506.05414
+  project_url: https://zijuncui02.github.io/SAVVY/
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.001_savvy.zip
+  archive_type: zip
+  local_dir_name: paper-1-SAVVY
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.001_savvy.zip
+  sha256: 2edb91c0816c40fa8734e871271e582ae21ecb66d25eb3f6437cc422534a6028
+  size_bytes: 2025393
+display:
+  palette_seed: ember-cyan-signal
+  art_style: cinematic-technical
+  accent_priority: high
+image_path: ../../../AISB/image/001_aisb.t3.001_savvy.jpg