@researai/deepscientist 1.5.17 → 1.6.0

package/AISB/catalog/aisb.t3.018_cotsynth.yaml

@@ -0,0 +1,206 @@
+schema_version: 1
+id: aisb.t3.018_cotsynth
+name: 'CoT-based Synthesizer: Enhancing LLM Performance through Answer Synthesis'
+version: 0.1.0
+one_line: 'Inference-time answer-synthesis benchmark: generate diverse CoT candidate
+  responses from 8B-scale policy models, then synthesize a superior answer via a trained
+  Synthesizer-8B, evaluating accuracy on GSM8k, MATH500, WikiTQ, and FeTaQA.
+
+  '
+task_description: 'This benchmark reproduces the CoT-based Synthesizer paper (ACL
+  2025). The core workflow is: (1) use a policy model (e.g. Llama3-8B-Instruct) to
+  sample N diverse candidate responses per question via temperature/top-p decoding,
+  (2) feed candidates into a Synthesizer model that performs CoT-based analysis and
+  synthesis to produce a refined final answer, and (3) evaluate on four benchmarks
+  — GSM8k (exact-match accuracy), MATH500 (exact-match accuracy via DART-Math evaluator),
+  WikiTQ (exact-match accuracy), and FeTaQA (ROUGE-L recall). The snapshot includes
+  bundled evaluation test data in the data/ folder, inference code using vLLM, evaluation
+  scripts, a fine-tuning pipeline, and a data-generation pipeline for creating synthesis
+  training data. A pre-trained Synthesizer-8B checkpoint and training data are available
+  on Hugging Face. The workflow is inference-heavy: each evaluation requires serving
+  an 8B model via vLLM and running multi-sample generation. MATH evaluation additionally
+  requires installing the dart-math package from GitHub. The benchmark compares against
+  Self-Consistency, Universal Self-Consistency, Best-of-N with reward models, and
+  LMCOR baselines.
+
+  '
+capability_tags:
+- research_code_optimization
+- large_language_models
+- answer_synthesis
+- mathematical_reasoning
+- table_qa
+- chain_of_thought
+- inference_scaling
+aisb_direction: T3
+track_fit:
+- paper_track
+- benchmark_track
+task_mode: evaluation_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: runnable
+support_level: advanced
+cost_band: high
+time_band: 1d+
+difficulty: hard
+data_access: public
+primary_outputs:
+- accuracy_gsm8k
+- accuracy_math500
+- accuracy_wikitq
+- rouge_l_fetaqa
+- synthesized_answers
+- vote_report
+launch_profiles:
+- id: quick_check
+  label: Quick Check
+  description: 'Run synthesis inference on a single dataset (e.g. MATH500 with pre-generated
+    candidate responses) and evaluate accuracy. Verifies that the vLLM serving and
+    evaluation pipeline work end-to-end.
+
+    '
+- id: synthesis_eval
+  label: Full Synthesis Evaluation
+  description: 'Run the complete answer-synthesis and evaluation workflow across all
+    four benchmarks (GSM8k, MATH500, WikiTQ, FeTaQA) using the Synthesizer-8B model
+    or Llama3.1-70B as the synthesizer. Requires generating candidate responses from
+    each policy model first.
+
+    '
+- id: data_pipeline
+  label: Data Generation Pipeline
+  description: 'Run the two-stage data generation pipeline (sampling.py → synthesizer.py
+    → filter) to produce training data for Synthesizer-8B. Requires a 70B response
+    LLM and is compute-intensive.
+
+    '
+dataset_download:
+  primary_method: mixed
+  sources:
+  - kind: bundled
+    url: null
+    access: public
+    note: 'Test sets for GSM8k, MATH500, WikiTQ, and FeTaQA are included in the data/
+      folder of the snapshot archive.
+
+      '
+  - kind: huggingface
+    url: https://huggingface.co/datasets/BoHanMint/Synthesizer-8B-math-train-data
+    access: public
+    note: 'Pre-generated synthesis training data (295k MATH, 87k WikiTQ). Only needed
+      if re-training Synthesizer-8B.
+
+      '
+  - kind: huggingface
+    url: https://huggingface.co/BoHanMint/Synthesizer-8B-math
+    access: public
+    note: 'Pre-trained Synthesizer-8B-math checkpoint. Required for inference unless
+      training from scratch.
+
+      '
+  - kind: github
+    url: https://github.com/hkust-nlp/dart-math
+    access: public
+    note: 'DART-Math evaluation library required for MATH500 exact-match scoring.
+      Must be pip-installed separately.
+
+      '
+  notes:
+  - Bundled test data is small (a few MB). Model checkpoints are ~16 GB for the 8B
+    model.
+  - If running baselines with Llama3.1-70B as synthesizer, ~140 GB of model weights
+    are needed.
+credential_requirements:
+  mode: optional
+  items:
+  - HuggingFace token (if gated model access is needed for Llama3 weights)
+  - OpenAI API key (only if evaluating GPT-4o as a policy model)
+  - GLM-4-Plus API key (only if evaluating GLM-4-Plus as a policy model)
+  notes:
+  - Core evaluation with open-source models requires no credentials.
+  - API keys are only needed to reproduce the full paper results with API-based policy
+    models.
+resources:
+  minimum:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 150
+    gpu_count: 1
+    gpu_vram_gb: 24
+  recommended:
+    cpu_cores: 32
+    ram_gb: 128
+    disk_gb: 300
+    gpu_count: 2
+    gpu_vram_gb: 48
+environment:
+  python: '3.11'
+  cuda: null
+  pytorch: null
+  flash_attn: null
+  key_packages:
+  - deepspeed==0.15.2
+  - vllm==0.5.3
+  - transformers==4.43.1
+  - rouge-score
+  - dart-math (pip install from GitHub)
+  notes:
+  - vLLM is the primary inference engine; synthesis_infer.py defaults to tensor_parallel_size=1
+    and max_model_len=8192.
+  - MATH evaluation requires dart-math installed via `pip install -e .` from the dart-math
+    repo clone.
+  - FeTaQA evaluation requires rouge-score package.
+  - The code hardcodes CUDA_VISIBLE_DEVICES in several scripts; adjust for your GPU
+    topology.
+  - See requirements.txt in the snapshot for the full dependency set.
+risk_flags:
+- external_package_dependency
+- large_model_weights
+- hardcoded_gpu_ids
+risk_notes:
+- MATH evaluation depends on the external dart-math package (GitHub clone + pip install
+  -e). If unavailable, MATH500 scoring will fail.
+- Scripts hardcode CUDA_VISIBLE_DEVICES (e.g. "0" in inference, "3" in eval). Must
+  be adjusted for multi-GPU or different hardware.
+- Full paper reproduction requires serving 70B models for candidate generation baselines,
+  which needs ≥2×80GB GPUs.
+- API-based policy model evaluation (GPT-4o, GLM-4-Plus) incurs real monetary cost
+  and requires API credentials.
+- No runtime execution was performed during packaging; metric values are not yet validated.
+recommended_when: 'Use this benchmark when you want an inference-heavy LLM evaluation
+  task focused on answer-synthesis strategies for mathematical reasoning and table
+  QA. Good fit for studying inference-time scaling, multi-response aggregation, and
+  CoT-based post-processing with 8B-class models. All evaluation data is bundled and
+  pre-trained checkpoints are publicly available on Hugging Face.
+
+  '
+not_recommended_when: 'Do not use this if you cannot serve 8B-class models on GPU
+  (minimum 24 GB VRAM), if you need a benchmark without model-serving overhead, or
+  if you need fully self-contained evaluation without any external package dependencies
+  (dart-math is required for MATH scoring).
+
+  '
+paper:
+  title: 'CoT-based Synthesizer: Enhancing LLM Performance through Answer Synthesis'
+  venue: ACL 2025
+  year: 2025
+  url: https://arxiv.org/abs/2501.01668
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.018_cotsynth.zip
+  archive_type: zip
+  local_dir_name: paper-18-CoTSynth
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.018_cotsynth.zip
+  sha256: 245a9c52e66e83cc77844c2375cf61a65cb6823fd84bdaf9142687559498a885
+  size_bytes: 925436
+commercial:
+  annual_fee: null
+display:
+  palette_seed: teal-gold-synthesis
+  art_style: benchmark-notebook
+  accent_priority: high
+image_path: ../image/018_aisb.t3.018_cotsynth.jpg

package/AISB/catalog/aisb.t3.018_cotsynth.zh.yaml

@@ -0,0 +1,162 @@
+schema_version: 1
+id: aisb.t3.018_cotsynth
+name: 'CoT合成器：通过答案合成提升大语言模型性能'
+version: 0.1.0
+one_line: '推理时答案合成基准测试：从8B规模策略模型生成多样化的CoT候选响应，然后通过训练好的Synthesizer-8B合成更优答案，在GSM8k、MATH500、WikiTQ和FeTaQA上评估准确率。
+
+'
+task_description: '该基准测试复现了CoT合成器论文（ACL 2025）。核心工作流程为：（1）使用策略模型（如Llama3-8B-Instruct）通过温度/top-p解码为每个问题采样N个多样化候选响应，（2）将候选响应输入合成器模型，执行基于CoT的分析与合成以产生精炼的最终答案，（3）在四个基准测试上评估——GSM8k（精确匹配准确率）、MATH500（通过DART-Math评估器的精确匹配准确率）、WikiTQ（精确匹配准确率）和FeTaQA（ROUGE-L召回率）。快照包含data/文件夹中的捆绑评估测试数据、使用vLLM的推理代码、评估脚本、微调流程以及用于创建合成训练数据的数据生成流程。预训练的Synthesizer-8B检查点及训练数据可在Hugging Face获取。该工作流程为推理密集型：每次评估需要通过vLLM服务8B模型并运行多样本生成。MATH评估还需从GitHub安装dart-math包。该基准测试与自洽性、通用自洽性、基于奖励模型的Best-of-N以及LMCOR基线进行比较。
+
+'
+capability_tags:
+- research_code_optimization
+- large_language_models
+- answer_synthesis
+- mathematical_reasoning
+- table_qa
+- chain_of_thought
+- inference_scaling
+aisb_direction: T3
+track_fit:
+- paper_track
+- benchmark_track
+task_mode: evaluation_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: runnable
+support_level: advanced
+cost_band: high
+time_band: 1d+
+difficulty: hard
+data_access: public
+primary_outputs:
+- accuracy_gsm8k
+- accuracy_math500
+- accuracy_wikitq
+- rouge_l_fetaqa
+- synthesized_answers
+- vote_report
+launch_profiles:
+- id: quick_check
+  label: 快速检查
+  description: '在单个数据集（如使用预生成候选响应的MATH500）上运行合成推理并评估准确率。验证vLLM服务和评估流程的端到端工作。
+
+'
+- id: synthesis_eval
+  label: 完整合成评估
+  description: '使用Synthesizer-8B模型或Llama3.1-70B作为合成器，在所有四个基准测试（GSM8k、MATH500、WikiTQ、FeTaQA）上运行完整的答案合成与评估工作流程。需要先从每个策略模型生成候选响应。
+
+'
+- id: data_pipeline
+  label: 数据生成流程
+  description: '运行两阶段数据生成流程（sampling.py → synthesizer.py → filter）以生成Synthesizer-8B的训练数据。需要70B响应LLM且计算密集。
+
+'
+dataset_download:
+  primary_method: mixed
+  sources:
+  - kind: bundled
+    url: null
+    access: public
+    note: 'GSM8k、MATH500、WikiTQ和FeTaQA的测试集包含在快照压缩包的data/文件夹中。
+
+'
+  - kind: huggingface
+    url: https://huggingface.co/datasets/BoHanMint/Synthesizer-8B-math-train-data
+    access: public
+    note: '预生成的合成训练数据（295k MATH、87k WikiTQ）。仅在重新训练Synthesizer-8B时需要。
+
+'
+  - kind: huggingface
+    url: https://huggingface.co/BoHanMint/Synthesizer-8B-math
+    access: public
+    note: '预训练的Synthesizer-8B-math检查点。除非从头训练，否则推理时必需。
+
+'
+  - kind: github
+    url: https://github.com/hkust-nlp/dart-math
+    access: public
+    note: 'MATH500精确匹配评分所需的DART-Math评估库。必须单独pip安装。
+
+'
+  notes:
+  - 捆绑测试数据很小（仅几MB）。模型检查点约16 GB（8B模型）。
+  - 如使用Llama3.1-70B作为合成器运行基线，需要约140 GB的模型权重。
+credential_requirements:
+  mode: optional
+  items:
+  - HuggingFace token（如需访问门控模型以获取Llama3权重）
+  - OpenAI API key（仅在评估GPT-4o作为策略模型时需要）
+  - GLM-4-Plus API key（仅在评估GLM-4-Plus作为策略模型时需要）
+  notes:
+  - 使用开源模型的核心评估无需凭据。
+  - 仅在复现API基策略模型的完整论文结果时需要API密钥。
+resources:
+  minimum:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 150
+    gpu_count: 1
+    gpu_vram_gb: 24
+  recommended:
+    cpu_cores: 32
+    ram_gb: 128
+    disk_gb: 300
+    gpu_count: 2
+    gpu_vram_gb: 48
+environment:
+  python: '3.11'
+  cuda: null
+  pytorch: null
+  flash_attn: null
+  key_packages:
+  - deepspeed==0.15.2
+  - vllm==0.5.3
+  - transformers==4.43.1
+  - rouge-score
+  - dart-math (pip install from GitHub)
+  notes:
+  - vLLM是主要推理引擎；synthesis_infer.py默认tensor_parallel_size=1，max_model_len=8192。
+  - MATH评估需要通过`pip install -e .`从dart-math仓库克隆并安装。
+  - FeTaQA评估需要rouge-score包。
+  - 代码在多个脚本中硬编码了CUDA_VISIBLE_DEVICES；请根据您的GPU拓扑进行调整。
+  - 快照中的requirements.txt包含完整的依赖列表。
+risk_flags:
+- external_package_dependency
+- large_model_weights
+- hardcoded_gpu_ids
+risk_notes:
+- MATH评估依赖外部dart-math包（GitHub克隆 + pip install -e）。如不可用，MATH500评分将失败。
+- 脚本硬编码了CUDA_VISIBLE_DEVICES（如推理中为"0"，评估中为"3"）。在多GPU或不同硬件上必须调整。
+- 完整论文复现需要服务70B模型以生成候选响应基线，需要≥2×80GB GPU。
+- 基于API的策略模型评估（GPT-4o、GLM-4-Plus）会产生实际货币成本，需要API凭据。
+- 打包过程中未执行运行时验证；指标值尚未确认。
+recommended_when: '当您需要一个推理密集型的LLM评估任务，重点关注数学推理和表格问答的答案合成策略时使用此基准测试。非常适合研究推理时扩展、多响应聚合以及基于CoT的8B类模型后处理。所有评估数据已捆绑，预训练检查点可在Hugging Face公开获取。
+
+'
+not_recommended_when: '如果无法在GPU上服务8B类模型（最低24 GB显存）、需要无模型服务开销的基准测试，或需要完全自包含的评估而不依赖任何外部包依赖项（dart-math是MATH评分必需的），请勿使用此基准测试。
+
+'
+paper:
+  title: 'CoT-based Synthesizer: Enhancing LLM Performance through Answer Synthesis'
+  venue: ACL 2025
+  year: 2025
+  url: https://arxiv.org/abs/2501.01668
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.018_cotsynth.zip
+  archive_type: zip
+  local_dir_name: paper-18-CoTSynth
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.018_cotsynth.zip
+  sha256: 245a9c52e66e83cc77844c2375cf61a65cb6823fd84bdaf9142687559498a885
+  size_bytes: 925436
+commercial:
+  annual_fee: null
+display:
+  palette_seed: teal-gold-synthesis
+  art_style: benchmark-notebook
+  accent_priority: high
+image_path: ../image/018_aisb.t3.018_cotsynth.jpg

package/AISB/catalog/aisb.t3.019_dyscaleut.yaml

@@ -0,0 +1,211 @@
+schema_version: 1
+id: aisb.t3.019_dyscaleut
+name: Dynamic Scaling of Unit Tests for Code Reward Modeling
+version: 0.1.0
+one_line: 'Generate and dynamically scale LLM-produced unit tests as reward signals
+  for best-of-N code solution selection, evaluated via pass@1 on HumanEval Plus, MBPP
+  Plus, and LiveCodeBench.
+
+  '
+task_description: 'This benchmark reproduces the CodeRM pipeline for improving code
+  generation through scaled unit-test reward modeling. The core workflow is: (1) a
+  policy LLM generates N candidate code solutions per programming problem, (2) a reward
+  LLM (or the fine-tuned CodeRM-8B unit test generator) produces M unit tests per
+  problem, (3) unit tests are executed against solutions inside a Docker sandbox,
+  and (4) the best solution is selected via variance-weighted majority voting over
+  execution results. A dynamic scaling mechanism allocates more unit tests to harder
+  problems using a trained difficulty classifier. The primary metric is pass@1 (best-of-N
+  accuracy) computed by evaluation/calculate_result.py. The local snapshot includes
+  the evaluation/execution code, bundled benchmark data, pre-computed inference results,
+  and a Docker image specification for sandboxed code execution. The inference and
+  preprocessing steps (generating new solutions/unit tests from scratch with policy/reward
+  LLMs) require external model access and are partially covered by bundled scripts
+  but may need adaptation. A pre-computed output.tar.gz from Google Drive can substitute
+  for the inference+execution steps.
+
+  '
+capability_tags:
+- research_code_optimization
+- code_generation
+- reward_modeling
+- unit_test_generation
+- evaluation
+aisb_direction: T3
+track_fit:
+- paper_track
+- benchmark_track
+task_mode: experiment_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: partial
+support_level: advanced
+cost_band: high
+time_band: 1d+
+difficulty: hard
+data_access: public
+primary_outputs:
+- pass_at_1
+- scaled_unit_tests
+- reward_scores
+launch_profiles:
+- id: quick_check
+  label: Quick Check
+  description: 'Run evaluation/calculate_result.py on the bundled or downloaded pre-computed
+    execution results to verify pass@1 on a single benchmark/model combination. No
+    GPU or Docker required.
+
+    '
+- id: code_reward_eval
+  label: Code Reward Eval
+  description: 'Pull the Docker sandbox image, execute unit tests against candidate
+    solutions using evaluation/evaluate.py, then compute pass@1 with evaluation/calculate_result.py.
+    Requires Docker and moderate compute. Uses bundled benchmark data and pre-generated
+    solutions/unit tests; does not require LLM inference.
+
+    '
+- id: full_pipeline
+  label: Full Pipeline (Inference + Eval)
+  description: 'Run end-to-end: multi-process LLM inference to generate solutions
+    and unit tests, preprocessing/merging, Docker-based execution, and final pass@1
+    calculation. Requires GPU access for LLM inference (CodeRM-8B or larger models)
+    and Docker for execution.
+
+    '
+dataset_download:
+  primary_method: mixed
+  sources:
+  - kind: huggingface
+    url: https://huggingface.co/datasets/KAKA22/CodeRM-UnitTest
+    access: public
+    note: 60k synthetic Python unit tests used to train CodeRM-8B.
+  - kind: huggingface
+    url: https://huggingface.co/KAKA22/CodeRM-8B
+    access: public
+    note: Fine-tuned 8B unit test generator model weights.
+  - kind: google_drive
+    url: https://drive.google.com/drive/folders/1-wUvy9Ox49V5CY38TMjCr5RlLysapyyj?usp=sharing
+    access: public
+    note: 'Pre-computed execution output (output.tar.gz) that can replace Steps 1-3
+      of the pipeline.
+
+      '
+  - kind: bundled
+    url: null
+    access: local
+    note: 'Benchmark data (HumanEval Plus, MBPP Plus, LiveCodeBench) and pre-generated
+      inference results are included under data/benchmark/ and data/result/ in the
+      snapshot.
+
+      '
+  notes:
+  - Training dataset is ~60k examples; model weights are ~16 GB; pre-computed outputs
+    vary by benchmark.
+  - Benchmarks themselves (HumanEval Plus, MBPP Plus, LiveCodeBench) are public.
+credential_requirements:
+  mode: none
+  items: []
+  notes:
+  - No API keys required if using bundled data and CodeRM-8B for inference.
+  - If replicating GPT-4o-mini or GPT-3.5 policy/reward experiments, OpenAI API keys
+    are needed.
+resources:
+  minimum:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 150
+    gpu_count: 1
+    gpu_vram_gb: 24
+  recommended:
+    cpu_cores: 32
+    ram_gb: 128
+    disk_gb: 300
+    gpu_count: 2
+    gpu_vram_gb: 48
+environment:
+  python: null
+  cuda: null
+  pytorch: null
+  flash_attn: null
+  key_packages:
+  - vllm
+  - transformers
+  - docker
+  notes:
+  - 'The snapshot includes a dedicated Docker execution environment (kaka0605/exec_unit_test:24.12.30)
+    for sandboxed large-scale code execution of generated unit tests.
+
+    '
+  - See docker_source/Dockerfile and docker_source/requirements.txt for the sandbox
+    dependencies.
+  - See bundled README and requirements for the host-side dependency set.
+  - Inference uses multi-process Python (inference/inference_mp.py) and likely requires
+    vLLM or similar for efficient serving.
+risk_flags:
+- docker_required
+- partial_snapshot
+- external_model_for_full_replication
+- code_execution_sandbox
+risk_notes:
+- 'The Docker sandbox (kaka0605/exec_unit_test:24.12.30) must be pulled or built locally
+  before running evaluation/evaluate.py. Without Docker, only the final calculate_result.py
+  step works on pre-computed outputs.
+
+  '
+- 'The preprocessing scripts (preprocess/) are present but the full inference pipeline
+  requires serving a policy LLM and a reward LLM, which is not fully automated in
+  the snapshot.
+
+  '
+- 'exec_main.py executes arbitrary generated Python code with a timeout mechanism
+  via signal.SIGALRM; this should only be run inside the provided Docker sandbox or
+  an equivalent isolated environment.
+
+  '
+- 'No benchmark execution was performed during the packaging pass; metric values are
+  not yet validated.
+
+  '
+recommended_when: 'Use this benchmark when you want to study how scaling LLM-generated
+  unit tests improves code reward signal quality and best-of-N code selection, or
+  when you need a code-generation task with real unit-test execution in the evaluation
+  loop. Also suitable for evaluating lightweight unit test generators against larger
+  teacher models.
+
+  '
+not_recommended_when: 'Do not use this if you cannot provide Docker-based containerized
+  code execution, if you need a text-only reward-model benchmark without code execution,
+  or if you lack GPU resources for LLM inference and only need a quick metric check
+  (use the quick_check profile with pre-computed outputs instead).
+
+  '
+paper:
+  title: Dynamic Scaling of Unit Tests for Code Reward Modeling
+  authors:
+  - Zeyao Ma
+  - Xiaokang Zhang
+  - Jing Zhang
+  - Jifan Yu
+  - Sijia Luo
+  - Jie Tang
+  venue: ACL 2025
+  year: 2025
+  url: https://arxiv.org/abs/2501.01054
+  homepage: https://code-reward-model.github.io/
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.019_dyscaleut.zip
+  archive_type: zip
+  local_dir_name: paper-19-DyScaleUT
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.019_dyscaleut.zip
+  sha256: 2eee5573353ade5e13c254f7372a3294b71459ee7c668205f27f2852347c141f
+  size_bytes: 60766
+commercial:
+  annual_fee: null
+display:
+  palette_seed: olive-ink-runtime
+  art_style: code-lab
+  accent_priority: high
+image_path: ../image/019_aisb.t3.019_dyscaleut.jpg