@researai/deepscientist 1.5.16 → 1.6.0

package/AISB/catalog/aisb.t3.049_hogwildinference.yaml

@@ -0,0 +1,183 @@
+schema_version: 1
+id: aisb.t3.049_hogwildinference
+name: 'Hogwild! Inference: Parallel LLM Generation via Concurrent Attention'
+version: 0.1.0
+one_line: 'Benchmark and optimize parallel LLM inference where multiple workers share
+  a concurrent KV cache, measuring tokens-per-second throughput and custom CUDA kernel
+  latency on L40-class GPUs.
+
+  '
+task_description: 'This benchmark evaluates a parallel LLM generation system called
+  Hogwild! Inference, where multiple LLM "workers" run concurrently with a shared
+  Key-Value attention cache, allowing instant cross-attention between workers via
+  RoPE position re-rotation. The optimization task is to maximize multi-worker inference
+  throughput (tokens_per_second) without breaking correctness or stability of the
+  concurrent cache mechanism.
+
+  The package provides two execution routes: (1) Jupyter notebooks (basic_example,
+  full_example, hogwild_with_fast_kernels) that run 2-agent parallel inference on
+  open-source reasoning LLMs (e.g. QwQ, DeepSeek-R1) using the Python `shared_cache`
+  library and `generation.py`, and (2) a standalone C++/CUDA kernel benchmark (`inference_lib/src/benchmark.cpp`)
+  that measures raw attention-kernel latency in microseconds for the custom Flash-Decoding-inspired
+  paged attention with per-block query rotation.
+
+  The fast kernels require compilation via CMake with nvcc (CUDA architectures 80/86/89)
+  and were tuned for L40 GPUs. The primary metric `tokens_per_second` is route-backed
+  through the kernel benchmark executable but has not been verified end-to-end in
+  this packaging pass; runtime execution is required before trusting reported values.
+  Model weights (e.g. Llama-3.2-3B for Colab, or larger reasoning models like QwQ-32B)
+  must be downloaded from HuggingFace at inference time. No training is involved—this
+  is purely an inference-time systems benchmark.
+
+  '
+capability_tags:
+- research_code_optimization
+- large_language_models
+- parallel_inference
+- systems_efficiency
+- cuda_kernels
+aisb_direction: T3
+track_fit:
+- paper_track
+- benchmark_track
+task_mode: evaluation_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: partial
+support_level: advanced
+time_band: 1-2d
+cost_band: high
+difficulty: hard
+data_access: public
+primary_outputs:
+- tokens_per_second
+- kernel_latency_us
+- benchmark_logs
+launch_profiles:
+- id: notebook_inference
+  label: Notebook Inference
+  description: 'Run the bundled Jupyter notebooks (basic_example, full_example, hogwild_with_fast_kernels)
+    to perform 2-agent parallel LLM inference with a shared KV cache. Requires a GPU
+    with sufficient VRAM for the chosen model. Outputs generation traces and informal
+    throughput.
+
+    '
+- id: kernel_benchmark
+  label: CUDA Kernel Benchmark
+  description: 'Build the C++ benchmark executable from inference_lib via CMake +
+    nvcc, then run it to measure raw attention kernel latency. Produces microsecond-level
+    timing for the custom paged-attention kernel with query rotation. Throughput conversion
+    to tokens_per_second must be verified manually against end-to-end generation.
+
+    '
+dataset_download:
+  primary_method: huggingface
+  sources:
+  - kind: huggingface
+    url: https://huggingface.co/models
+    access: public
+    note: 'Model weights (e.g. Llama-3.2-3B, QwQ-32B, DeepSeek-R1) are downloaded
+      at inference time via HuggingFace transformers. No separate dataset download
+      is required; the benchmark operates on prompted reasoning tasks defined in the
+      notebooks.
+
+      '
+  notes:
+  - No large offline dataset is needed. Storage is dominated by model weights (7–65
+    GB depending on model choice).
+  - The evals/ directory contains evaluation scripts for LIMO, LiveCodeBench, and
+    OlympiadBench but these are auxiliary.
+credential_requirements:
+  mode: none
+  items: []
+  notes:
+  - A HuggingFace token may be needed for gated models like Llama-3.2 but is not required
+    for all model choices.
+resources:
+  minimum:
+    cpu_cores: 8
+    ram_gb: 32
+    disk_gb: 80
+    gpu_count: 1
+    gpu_vram_gb: 24
+  recommended:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 150
+    gpu_count: 1
+    gpu_vram_gb: 48
+environment:
+  python: '3.10'
+  cuda: '11.8'
+  pytorch: 2.6.0
+  key_packages:
+  - transformers==4.51.3
+  - numpy
+  - CLI11 (C++ dep, fetched by CMake)
+  notes:
+  - The fast inference kernels require nvcc in PATH (or CUDACXX set). CMake >= 3.22
+    is needed.
+  - CUDA architectures 80, 86, 89 are targeted (A100, L40, RTX 4090 class).
+  - Kernels were specifically optimized for L40 and similar GPUs; performance on other
+    architectures may differ.
+  - The Colab notebook demonstrates a minimal route with Llama-3.2-3B and limited
+    collaboration.
+  - pip install -e . inside inference_lib/ builds the Python-bound CUDA extension.
+risk_flags:
+- metric_not_verified_e2e
+- requires_cuda_compilation
+- model_download_at_runtime
+risk_notes:
+- 'The primary metric tokens_per_second is route-backed via the C++ kernel benchmark
+  but no end-to-end benchmark execution was performed during packaging. Runtime verification
+  is required before trusting metric values.
+
+  '
+- 'Building the fast kernels requires a working nvcc toolchain and compatible GPU
+  architecture. Compilation failures are possible on non-standard CUDA setups.
+
+  '
+- 'Larger reasoning models (QwQ-32B, DeepSeek-R1) require 48+ GB VRAM; the minimum
+  24 GB spec only supports smaller models like Llama-3.2-3B with limited collaboration
+  quality.
+
+  '
+- 'The evals/ directory references external evaluation suites (LIMO, LiveCodeBench,
+  OlympiadBench) that may require additional setup not fully documented in the snapshot.
+
+  '
+recommended_when: 'Use this benchmark when you want an LLM-systems optimization task
+  focused on parallel inference throughput, concurrent KV cache design, and custom
+  CUDA kernel performance. Suitable for researchers working on inference-time parallelism,
+  attention kernel optimization, or multi-agent LLM coordination at the systems level.
+
+  '
+not_recommended_when: 'Do not use this if you lack access to a modern CUDA GPU (Ampere
+  or newer), if you need a training-oriented benchmark, or if you are looking for
+  a self-contained accuracy evaluation with automated scoring. The metric surface
+  requires manual verification and the kernel build adds significant setup complexity.
+
+  '
+paper:
+  title: 'Hogwild! Inference: Parallel LLM Generation via Concurrent Attention'
+  venue: arXiv preprint
+  year: 2025
+  url: https://arxiv.org/abs/2504.06261
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.049_hogwildinference.zip
+  archive_type: zip
+  local_dir_name: paper-49-HogwildInference
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.049_hogwildinference.zip
+  sha256: b5cc4e8d0a697683307a664b948af23324ca548c6ce7b8380035fe7410dabcdd
+  size_bytes: 226660
+commercial:
+  annual_fee: null
+display:
+  palette_seed: orange-slate-cache
+  art_style: concurrent-systems
+  accent_priority: high
+image_path: ../image/049_aisb.t3.049_hogwildinference.jpg

package/AISB/catalog/aisb.t3.049_hogwildinference.zh.yaml

@@ -0,0 +1,138 @@
+schema_version: 1
+id: aisb.t3.049_hogwildinference
+name: 'Hogwild! Inference: Parallel LLM Generation via Concurrent Attention'
+version: 0.1.0
+one_line: '在多个工作进程共享并发 KV 缓存的并行 LLM 推理场景下进行基准测试和优化，测量 L40 级 GPU 上的每秒令牌吞吐量和自定义 CUDA 内核延迟。
+
+  '
+task_description: '该基准测试评估名为 Hogwild! Inference 的并行 LLM 生成系统，其中多个 LLM"工作进程"通过共享键值注意力缓存并发运行，通过 RoPE 位置重旋转实现工作进程间的即时交叉注意力。优化任务是在不破坏并发缓存机制正确性或稳定性的前提下，最大化多工作进程推理吞吐量（tokens_per_second）。
+
+  该软件包提供两条执行路径：（1）Jupyter 笔记本（basic_example、full_example、hogwild_with_fast_kernels），使用 Python `shared_cache` 库和 `generation.py` 在开源推理 LLM（如 QwQ、DeepSeek-R1）上运行 2 智能体并行推理；（2）独立 C++/CUDA 内核基准测试（`inference_lib/src/benchmark.cpp`），用于测量自定义 Flash-Decoding 启发的分页注意力（带逐块查询旋转）的原始注意力内核延迟（微秒级）。
+
+  快速内核需要通过 CMake + nvcc 编译（CUDA 架构 80/86/89），并针对 L40 GPU 进行了优化。主要指标 `tokens_per_second` 通过内核基准测试可执行文件进行了路由支持，但在本次打包过程中未进行端到端验证，需要运行时执行后才能信任报告值。模型权重（如 Colab 用的 Llama-3.2-3B，或更大的推理模型如 QwQ-32B）需要在推理时从 HuggingFace 下载。无需训练——这是纯推理时系统基准测试。
+
+  '
+capability_tags:
+- research_code_optimization
+- large_language_models
+- parallel_inference
+- systems_efficiency
+- cuda_kernels
+aisb_direction: T3
+track_fit:
+- paper_track
+- benchmark_track
+task_mode: evaluation_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: partial
+support_level: advanced
+time_band: 1-2d
+cost_band: high
+difficulty: hard
+data_access: public
+primary_outputs:
+- tokens_per_second
+- kernel_latency_us
+- benchmark_logs
+launch_profiles:
+- id: notebook_inference
+  label: 笔记本推理
+  description: '运行捆绑的 Jupyter 笔记本（basic_example、full_example、hogwild_with_fast_kernels），使用共享 KV 缓存执行 2 智能体并行 LLM 推理。需要具有足够 VRAM 的 GPU 用于所选模型。输出生成轨迹和非正式吞吐量。
+
+    '
+- id: kernel_benchmark
+  label: CUDA 内核基准测试
+  description: '通过 CMake + nvcc 从 inference_lib 构建 C++ 基准测试可执行文件，然后运行它来测量原始注意力内核延迟。为自定义分页注意力内核（带查询旋转）生成微秒级计时。吞吐量转换为 tokens_per_second 必须手动验证以对抗端到端生成。
+
+    '
+dataset_download:
+  primary_method: huggingface
+  sources:
+  - kind: huggingface
+    url: https://huggingface.co/models
+    access: public
+    note: '模型权重（如 Llama-3.2-3B、QwQ-32B、DeepSeek-R1）在推理时通过 HuggingFace transformers 下载。不需要单独的数据集下载；基准测试操作在笔记本中定义的提示推理任务。
+
+      '
+  notes:
+  - 不需要大型离线数据集。存储主要由模型权重决定（根据模型选择为 7–65 GB）。
+  - evals/ 目录包含 LIMO、LiveCodeBench 和 OlympiadBench 的评估脚本，但这些是辅助性的。
+credential_requirements:
+  mode: none
+  items: []
+  notes:
+  - 对于 Llama-3.2 等门控模型可能需要 HuggingFace 令牌，但并非所有模型选择都需要。
+resources:
+  minimum:
+    cpu_cores: 8
+    ram_gb: 32
+    disk_gb: 80
+    gpu_count: 1
+    gpu_vram_gb: 24
+  recommended:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 150
+    gpu_count: 1
+    gpu_vram_gb: 48
+environment:
+  python: '3.10'
+  cuda: '11.8'
+  pytorch: 2.6.0
+  key_packages:
+  - transformers==4.51.3
+  - numpy
+  - CLI11 (C++ 依赖，由 CMake 获取)
+  notes:
+  - 快速推理内核需要 nvcc 在 PATH 中（或设置 CUDACXX）。需要 CMake >= 3.22。
+  - 目标是 CUDA 架构 80、86、89（A100、L40、RTX 4090 类）。
+  - 内核专门针对 L40 和类似 GPU 进行了优化；其他架构上的性能可能有所不同。
+  - Colab 笔记本演示了使用 Llama-3.2-3B 和有限协作的最小路径。
+  - 在 inference_lib/ 中运行 pip install -e . 可构建绑定 Python 的 CUDA 扩展。
+risk_flags:
+- metric_not_verified_e2e
+- requires_cuda_compilation
+- model_download_at_runtime
+risk_notes:
+- '主要指标 tokens_per_second 通过 C++ 内核基准测试进行了路由支持，但在打包期间未执行端到端基准测试。在信任指标值之前需要运行时验证。
+
+  '
+- '构建快速内核需要可用的 nvcc 工具链和兼容的 GPU 架构。在非标准 CUDA 设置上可能出现编译失败。
+
+  '
+- '更大的推理模型（QwQ-32B、DeepSeek-R1）需要 48+ GB VRAM；最低 24 GB 规格仅支持较小的模型（如 Llama-3.2-3B），协作质量有限。
+
+  '
+- 'evals/ 目录引用了外部评估套件（LIMO、LiveCodeBench、OlympiadBench），这些可能需要快照中未完全记录的额外设置。
+
+  '
+recommended_when: '当您需要一个专注于并行推理吞吐量、并发 KV 缓存设计和自定义 CUDA 内核性能的 LLM 系统优化任务时使用此基准测试。适用于从事推理时并行性、注意力内核优化或在系统级别进行多智能体 LLM 协调的研究人员。
+
+  '
+not_recommended_when: '如果您无法访问现代 CUDA GPU（安培或更新），如果您需要以训练为导向的基准测试，或者如果您正在寻找具有自动评分的自包含准确性评估，请不要使用此基准测试。指标表面需要手动验证，内核构建增加了显著的设置复杂性。
+
+  '
+paper:
+  title: 'Hogwild! Inference: Parallel LLM Generation via Concurrent Attention'
+  venue: arXiv preprint
+  year: 2025
+  url: https://arxiv.org/abs/2504.06261
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.049_hogwildinference.zip
+  archive_type: zip
+  local_dir_name: paper-49-HogwildInference
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.049_hogwildinference.zip
+  sha256: b5cc4e8d0a697683307a664b948af23324ca548c6ce7b8380035fe7410dabcdd
+  size_bytes: 226660
+commercial:
+  annual_fee: null
+display:
+  palette_seed: orange-slate-cache
+  art_style: concurrent-systems
+  accent_priority: high
+image_path: ../image/049_aisb.t3.049_hogwildinference.jpg

package/AISB/catalog/aisb.t3.050_causalpfn.yaml

@@ -0,0 +1,214 @@
+id: aisb.t3.050_causalpfn
+name: 'CausalPFN: Amortized Causal Effect Estimation via In-Context Learning'
+version: 0.1.0
+one_line: Transformer-based amortized causal effect estimation using PFN-style in-context
+  learning with calibrated uncertainty quantification.
+task_description: 'CausalPFN is a transformer model that amortizes causal effect estimation
+  by training once on a large library of simulated data-generating processes (DGPs)
+  satisfying the ignorability assumption. Given a new observational dataset, it infers
+  causal effects out of the box via in-context learning without task-specific tuning,
+  fine-tuning, or hyperparameter optimization. The model estimates both conditional
+  average treatment effects (CATE) and average treatment effects (ATE), with a principled
+  uncertainty quantification framework producing finite-sample calibrated credible
+  intervals. Amortization shifts computational burden from inference to pre-training,
+  enabling fast forward-pass-only evaluation on new datasets.
+
+  '
+task_mode: experiment_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: external_eval_required
+support_level: recovery
+time_band: 2-8h
+cost_band: medium
+difficulty: medium
+data_access: public
+primary_outputs:
+- ihdp_pehe
+- ihdp_ate_rel_err
+- causal_predictions
+- bootstrap_report
+launch_profiles:
+- id: quick_eval
+  label: Quick Eval
+  description: Run CausalPFN evaluation on a single supported dataset (e.g., IHDP)
+    with default settings. Suitable for verifying installation and basic functionality.
+- id: full_benchmark
+  label: Full Benchmark
+  description: Run the complete amortized causal-effect benchmark across IHDP, ACIC,
+    and Lalonde datasets with bootstrap ensemble aggregation and temperature scaling.
+dataset_download:
+  primary_method: pypi
+  sources:
+  - name: PyPI
+    url: https://pypi.org/project/causalpfn/
+    archive_type: null
+    local_dir_name: null
+  - name: HuggingFace Hub
+    url: https://huggingface.co/vdblm/causalpfn
+    archive_type: null
+    local_dir_name: null
+  notes:
+  - Model weights are downloaded from HuggingFace Hub at runtime.
+  - Install via: pip install causalpfn
+  - Optional dev dependencies: pip install causalpfn[dev]
+credential_requirements:
+  mode: none
+  items: []
+  notes:
+  - HuggingFace Hub access requires network connectivity for model download.
+  - Optional HF_TOKEN for rate-limited or private model access.
+resources:
+  minimum:
+    cpu_cores: 8
+    ram_gb: 32
+    disk_gb: 50
+    gpu_count: 0
+    gpu_vram_gb: 0
+    notes: CPU-only execution is supported via causalpfn Python API with torch.device('cpu').
+  recommended:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 100
+    gpu_count: 1
+    gpu_vram_gb: 16
+    notes: GPU acceleration recommended for faster inference on large datasets.
+environment:
+  python: '3.10'
+  cuda: '11.8'
+  pytorch: '2.3'
+  flash_attn: null
+  key_packages:
+  - causalpfn>=1.0.0
+  - torch>=2.3.0
+  - numpy>=1.24.0
+  - scikit-learn>=1.3.0
+  - faiss-cpu==1.9.0
+  - huggingface_hub>=0.20.0
+  - tqdm>=4.65.0
+  notes:
+  - CPU-only execution is viable for minimum resource route.
+  - GPU acceleration available via CUDA for inference speedup.
+  - Full dependency set available in bundled requirements.txt and pyproject.toml.
+risk_flags:
+- external_model_download
+- bootstrap_variance
+risk_notes:
+- Model weights are fetched from HuggingFace Hub at runtime; ensure network access.
+- Bootstrap ensemble introduces variance across seeds; use provided bootstrap_report
+  for diagnostics.
+recommended_when: 'Use this benchmark for causal inference tasks requiring tractable
+  local runs, clear effect-estimation metrics (PEHE, ATE relative error), and uncertainty
+  quantification. Ideal for uplift modeling, treatment effect estimation from observational
+  data, and comparison with specialized causal estimators. Recommended when you need
+  a ready-to-use estimator without manual tuning.
+
+  '
+not_recommended_when: 'Do not use this benchmark if you need a giant multimodal benchmark,
+  raw language generation tasks, or image/video understanding. Not suitable when treatment
+  assignment is not ignorable or when unobserved confounding is present in the data.
+
+  '
+paper:
+  title: 'CausalPFN: Amortized Causal Effect Estimation via In-Context Learning'
+  venue: NeurIPS 2025
+  year: 2025
+  url: https://arxiv.org/abs/2506.07918
+  authors:
+  - Vahid Balazadeh
+  - Hamidreza Kamkari
+  - Valentin Thomas
+  - Benson Li
+  - Junwei Ma
+  - Jesse C. Cresswell
+  - Rahul G. Krishnan
+  institutions:
+  - University of Toronto
+  - Vector Institute
+  - Layer 6 AI
+display:
+  palette_seed: mint-charcoal-causal
+  art_style: causal-analytics
+  accent_priority: medium
+image_path: ../image/050_aisb.t3.050_causalpfn.jpg
+capability_tags:
+- causal_inference
+- in_context_learning
+- tabular_ml
+- uncertainty_estimation
+- uplift_modeling
+- treatment_effect_estimation
+aisb_direction: T3
+track_fit:
+- paper_track
+- benchmark_track
+code_anchors:
+  evaluation:
+  - path: eval.py
+    description: Main evaluation script for IHDP benchmark with multi-seed bootstrap
+      ensemble
+  - path: src/causalpfn/evaluation.py
+    description: Core evaluation metrics including PEHE and calibration curve calculations
+  estimators:
+  - path: src/causalpfn/causal_estimator.py
+    description: CATEEstimator and ATEEstimator classes with confidence interval estimation
+  benchmarks:
+  - path: benchmarks/ihdp.py
+    description: IHDP benchmark implementation
+  - path: benchmarks/acic2016.py
+    description: ACIC 2016 benchmark implementation
+  - path: benchmarks/lalonde.py
+    description: Lalonde benchmark implementation
+  - path: benchmarks/hillstrom.py
+    description: Hillstrom uplift modeling benchmark
+metric_contract:
+  ihdp_pehe:
+    origin_path: eval.py
+    source_ref: calculate_pehe
+    evaluation_protocol:
+      code_paths:
+      - eval.py
+      - src/causalpfn/evaluation.py
+    status: code_backed
+    provisional: false
+  ihdp_ate_rel_err:
+    origin_path: eval.py
+    source_ref: calculate_ate_rel_error
+    evaluation_protocol:
+      code_paths:
+      - eval.py
+      - src/causalpfn/evaluation.py
+    status: code_backed
+    provisional: false
+evaluation_details:
+  bootstrap_ensemble:
+    seeds: 2
+    subsamples: 3
+    subsample_fraction: 0.92
+    temperature_values:
+    - 0.3
+    - 0.5
+    - 0.7
+    - 0.9
+    - 2.0
+    - 4.0
+    - 8.0
+    total_predictions_per_realization: 42
+  s_learner:
+    regressor: Ridge
+    alpha: 0.001
+    mu_features: true
+  nearest_neighbors: 85
+commercial:
+  annual_fee: null
+  notes: Apache 2.0 License
+download:
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.050_causalpfn.zip
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.050_causalpfn.zip
+  archive_type: zip
+  sha256: d7ba4ba97509f90f00b4f9953b10d76fc5bc56ea116a7141a74a11990ee99a99
+  size_bytes: 48053403