@researai/deepscientist 1.5.17 → 1.6.0

package/AISB/catalog/aisb.t3.009_scoremissing.yaml

@@ -0,0 +1,179 @@
+schema_version: 1
+id: aisb.t3.009_scoremissing
+name: Score Matching with Missing Data
+version: 0.1.0
+one_line: 'Learn full score functions from partially observed tabular data using importance-weighting
+  and variational marginal score matching, evaluated on simulated GGM/ICA/normal-estimation
+  tasks and real-world S&P 100 and yeast graphical model recovery.
+
+  '
+task_description: 'This benchmark packages the full experimental pipeline from the
+  ICML 2025 Spotlight paper "Score Matching with Missing Data". The core task is to
+  estimate the score (gradient of the log-density) of a multivariate distribution
+  when training samples have missing coordinates under a Missing Completely At Random
+  (MCAR) mechanism. Two complementary methods are provided: (1) marginal importance-weighting
+  (IW) score matching, which excels in lower-dimensional and small-sample settings,
+  and (2) marginal variational score matching, which is stronger in high-dimensional
+  problems such as Gaussian Graphical Model (GGM) estimation.
+
+  The experiment suite spans five sub-tasks run from Python scripts in HPC/: GGM structure
+  recovery (simulated), ICA parameter estimation, truncated/untruncated normal parameter
+  estimation, S&P 100 GGM estimation, and yeast GGM estimation. Each script accepts
+  a repeat count and a parameter-grid index as CLI arguments. Metrics (AUC, accuracy,
+  TPR, FPR) are computed inside the scripts via MSM/utils/data.py helper functions
+  and aggregated in Jupyter notebooks under plotcode/. Real-world data (yeast tensor,
+  S&P 100 CSV) is bundled in real_world_experiments/RealData/. No external evaluation
+  service or API credentials are needed; the benchmark is fully self-contained.
+
+  '
+capability_tags:
+- research_code_optimization
+- missing_data
+- probabilistic_modeling
+- score_matching
+- tabular_ml
+- graphical_model_estimation
+aisb_direction: T3
+track_fit:
+- paper_track
+- benchmark_track
+task_mode: experiment_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: runnable
+support_level: advanced
+cost_band: medium
+time_band: 6-24h
+difficulty: hard
+data_access: public
+primary_outputs:
+- auc
+- accuracy
+- tpr
+- fpr
+- structure_recovery_report
+launch_profiles:
+- id: quick_check
+  label: Quick Check
+  description: 'Run a single yeast or GGM experiment script with nrep=1 and one parameter-grid
+    index to verify the environment and produce a small set of AUC/accuracy values.
+    Expected wall-time ~10–30 minutes on CPU.
+
+    '
+- id: full_experiments
+  label: Full Experiments
+  description: 'Sweep all HPC experiment scripts (GGM, ICA, NormalEstimation, snp100,
+    yeast) across the full parameter grids using the bash scripts in HPC/bashscripts/.
+    Each script runs 10 000 training epochs per configuration. Expect 6–24 hours on
+    a 16-core CPU or faster with a single GPU.
+
+    '
+- id: plot_reproduction
+  label: Plot Reproduction
+  description: 'After full experiments, run the Jupyter notebooks in plotcode/ to
+    regenerate all paper figures and metric summaries from saved results.
+
+    '
+dataset_download:
+  primary_method: bundled
+  sources:
+  - kind: archive
+    url: https://deepscientist.cc/AISB/009_scoremissing
+    access: public
+    note: 'Full snapshot including real-world data tensors (yeast, S&P 100) in real_world_experiments/RealData/.
+      Simulated data is generated on-the-fly by the experiment scripts. Archive is
+      a zip file.
+
+      '
+  notes:
+  - All datasets are either bundled or generated programmatically; no additional downloads
+    required.
+  - Total disk footprint after extraction is modest (well under 10 GB including results).
+credential_requirements:
+  mode: none
+  items: []
+  notes: []
+resources:
+  minimum:
+    cpu_cores: 8
+    ram_gb: 32
+    disk_gb: 80
+    gpu_count: 0
+    gpu_vram_gb: 0
+  recommended:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 150
+    gpu_count: 1
+    gpu_vram_gb: 16
+environment:
+  python: '3.10'
+  cuda: null
+  pytorch: 1.12.1
+  flash_attn: null
+  key_packages:
+  - jax==0.3.25
+  - jaxlib==0.3.25
+  - tensorflow==2.11.0
+  - scikit-learn
+  - tqdm
+  notes:
+  - CPU-only execution is plausible for the minimum route; GPU accelerates PyTorch
+    training loops.
+  - The MSM package is PyTorch-based; JAX/TensorFlow dependencies support specific
+    experiment variants.
+  - An environment.yml is bundled in the snapshot for conda-based setup.
+  - See the bundled README and requirements for the full dependency set.
+risk_flags:
+- mixed_framework_dependencies
+- long_sweep_walltime
+risk_notes:
+- 'The project mixes PyTorch, JAX, and TensorFlow dependencies. Resolving compatible
+  versions in one environment may require careful pinning or separate conda environments.
+
+  '
+- 'Full experiment sweeps (all five sub-tasks × all parameter-grid indices × multiple
+  repeats) can take many hours. Plan for at least 6 hours wall-time on recommended
+  hardware.
+
+  '
+- 'No benchmark execution was performed during the packaging pass; metrics are code-backed
+  but not yet runtime-verified.
+
+  '
+recommended_when: 'Use this benchmark when you want a self-contained probabilistic
+  missing-data task combining theoretical score-matching methods with both synthetic
+  and real-world graphical model evaluation. Good fit for agents that can navigate
+  multi-script experiment pipelines, handle CLI-parameterised sweeps, and interpret
+  structure-recovery metrics (AUC, TPR, FPR).
+
+  '
+not_recommended_when: 'Avoid if you need a lightweight single-script benchmark, a
+  task centred on large pretrained foundation models, or a benchmark with GPU-intensive
+  deep learning. Also not suitable if you cannot install mixed PyTorch/JAX/TensorFlow
+  environments.
+
+  '
+paper:
+  title: Score Matching with Missing Data
+  venue: ICML 2025 Spotlight
+  year: 2025
+  url: https://arxiv.org/abs/2506.00557
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.009_scoremissing.zip
+  archive_type: zip
+  local_dir_name: paper-9-ScoreMissing
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.009_scoremissing.zip
+  sha256: bed639544fdb6e317d4a5b2c2663c8765aa508e39566551817d1586541cccb27
+  size_bytes: 27750054
+commercial:
+  annual_fee: null
+display:
+  palette_seed: moss-indigo-density
+  art_style: statistical-notebook
+  accent_priority: medium
+image_path: ../image/009_aisb.t3.009_scoremissing.jpg

package/AISB/catalog/aisb.t3.009_scoremissing.zh.yaml

@@ -0,0 +1,119 @@
+schema_version: 1
+id: aisb.t3.009_scoremissing
+name: 缺失数据的分数匹配
+version: 0.1.0
+one_line: '从部分观测的表格数据中学习完整分数函数，采用重要性加权和变分边际分数匹配方法，在模拟的GGM/ICA/正态估计任务以及真实的S&P 100和酵母图形模型恢复任务上进行评估。'
+task_description: '该基准测试打包了ICML 2025 Spotlight论文"Score Matching with Missing Data"的完整实验流程。核心任务是在训练样本存在缺失坐标且缺失机制为完全随机缺失（MCAR）的条件下，估计多元分布的分数（对数密度梯度）。提供了两种互补方法：（1）边际重要性加权（IW）分数匹配，在低维和小样本场景表现优异；（2）边际变分分数匹配，在高维问题（如高斯图形模型GGM估计）上更强。
+
+实验套件包含五个子任务，从HPC/目录下的Python脚本运行：GGM结构恢复（模拟）、ICA参数估计、截断/非截断正态参数估计、S&P 100 GGM估计和酵母GGM估计。每个脚本接受重复次数和参数网格索引作为命令行参数。指标（AUC、准确率、TPR、FPR）通过MSM/utils/data.py辅助函数在脚本内计算，并在plotcode/目录下的Jupyter笔记本中聚合。真实数据（酵母张量、S&P 100 CSV）打包在real_world_experiments/RealData/中。无需外部评估服务或API凭据；该基准测试完全自包含。'
+capability_tags:
+- research_code_optimization
+- missing_data
+- probabilistic_modeling
+- score_matching
+- tabular_ml
+- graphical_model_estimation
+aisb_direction: T3
+track_fit:
+- paper_track
+- benchmark_track
+task_mode: experiment_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: runnable
+support_level: advanced
+cost_band: medium
+time_band: 6-24h
+difficulty: hard
+data_access: public
+primary_outputs:
+- auc
+- accuracy
+- tpr
+- fpr
+- structure_recovery_report
+launch_profiles:
+- id: quick_check
+  label: 快速检查
+  description: '运行单个酵母或GGM实验脚本，nrep=1和一个参数网格索引，以验证环境并生成少量AUC/准确率值。预期CPU运行时间约10-30分钟。'
+- id: full_experiments
+  label: 完整实验
+  description: '使用HPC/bashscripts/目录下的bash脚本，对所有HPC实验脚本（GGM、ICA、NormalEstimation、snp100、yeast）进行完整参数网格扫描。每个脚本每个配置运行10000个训练轮次。预期在16核CPU或配备单GPU的更快硬件上需要6-24小时。'
+- id: plot_reproduction
+  label: 图表复现
+  description: '在完成完整实验后，运行plotcode/目录下的Jupyter笔记本，从保存的结果中重新生成所有论文图表和指标摘要。'
+dataset_download:
+  primary_method: bundled
+  sources:
+  - kind: archive
+    url: https://deepscientist.cc/AISB/009_scoremissing
+    access: public
+    note: '完整快照，包含真实世界数据张量（酵母、S&P 100）位于real_world_experiments/RealData/。模拟数据由实验脚本动态生成。压缩包为zip格式。'
+  notes:
+  - 所有数据集均为内置或程序生成，无需额外下载。
+  - 解压后磁盘占用很小（包括结果文件在内远低于10 GB）。
+credential_requirements:
+  mode: none
+  items: []
+  notes: []
+resources:
+  minimum:
+    cpu_cores: 8
+    ram_gb: 32
+    disk_gb: 80
+    gpu_count: 0
+    gpu_vram_gb: 0
+  recommended:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 150
+    gpu_count: 1
+    gpu_vram_gb: 16
+environment:
+  python: '3.10'
+  cuda: null
+  pytorch: 1.12.1
+  flash_attn: null
+  key_packages:
+  - jax==0.3.25
+  - jaxlib==0.3.25
+  - tensorflow==2.11.0
+  - scikit-learn
+  - tqdm
+  notes:
+  - 最小配置下支持纯CPU执行；GPU可加速PyTorch训练循环。
+  - MSM包基于PyTorch；JAX/TensorFlow依赖支持特定的实验变体。
+  - 快照中捆绑了conda环境配置文件environment.yml。
+  - 详见捆绑的README和requirements文件以了解完整依赖集。
+risk_flags:
+- mixed_framework_dependencies
+- long_sweep_walltime
+risk_notes:
+- '项目混合了PyTorch、JAX和TensorFlow依赖。在单一环境中解决兼容版本可能需要仔细固定版本或使用独立的conda环境。'
+- '完整实验扫描（所有五个子任务×所有参数网格索引×多次重复）可能需要数小时。在推荐硬件上至少预留6小时运行时间。'
+- '打包过程中未执行基准测试；指标有代码支持但尚未运行时验证。'
+recommended_when: '当你需要一个自包含的概率缺失数据任务，结合理论分数匹配方法与合成和真实世界图形模型评估时使用。非常适合能够处理多脚本实验流程、处理CLI参数化扫描以及解读结构恢复指标（AUC、TPR、FPR）的智能体。'
+not_recommended_when: '避免用于以下场景：需要轻量级单脚本基准测试、以大型预训练基础模型为中心的任务、或GPU密集型深度学习基准测试。如果无法安装混合PyTorch/JAX/TensorFlow环境，也不适用。'
+paper:
+  title: Score Matching with Missing Data
+  venue: ICML 2025 Spotlight
+  year: 2025
+  url: https://arxiv.org/abs/2506.00557
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.009_scoremissing.zip
+  archive_type: zip
+  local_dir_name: paper-9-ScoreMissing
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.009_scoremissing.zip
+  sha256: bed639544fdb6e317d4a5b2c2663c8765aa508e39566551817d1586541cccb27
+  size_bytes: 27750054
+commercial:
+  annual_fee: null
+display:
+  palette_seed: moss-indigo-density
+  art_style: statistical-notebook
+  accent_priority: medium
+image_path: ../image/009_aisb.t3.009_scoremissing.jpg

package/AISB/catalog/aisb.t3.010_suitabilityfilter.yaml

@@ -0,0 +1,221 @@
+schema_version: 1
+id: aisb.t3.010_suitabilityfilter
+name: 'Suitability Filter: A Statistical Framework for Classifier Evaluation in Real-World
+  Deployment Settings'
+version: 0.1.0
+one_line: 'Optimize a deployment-time statistical filter that uses non-inferiority
+  hypothesis testing on model-derived suitability signals to decide whether a classifier''s
+  accuracy has degraded beyond an acceptable margin on unlabeled, distribution-shifted
+  user data (evaluated on WILDS FMoW/RxRx1/CivilComments).
+
+  '
+task_description: 'This benchmark packages the Suitability Filter framework from ICML
+  2025. The core task is to improve a statistical decision system that determines
+  whether a pre-trained classifier remains safe to deploy under covariate shift, without
+  access to ground-truth labels on the target data.
+
+  The filter works by: (1) extracting suitability signals (max softmax, predictive
+  entropy, logit statistics, loss/energy features — 12 signals total) from both labeled
+  test data and unlabeled user data; (2) training a lightweight prediction-correctness
+  estimator (logistic regression by default, calibrated via CalibratedClassifierCV)
+  on a labeled holdout set to map signals to per-sample correctness probabilities
+  p_c; (3) comparing the p_c distributions between test and user data via a one-sided
+  non-inferiority test (Welch''s t-test or z-test) with configurable margin m and
+  significance level alpha; (4) outputting SUITABLE (reject H0) or INCONCLUSIVE.
+
+  The packaged evaluation route runs cached FMoW-WILDS features across multiple seeds
+  (0, 1, 2), fold configurations, feature subsets (confidence/logit/loss-energy),
+  and classifier hyperparameters. Primary metrics are the OOD suitability score (ROC-AUC
+  of the filter''s p-values against ground-truth suitability labels across ~29k experiments),
+  the raw p-value, and the binary suitability decision. Optimization targets include
+  the correctness estimator''s regularization (C sweep), signal subset selection,
+  fold count, calibration strategy, and Stouffer''s method for combining p-values
+  across feature subsets.
+
+  Pre-extracted feature caches and split indices are bundled under results/features/
+  and results/split_indices/. The WILDS datasets themselves (FMoW, RxRx1, CivilComments)
+  and pretrained model weights are needed only if regenerating features from scratch;
+  the default execution route uses cached features and does not require downloading
+  WILDS data.
+
+  '
+capability_tags:
+- research_code_optimization
+- classifier_evaluation
+- distribution_shift
+- statistical_testing
+- reliability
+- hypothesis_testing
+- deployment_monitoring
+aisb_direction: T3
+track_fit:
+- paper_track
+- benchmark_track
+task_mode: experiment_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: runnable
+support_level: turnkey
+cost_band: medium
+time_band: 6-24h
+difficulty: hard
+data_access: public
+primary_outputs:
+- ood_suitability_score
+- p_value
+- suitability_decision
+launch_profiles:
+- id: quick_check
+  label: Quick Check
+  description: 'Run the packaged suitability filter logic on cached FMoW features
+    for a single seed to verify statistical test outputs (p-value, suitability decision)
+    and basic ROC-AUC.
+
+    '
+- id: shift_eval
+  label: Full Shift Evaluation
+  description: 'Run the complete FMoW-WILDS deployment-shift evaluation across all
+    seeds (0, 1, 2), feature subsets, fold configurations, and classifier settings
+    using run_fmow.py. Reports OOD suitability ROC-AUC across ~29k experiment configurations.
+
+    '
+- id: lr_tune
+  label: LR Hyperparameter Sweep
+  description: 'Run run_fmow_lr_tune.py to sweep logistic regression regularization
+    C values (0.01, 0.1, 1.0, 5.0, 10.0) on OOD data for seed=0 to find optimal correctness
+    estimator configuration.
+
+    '
+- id: seed_search
+  label: Seed Search
+  description: 'Run run_fmow_seed_search.py variants to explore reproducibility and
+    seed sensitivity of the suitability filter decisions.
+
+    '
+dataset_download:
+  primary_method: bundled_cache
+  sources:
+  - kind: bundled
+    url: null
+    access: public
+    note: 'Pre-extracted feature caches (results/features/*.pkl) and split indices
+      (results/split_indices/*.pkl) are included in the snapshot. No download needed
+      for the default execution route.
+
+      '
+  - kind: pip_package
+    url: https://pypi.org/project/wilds/
+    access: public
+    note: 'WILDS 2.0.0 package needed for feature re-extraction from raw data. FMoW-WILDS,
+      RxRx1-WILDS, and CivilComments-WILDS datasets are downloaded on first use via
+      the wilds library. FMoW alone is ~50 GB; all three together can exceed 100 GB.
+
+      '
+  notes:
+  - Default route uses bundled cached features; WILDS download only needed for feature
+    regeneration.
+  - Pretrained model weights (ERM) are required for feature regeneration but not for
+    cached route.
+credential_requirements:
+  mode: none
+  items: []
+  notes:
+  - No API keys or credentials required for any execution route.
+resources:
+  minimum:
+    cpu_cores: 8
+    ram_gb: 32
+    disk_gb: 100
+    gpu_count: 1
+    gpu_vram_gb: 16
+  recommended:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 200
+    gpu_count: 1
+    gpu_vram_gb: 24
+environment:
+  python: 3.11.9
+  cuda: '11.7'
+  pytorch: 2.0.0
+  flash_attn: null
+  key_packages:
+  - transformers==4.41.2
+  - torch-geometric==2.5.3
+  - wilds==2.0.0
+  - scikit-learn
+  - scipy
+  - statsmodels
+  - pandas
+  - numpy
+  notes:
+  - See the bundled requirements.txt for the full conda environment specification.
+  - GPU is used for feature extraction (torch.device("cuda" if available)); the cached-feature
+    route runs statistical tests on CPU only (scikit-learn, scipy).
+  - The run_fmow.py scripts set sys.path to '/repo' — adjust if running outside the
+    expected container.
+risk_flags:
+- large_experiment_matrix
+- external_dataset_for_regeneration
+risk_notes:
+- 'The full evaluation route runs ~29k experiment configurations across seeds, folds,
+  feature subsets, and margins. Wall time can reach several hours on CPU.
+
+  '
+- 'Feature regeneration from raw WILDS data requires downloading FMoW-WILDS (~50 GB)
+  and pretrained ERM model weights, but this is not needed for the default cached
+  route.
+
+  '
+- 'The run scripts hardcode sys.path to ''/repo'' and os.chdir(''/repo''); execution
+  outside the expected container environment may require path adjustments.
+
+  '
+- 'No runtime execution was performed during the packaging pass; metric values should
+  be verified by running the benchmark.
+
+  '
+recommended_when: 'Use this benchmark when you want a model-monitoring optimization
+  task that combines statistical hypothesis testing (non-inferiority tests) with realistic
+  WILDS deployment shifts. Well-suited for research on deployment-time safety, distribution
+  shift detection, unsupervised accuracy estimation, and selective prediction. The
+  cached-feature route makes iteration fast without needing to re-run model inference.
+
+  '
+not_recommended_when: 'Not suitable if you need a tiny or quick-to-iterate toy benchmark,
+  cannot allocate 32+ GB RAM for the experiment matrices, or want to evaluate non-classification
+  tasks. Also not ideal if you need end-to-end training of the base classifier — this
+  benchmark focuses on the filter/monitoring layer, not the upstream model training.
+
+  '
+paper:
+  title: 'Suitability Filter: A Statistical Framework for Classifier Evaluation in
+    Real-World Deployment Settings'
+  authors:
+  - Angéline Pouget
+  - Mohammad Yaghini
+  - Stephan Rabanser
+  - Nicolas Papernot
+  venue: ICML 2025
+  year: 2025
+  url: https://arxiv.org/abs/2505.22356
+  volume: 267
+  publisher: PMLR
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.010_suitabilityfilter.zip
+  archive_type: zip
+  local_dir_name: paper-10-SuitabilityFilter
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.010_suitabilityfilter.zip
+  sha256: 146f3477fb91d43974f51dac9013768be094e9ad355d2196c820915f062645f1
+  size_bytes: 54606221
+commercial:
+  annual_fee: null
+display:
+  palette_seed: sand-graphite-verify
+  art_style: clean-statistical
+  accent_priority: high
+image_path: ../image/010_aisb.t3.010_suitabilityfilter.jpg

package/AISB/catalog/aisb.t3.010_suitabilityfilter.zh.yaml

@@ -0,0 +1,141 @@
+schema_version: 1
+id: aisb.t3.010_suitabilityfilter
+name: "适用性过滤器：真实世界部署场景下分类器评估的统计框架"
+version: 0.1.0
+one_line: "优化一个部署时的统计过滤器，使用非劣性假设检验对模型衍生的适用性信号进行测试，以判断分类器在无标签、分布偏移的用户数据上的准确率是否已退化到不可接受的阈值（基于WILDS FMoW/RxRx1/CivilComments评估）。"
+task_description: |
+  本基准测试打包了ICML 2025的适用性过滤器框架。核心任务是改进一个统计决策系统，在无法获得目标数据真实标签的情况下，判断预训练分类器在协变量偏移下是否仍然安全可部署。
+
+  过滤器的工作流程为：(1) 从带标签的测试数据和无标签的用户数据中提取适用性信号（最大softmax值、预测熵、logit统计量、损失/能量特征——共12个信号）；(2) 在带标签的保留集上训练一个轻量级的预测正确性估计器（默认使用逻辑回归，通过CalibratedClassifierCV进行校准），将信号映射到每个样本的正确概率p_c；(3) 通过单侧非劣性检验（Welch''s t检验或z检验，可配置边际m和显著性水平alpha）比较测试数据和用户数据之间的p_c分布；(4) 输出SUITABLE（拒绝H0）或INCONCLUSIVE。
+
+  打包的评估流程跨多个随机种子（0、1、2）、折数配置、特征子集（置信度/logit/损失-能量）和分类器超参数运行缓存的FMoW-WILDS特征。主要指标包括：OOD适用性得分（过滤器p值相对于真实适用性标签的ROC-AUC，跨约29k个实验）、原始p值和二元适用性决策。优化目标包括正确性估计器的正则化（C扫描）、信号子集选择、折数、校准策略，以及用于组合特征子集p值的Stouffer方法。
+
+  预提取的特征缓存和分割索引分别打包在results/features/和results/split_indices/下。仅在从头重新生成特征时才需要WILDS数据集本身（FMoW、RxRx1、CivilComments）和预训练模型权重；默认执行流程使用缓存特征，无需下载WILDS数据。
+capability_tags:
+- 研究代码优化
+- 分类器评估
+- 分布偏移
+- 统计检验
+- 可靠性
+- 假设检验
+- 部署监控
+aisb_direction: T3
+track_fit:
+- paper_track
+- benchmark_track
+task_mode: experiment_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: runnable
+support_level: turnkey
+cost_band: medium
+time_band: 6-24h
+difficulty: hard
+data_access: public
+primary_outputs:
+- ood_suitability_score
+- p_value
+- suitability_decision
+launch_profiles:
+- id: quick_check
+  label: 快速检查
+  description: "在缓存的FMoW特征上运行打包的适用性过滤器逻辑，使用单个种子验证统计检验输出（p值、适用性决策）和基本ROC-AUC。"
+- id: shift_eval
+  label: 完整偏移评估
+  description: "跨所有种子（0、1、2）、特征子集、折数配置和分类器设置运行完整的FMoW-WILDS部署偏移评估。使用run_fmow.py，报告约29k个实验配置下的OOD适用性ROC-AUC。"
+- id: lr_tune
+  label: LR超参数扫描
+  description: "运行run_fmow_lr_tune.py，在OOD数据上扫描逻辑回归正则化C值（0.01、0.1、1.0、5.0、10.0），种子=0，以找到最优的正确性估计器配置。"
+- id: seed_search
+  label: 种子搜索
+  description: "运行run_fmow_seed_search.py变体，探索适用性过滤器决策的可重复性和种子敏感性。"
+dataset_download:
+  primary_method: bundled_cache
+  sources:
+  - kind: bundled
+    url: null
+    access: public
+    note: "预提取的特征缓存（results/features/*.pkl）和分割索引（results/split_indices/*.pkl）已包含在快照中。默认执行流程无需下载。"
+  - kind: pip_package
+    url: https://pypi.org/project/wilds/
+    access: public
+    note: "WILDS 2.0.0包用于从原始数据重新提取特征。FMoW-WILDS、RxRx1-WILDS和CivilComments-WILDS数据集通过wilds库首次使用时下载。仅FMoW约50GB；三个数据集合计可超过100GB。"
+notes:
+- 默认流程使用打包的缓存特征；仅在需要重新生成特征时才需要下载WILDS。
+- 预训练模型权重（ERM）用于特征重新生成，缓存流程无需此权重。
+credential_requirements:
+  mode: none
+  items: []
+  notes:
+  - 任何执行流程都无需API密钥或凭据。
+resources:
+  minimum:
+    cpu_cores: 8
+    ram_gb: 32
+    disk_gb: 100
+    gpu_count: 1
+    gpu_vram_gb: 16
+  recommended:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 200
+    gpu_count: 1
+    gpu_vram_gb: 24
+environment:
+  python: 3.11.9
+  cuda: "11.7"
+  pytorch: 2.0.0
+  flash_attn: null
+  key_packages:
+  - transformers==4.41.2
+  - torch-geometric==2.5.3
+  - wilds==2.0.0
+  - scikit-learn
+  - scipy
+  - statsmodels
+  - pandas
+  - numpy
+  notes:
+  - 完整的conda环境规范请参见打包的requirements.txt。
+  - GPU用于特征提取（torch.device("cuda" if available)）；缓存特征流程仅在CPU上运行统计检验（scikit-learn、scipy）。
+  - run_fmow.py脚本设置sys.path为'/repo'——如在预期容器外运行需调整路径。
+risk_flags:
+- large_experiment_matrix
+- external_dataset_for_regeneration
+risk_notes:
+- "完整评估流程跨种子、折数、特征子集和边际运行约29k个实验配置。CPU上运行时间可达数小时。"
+- "从原始WILDS数据重新生成特征需要下载FMoW-WILDS（约50GB）和预训练ERM模型权重，但默认缓存流程无需此操作。"
+- "运行脚本硬编码sys.path为'/repo'和os.chdir('/repo')；在预期容器环境外执行可能需要调整路径。"
+- "打包过程中未执行运行时测试；指标值应通过运行基准测试进行验证。"
+recommended_when: "当您需要一个结合统计假设检验（非劣性检验）与真实WILDS部署偏移的模型监控优化任务时使用此基准测试。非常适合研究部署时安全性、分布偏移检测、无监督准确率估计和选择性预测。缓存特征流程使迭代快速，无需重新运行模型推理。"
+not_recommended_when: "如果需要一个小型的或快速迭代的玩具基准测试、无法分配32GB以上内存用于实验矩阵，或需要评估非分类任务，则不适合使用。此外，如果您需要端到端训练基础分类器——本基准测试专注于过滤器/监控层，而非上游模型训练——也不太理想。"
+paper:
+  title: "Suitability Filter: A Statistical Framework for Classifier Evaluation in Real-World Deployment Settings"
+  authors:
+  - Angéline Pouget
+  - Mohammad Yaghini
+  - Stephan Rabanser
+  - Nicolas Papernot
+  venue: ICML 2025
+  year: 2025
+  url: https://arxiv.org/abs/2505.22356
+  volume: 267
+  publisher: PMLR
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.010_suitabilityfilter.zip
+  archive_type: zip
+  local_dir_name: paper-10-SuitabilityFilter
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.010_suitabilityfilter.zip
+  sha256: 146f3477fb91d43974f51dac9013768be094e9ad355d2196c820915f062645f1
+  size_bytes: 54606221
+commercial:
+  annual_fee: null
+display:
+  palette_seed: sand-graphite-verify
+  art_style: clean-statistical
+  accent_priority: high
+image_path: ../image/010_aisb.t3.010_suitabilityfilter.jpg