@researai/deepscientist 1.5.17 → 1.6.0

package/AISB/catalog/aisb.t3.036_weakstrongpref.yaml

@@ -0,0 +1,210 @@
+id: aisb.t3.036_weakstrongpref
+name: COWEST
+version: 0.1.0
+one_line: Weak-strong model collaboration via DPO, SFT, and preference-aligned candidate
+  aggregation to improve response quality on specialized QA tasks.
+task_description: 'This benchmark implements COWEST (Collaboration method between
+  Weak and Strong models), a framework that pairs a specialized weak model with a
+  general strong model to tackle reasoning tasks requiring domain-specific knowledge.
+  During inference, the weak model generates initial drafts and background information
+  while the strong model refines them using advanced reasoning. During training, the
+  framework quantifies the weak model''s contribution influence and constructs preference
+  pairs to guide DPO fine-tuning of the weak model. The pipeline supports both individual
+  model baselines and the full collaborative workflow.
+
+  '
+task_mode: experiment_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: external_eval_required
+support_level: advanced
+time_band: 1d+
+cost_band: high
+difficulty: hard
+data_access: public
+primary_outputs:
+- f1_ifqa
+- em_ifqa
+- preference_checkpoint
+- aligned_candidates
+launch_profiles:
+- id: quick_eval
+  label: Quick Eval
+  description: Run one packaged evaluation route on the weak-strong preference stack
+    using pretrained checkpoints.
+- id: full_alignment
+  label: Full Alignment
+  description: Run the complete SFT, DPO, and preference-aligned aggregation workflow
+    from scratch.
+- id: inference_only
+  label: Inference Only
+  description: Run inference pipelines for weak-only, strong-only, and weak-strong
+    collaborative modes without training.
+dataset_download:
+  primary_method: mixed
+  sources:
+  - type: url
+    url: https://deepscientist.cc/AISB/036_weakstrongpref
+    format: zip
+  - type: huggingface
+    note: Some datasets may require Hugging Face authentication for IFQA
+  notes:
+  - Archive extracts to paper-36-WeakStrongPref
+  - IFQA dataset components included in bundle
+credential_requirements:
+  mode: partial
+  items:
+  - HuggingFace token for Llama-3-8B-Instruct access
+  - Azure OpenAI API key for GPT-35-turbo (used as strong model in inference)
+  notes:
+  - Strong model can be swapped for open-weight alternatives to avoid proprietary
+    API requirements
+environment:
+  python: '3.10'
+  cuda: '11.8'
+  pytorch: 2.0.1
+  flash_attn: null
+  key_packages:
+  - bitsandbytes==0.41.1
+  - peft
+  - transformers
+  - datasets
+  - sklearn
+  - torch>=2.0.0
+  notes:
+  - Bundled requirements pin torch to <=2.0.1 and rely on a custom Transformers fork
+    for left-padding behavior
+  - See bundled README/requirements.txt for the full dependency set
+  - Requires multi-GPU setup for full training pipeline
+resources:
+  minimum:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 150
+    gpu_count: 1
+    gpu_vram_gb: 24
+    notes: Single GPU feasible only for inference; training requires multi-GPU for
+      LoRA with 8B models
+  recommended:
+    cpu_cores: 32
+    ram_gb: 128
+    disk_gb: 300
+    gpu_count: 4
+    gpu_vram_gb: 24
+    notes: SFT script configured for 4x24GB GPUs (WORLD_SIZE=4); DPO uses 3-GPU setup
+risk_flags:
+- proprietary_model_dependency
+- multi_gpu_training_complexity
+- dataset_size
+risk_notes:
+- Full pipeline requires access to Llama-3-8B-Instruct via HuggingFace
+- DPO training uses RMSProp optimizer (non-standard choice)
+- Azure OpenAI dependency for strong model inference can introduce API latency and
+  cost
+recommended_when: 'Use this benchmark when you want to study weak-strong model collaboration
+  with alignment objectives, explore preference data construction from collaborative
+  feedback, or evaluate alignment methods on knowledge-intensive QA tasks requiring
+  specialized domain knowledge.
+
+  '
+not_recommended_when: 'Do not use this if you cannot support LLaMA-class 8B model
+  training with LoRA, or if you need a lightweight benchmark without DPO-style optimization.
+  Not recommended when Azure API access is unavailable and no alternative strong model
+  is configured.
+
+  '
+paper:
+  title: Synergistic Weak-Strong Collaboration by Aligning Preferences
+  venue: ACL 2025
+  year: 2025
+  url: https://arxiv.org/abs/2504.15188
+  authors:
+  - Yizhu Jiao
+  - Xuchao Zhang
+  - Zhaoyang Wang
+  - Yubo Ma
+  - Zhun Deng
+  - Rujia Wang
+  - Chetan Bansal
+  - Saravan Rajmohan
+  - Jiawei Han
+  - Huaxiu Yao
+  institutions:
+  - Microsoft Research
+  - University of Illinois Urbana-Champaign
+  - University of North Carolina at Chapel Hill
+  - Nanyang Technological University
+display:
+  palette_seed: coral-ink-alignment
+  art_style: preference-dashboard
+  accent_priority: high
+image_path: ../image/036_aisb.t3.036_weakstrongpref.jpg
+capability_tags:
+- research_code_optimization
+- alignment
+- preference_optimization
+- large_language_models
+- evaluation
+- weak_to_strong_generalization
+- collaborative_inference
+aisb_direction: T3
+track_fit:
+- paper_track
+- benchmark_track
+execution_notes:
+  stages:
+  - id: sft
+    script: sft.sh
+    model: meta-llama/Meta-Llama-3-8B-Instruct
+    dataset: ifqa_sft
+    epochs: 3
+    batch_per_device: 1
+    gradient_accumulation: 8
+    lora_r: 16
+    lora_alpha: 16
+    lr: 1.41e-05
+    output: ./lora_weights/llama3-8b-sft-ifqa
+  - id: dpo
+    script: dpo.sh
+    model: meta-llama/Meta-Llama-3-8B-Instruct
+    dataset: ifqa_dpo
+    epochs: 1
+    batch_per_device: 1
+    gradient_accumulation: 16
+    lora_r: 16
+    lora_alpha: 16
+    lr: 1.41e-05
+    optimizer: rmsprop
+    output: ./lora_weights/llama3-8b-dpo-ifqa
+  - id: inference
+    script: inference_weak_strong_models.py
+    weak_model: local/lora_adapters
+    strong_model: gpt-35-turbo (or open-weight alternative)
+    modes:
+    - weak_only
+    - strong_only
+    - weak_strong_collaborative
+  metric_protocol:
+    f1_ifqa:
+      origin: code-backed
+      source: evaluation.py
+      description: F1 score computed over question-level answers for IFQA dataset
+    em_ifqa:
+      origin: code-backed
+      source: evaluation.py
+      description: Exact match metric for IFQA question answering
+  caveats:
+  - No benchmark execution performed in packaging pass; metric values require trusted
+    runtime output
+  - Static audit confirms executable anchors exist for all staged metrics
+  - README absent in current snapshot; follow AGENTS.md code anchors
+download:
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.036_weakstrongpref.zip
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.036_weakstrongpref.zip
+  archive_type: zip
+  sha256: e98abaf4d388b77c120e970e947bd56f2be0276171180bf6c9ee3d5e144a77e7
+  size_bytes: 55245

package/AISB/catalog/aisb.t3.036_weakstrongpref.zh.yaml

@@ -0,0 +1,194 @@
+id: aisb.t3.036_weakstrongpref
+name: COWEST
+version: 0.1.0
+one_line: 通过DPO、SFT和偏好对齐候选聚合实现弱强模型协作，提升专业问答任务的回复质量。
+task_description: >
+  本基准测试实现了COWEST（弱强模型协作方法），该框架将专业弱模型与通用强模型配对，
+  以处理需要领域特定知识的推理任务。在推理阶段，弱模型生成初始草稿和背景信息，
+  强模型则利用高级推理能力对其进行精炼。在训练阶段，框架量化弱模型的贡献影响，
+  并构建偏好对来指导对弱模型的DPO微调。该流程支持独立模型基线和完整协作工作流。
+
+task_mode: experiment_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: external_eval_required
+support_level: advanced
+time_band: 1d+
+cost_band: high
+difficulty: hard
+data_access: public
+primary_outputs:
+- f1_ifqa
+- em_ifqa
+- preference_checkpoint
+- aligned_candidates
+launch_profiles:
+- id: quick_eval
+  label: 快速评估
+  description: 使用预训练检查点在弱强偏好堆栈上运行一个打包的评估流程。
+- id: full_alignment
+  label: 完整对齐
+  description: 从零开始运行完整的SFT、DPO和偏好对齐聚合工作流。
+- id: inference_only
+  label: 仅推理
+  description: 运行弱模型仅、强模型仅和弱强协作模式的推理流程，无需训练。
+dataset_download:
+  primary_method: mixed
+  sources:
+  - type: url
+    url: https://deepscientist.cc/AISB/036_weakstrongpref
+    format: zip
+  - type: huggingface
+    note: 部分数据集可能需要Hugging Face认证以访问IFQA
+  notes:
+  - 压缩包解压至paper-36-WeakStrongPref
+  - IFQA数据集组件已包含在包中
+credential_requirements:
+  mode: partial
+  items:
+  - HuggingFace token用于访问Llama-3-8B-Instruct
+  - Azure OpenAI API密钥用于GPT-35-turbo（在推理中作为强模型）
+  notes:
+  - 强模型可替换为开源权重替代方案以避免使用专有API
+environment:
+  python: '3.10'
+  cuda: '11.8'
+  pytorch: 2.0.1
+  flash_attn: null
+  key_packages:
+  - bitsandbytes==0.41.1
+  - peft
+  - transformers
+  - datasets
+  - sklearn
+  - torch>=2.0.0
+  notes:
+  - 打包的依赖项将torch固定在<=2.0.1，并依赖自定义Transformers分支以实现左填充行为
+  - 详见打包的README/requirements.txt了解完整依赖集
+  - 完整训练流程需要多GPU配置
+resources:
+  minimum:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 150
+    gpu_count: 1
+    gpu_vram_gb: 24
+    notes: 单GPU仅可用于推理；训练需要多GPU以支持8B模型的LoRA
+  recommended:
+    cpu_cores: 32
+    ram_gb: 128
+    disk_gb: 300
+    gpu_count: 4
+    gpu_vram_gb: 24
+    notes: SFT脚本配置为4张24GB GPU（WORLD_SIZE=4）；DPO使用3-GPU配置
+risk_flags:
+- proprietary_model_dependency
+- multi_gpu_training_complexity
+- dataset_size
+risk_notes:
+- 完整流程需要通过HuggingFace访问Llama-3-8B-Instruct
+- DPO训练使用RMSProp优化器（非标准选择）
+- 强模型推理依赖Azure OpenAI可能引入API延迟和成本
+recommended_when: >
+  当您希望研究带对齐目标的弱强模型协作、探索从协作反馈构建偏好数据，
+  或在需要专业领域知识的知识密集型问答任务上评估对齐方法时使用此基准测试。
+
+not_recommended_when: >
+  如果您无法支持LLaMA类8B模型的LoRA训练，或需要不含DPO风格优化的轻量级基准测试，
+  请勿使用。当Azure API不可用且未配置替代强模型时也不推荐使用。
+
+paper:
+  title: Synergistic Weak-Strong Collaboration by Aligning Preferences
+  venue: ACL 2025
+  year: 2025
+  url: https://arxiv.org/abs/2504.15188
+  authors:
+  - Yizhu Jiao
+  - Xuchao Zhang
+  - Zhaoyang Wang
+  - Yubo Ma
+  - Zhun Deng
+  - Rujia Wang
+  - Chetan Bansal
+  - Saravan Rajmohan
+  - Jiawei Han
+  - Huaxiu Yao
+  institutions:
+  - Microsoft Research
+  - University of Illinois Urbana-Champaign
+  - University of North Carolina at Chapel Hill
+  - Nanyang Technological University
+display:
+  palette_seed: coral-ink-alignment
+  art_style: preference-dashboard
+  accent_priority: high
+image_path: ../image/036_aisb.t3.036_weakstrongpref.jpg
+capability_tags:
+- research_code_optimization
+- alignment
+- preference_optimization
+- large_language_models
+- evaluation
+- weak_to_strong_generalization
+- collaborative_inference
+aisb_direction: T3
+track_fit:
+- paper_track
+- benchmark_track
+execution_notes:
+  stages:
+  - id: sft
+    script: sft.sh
+    model: meta-llama/Meta-Llama-3-8B-Instruct
+    dataset: ifqa_sft
+    epochs: 3
+    batch_per_device: 1
+    gradient_accumulation: 8
+    lora_r: 16
+    lora_alpha: 16
+    lr: 1.41e-05
+    output: ./lora_weights/llama3-8b-sft-ifqa
+  - id: dpo
+    script: dpo.sh
+    model: meta-llama/Meta-Llama-3-8B-Instruct
+    dataset: ifqa_dpo
+    epochs: 1
+    batch_per_device: 1
+    gradient_accumulation: 16
+    lora_r: 16
+    lora_alpha: 16
+    lr: 1.41e-05
+    optimizer: rmsprop
+    output: ./lora_weights/llama3-8b-dpo-ifqa
+  - id: inference
+    script: inference_weak_strong_models.py
+    weak_model: local/lora_adapters
+    strong_model: gpt-35-turbo (or open-weight alternative)
+    modes:
+    - weak_only
+    - strong_only
+    - weak_strong_collaborative
+  metric_protocol:
+    f1_ifqa:
+      origin: code-backed
+      source: evaluation.py
+      description: F1 score computed over question-level answers for IFQA dataset
+    em_ifqa:
+      origin: code-backed
+      source: evaluation.py
+      description: Exact match metric for IFQA question answering
+  caveats:
+  - No benchmark execution performed in packaging pass; metric values require trusted
+    runtime output
+  - Static audit confirms executable anchors exist for all staged metrics
+  - README absent in current snapshot; follow AGENTS.md code anchors
+download:
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.036_weakstrongpref.zip
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.036_weakstrongpref.zip
+  archive_type: zip
+  sha256: e98abaf4d388b77c120e970e947bd56f2be0276171180bf6c9ee3d5e144a77e7
+  size_bytes: 55245

package/AISB/catalog/aisb.t3.037_dementiamask.yaml

@@ -0,0 +1,172 @@
+id: aisb.t3.037_dementiamask
+name: DementiaMask
+version: 0.1.0
+one_line: Debiasing benchmark for speech-based dementia detection using weight masking
+  to isolate gender-related confounding in transformer models.
+task_description: 'This packaged benchmark covers gender-confound mitigation in speech-based
+  dementia detection using two weight masking methods: Extended Confounding Filter
+  (ECF) and Dual Filter (DF). The workflow trains a BERT-based classifier on speech-derived
+  transcripts, then applies gradient-based weight tracking to identify and ablate
+  parameters encoding confounding information. Primary evaluation measures AUPRC and
+  delta-FPR (false positive rate gap between demographic groups) to assess both diagnostic
+  performance and fairness under distribution shift. The benchmark generates train/eval/test
+  splits with controlled confounder distributions via cf_train_test_split, supporting
+  configurable confounder ratios and test set imbalances.
+
+  '
+task_mode: experiment_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: runnable
+support_level: advanced
+time_band: 2-6h
+cost_band: medium
+difficulty: medium
+data_access: restricted
+primary_outputs:
+- diagnostic_scores
+- debiasing_report
+- delta_fpr
+launch_profiles:
+- id: quick_eval
+  label: Quick Eval
+  description: Run a single DualFilter evaluation path with preconfigured hyperparameters
+    (alpha_train=0.2, mask_ratio=0.10, 5 repetitions).
+- id: full_train_eval
+  label: Full Train + Eval
+  description: Run the complete masking-based debiasing workflow with configurable
+    confounder ratios via run_df.sh, including downstream fairness evaluation.
+- id: cf_evaluation
+  label: Confounding Filter Eval
+  description: Run the original Confounding Filter baseline via run_cf.sh for comparison.
+- id: ecf_evaluation
+  label: Extended Confounding Filter Eval
+  description: Run ECF with sequential layer unfreezing via run_ecf.sh.
+dataset_download:
+  primary_method: mixed
+  sources:
+  - deepscientist_url
+  notes:
+  - Archive contains pitts speech dataset with controlled confounder splits
+  - Data split parameters: train_pos_z0, train_pos_z1, alpha_test control confounder
+      distributions
+credential_requirements:
+  mode: external_access
+  items:
+  - deepscientist_download_token
+  notes:
+  - Requires ACL 2025 artifact access
+resources:
+  minimum:
+    cpu_cores: 8
+    ram_gb: 16
+    disk_gb: 30
+    gpu_count: 1
+    gpu_vram_gb: 8
+  recommended:
+    cpu_cores: 16
+    ram_gb: 32
+    disk_gb: 80
+    gpu_count: 1
+    gpu_vram_gb: 16
+environment:
+  python: '3.9'
+  cuda: '11.8'
+  pytorch: 2.1.0
+  flash_attn: null
+  key_packages:
+  - transformers
+  - torch
+  - numpy
+  - pandas
+  notes:
+  - Model weights (bert-base-uncased) must be available locally or downloaded
+  - See bundled requirements.txt for full dependency set
+risk_flags:
+- distribution_shift
+- demographic_disparity
+risk_notes:
+- Fairness-accuracy trade-off observed: debiasing may slightly reduce dementia detection
+    AUPRC
+- Distribution shift between train and test confounder distributions can affect delta_FPR
+- Gender confounder mitigation effectiveness varies with mask_ratio hyperparameter
+recommended_when: 'Use this benchmark when evaluating bias mitigation techniques in
+  clinical NLP, studying gender confounding in dementia detection, or comparing weight
+  masking approaches (ECF vs DF vs naive baselines) on speech-derived medical text
+  classification.
+
+  '
+not_recommended_when: 'Do not use when access to speech or clinical datasets is unavailable,
+  or when pure diagnostic accuracy without fairness considerations is the sole objective.
+
+  '
+paper:
+  title: Mitigating Confounding in Speech-Based Dementia Detection through Weight
+    Masking
+  authors: Sheng, Ding, Hur, Li, Cohen, Pakhomov
+  venue: ACL 2025 Main
+  year: 2025
+  url: https://arxiv.org/abs/2506.05610
+  pages: 8
+  abstract: 'Deep transformer models detect linguistic anomalies in patient transcripts
+    for early Alzheimer''s disease screening. This work addresses gender confounding
+    via Extended Confounding Filter (ECF) and Dual Filter (DF), which isolate and
+    ablate gender-associated weights. Results show transformer models tend to overfit
+    training distributions; disrupting gender-related weights yields a deconfounded
+    classifier with trade-off of slightly reduced dementia detection performance.
+
+    '
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.037_dementiamask.zip
+  archive_type: zip
+  local_dir_name: paper-37-DementiaMask
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.037_dementiamask.zip
+  sha256: d759476e5d109aae5e6177cd9e294126e5d9fd1e1e21af1b40d16f855455ba5c
+  size_bytes: 98039
+display:
+  palette_seed: wine-silver-clinic
+  art_style: clinical-notebook
+  accent_priority: medium
+image_path: ../image/037_aisb.t3.037_dementiamask.jpg
+metric_evidence:
+- metric: auprc
+  origin: run_experiment.py
+  status: code_backed
+  protocol: 'Average Precision Score (APS) computed per group (full, confounder_0,
+    confounder_1) after DualFilter training and ablation. Reported per-group and aggregated.
+
+    '
+- metric: delta_fpr
+  origin: run_experiment.py
+  status: code_backed
+  protocol: 'Delta FPR = FPR_confounder_1 - FPR_confounder_0, where FPR = 1 - recall_neg.
+    Measures fairness gap between demographic groups after debiasing.
+
+    '
+execution_anchors:
+  primary: run_experiment.py
+  shell_scripts:
+  - run_df.sh
+  - run_cf.sh
+  - run_ecf.sh
+  model_modules:
+  - src/model/weights_filter.py
+  preprocessing: src/preprocessing/data_generator.py
+hardware_notes: 'Single GPU execution sufficient. bert-base-uncased requires ~1.1GB
+  GPU memory for inference. Training with 5 repetitions and 30 epochs per rep on full
+  dataset may require 8+ GB VRAM with gradient accumulation. CPU-only execution not
+  recommended for training.
+
+  '
+dataset_notes: 'pitts speech dataset contains first-person narrative transcripts from
+  dementia patients and healthy controls. cf_train_test_split generates controlled
+  confounder distributions with configurable P(Y=1|Z) for train and test sets. Key
+  parameters: train_pos_z0 (baseline dementia rate for Z=0), train_pos_z1 (baseline
+  dementia rate for Z=1), alpha_test (test set imbalance ratio). Test set fixed at
+  150 samples, validation at 120 samples.
+
+  '

package/AISB/catalog/aisb.t3.037_dementiamask.zh.yaml

@@ -0,0 +1,132 @@
+id: aisb.t3.037_dementiamask
+name: 痴呆症掩码
+version: 0.1.0
+one_line: 基于权重掩码的语音痴呆检测去偏基准，用于隔离Transformer模型中与性别相关的混杂因素。
+task_description: |
+  此打包基准测试涵盖使用两种权重掩码方法进行语音痴呆检测中的性别混杂因素缓解：扩展混杂过滤器（ECF）和双过滤器（DF）。工作流程在语音衍生转录文本上训练基于BERT的分类器，然后应用基于梯度的权重追踪来识别并消融编码混杂信息的参数。主要评估指标包括AUPRC和delta-FPR（不同人口统计学组之间的假阳性率差距），以评估分布偏移下的诊断性能和公平性。基准测试通过cf_train_test_split生成训练/评估/测试集划分，并对混杂因素分布进行控制，支持可配置的混杂因素比例和测试集不平衡。
+task_mode: experiment_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: runnable
+support_level: advanced
+time_band: 2-6h
+cost_band: medium
+difficulty: medium
+data_access: restricted
+primary_outputs:
+- diagnostic_scores
+- debiasing_report
+- delta_fpr
+launch_profiles:
+- id: quick_eval
+  label: 快速评估
+  description: 使用预配置超参数运行单一DualFilter评估路径（alpha_train=0.2, mask_ratio=0.10, 5次重复）。
+- id: full_train_eval
+  label: 完整训练+评估
+  description: 通过run_df.sh运行完整的基于掩码的去偏工作流，支持可配置的混杂因素比例，包括下游公平性评估。
+- id: cf_evaluation
+  label: 混杂过滤器评估
+  description: 通过run_cf.sh运行原始Confounding Filter基线进行对比。
+- id: ecf_evaluation
+  label: 扩展混杂过滤器评估
+  description: 通过run_ecf.sh运行ECF，使用逐层解冻策略。
+dataset_download:
+  primary_method: mixed
+  sources:
+  - deepscientist_url
+  notes:
+  - 压缩包包含pitts语音数据集及受控的混杂因素划分
+  - 数据划分参数：train_pos_z0、train_pos_z1、alpha_test控制混杂因素分布
+credential_requirements:
+  mode: external_access
+  items:
+  - deepscientist_download_token
+  notes:
+  - 需要ACL 2025论文工件访问权限
+resources:
+  minimum:
+    cpu_cores: 8
+    ram_gb: 16
+    disk_gb: 30
+    gpu_count: 1
+    gpu_vram_gb: 8
+  recommended:
+    cpu_cores: 16
+    ram_gb: 32
+    disk_gb: 80
+    gpu_count: 1
+    gpu_vram_gb: 16
+environment:
+  python: '3.9'
+  cuda: '11.8'
+  pytorch: 2.1.0
+  flash_attn: null
+  key_packages:
+  - transformers
+  - torch
+  - numpy
+  - pandas
+  notes:
+  - 模型权重（bert-base-uncased）必须在本地可用或已下载
+  - 完整的依赖项列表见捆绑的requirements.txt
+risk_flags:
+- distribution_shift
+- demographic_disparity
+risk_notes:
+- 观察到公平性-准确性权衡：去偏可能略微降低痴呆检测AUPRC
+- 训练和测试混杂因素分布之间的分布偏移会影响delta_FPR
+- 性别混杂因素缓解效果随mask_ratio超参数变化
+recommended_when: |
+  在评估临床NLP中的偏差缓解技术、研究痴呆检测中的性别混杂因素，或比较语音衍生医学文本分类中的权重掩码方法（ECF vs DF vs 朴素基线）时使用此基准测试。
+not_recommended_when: |
+  当无法获得语音或临床数据集访问权限，或当纯粹追求诊断准确性而不考虑公平性时，不建议使用此基准测试。
+paper:
+  title: Mitigating Confounding in Speech-Based Dementia Detection through Weight
+    Masking
+  authors: Sheng, Ding, Hur, Li, Cohen, Pakhomov
+  venue: ACL 2025 Main
+  year: 2025
+  url: https://arxiv.org/abs/2506.05610
+  pages: 8
+  abstract: |
+    深度Transformer模型通过检测患者转录文本中的语言异常来进行早期阿尔茨海默病筛查。本工作通过扩展混杂过滤器（ECF）和双过滤器（DF）处理性别混杂因素，这两种方法可隔离并消融性别相关权重。结果表明Transformer模型倾向于过度拟合训练分布；破坏性别相关权重会产生去偏分类器，代价是痴呆检测性能略有下降。
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.037_dementiamask.zip
+  archive_type: zip
+  local_dir_name: paper-37-DementiaMask
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.037_dementiamask.zip
+  sha256: d759476e5d109aae5e6177cd9e294126e5d9fd1e1e21af1b40d16f855455ba5c
+  size_bytes: 98039
+display:
+  palette_seed: wine-silver-clinic
+  art_style: clinical-notebook
+  accent_priority: medium
+image_path: ../image/037_aisb.t3.037_dementiamask.jpg
+metric_evidence:
+- metric: auprc
+  origin: run_experiment.py
+  status: code_backed
+  protocol: |
+    在DualFilter训练和消融后，按组（full、confounder_0、confounder_1）计算平均精度分数（APS）。报告每组及聚合结果。
+- metric: delta_fpr
+  origin: run_experiment.py
+  status: code_backed
+  protocol: |
+    Delta FPR = FPR_confounder_1 - FPR_confounder_0，其中FPR = 1 - recall_neg。衡量去偏后不同人口统计学组之间的公平性差距。
+execution_anchors:
+  primary: run_experiment.py
+  shell_scripts:
+  - run_df.sh
+  - run_cf.sh
+  - run_ecf.sh
+  model_modules:
+  - src/model/weights_filter.py
+  preprocessing: src/preprocessing/data_generator.py
+hardware_notes: |
+  单GPU执行足够。bert-base-uncased推理需要约1.1GB GPU内存。在完整数据集上用5次重复和每次30个epoch进行训练，配合梯度累积可能需要8GB以上显存。不建议CPU-only执行训练。
+dataset_notes: |
+  pitts语音数据集包含来自痴呆症患者和健康对照组的第一人称叙事转录文本。cf_train_test_split生成受控的混杂因素分布，支持可配置的P(Y=1|Z)用于训练和测试集。关键参数：train_pos_z0（Z=0的基线痴呆率）、train_pos_z1（Z=1的基线痴呆率）、alpha_test（测试集不平衡比例）。测试集固定150个样本，验证集120个样本。