@researai/deepscientist 1.5.16 → 1.6.0

package/AISB/catalog/aisb.t3.027_citeeval.yaml

@@ -0,0 +1,182 @@
+schema_version: 1
+id: aisb.t3.027_citeeval
+name: 'CiteEval: Principle-Driven Citation Evaluation for Source Attribution'
+version: 0.1.0
+one_line: 'Evaluate and optimize citation quality metrics on the CiteBench benchmark
+  using CiteEval-Auto, a principle-driven framework that assesses citations against
+  full retrieval context, user queries, and generated text — measuring statement-level
+  Pearson correlation with human judgments.
+
+  '
+task_description: 'This packaged benchmark covers principle-driven citation evaluation
+  for retrieval-augmented generation (RAG) systems. The core task is to run the CiteEval-Auto
+  metric suite — comprising context attribution (CA), citation editing (CE), and citation
+  rating (CR via IterCoE and EditDist) modules — against the CiteBench dataset, which
+  contains multi-domain queries (ASQA, ELI5, MS MARCO, LFRQA) with statement-level
+  human annotations of citation quality on a 1-5 Likert scale. The primary metric
+  is statement-level Pearson correlation between predicted and human citation ratings.
+  The evaluation goes beyond simple NLI-based supportiveness by considering full retrieval
+  sources, user context, response context, and parametric knowledge. The benchmark
+  supports two evaluation scenarios: "Full" (all citable statements, uncited ones
+  penalized) and "Cited" (only cited statements evaluated). Execution requires an
+  LLM API (OpenAI or DeepSeek) for the CiteEval-Auto modules, which use prompted LLM
+  calls for context attribution and citation rating. The CiteBench dataset must be
+  downloaded separately from Google Drive. Pre-computed metric outputs for baselines
+  (AutoAIS, LQAC, AttriScore) are bundled in the snapshot.
+
+  '
+capability_tags:
+- research_code_optimization
+- citation_evaluation
+- retrieval_augmented_generation
+- source_attribution
+- evaluation
+- meta_evaluation
+aisb_direction: T3
+track_fit:
+- paper_track
+- benchmark_track
+task_mode: evaluation_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: runnable
+support_level: advanced
+cost_band: medium
+time_band: 2-6h
+difficulty: medium
+data_access: public
+primary_outputs:
+- statement_level_pearson
+- context_attribution_correlation
+- citation_rating_correlation
+- citation_eval_report
+- principle_judgments
+launch_profiles:
+- id: quick_check
+  label: Quick Check
+  description: 'Run the packaged CiteEval-Auto evaluation on a small example batch
+    or the pre-computed metric outputs to verify the pipeline and reproduce baseline
+    correlation numbers.
+
+    '
+- id: full_eval
+  label: Full Metric Evaluation
+  description: 'Run the full CiteEval-Auto pipeline (CA, CE, CR modules) over the
+    CiteBench metric test set from scratch using an LLM API, then compute human correlation
+    metrics against statement-level and response-level human annotations.
+
+    '
+- id: system_eval
+  label: System Evaluation
+  description: 'Evaluate citation quality of a custom RAG system''s outputs using
+    CiteEval-Auto. Requires converting system output to .citeeval format, running
+    the metric suite, and printing results with run_system_eval.sh. Supports --cited
+    and full scenarios.
+
+    '
+dataset_download:
+  primary_method: manual
+  sources:
+  - kind: google_drive
+    url: https://drive.google.com/drive/folders/12Evj0f92wKz_7OGuuwq3KShTdSM8eu4v?usp=drive_link
+    access: public
+    note: 'CiteBench dataset including metric_dev, metric_test, full dev, and full
+      test splits. Must be downloaded manually and placed under data/ in the project
+      root.
+
+      '
+  notes:
+  - Pre-computed metric outputs for baseline metrics (AutoAIS, LQAC, AttriScore) are
+    bundled in data/metric_eval_outputs/.
+  - Dataset size is moderate (thousands of queries with annotations); exact download
+    size not documented but expected under 1 GB.
+credential_requirements:
+  mode: api_key
+  items:
+  - OPENAI_API_KEY (for GPT-4o-based CiteEval-Auto runs, or substitute with DeepSeek
+    API)
+  notes:
+  - The default config in run_citeeval.sh uses model=deepseek-chat; run_system_eval.sh
+    references gpt-4o.
+  - API costs depend on the number of statements evaluated and the chosen model. The
+    metric test set has ~1000 responses.
+  - CPU-only execution of the evaluation scripts is possible but still requires an
+    LLM API for the CiteEval-Auto modules.
+resources:
+  minimum:
+    cpu_cores: 8
+    ram_gb: 32
+    disk_gb: 50
+    gpu_count: 0
+    gpu_vram_gb: 0
+  recommended:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 100
+    gpu_count: 1
+    gpu_vram_gb: 16
+environment:
+  python: '3.10'
+  cuda: '11.7'
+  pytorch: 1.13.0
+  key_packages:
+  - openai==1.16.2
+  - transformers==4.38.2
+  notes:
+  - CPU-only execution is plausible for the minimum route; GPU is only needed if running
+    local NLI models for baselines.
+  - The requirements.txt file has a typo in the README (requirments.txt) — use the
+    actual file in the snapshot.
+  - CITEEVAL_ROOT and PYTHONPATH environment variables must be set as described in
+    README.
+risk_flags:
+- api_dependency
+- external_dataset_download
+- api_cost_variable
+risk_notes:
+- CiteEval-Auto modules require live LLM API calls (OpenAI or DeepSeek). Without an
+  API key, only pre-computed metric outputs can be evaluated.
+- CiteBench data must be manually downloaded from Google Drive; it is not bundled
+  in the snapshot.
+- API costs scale with the number of evaluated statements and the LLM model used.
+  The metric test set has ~1000 responses with multiple statements each.
+- The default model in run_citeeval.sh is deepseek-chat, not GPT-4o as used in the
+  paper's main results. Reproducing paper numbers requires GPT-4o.
+- No benchmark execution was performed during the packaging pass; metric values should
+  be verified at runtime.
+recommended_when: 'Use this benchmark when you want a citation-quality metric task
+  that evaluates full retrieval context, user queries, and generated text rather than
+  simple NLI-based supportiveness proxies. Ideal for research on improving RAG citation
+  quality, developing better automatic citation metrics, or benchmarking citation
+  quality of new RAG systems against human judgments.
+
+  '
+not_recommended_when: 'Do not use this if you need a fully self-contained benchmark
+  with no external API calls, if you lack access to OpenAI or DeepSeek APIs, or if
+  your task does not involve retrieval passages and source attribution. Not suitable
+  if you need a benchmark that can run entirely offline without any LLM API dependency.
+
+  '
+paper:
+  title: 'CiteEval: Principle-Driven Citation Evaluation for Source Attribution'
+  venue: arXiv preprint
+  year: 2025
+  url: https://arxiv.org/abs/2506.01829
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.027_citeeval.zip
+  archive_type: zip
+  local_dir_name: paper-27-CiteEval
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.027_citeeval.zip
+  sha256: 5eb48e11b91ec4856d18461899c236f658b110dc49ff29487205908230ac8d0b
+  size_bytes: 83173
+commercial:
+  annual_fee: null
+display:
+  palette_seed: cream-navy-citation
+  art_style: reference-audit
+  accent_priority: high
+image_path: ../image/027_aisb.t3.027_citeeval.jpg

package/AISB/catalog/aisb.t3.027_citeeval.zh.yaml

@@ -0,0 +1,135 @@
+schema_version: 1
+id: aisb.t3.027_citeeval
+name: 'CiteEval: 基于原则驱动的引用评估用于来源归属'
+version: 0.1.0
+one_line: '在CiteBench基准上使用CiteEval-Auto（一个原则驱动的框架）评估和优化引用质量指标，该框架根据完整检索上下文、用户查询和生成文本对引用进行评估——衡量语句级皮尔逊相关系数与人工判断的一致性。
+
+  '
+task_description: '该打包基准测试涵盖用于检索增强生成（RAG）系统的原则驱动引用评估。核心任务是在CiteBench数据集上运行CiteEval-Auto指标套件——包括上下文归因（CA）、引用编辑（CE）和引用评分（CR，通过IterCoE和EditDist）模块。该数据集包含多领域查询（ASQA、ELI5、MS MARCO、LFRQA），带有引用质量的语句级人工标注（1-5李克特量表）。主要指标是预测引用评分与人工引用评分之间的语句级皮尔逊相关系数。该评估超越了简单的基于NLI的支持度考量，考虑了完整检索来源、用户上下文、响应上下文和参数知识。基准测试支持两种评估场景："完整"（所有可引用语句，未引用者受罚）和"已引用"（仅评估已引用的语句）。执行需要LLM API（OpenAI或DeepSeek）来运行CiteEval-Auto模块，这些模块使用提示的LLM调用进行上下文归因和引用评分。CiteBench数据集需从Google Drive单独下载。基线（AutoAIS、LQAC、AttriScore）的预计算指标输出已捆绑在快照中。
+
+  '
+capability_tags:
+- research_code_optimization
+- citation_evaluation
+- retrieval_augmented_generation
+- source_attribution
+- evaluation
+- meta_evaluation
+aisb_direction: T3
+track_fit:
+- paper_track
+- benchmark_track
+task_mode: evaluation_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: runnable
+support_level: advanced
+cost_band: medium
+time_band: 2-6h
+difficulty: medium
+data_access: public
+primary_outputs:
+- statement_level_pearson
+- context_attribution_correlation
+- citation_rating_correlation
+- citation_eval_report
+- principle_judgments
+launch_profiles:
+- id: quick_check
+  label: 快速检查
+  description: '在小样本批次或预计算指标输出上运行打包的CiteEval-Auto评估，以验证流程并复现基线相关系数。
+
+    '
+- id: full_eval
+  label: 完整指标评估
+  description: '使用LLM API从头开始对CiteBench指标测试集运行完整的CiteEval-Auto流程（CA、CE、CR模块），然后根据语句级和响应级人工标注计算与人工的相关性指标。
+
+    '
+- id: system_eval
+  label: 系统评估
+  description: '使用CiteEval-Auto评估自定义RAG系统输出的引用质量。需要将系统输出转换为.citeeval格式，运行指标套件，并使用run_system_eval.sh打印结果。支持--cited和完整场景。
+
+    '
+dataset_download:
+  primary_method: manual
+  sources:
+  - kind: google_drive
+    url: https://drive.google.com/drive/folders/12Evj0f92wKz_7OGuuwq3KShTdSM8eu4v?usp=drive_link
+    access: public
+    note: 'CiteBench数据集，包含metric_dev、metric_test、full dev和full test划分。必须手动下载并放置在项目根目录下的data/中。
+
+      '
+  notes:
+  - 基线指标（AutoAIS、LQAC、AttriScore）的预计算指标输出已捆绑在data/metric_eval_outputs/中。
+  - 数据集大小适中（数千个带标注的查询）；确切下载大小未记录，但预计在1GB以下。
+credential_requirements:
+  mode: api_key
+  items:
+  - OPENAI_API_KEY（用于基于GPT-4o的CiteEval-Auto运行，或使用DeepSeek API替代）
+  notes:
+  - run_citeeval.sh中的默认配置使用model=deepseek-chat；run_system_eval.sh引用gpt-4o。
+  - API成本取决于评估的语句数量和所选模型。指标测试集约有1000个响应。
+  - 评估脚本可以在仅CPU模式下执行，但仍需要LLM API来运行CiteEval-Auto模块。
+resources:
+  minimum:
+    cpu_cores: 8
+    ram_gb: 32
+    disk_gb: 50
+    gpu_count: 0
+    gpu_vram_gb: 0
+  recommended:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 100
+    gpu_count: 1
+    gpu_vram_gb: 16
+environment:
+  python: '3.10'
+  cuda: '11.7'
+  pytorch: 1.13.0
+  key_packages:
+  - openai==1.16.2
+  - transformers==4.38.2
+  notes:
+  - 最低规格路线可以仅用CPU执行；仅在运行本地NLI模型进行基线评估时才需要GPU。
+  - requirements.txt文件在README中有拼写错误（requirments.txt）——请使用快照中的实际文件。
+  - 必须按README中的说明设置CITEEVAL_ROOT和PYTHONPATH环境变量。
+risk_flags:
+- api_dependency
+- external_dataset_download
+- api_cost_variable
+risk_notes:
+- CiteEval-Auto模块需要实时LLM API调用（OpenAI或DeepSeek）。如果没有API密钥，只能评估预计算的指标输出。
+- CiteBench数据必须从Google Drive手动下载；未捆绑在快照中。
+- API成本随评估的语句数量和使用的LLM模型而增加。指标测试集约有1000个响应，每个响应包含多个语句。
+- run_citeeval.sh中的默认模型是deepseek-chat，而非论文主要结果中使用的GPT-4o。复现论文数据需要GPT-4o。
+- 打包过程中未执行基准测试；指标值应在运行时验证。
+recommended_when: '当您需要一个评估完整检索上下文、用户查询和生成文本的引用质量指标任务，而非简单的基于NLI的支持度代理时，使用此基准测试。非常适合研究改进RAG引用质量、开发更好的自动引用指标，或根据人工判断基准测试新RAG系统的引用质量。
+
+  '
+not_recommended_when: '如果您需要一个完全自包含、无外部API调用的基准测试，或者您无法访问OpenAI或DeepSeek API，或者您的任务不涉及检索段落和来源归属，请勿使用此基准测试。如果您需要一个可以完全离线运行、没有任何LLM API依赖的基准测试，则不适合。
+
+  '
+paper:
+  title: 'CiteEval: Principle-Driven Citation Evaluation for Source Attribution'
+  venue: arXiv preprint
+  year: 2025
+  url: https://arxiv.org/abs/2506.01829
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.027_citeeval.zip
+  archive_type: zip
+  local_dir_name: paper-27-CiteEval
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.027_citeeval.zip
+  sha256: 5eb48e11b91ec4856d18461899c236f658b110dc49ff29487205908230ac8d0b
+  size_bytes: 83173
+commercial:
+  annual_fee: null
+display:
+  palette_seed: cream-navy-citation
+  art_style: reference-audit
+  accent_priority: high
+image_path: ../image/027_aisb.t3.027_citeeval.jpg

package/AISB/catalog/aisb.t3.028_sbam.yaml

@@ -0,0 +1,206 @@
+schema_version: 1
+id: aisb.t3.028_sbam
+name: Segment-Based Attention Masking for GPTs
+version: 0.1.0
+one_line: Fine-tune and evaluate Masked Attention by Segment (MAS) on Llama-3.2-1B
+  across eight commonsense reasoning datasets, comparing segment-aware prefill masking
+  against standard causal masking.
+task_description: 'This benchmark evaluates the Masked Attention by Segment (MAS)
+  technique, which modifies the causal attention mask of decoder-only GPTs so that
+  tokens within each prefill segment (system prompt, user prompt) can attend bidirectionally,
+  while the autoregressive generation phase retains standard causal masking. The packaged
+  task uses a LoRA-adapted Llama-3.2-1B model and evaluates on eight commonsense reasoning
+  datasets: BoolQ, PIQA, SocialIQA, HellaSwag, WinoGrande, ARC-Challenge, ARC-Easy,
+  and OpenBookQA. A pre-trained MAS LoRA checkpoint is bundled (trained_models_and_results/Llama-3.2-1B_epoch3_MAS),
+  so evaluation can proceed without re-training. The evaluation script (run_eval_all.sh)
+  parallelizes across two GPUs and reports per-dataset accuracy plus an average. The
+  base Llama-3.2-1B model must be downloaded from HuggingFace (gated access). The
+  fine-tuning dataset (170k commonsense reasoning samples in chat-template format)
+  is referenced via ft_utils.py but is not bundled; re-training requires sourcing
+  it externally. The core evaluation route is self-contained given the bundled LoRA
+  weights and base model access.
+
+  '
+capability_tags:
+- research_code_optimization
+- large_language_models
+- transformers
+- prompt_engineering
+- attention_analysis
+- commonsense_reasoning
+aisb_direction: T3
+track_fit:
+- paper_track
+- benchmark_track
+task_mode: evaluation_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: runnable
+support_level: turnkey
+cost_band: medium
+time_band: 6-24h
+difficulty: medium
+data_access: public
+primary_outputs:
+- average_accuracy
+- arc_c_accuracy
+- obqa_accuracy
+- boolq_accuracy
+- piqa_accuracy
+- siqa_accuracy
+- hellaswag_accuracy
+- winogrande_accuracy
+- arc_e_accuracy
+- masked_attention_report
+- prompt_variant_scores
+launch_profiles:
+- id: quick_check
+  label: Quick Check
+  description: 'Run mas_eval.py on a single dataset (e.g. ARC-Challenge) with the
+    bundled LoRA checkpoint to verify the evaluation pipeline works end-to-end on
+    one GPU.
+
+    '
+- id: full_eval
+  label: Full Eval
+  description: 'Run run_eval_all.sh to evaluate the bundled MAS LoRA checkpoint on
+    all eight commonsense reasoning datasets in parallel across two GPUs, producing
+    per-dataset and average accuracy.
+
+    '
+- id: retrain_and_eval
+  label: Retrain + Eval
+  description: 'Re-run LoRA fine-tuning with MAS masking on the 170k commonsense dataset
+    (must be sourced externally), then evaluate. This reproduces the full paper pipeline.
+
+    '
+dataset_download:
+  primary_method: mixed
+  sources:
+  - kind: huggingface
+    url: https://huggingface.co/meta-llama/Llama-3.2-1B
+    access: gated
+    note: 'Base Llama-3.2-1B model weights. Requires HuggingFace account and Meta
+      license acceptance. ~2.5 GB download.
+
+      '
+  - kind: bundled
+    url: null
+    access: local
+    note: 'Pre-trained MAS LoRA weights are bundled in trained_models_and_results/Llama-3.2-1B_epoch3_MAS.
+      A standard causal LoRA checkpoint is in Llama-3.2-1B_epoch3.
+
+      '
+  - kind: external
+    url: null
+    access: public
+    note: 'The 170k commonsense reasoning training dataset (referenced by ft_utils.py)
+      is not bundled. It follows the setup from Liu et al. (2024) / Hu et al. (2023).
+      Only needed if re-training.
+
+      '
+  - kind: external
+    url: null
+    access: public
+    note: 'Test splits for BoolQ, PIQA, SocialIQA, HellaSwag, WinoGrande, ARC-Challenge,
+      ARC-Easy, OpenBookQA are loaded via ft_utils.py at eval time. These are public
+      datasets typically fetched from HuggingFace datasets hub.
+
+      '
+  notes:
+  - Total disk usage with base model and evaluation datasets is approximately 10-20
+    GB.
+  - If re-training, the 170k training samples add several GB.
+credential_requirements:
+  mode: api_key
+  items:
+  - HuggingFace token with Meta Llama license acceptance (for base model download)
+  notes:
+  - No other API keys required for evaluation.
+  - If base model is pre-cached locally, no credentials are needed at runtime.
+resources:
+  minimum:
+    cpu_cores: 8
+    ram_gb: 32
+    disk_gb: 50
+    gpu_count: 1
+    gpu_vram_gb: 12
+  recommended:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 120
+    gpu_count: 2
+    gpu_vram_gb: 24
+environment:
+  python: '3.10'
+  cuda: '11.8'
+  pytorch: 2.1.0
+  flash_attn: null
+  key_packages:
+  - transformers==4.47.0
+  - peft
+  - torch
+  - tqdm
+  notes:
+  - MAS implementation uses eager attention (attn_implementation="eager"), not flash
+    attention.
+  - See bundled requirements.txt for full dependency set.
+  - CUDA is required; the code checks torch.cuda.is_available() and defaults to "cuda".
+risk_flags:
+- gated_model_dependency
+- training_data_not_bundled
+- eager_attention_only
+risk_notes:
+- Base Llama-3.2-1B requires gated HuggingFace access; evaluation cannot proceed without
+  it.
+- The 170k commonsense training dataset is not bundled; only the eval route with pre-trained
+  LoRA is self-contained.
+- MAS currently only supports Llama models with eager attention mode; flash_attn is
+  not compatible.
+- run_eval_all.sh hardcodes paths (/models/Llama-3.2-1B, /repo/); these must be adjusted
+  to local layout.
+- No benchmark execution was performed during the packaging pass; metric values are
+  not pre-validated.
+- Batch size is set to 1 in run_eval_all.sh; full eval on 8 datasets takes several
+  hours.
+recommended_when: 'Use this benchmark when you want a manageable, single-GPU evaluation
+  of a novel attention masking strategy for decoder-only LLMs on commonsense reasoning.
+  Good for studying how segment-level bidirectional attention during prefill affects
+  downstream accuracy compared to standard causal masking, with lightweight LoRA fine-tuning.
+
+  '
+not_recommended_when: 'Do not use this if you need a large-scale multi-billion parameter
+  LLM training benchmark, a non-transformer architecture task, or a benchmark with
+  fully bundled training data. Also not suitable if you cannot obtain gated access
+  to Meta Llama models.
+
+  '
+paper:
+  title: Segment-Based Attention Masking for GPTs
+  authors:
+  - Shahar Katz
+  - Liran Ringel
+  - Yaniv Romano
+  - Lior Wolf
+  venue: arXiv preprint
+  year: 2024
+  url: https://arxiv.org/abs/2412.18487
+  doi: null
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.028_sbam.zip
+  archive_type: zip
+  local_dir_name: paper-28-SBAM
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.028_sbam.zip
+  sha256: e6a4ea39b6fa60b49a9ed55f8386355bae8ed0b92188064bc351a5f71c8f9066
+  size_bytes: 104097
+commercial:
+  annual_fee: null
+display:
+  palette_seed: violet-graphite-mask
+  art_style: attention-map
+  accent_priority: medium
+image_path: ../image/028_aisb.t3.028_sbam.jpg

package/AISB/catalog/aisb.t3.028_sbam.zh.yaml

@@ -0,0 +1,166 @@
+schema_version: 1
+id: aisb.t3.028_sbam
+name: 基于分段注意力的GPT掩码技术
+version: 0.1.0
+one_line: 在Llama-3.2-1B上对分段掩码注意力(MAS)进行微调和评估，在八个常识推理数据集上比较分段感知的预填充掩码与标准因果掩码。
+task_description: '该基准测试评估分段掩码注意力(MAS)技术，该技术修改了解码器专用GPT的因果注意力掩码，使每个预填充分段（系统提示、用户提示）内的token能够双向注意力，而自回归生成阶段保留标准因果掩码。打包的任务使用LoRA适配的Llama-3.2-1B模型，并在八个常识推理数据集上进行评估：BoolQ、PIQA、SocialIQA、HellaSwag、WinoGrande、ARC-Challenge、ARC-Easy和OpenBookQA。预训练的MAS LoRA检查点已打包（trained_models_and_results/Llama-3.2-1B_epoch3_MAS），因此评估可以在不重新训练的情况下进行。评估脚本（run_eval_all.sh）在两个GPU上并行处理，并报告每个数据集的准确率及平均值。基础Llama-3.2-1B模型必须从HuggingFace下载（需要 gated 访问）。微调数据集（170k条聊天模板格式的常识推理样本）通过ft_utils.py引用，但未打包；如需重新训练需要外部获取。核心评估流程在有打包的LoRA权重和基础模型访问权限的情况下是自包含的。
+
+  '
+capability_tags:
+- research_code_optimization
+- large_language_models
+- transformers
+- prompt_engineering
+- attention_analysis
+- commonsense_reasoning
+aisb_direction: T3
+track_fit:
+- paper_track
+- benchmark_track
+task_mode: evaluation_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: runnable
+support_level: turnkey
+cost_band: medium
+time_band: 6-24h
+difficulty: medium
+data_access: public
+primary_outputs:
+- average_accuracy
+- arc_c_accuracy
+- obqa_accuracy
+- boolq_accuracy
+- piqa_accuracy
+- siqa_accuracy
+- hellaswag_accuracy
+- winogrande_accuracy
+- arc_e_accuracy
+- masked_attention_report
+- prompt_variant_scores
+launch_profiles:
+- id: quick_check
+  label: 快速检查
+  description: '使用打包的LoRA检查点在单个数据集（例如ARC-Challenge）上运行mas_eval.py，以验证评估流程在单个GPU上端到端工作。
+
+    '
+- id: full_eval
+  label: 完整评估
+  description: '运行run_eval_all.sh在两个GPU上并行评估打包的MAS LoRA检查点在所有八个常识推理数据集上的表现，生成每个数据集的准确率和平均准确率。
+
+    '
+- id: retrain_and_eval
+  label: 重新训练 + 评估
+  description: '使用MAS掩码在170k常识数据集上重新运行LoRA微调（必须从外部获取），然后进行评估。这将重现完整的论文流程。
+
+    '
+dataset_download:
+  primary_method: mixed
+  sources:
+  - kind: huggingface
+    url: https://huggingface.co/meta-llama/Llama-3.2-1B
+    access: gated
+    note: '基础Llama-3.2-1B模型权重。需要HuggingFace账户和Meta许可协议接受。约2.5GB下载量。
+
+      '
+  - kind: bundled
+    url: null
+    access: local
+    note: '预训练的MAS LoRA权重打包在trained_models_and_results/Llama-3.2-1B_epoch3_MAS中。标准因果LoRA检查点在Llama-3.2-1B_epoch3中。
+
+      '
+  - kind: external
+    url: null
+    access: public
+    note: '170k常识推理训练数据集（由ft_utils.py引用）未打包。它遵循Liu等人(2024)/Hu等人(2023)的设置。仅在重新训练时需要。
+
+      '
+  - kind: external
+    url: null
+    access: public
+    note: 'BoolQ、PIQA、SocialIQA、HellaSwag、WinoGrande、ARC-Challenge、ARC-Easy、OpenBookQA的测试集通过ft_utils.py在评估时加载。这些是通常从HuggingFace数据集中心获取的公共数据集。
+
+      '
+  notes:
+  - 包含基础模型和评估数据集的总磁盘使用量约为10-20GB。
+  - 如果重新训练，170k训练样本会增加数GB。
+credential_requirements:
+  mode: api_key
+  items:
+  - 具有Meta Llama许可协议接受权限的HuggingFace令牌（用于基础模型下载）
+  notes:
+  - 评估不需要其他API密钥。
+  - 如果基础模型已在本地缓存，运行时不需要凭据。
+resources:
+  minimum:
+    cpu_cores: 8
+    ram_gb: 32
+    disk_gb: 50
+    gpu_count: 1
+    gpu_vram_gb: 12
+  recommended:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 120
+    gpu_count: 2
+    gpu_vram_gb: 24
+environment:
+  python: '3.10'
+  cuda: '11.8'
+  pytorch: 2.1.0
+  flash_attn: null
+  key_packages:
+  - transformers==4.47.0
+  - peft
+  - torch
+  - tqdm
+  notes:
+  - MAS实现使用eager注意力（attn_implementation="eager"），而非flash注意力。
+  - 有关完整的依赖项列表，请参阅打包的requirements.txt。
+  - 需要CUDA；代码检查torch.cuda.is_available()并默认为"cuda"。
+risk_flags:
+- gated_model_dependency
+- training_data_not_bundled
+- eager_attention_only
+risk_notes:
+- 基础Llama-3.2-1B需要gated HuggingFace访问；没有它评估无法进行。
+- 170k常识训练数据集未打包；只有带有预训练LoRA的评估流程是自包含的。
+- MAS目前仅支持使用eager注意力模式的Llama模型；flash_attn不兼容。
+- run_eval_all.sh硬编码路径（/models/Llama-3.2-1B、/repo/）；必须调整为本地布局。
+- 打包过程中未执行基准测试；指标值未预先验证。
+- run_eval_all.sh中批大小设置为1；在8个数据集上完整评估需要数小时。
+recommended_when: '当您想要对解码器专用LLM的新型注意力掩码策略在常识推理上进行可管理的单GPU评估时使用此基准测试。适合研究预填充期间分段级双向注意力如何影响下游准确率（与标准因果掩码相比），并使用轻量级LoRA微调。
+
+  '
+not_recommended_when: '如果需要数十亿参数LLM的大规模训练基准测试、非Transformer架构任务或完全打包训练数据的基准测试，请勿使用此基准测试。如果无法获得Meta Llama模型的gated访问权限，也不适用。
+
+  '
+paper:
+  title: Segment-Based Attention Masking for GPTs
+  authors:
+  - Shahar Katz
+  - Liran Ringel
+  - Yaniv Romano
+  - Lior Wolf
+  venue: arXiv preprint
+  year: 2024
+  url: https://arxiv.org/abs/2412.18487
+  doi: null
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.028_sbam.zip
+  archive_type: zip
+  local_dir_name: paper-28-SBAM
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.028_sbam.zip
+  sha256: e6a4ea39b6fa60b49a9ed55f8386355bae8ed0b92188064bc351a5f71c8f9066
+  size_bytes: 104097
+commercial:
+  annual_fee: null
+display:
+  palette_seed: violet-graphite-mask
+  art_style: attention-map
+  accent_priority: medium
+image_path: ../image/028_aisb.t3.028_sbam.jpg