@researai/deepscientist 1.5.16 → 1.6.0

package/AISB/catalog/aisb.t3.023_guidedembed.yaml

@@ -0,0 +1,211 @@
+id: aisb.t3.023_guidedembed
+name: 'GSTransform: Guided Space Transformation for Instruction-Following Text Embeddings'
+version: 0.1.0
+one_line: Lightweight transformation of pretrained text embeddings to follow user
+  instructions without corpus re-encoding, achieving 6-300x speedup.
+task_description: 'This packaged benchmark covers GSTransform, a novel instruction-following
+  text embedding framework. Unlike prior approaches that require re-encoding the entire
+  corpus for each instruction (O(m×n) forward passes), GSTransform transforms pre-computed
+  embeddings in real time using a lightweight model. The two-stage pipeline consists
+  of: (1) Instruction-based Label Construction where an LLM generates instruction-aware
+  labels from a small sampled corpus, and (2) Label-guided Embedding Transformation
+  where a transformation model maps generic embeddings to an instruction-aligned semantic
+  space using contrastive and reconstruction losses. The benchmark evaluates on three
+  instruction-awareness tasks—clustering, semantic textual similarity (STS), and triplet
+  alignment—across nine real-world datasets. Evaluation requires configuring the dataset
+  in data.py, the instruction in label_construction.py, and an OpenAI API key in util.py.
+
+  '
+task_mode: experiment_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: runnable
+support_level: advanced
+time_band: 6-24h
+cost_band: medium
+difficulty: medium
+data_access: public
+primary_outputs:
+- v_measure
+- nmi
+- sts_score
+- triplet_score
+- embedding_checkpoint
+launch_profiles:
+- id: quick_check
+  label: Quick Check
+  description: 'Run the clustering-oriented evaluation route on a prepared embedding
+    setup. Skips label construction and training; requires pre-computed embeddings
+    and labeled classification results from prior runs.
+
+    '
+  estimated_time: 10-30m
+- id: full_pipeline
+  label: Full Pipeline
+  description: 'Construct instruction-aware labels using an LLM, train the GSTransform
+    transformation model, and run the full packaged evaluation pipeline including
+    clustering, STS, and triplet evaluations.
+
+    '
+  estimated_time: 4-12h
+  requires_api_key: true
+dataset_download:
+  primary_method: direct_archive
+  sources:
+  - url: https://deepscientist.cc/AISB/023_guidedembed
+    archive_type: zip
+  notes:
+  - Download the paper-23-GuidedEmbed archive
+  - Contains pretrained UAE-Large-V1 model under uae_model/
+  - Configure dataset path in data.py (line 20) before running
+  - Configure instruction in label_construction.py (line 289)
+  - Requires OpenAI API key in util.py (line 8) for label construction
+credential_requirements:
+  mode: api_key_required
+  items:
+  - name: openai_api_key
+    description: OpenAI API key for LLM-based label construction and taxonomy generation
+    scope: label_construction.py
+  notes:
+  - API key only needed during label_construction stage
+  - LLM calls are limited to a small annotated subset (5000 samples recommended)
+  - Cost is minimal compared to full corpus re-encoding approaches
+resources:
+  minimum:
+    cpu_cores: 8
+    ram_gb: 32
+    disk_gb: 50
+    gpu_count: 1
+    gpu_vram_gb: 12
+  recommended:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 100
+    gpu_count: 1
+    gpu_vram_gb: 24
+environment:
+  python: '3.9'
+  cuda: '11.8'
+  pytorch: 2.0.0
+  flash_attn: null
+  key_packages:
+  - name: torch
+    version: '>=2.0'
+  - name: numpy
+    version: '>=1.24'
+  - name: scikit-learn
+    version: '>=1.3'
+  - name: sentence-transformers
+    version: '>=2.2'
+  - name: openai
+    version: '>=1.0'
+  notes:
+  - Full dependency set in bundled requirements.txt
+  - UAE-Large-V1 pretrained encoder bundled in uae_model/ directory
+  - CUDA required for efficient embedding transformation and evaluation
+risk_flags:
+- api_cost
+- disk_io_bottleneck
+risk_notes:
+- API key consumption limited to label construction phase only
+- Embedding cache (embeddings.pkl) can be large; ensure adequate disk space
+- Training uses contrastive loss with margin; adjust MARGIN (default 1.0) in train.py
+  for stability
+recommended_when: 'Use this benchmark when you need a compact NLP training task around
+  instruction-aware embeddings with clustering, STS, or triplet evaluation metrics.
+  Ideal for exploring efficient alternatives to full corpus re-encoding methods like
+  Instructor or InBedder.
+
+  '
+not_recommended_when: 'Do not use this if you need a very large multimodal benchmark,
+  full LLM finetuning workflows, or evaluation scenarios without any API access. This
+  benchmark requires at least one GPU and an OpenAI API key for the label construction
+  pipeline.
+
+  '
+paper:
+  title: 'Don''t Reinvent the Wheel: Efficient Instruction-Following Text Embedding
+    based on Guided Space Transformation'
+  venue: ACL 2025 Oral
+  year: 2025
+  url: https://arxiv.org/abs/2505.24754
+  code_url: https://github.com/YingchaojieFeng/GSTransform
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.023_guidedembed.zip
+  archive_type: zip
+  local_dir_name: paper-23-GuidedEmbed
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.023_guidedembed.zip
+  sha256: 69f98b98d8db4889e5b7063ee007a61b2d823f1ac03114c784e6f02418cc7640
+  size_bytes: 1470047
+display:
+  palette_seed: sage-indigo-embed
+  art_style: semantic-minimal
+  accent_priority: medium
+image_path: ../image/023_aisb.t3.023_guidedembed.jpg
+metric_evidence:
+  v_measure:
+    origin: run_eval_pipeline.py
+    source_ref: FINAL V-MEASURE output line
+    evaluation_protocol: clustering_with_oracle_labels
+  nmi:
+    origin: evaluate_cluster.py
+    source_ref: sklearn.metrics.normalized_mutual_info_score
+    evaluation_protocol: kmeans_clustering
+  sts_score:
+    origin: evaluate_sts.py
+    source_ref: cosine_similarity_comparison
+    evaluation_protocol: semantic_textual_similarity_benchmark
+  triplet_score:
+    origin: evaluate_triplet.py
+    source_ref: triplet_accuracy_evaluation
+    evaluation_protocol: instruction_aware_triplet_alignment
+execution_notes:
+  pipeline_order:
+  - step: data.py
+    description: Load and preprocess dataset; configure DATASET_PATH
+  - step: label_construction.py
+    description: Generate instruction-aware taxonomy and labels using LLM
+  - step: train.py
+    description: Train GSTransform model on labeled embeddings
+  - step: evaluate_cluster.py
+    description: Evaluate on clustering task (primary V-measure, NMI)
+  - step: evaluate_sts.py
+    description: Evaluate on semantic textual similarity task
+  - step: evaluate_triplet.py
+    description: Evaluate on triplet alignment task
+  checkpoints:
+  - path: cache/model.pth
+    description: Trained GSTransform transformation model
+  - path: cache/embeddings.pkl
+    description: Pre-computed and cached base embeddings
+  - path: cache/classifications_xxx.json
+    description: LLM-generated instruction-aware labels
+capability_tags:
+- research_code_optimization
+- text_embeddings
+- representation_learning
+- instruction_following
+- nlp
+- efficient_inference
+- contrastive_learning
+aisb_direction: T3
+track_fit:
+- paper_track
+- benchmark_track
+commercial:
+  annual_fee: null
+metadata:
+  base_encoder: WhereIsAI/UAE-Large-V1
+  transformation_output_dim: 128
+  training_batch_size: 512
+  default_epochs: 500
+  default_learning_rate: 0.0001
+  default_margin: 1.0
+  patience: 50
+  n_train_samples: 5000
+  datasets_evaluated: 9
+  speedup_factor: 6-300x

package/AISB/catalog/aisb.t3.023_guidedembed.zh.yaml

@@ -0,0 +1,189 @@
+id: aisb.t3.023_guidedembed
+name: 'GSTransform：面向指令跟随的文本嵌入引导式空间变换'
+version: 0.1.0
+one_line: 轻量级预训练文本嵌入变换，无需语料库重编码即可跟随用户指令，实现6-300倍加速。
+task_description: '本打包基准测试涵盖GSTransform，一个新颖的指令跟随文本嵌入框架。与先前需要为每条指令重编码整个语料库的方法（O(m×n)次前向传播）不同，GSTransform使用轻量级模型实时变换预计算嵌入。两阶段流程包括：（1）基于指令的标签构建，其中LLM从小规模采样语料库生成指令感知标签；（2）标签引导的嵌入变换，其中变换模型使用对比损失和重构损失将通用嵌入映射到指令对齐的语义空间。基准测试在三个指令感知任务上评估——聚类、语义文本相似度（STS）和三元组对齐——涵盖九个真实世界数据集。评估需要在data.py中配置数据集，在label_construction.py中配置指令，在util.py中配置OpenAI API密钥。
+
+  '
+task_mode: experiment_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: runnable
+support_level: advanced
+time_band: 6-24h
+cost_band: medium
+difficulty: medium
+data_access: public
+primary_outputs:
+- v_measure
+- nmi
+- sts_score
+- triplet_score
+- embedding_checkpoint
+launch_profiles:
+- id: quick_check
+  label: 快速检查
+  description: '在已准备好的嵌入设置上运行面向聚类的评估流程。跳过标签构建和训练阶段，需要来自先前运行预计算的嵌入和带标签的分类结果。
+
+    '
+  estimated_time: 10-30m
+- id: full_pipeline
+  label: 完整流程
+  description: '使用LLM构建指令感知标签，训练GSTransform变换模型，并运行完整的打包评估流程，包括聚类、STS和三元组评估。
+
+    '
+  estimated_time: 4-12h
+  requires_api_key: true
+dataset_download:
+  primary_method: direct_archive
+  sources:
+  - url: https://deepscientist.cc/AISB/023_guidedembed
+    archive_type: zip
+  notes:
+  - 下载paper-23-GuidedEmbed压缩包
+  - 包含u ae_model/目录下的预训练UAE-Large-V1模型
+  - 运行前在data.py（第20行）配置数据集路径
+  - 在label_construction.py（第289行）配置指令
+  - 在util.py（第8行）需要OpenAI API密钥用于标签构建
+credential_requirements:
+  mode: api_key_required
+  items:
+  - name: openai_api_key
+    description: 用于LLM标签构建和分类生成的OpenAI API密钥
+    scope: label_construction.py
+  notes:
+  - API密钥仅在标签构建阶段需要
+  - LLM调用仅限于小规模标注子集（建议5000个样本）
+  - 相比完整语料库重编码方法，成本极低
+resources:
+  minimum:
+    cpu_cores: 8
+    ram_gb: 32
+    disk_gb: 50
+    gpu_count: 1
+    gpu_vram_gb: 12
+  recommended:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 100
+    gpu_count: 1
+    gpu_vram_gb: 24
+environment:
+  python: '3.9'
+  cuda: '11.8'
+  pytorch: 2.0.0
+  flash_attn: null
+  key_packages:
+  - name: torch
+    version: '>=2.0'
+  - name: numpy
+    version: '>=1.24'
+  - name: scikit-learn
+    version: '>=1.3'
+  - name: sentence-transformers
+    version: '>=2.2'
+  - name: openai
+    version: '>=1.0'
+  notes:
+  - 完整依赖项在打包的requirements.txt中
+  - UAE-Large-V1预训练编码器打包在u ae_model/目录中
+  - 需要CUDA以支持高效的嵌入变换和评估
+risk_flags:
+- api_cost
+- disk_io_bottleneck
+risk_notes:
+- API密钥消耗仅限于标签构建阶段
+- 嵌入缓存（embeddings.pkl）可能较大，请确保有足够的磁盘空间
+- 训练使用带边缘的对比损失；可在train.py中调整MARGIN（默认1.0）以提高稳定性
+recommended_when: '当需要一个围绕指令感知嵌入的紧凑NLP训练任务时使用本基准测试，包含聚类、STS或三元组评估指标。非常适合探索Instructor或InBedder等完整语料库重编码方法的高效替代方案。
+
+  '
+not_recommended_when: '如果需要超大规模多模态基准测试、完整LLM微调工作流或完全没有API访问权限的评估场景，请勿使用本基准测试。本基准测试至少需要一个GPU和OpenAI API密钥用于标签构建流程。
+
+  '
+paper:
+  title: 'Don''t Reinvent the Wheel: Efficient Instruction-Following Text Embedding
+    based on Guided Space Transformation'
+  venue: ACL 2025 Oral
+  year: 2025
+  url: https://arxiv.org/abs/2505.24754
+  code_url: https://github.com/YingchaojieFeng/GSTransform
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.023_guidedembed.zip
+  archive_type: zip
+  local_dir_name: paper-23-GuidedEmbed
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.023_guidedembed.zip
+  sha256: 69f98b98d8db4889e5b7063ee007a61b2d823f1ac03114c784e6f02418cc7640
+  size_bytes: 1470047
+display:
+  palette_seed: sage-indigo-embed
+  art_style: semantic-minimal
+  accent_priority: medium
+image_path: ../image/023_aisb.t3.023_guidedembed.jpg
+metric_evidence:
+  v_measure:
+    origin: run_eval_pipeline.py
+    source_ref: FINAL V-MEASURE output line
+    evaluation_protocol: clustering_with_oracle_labels
+  nmi:
+    origin: evaluate_cluster.py
+    source_ref: sklearn.metrics.normalized_mutual_info_score
+    evaluation_protocol: kmeans_clustering
+  sts_score:
+    origin: evaluate_sts.py
+    source_ref: cosine_similarity_comparison
+    evaluation_protocol: semantic_textual_similarity_benchmark
+  triplet_score:
+    origin: evaluate_triplet.py
+    source_ref: triplet_accuracy_evaluation
+    evaluation_protocol: instruction_aware_triplet_alignment
+execution_notes:
+  pipeline_order:
+  - step: data.py
+    description: 加载和预处理数据集；配置DATASET_PATH
+  - step: label_construction.py
+    description: 使用LLM生成指令感知分类和标签
+  - step: train.py
+    description: 在带标签的嵌入上训练GSTransform模型
+  - step: evaluate_cluster.py
+    description: 在聚类任务上评估（主要V-measure、NMI）
+  - step: evaluate_sts.py
+    description: 在语义文本相似度任务上评估
+  - step: evaluate_triplet.py
+    description: 在三元组对齐任务上评估
+  checkpoints:
+  - path: cache/model.pth
+    description: 训练好的GSTransform变换模型
+  - path: cache/embeddings.pkl
+    description: 预计算并缓存的基础嵌入
+  - path: cache/classifications_xxx.json
+    description: LLM生成的指令感知标签
+capability_tags:
+- research_code_optimization
+- text_embeddings
+- representation_learning
+- instruction_following
+- nlp
+- efficient_inference
+- contrastive_learning
+aisb_direction: T3
+track_fit:
+- paper_track
+- benchmark_track
+commercial:
+  annual_fee: null
+metadata:
+  base_encoder: WhereIsAI/UAE-Large-V1
+  transformation_output_dim: 128
+  training_batch_size: 512
+  default_epochs: 500
+  default_learning_rate: 0.0001
+  default_margin: 1.0
+  patience: 50
+  n_train_samples: 5000
+  datasets_evaluated: 9
+  speedup_factor: 6-300x

package/AISB/catalog/aisb.t3.024_outputcentric.yaml

@@ -0,0 +1,148 @@
+id: aisb.t3.024_outputcentric
+name: Enhancing Automated Interpretability with Output-Centric Feature Descriptions
+version: 0.1.0
+one_line: Two-faceted interpretability evaluation pipeline describing LLM features
+  through input-activation and output-behavior lenses.
+task_description: 'This packaged benchmark covers an interpretability evaluation pipeline
+  that describes features through their output behavior. The task characterizes latent
+  features in large language models by what they do at the model output level, proposing
+  output-centric methods (VocabProj, TokenChange) alongside the standard MaxAct approach.
+  The benchmark evaluates feature descriptions along two complementary dimensions:
+  input-based evaluation (how accurately descriptions identify activating inputs)
+  and output-based evaluation (how effectively descriptions capture causal impact
+  on model generation). Experiments span neuron-aligned and sparse autoencoder (SAE)
+  features from residual and MLP layers across multiple LLMs including GPT-small,
+  LLaMA-3.1-8B, and Gemma-2-2B.
+
+  '
+task_mode: evaluation_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: runnable
+support_level: turnkey
+time_band: 1-2h
+cost_band: low
+difficulty: medium
+data_access: public
+primary_outputs:
+- output_based_evaluation_score
+- input_based_evaluation_score
+- feature_descriptions
+launch_profiles:
+- id: notebook_eval
+  label: Notebook Eval
+  description: Run the packaged feature_descriptions_pipeline.ipynb notebook for output-centric
+    feature description evaluation across all model variants.
+dataset_download:
+  primary_method: direct_archive
+  sources:
+  - url: https://deepscientist.cc/AISB/024_outputcentric
+    type: primary_archive
+  notes:
+  - Archive contains pre-computed feature descriptions in descriptions/ directory
+  - Includes CSV files for gpt-small, llama-3.1-8b, llama-3.1-8b-it, and gemma-2-2b
+  - CPU-only execution feasible for minimum route
+credential_requirements:
+  mode: optional_api_keys
+  items:
+  - name: HF_TOKEN
+    description: Hugging Face token for model access
+    required: false
+  - name: GAI_KEY
+    description: Gemini API key for description generation
+    required: false
+  notes:
+  - OPENAI key expected in environment variable if used
+  - Evaluation can proceed without API keys using pre-computed descriptions
+resources:
+  minimum:
+    cpu_cores: 8
+    ram_gb: 16
+    disk_gb: 20
+    gpu_count: 0
+    gpu_vram_gb: 0
+    notes: CPU-only execution supported for minimum route
+  recommended:
+    cpu_cores: 16
+    ram_gb: 32
+    disk_gb: 80
+    gpu_count: 1
+    gpu_vram_gb: 16
+    notes: GPU acceleration recommended for full pipeline evaluation
+environment:
+  python: '3.10'
+  cuda: '11.8'
+  pytorch: 2.1.0
+  flash_attn: null
+  key_packages:
+  - transformers
+  - torch
+  - pandas
+  - numpy
+  - jupyter
+  notes:
+  - See bundled README/requirements.txt for full dependency set
+  - CPU-only execution plausible for minimum route
+  - Additional packages may be required for SAE feature evaluation
+risk_flags:
+- metric_provisional
+- api_key_optional
+risk_notes:
+- Static code audit confirms executable anchors for all staged metrics
+- No benchmark execution was performed in packaging pass
+- Runtime execution required before trusting metric values
+- API keys optional; evaluation can use pre-computed feature descriptions
+recommended_when: 'Use this benchmark when conducting interpretability research on
+  LLM features, developing automated description pipelines, comparing input-centric
+  versus output-centric description methods, evaluating SAE features, or assessing
+  feature description quality for steering applications. Ideal for benchmarking description
+  faithfulness across both input-activation and output-behavior dimensions.
+
+  '
+not_recommended_when: 'Do not use this benchmark if you require a large-scale supervised
+  training benchmark, dense multimodal data processing, or retraining-focused workflows.
+  Not suitable for tasks requiring dense annotation pipelines or non-LLM model interpretability.
+
+  '
+paper:
+  title: Enhancing Automated Interpretability with Output-Centric Feature Descriptions
+  authors:
+  - Yoav Gur-Arieh
+  - Roy Mayan
+  - Chen Agassy
+  - Atticus Geiger
+  - Mor Geva
+  venue: arXiv preprint
+  year: 2025
+  eprint: '2501.08319'
+  primary_class: cs.CL
+  url: https://arxiv.org/abs/2501.08319
+  code_url: https://github.com/yoavgur/Feature-Descriptions
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.024_outputcentric.zip
+  archive_type: zip
+  local_dir_name: paper-24-OutputCentric
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.024_outputcentric.zip
+  sha256: 98fbc91e990b4a6c440895a60947c3f09069c8e5d5cce6283f925f6f27876614
+  size_bytes: 3484257
+display:
+  palette_seed: ivory-rose-neuron
+  art_style: interpretability-atlas
+  accent_priority: medium
+image_path: ../image/024_aisb.t3.024_outputcentric.jpg
+capability_tags:
+- research_code_optimization
+- mechanistic_interpretability
+- automated_evaluation
+- large_language_models
+- feature_analysis
+- sparse_autoencoders
+- model_steering
+aisb_direction: T3
+track_fit:
+- paper_track
+- benchmark_track

package/AISB/catalog/aisb.t3.024_outputcentric.zh.yaml

@@ -0,0 +1,131 @@
+id: aisb.t3.024_outputcentric
+name: 通过以输出为中心的特征描述增强自动化可解释性
+version: 0.1.0
+one_line: 双角度可解释性评估流程，从输入激活和输出行为两个视角描述LLM特征。
+task_description: '本基准测试包含一个可解释性评估流程，通过输出行为来描述特征。该任务通过模型输出层面描述大型语言模型中的潜在特征，提出了以输出为中心的方法（VocabProj、TokenChange）以及标准的MaxAct方法。基准测试从两个互补维度评估特征描述：基于输入的评估（描述如何准确识别激活输入）和基于输出的评估（描述如何有效捕捉对模型生成的因果影响）。实验涵盖来自残差层和MLP层的神经元对齐特征和稀疏自编码器（SAE）特征，涉及GPT-small、LLaMA-3.1-8B和Gemma-2-2B等多个LLM。
+
+  '
+task_mode: evaluation_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: runnable
+support_level: turnkey
+time_band: 1-2h
+cost_band: low
+difficulty: medium
+data_access: public
+primary_outputs:
+- output_based_evaluation_score
+- input_based_evaluation_score
+- feature_descriptions
+launch_profiles:
+- id: notebook_eval
+  label: 笔记本评估
+  description: 运行打包好的feature_descriptions_pipeline.ipynb笔记本，对所有模型变体进行以输出为中心的特征描述评估。
+dataset_download:
+  primary_method: direct_archive
+  sources:
+  - url: https://deepscientist.cc/AISB/024_outputcentric
+    type: primary_archive
+  notes:
+  - 压缩包包含descriptions/目录下的预计算特征描述
+  - 包含gpt-small、llama-3.1-8b、llama-3.1-8b-it和gemma-2-2b的CSV文件
+  - 最低配置可仅使用CPU执行
+credential_requirements:
+  mode: optional_api_keys
+  items:
+  - name: HF_TOKEN
+    description: Hugging Face令牌，用于模型访问
+    required: false
+  - name: GAI_KEY
+    description: Gemini API密钥，用于生成描述
+    required: false
+  notes:
+  - 如需使用则应在环境变量中配置OPENAI密钥
+  - 无需API密钥即可使用预计算描述进行评估
+resources:
+  minimum:
+    cpu_cores: 8
+    ram_gb: 16
+    disk_gb: 20
+    gpu_count: 0
+    gpu_vram_gb: 0
+    notes: 最低配置支持纯CPU执行
+  recommended:
+    cpu_cores: 16
+    ram_gb: 32
+    disk_gb: 80
+    gpu_count: 1
+    gpu_vram_gb: 16
+    notes: 建议使用GPU加速以完成完整流程评估
+environment:
+  python: '3.10'
+  cuda: '11.8'
+  pytorch: 2.1.0
+  flash_attn: null
+  key_packages:
+  - transformers
+  - torch
+  - pandas
+  - numpy
+  - jupyter
+  notes:
+  - 详见捆绑的README/requirements.txt获取完整依赖列表
+  - 最低配置可仅使用CPU执行
+  - SAE特征评估可能需要额外包
+risk_flags:
+- metric_provisional
+- api_key_optional
+risk_notes:
+- 静态代码审查确认所有阶段性指标均有可执行锚点
+- 打包过程中未执行基准测试
+- 使用前需进行运行时执行验证指标值
+- API密钥可选；可使用预计算特征描述进行评估
+recommended_when: '当您进行LLM特征可解释性研究、开发自动化描述流程、比较输入中心与输出中心描述方法、评估SAE特征，或为 steering 应用评估特征描述质量时，可以使用此基准测试。该基准测试非常适合在输入激活和输出行为两个维度上基准测试描述的忠实度。
+
+  '
+not_recommended_when: '如果您需要大规模监督训练基准测试、密集多模态数据处理或以重训练为重点的工作流程，请勿使用此基准测试。不适用于需要密集标注流程或非LLM模型可解释性的任务。
+
+  '
+paper:
+  title: Enhancing Automated Interpretability with Output-Centric Feature Descriptions
+  authors:
+  - Yoav Gur-Arieh
+  - Roy Mayan
+  - Chen Agassy
+  - Atticus Geiger
+  - Mor Geva
+  venue: arXiv preprint
+  year: 2025
+  eprint: '2501.08319'
+  primary_class: cs.CL
+  url: https://arxiv.org/abs/2501.08319
+  code_url: https://github.com/yoavgur/Feature-Descriptions
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.024_outputcentric.zip
+  archive_type: zip
+  local_dir_name: paper-24-OutputCentric
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.024_outputcentric.zip
+  sha256: 98fbc91e990b4a6c440895a60947c3f09069c8e5d5cce6283f925f6f27876614
+  size_bytes: 3484257
+display:
+  palette_seed: ivory-rose-neuron
+  art_style: interpretability-atlas
+  accent_priority: medium
+image_path: ../image/024_aisb.t3.024_outputcentric.jpg
+capability_tags:
+- research_code_optimization
+- mechanistic_interpretability
+- automated_evaluation
+- large_language_models
+- feature_analysis
+- sparse_autoencoders
+- model_steering
+aisb_direction: T3
+track_fit:
+- paper_track
+- benchmark_track