@researai/deepscientist 1.5.17 → 1.6.0

package/AISB/catalog/aisb.t3.047_timepfn.yaml

@@ -0,0 +1,156 @@
+schema_version: 1
+id: aisb.t3.047_timepfn
+name: TimePFN
+version: 0.1.0
+one_line: PFN-style multivariate time-series forecasting benchmark with synthetic
+  pretraining and schedule-sensitive fine-tuning.
+task_description: 'This packaged benchmark covers the full TimePFN workflow: synthetic
+  multivariate time-series data generation via Gaussian process kernel composition
+  and the linear coregionalization method (LMC-Synth), transformer pretraining on
+  those synthetic datasets, and fine-tuning plus evaluation on standard downstream
+  forecasting benchmarks (e.g. ETT, Weather, Exchange). The task is to transfer PFN-style
+  in-context Bayesian generalisation into practical zero-shot and few-shot forecasting
+  quality on public benchmark datasets. Primary metrics are MSE and MAE, both code-backed
+  through run.py and utils/metrics.py. No trusted execution output exists yet; runtime
+  execution is required before metric values can be considered verified.
+
+  '
+task_mode: experiment_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: runnable
+support_level: advanced
+time_band: 6-24h
+cost_band: medium
+difficulty: hard
+data_access: public
+primary_outputs:
+- mse
+- mae
+- pfn_checkpoint
+launch_profiles:
+- id: quick_eval
+  label: Quick Eval
+  description: 'Run one downstream TimePFN forecasting evaluation route using the
+    provided model checkpoint and test scripts under pfn_scripts/test/. No pretraining
+    required.
+
+    '
+- id: pretrain_and_eval
+  label: Pretrain + Eval
+  description: 'Run the full packaged workflow: generate synthetic MTS data via LMC-Synth,
+    pretrain the TimePFN transformer on synthetic data, fine-tune on a downstream
+    benchmark dataset, then evaluate MSE and MAE.
+
+    '
+dataset_download:
+  primary_method: mixed
+  sources:
+  - label: Downstream benchmark datasets (ETT, Weather, Exchange, etc.)
+    url: https://drive.google.com/file/d/1l51QsKvQPcqILT3DwfjCgx8Dsg2rpjot/view?usp=sharing
+    format: zip
+    local_path: ./datasets
+    notes: Provided via iTransformer's datasets.zip Google Drive link. Download and
+      unzip into ./datasets before running evaluation scripts.
+  - label: Synthetic pretraining data
+    url: null
+    format: generated
+    local_path: ./synthetic_data_generation
+    notes: Generated locally via LMC-Synth.py and kernel-synth.py. Read bash script
+      comments and directives before running.
+  - label: Pretrained model checkpoint
+    url: https://deepscientist.cc/AISB/047_timepfn
+    format: zip
+    local_path: paper-47-TimePFN
+    notes: Official AISB archive download. Contains checkpoint, training, fine-tuning,
+      and testing scripts under pfn_scripts/.
+  notes:
+  - Downstream benchmark datasets must be placed under ./datasets before any evaluation
+    script is invoked.
+  - Synthetic data generation is optional if using the provided pretrained checkpoint
+    for quick_eval.
+credential_requirements:
+  mode: none
+  items: []
+  notes:
+  - Google Drive download for datasets.zip does not require authentication but may
+    require manual download if automated fetch is rate-limited.
+resources:
+  minimum:
+    cpu_cores: 8
+    ram_gb: 32
+    disk_gb: 100
+    gpu_count: 1
+    gpu_vram_gb: 16
+  recommended:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 200
+    gpu_count: 1
+    gpu_vram_gb: 24
+environment:
+  python: '3.10'
+  cuda: null
+  pytorch: 2.2.1
+  flash_attn: null
+  key_packages:
+  - transformers==4.41.1
+  - pynvml
+  - numpy
+  notes:
+  - Full dependency list is in requirements.txt. Use a conda virtual environment as
+    recommended by the README.
+  - run.py uses pynvml for GPU memory polling; pynvml must be installed and NVML must
+    be accessible.
+  - CUDA version not pinned in source; verify compatibility with PyTorch 2.2.1 and
+    your driver.
+risk_flags:
+- dataset_manual_download
+- no_trusted_execution_output
+risk_notes:
+- Downstream benchmark datasets require a manual Google Drive download step; automated
+  pipelines may need a workaround.
+- No benchmark execution was performed during the packaging pass. Metric values in
+  metric_contract.json are provisional until runtime execution confirms them.
+- Synthetic data generation volume (millions of samples) can be time- and disk-intensive;
+  read bash script directives carefully before launching.
+recommended_when: 'Use this benchmark when evaluating foundation-model-style multivariate
+  time-series forecasting with synthetic pretraining, or when studying zero-shot and
+  few-shot generalisation on standard public forecasting benchmarks (ETT, Weather,
+  Exchange). Also appropriate when comparing against PFN-based or Chronos-style forecasting
+  baselines.
+
+  '
+not_recommended_when: 'Do not use this benchmark if you need a simple or trivial forecasting
+  baseline, if GPU training is unavailable, if automated dataset download is a hard
+  requirement, or if you need verified execution outputs before trusting metric values
+  (runtime execution must be completed first).
+
+  '
+paper:
+  title: 'TimePFN: Effective Multivariate Time Series Forecasting with Synthetic Data'
+  venue: AAAI 2025
+  year: 2025
+  url: https://arxiv.org/abs/2502.16294
+  authors:
+  - Ege Onur Taga
+  - M. Emrullah Ildiz
+  - Samet Oymak
+  affiliation: University of Michigan, Ann Arbor
+  citation_key: taga2025timepfn
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.047_timepfn.zip
+  archive_type: zip
+  local_dir_name: paper-47-TimePFN
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.047_timepfn.zip
+  sha256: b295b1371d836aeeec199601aa42427fcd6cbbf1f73349b5ebdbfd3a5977ab18
+  size_bytes: 73341
+display:
+  palette_seed: jade-navy-pfn
+  art_style: foundation-signal
+  accent_priority: high
+image_path: ../image/047_aisb.t3.047_timepfn.jpg

package/AISB/catalog/aisb.t3.047_timepfn.zh.yaml

@@ -0,0 +1,124 @@
+schema_version: 1
+id: aisb.t3.047_timepfn
+name: TimePFN
+version: 0.1.0
+one_line: 基于PFN风格的多变量时间序列预测基准，采用合成数据预训练和调度敏感的微调策略。
+task_description: '该打包基准测试涵盖完整的TimePFN工作流程：通过高斯过程核组合和线性协同区域化方法（LMC-Synth）生成合成多变量时间序列数据，在合成数据集上进行Transformer预训练，并在标准下游预测基准（如ETT、Weather、Exchange）上进行微调和评估。任务是将PFN风格的上下文贝叶斯泛化能力迁移到公共基准数据集上的实际零样本和少样本预测质量。主要指标为MSE和MAE，均通过run.py和utils/metrics.py提供代码支持。目前尚无可信执行输出；需要运行时执行后才能将指标值视为已验证。
+
+  '
+task_mode: experiment_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: runnable
+support_level: advanced
+time_band: 6-24h
+cost_band: medium
+difficulty: hard
+data_access: public
+primary_outputs:
+- mse
+- mae
+- pfn_checkpoint
+launch_profiles:
+- id: quick_eval
+  label: 快速评估
+  description: '使用提供的模型检查点和pfn_scripts/test/下的测试脚本运行一个下游TimePFN预测评估流程。无需预训练。
+
+    '
+- id: pretrain_and_eval
+  label: 预训练 + 评估
+  description: '运行完整的打包工作流程：通过LMC-Synth生成合成MTS数据，在合成数据上预训练TimePFN transformer，在下游基准数据集上微调，然后评估MSE和MAE。
+
+    '
+dataset_download:
+  primary_method: mixed
+  sources:
+  - label: 下游基准数据集（ETT、Weather、Exchange等）
+    url: https://drive.google.com/file/d/1l51QsKvQPcqILT3DwfjCgx8Dsg2rpjot/view?usp=sharing
+    format: zip
+    local_path: ./datasets
+    notes: 通过iTransformer的datasets.zip Google Drive链接提供。在运行评估脚本前下载并解压到./datasets目录。
+  - label: 合成预训练数据
+    url: null
+    format: generated
+    local_path: ./synthetic_data_generation
+    notes: 通过LMC-Synth.py和kernel-synth.py本地生成。运行前请阅读bash脚本中的注释和指令。
+  - label: 预训练模型检查点
+    url: https://deepscientist.cc/AISB/047_timepfn
+    format: zip
+    local_path: paper-47-TimePFN
+    notes: 官方AISB归档下载。包含检查点、训练、微调和pfn_scripts/下的测试脚本。
+  notes:
+  - 在调用任何评估脚本前，必须将下游基准数据集放置在./datasets目录下。
+  - 如果使用提供的预训练检查点进行快速评估，合成数据生成是可选的。
+credential_requirements:
+  mode: none
+  items: []
+  notes:
+  - 用于datasets.zip的Google Drive下载不需要身份验证，但如果自动获取受限，可能需要手动下载。
+resources:
+  minimum:
+    cpu_cores: 8
+    ram_gb: 32
+    disk_gb: 100
+    gpu_count: 1
+    gpu_vram_gb: 16
+  recommended:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 200
+    gpu_count: 1
+    gpu_vram_gb: 24
+environment:
+  python: '3.10'
+  cuda: null
+  pytorch: 2.2.1
+  flash_attn: null
+  key_packages:
+  - transformers==4.41.1
+  - pynvml
+  - numpy
+  notes:
+  - 完整的依赖列表在requirements.txt中。建议按照README中的建议使用conda虚拟环境。
+  - run.py使用pynvml进行GPU内存轮询；必须安装pynvml且NVML必须可访问。
+  - 源代码中未固定CUDA版本；请验证其与PyTorch 2.2.1及您的驱动的兼容性。
+risk_flags:
+- dataset_manual_download
+- no_trusted_execution_output
+risk_notes:
+- 下游基准数据集需要手动从Google Drive下载；自动化流程可能需要变通方案。
+- 打包过程中未执行基准测试。metric_contract.json中的指标值为临时值，需待运行时执行确认。
+- 合成数据生成量（百万级样本）可能耗时且占用大量磁盘空间；启动前请仔细阅读bash脚本指令。
+recommended_when: '在评估基于合成数据预训练的基础模型风格多变量时间序列预测时使用此基准，或在标准公共预测基准（ETT、Weather、Exchange）上研究零样本和少样本泛化时使用此基准。在与基于PFN或Chronos风格的预测基线进行比较时也适用。
+
+  '
+not_recommended_when: '如果需要简单或基础的时间序列预测基线、如果GPU训练不可用、如果自动化数据集下载是硬性要求、或者在信任指标值之前需要已验证的执行输出（必须先完成运行时执行），请勿使用此基准。
+
+  '
+paper:
+  title: 'TimePFN: Effective Multivariate Time Series Forecasting with Synthetic Data'
+  venue: AAAI 2025
+  year: 2025
+  url: https://arxiv.org/abs/2502.16294
+  authors:
+  - Ege Onur Taga
+  - M. Emrullah Ildiz
+  - Samet Oymak
+  affiliation: University of Michigan, Ann Arbor
+  citation_key: taga2025timepfn
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.047_timepfn.zip
+  archive_type: zip
+  local_dir_name: paper-47-TimePFN
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.047_timepfn.zip
+  sha256: b295b1371d836aeeec199601aa42427fcd6cbbf1f73349b5ebdbfd3a5977ab18
+  size_bytes: 73341
+display:
+  palette_seed: jade-navy-pfn
+  art_style: foundation-signal
+  accent_priority: high
+image_path: ../image/047_aisb.t3.047_timepfn.jpg

package/AISB/catalog/aisb.t3.048_proxyspex.yaml

@@ -0,0 +1,148 @@
+id: aisb.t3.048_proxyspex
+name: 'Proxy-SPEX: Sample-Efficient Interpretability via Sparse Feature Interactions
+  in LLMs'
+version: 0.1.0
+one_line: Shapley-style sparse feature-interaction approximations for identifying
+  compact, meaningful interaction structure in LLM predictions.
+task_description: 'This packaged benchmark covers sparse feature-interaction explanations
+  for language models using Shapley-style approximation via the ProxySPEX (Proxy SParse
+  EXplainer) method exposed within the shapiq Python package. The task evaluates the
+  quality of interaction discovery using budget-limited game evaluations to identify
+  compact, meaningful interaction structure behind model behavior. The shapiq package
+  extends Shapley value methodology beyond feature attributions to any-order Shapley
+  interactions, providing state-of-the-art approximation algorithms and a benchmarking
+  suite with pre-computed games and ground-truth values across 10+ application domains.
+
+  '
+task_mode: evaluation_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: runnable
+support_level: turnkey
+time_band: 2-6h
+cost_band: medium
+difficulty: medium
+data_access: public
+primary_outputs:
+- precision_at_10
+- precision_at_5
+- feature_interaction_rankings
+- mse
+- mae
+- explanation_report
+launch_profiles:
+- id: quick_eval
+  label: Quick Eval
+  description: 'Run the packaged sparse interaction explanation route on a single
+    target setup using the ProxySPEX approximator with StratifiedBySize sampling on
+    the SOUM (Set- Oriented Utility Model) synthetic game. Executes ensemble of 10
+    runs per seed with budget=200 and k-SII index for primary evaluation.
+
+    '
+- id: full_benchmark
+  label: Full Benchmark Suite
+  description: 'Run the complete shapiq benchmarking suite via eval_benchmark_results.py,
+    evaluating all approximator configurations across pre-computed games including
+    LocalExplanation, GlobalExplanation, FeatureSelection, DataValuation, DatasetValuation,
+    and EnsembleSelection domains.
+
+    '
+dataset_download:
+  primary_method: archive
+  sources:
+  - url: https://deepscientist.cc/AISB/048_proxyspex
+    type: primary_archive
+  notes:
+  - Archive contains the broader shapiq package (local_dir_name: paper-48-Shapiq)
+  - ProxySPEX is one method exposed within the shapiq package API
+  - Pre-computed game data and benchmark results included in data/precomputed_games/
+  - Additional datasets included: adult_census.csv, bike.csv, california_housing.csv
+credential_requirements:
+  mode: none
+  items: []
+  notes: []
+resources:
+  minimum:
+    cpu_cores: 8
+    ram_gb: 32
+    disk_gb: 50
+    gpu_count: 0
+    gpu_vram_gb: 0
+  recommended:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 100
+    gpu_count: 1
+    gpu_vram_gb: 24
+environment:
+  python: '3.12'
+  cuda: null
+  pytorch: null
+  flash_attn: null
+  key_packages:
+  - shapiq
+  - numpy
+  - scipy
+  - pandas
+  - scikit-learn
+  - joblib
+  - tqdm
+  - sparse-transform
+  - galois
+  - matplotlib
+  - networkx
+  notes:
+  - CPU-only execution is fully supported for the minimum route
+  - GPU acceleration optional but not required; recommended for large-scale LLM explanations
+  - The shapiq package is the containing module; ProxySPEX is accessed via shapiq.ProxySPEX
+    or shapiq.explainer with approximator="proxyspex"
+  - See pyproject.toml and bundled requirements for full dependency specification
+risk_flags: []
+risk_notes:
+- Metric values are code-backed but no benchmark execution was performed in this packaging
+  pass
+- Runtime validation required before trusting quantitative outputs
+recommended_when: 'Use this benchmark when you need interpretability tasks focused
+  on sparse interaction discovery in language models, want to evaluate approximation
+  quality against ground-truth Shapley interactions, or need a CPU-friendly explainability
+  evaluation framework with support for any-order interaction quantification.
+
+  '
+not_recommended_when: 'Do not use this benchmark for pure text generation evaluation,
+  full model fine-tuning workflows, or when you require GPU-accelerated training.
+  Not suitable when you cannot support repeated explanation runs with varying budget
+  constraints.
+
+  '
+paper:
+  title: 'Proxy-SPEX: Sample-Efficient Interpretability via Sparse Feature Interactions
+    in LLMs'
+  venue: NeurIPS 2025
+  year: 2025
+  url: https://openreview.net/forum?id=KI8qan2EA7
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.048_proxyspex.zip
+  archive_type: zip
+  local_dir_name: paper-48-Shapiq
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.048_proxyspex.zip
+  sha256: 3fea6e582a4001554d86ebcb7b95b55a4d537689f6279c561be43ba0e8302778
+  size_bytes: 135005330
+display:
+  palette_seed: umber-cyan-interact
+  art_style: interaction-map
+  accent_priority: high
+image_path: ../image/048_aisb.t3.048_proxyspex.jpg
+capability_tags:
+- research_code_optimization
+- mechanistic_interpretability
+- shapley_interactions
+- large_language_models
+- explanation_methods
+aisb_direction: T3
+track_fit:
+- paper_track
+- benchmark_track

package/AISB/catalog/aisb.t3.048_proxyspex.zh.yaml

@@ -0,0 +1,125 @@
+id: aisb.t3.048_proxyspex
+name: 'Proxy-SPEX: Sample-Efficient Interpretability via Sparse Feature Interactions
+  in LLMs'
+version: 0.1.0
+one_line: >-
+  基于Shapley风格的稀疏特征交互近似方法，用于识别LLM预测中紧凑且有意义的交互结构。
+task_description: >-
+  该打包基准测试涵盖使用ProxySPEX（Proxy SParse EXplainer）方法通过Shapley风格近似进行语言模型的稀疏特征交互解释。该任务通过预算限制的博弈评估来评估交互发现的质量，以识别模型行为背后紧凑且有意义的交互结构。shapiq包将Shapley值方法从特征归因扩展到任意阶Shapley交互，提供最先进的近似算法以及包含预计算博弈和真值的基准测试套件，涵盖10多个应用领域。
+task_mode: evaluation_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: runnable
+support_level: turnkey
+time_band: 2-6h
+cost_band: medium
+difficulty: medium
+data_access: public
+primary_outputs:
+- precision_at_10
+- precision_at_5
+- feature_interaction_rankings
+- mse
+- mae
+- explanation_report
+launch_profiles:
+- id: quick_eval
+  label: Quick Eval
+  label_zh: 快速评估
+  description: >-
+    使用ProxySPEX近似器在SOUM（集合导向效用模型）合成博弈上运行打包的稀疏交互解释流程，采用StratifiedBySize采样。针对主要评估使用budget=200和k-SII索引执行每个种子10次运行的集成。
+- id: full_benchmark
+  label: Full Benchmark Suite
+  label_zh: 完整基准测试套件
+  description: >-
+    通过eval_benchmark_results.py运行完整的shapiq基准测试套件，评估所有近似器配置在预计算博弈上的表现，包括LocalExplanation、GlobalExplanation、FeatureSelection、DataValuation、DatasetValuation和EnsembleSelection领域。
+dataset_download:
+  primary_method: archive
+  sources:
+  - url: https://deepscientist.cc/AISB/048_proxyspex
+    type: primary_archive
+  notes:
+  - 压缩包包含更广泛的shapiq包（local_dir_name: paper-48-Shapiq）
+  - ProxySPEX是shapiq包API中暴露的一种方法
+  - 预计算的博弈数据和基准测试结果包含在data/precomputed_games/中
+  - 附加数据集包括：adult_census.csv、bike.csv、california_housing.csv
+credential_requirements:
+  mode: none
+  items: []
+  notes: []
+resources:
+  minimum:
+    cpu_cores: 8
+    ram_gb: 32
+    disk_gb: 50
+    gpu_count: 0
+    gpu_vram_gb: 0
+  recommended:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 100
+    gpu_count: 1
+    gpu_vram_gb: 24
+environment:
+  python: '3.12'
+  cuda: null
+  pytorch: null
+  flash_attn: null
+  key_packages:
+  - shapiq
+  - numpy
+  - scipy
+  - pandas
+  - scikit-learn
+  - joblib
+  - tqdm
+  - sparse-transform
+  - galois
+  - matplotlib
+  - networkx
+  notes:
+  - 最小化路线完全支持CPU执行
+  - GPU加速可选但非必需；大规模LLM解释推荐使用GPU
+  - shapiq包是包含模块；ProxySPEX通过shapiq.ProxySPEX或shapiq.explainer
+    with approximator="proxyspex"访问
+  - 参见pyproject.toml和捆绑的requirements以获取完整的依赖规范
+risk_flags: []
+risk_notes:
+- 指标值有代码支持，但本次打包过程中未执行基准测试
+- 在信任定量输出之前需要运行时验证
+recommended_when: >-
+  当您需要专注于语言模型中稀疏交互发现的可解释性任务时使用此基准测试；想要根据真值Shapley交互评估近似质量时使用；或者需要一个支持任意阶交互量化的CPU友好的可解释性评估框架时使用。
+not_recommended_when: >-
+  不要将此基准测试用于纯文本生成评估、完整模型微调工作流程，或当您需要GPU加速训练时。如果无法支持具有不同预算约束的重复解释运行，则不适合使用。
+paper:
+  title: 'Proxy-SPEX: Sample-Efficient Interpretability via Sparse Feature Interactions
+    in LLMs'
+  venue: NeurIPS 2025
+  year: 2025
+  url: https://openreview.net/forum?id=KI8qan2EA7
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.048_proxyspex.zip
+  archive_type: zip
+  local_dir_name: paper-48-Shapiq
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.048_proxyspex.zip
+  sha256: 3fea6e582a4001554d86ebcb7b95b55a4d537689f6279c561be43ba0e8302778
+  size_bytes: 135005330
+display:
+  palette_seed: umber-cyan-interact
+  art_style: interaction-map
+  accent_priority: high
+image_path: ../image/048_aisb.t3.048_proxyspex.jpg
+capability_tags:
+- research_code_optimization
+- mechanistic_interpretability
+- shapley_interactions
+- large_language_models
+- explanation_methods
+aisb_direction: T3
+track_fit:
+- paper_track
+- benchmark_track