@researai/deepscientist 1.5.16 → 1.6.0

package/AISB/catalog/aisb.t3.039_calf.yaml

@@ -0,0 +1,224 @@
+id: aisb.t3.039_calf
+name: 'CALF: Aligning LLMs for Time Series Forecasting via Cross-modal Fine-Tuning'
+version: 0.1.0
+one_line: Cross-modal fine-tuning framework that aligns time series distributions
+  with LLM textual representations for state-of-the-art multivariate forecasting.
+task_description: 'This packaged benchmark covers CALF (Cross-ModAl LLM Fine-Tuning),
+  a framework that bridges the distribution discrepancy between temporal data and
+  the textual nature of LLMs. The task involves adapting a language-model-style architecture
+  to time-series inputs through three cross-modal fine-tuning techniques: (1) Cross-Modal
+  Match Module for input distribution alignment via principal word embedding extraction
+  and cross-attention; (2) Feature Regularization Loss for intermediate layer alignment
+  and gradient guidance; (3) Output Consistency Loss for resolving representation
+  space discrepancies. Supports both long-term (ETT-small, electricity, traffic, weather)
+  and short-term (M4) forecasting tasks with seq_len/pred_len configurations.
+
+  '
+task_mode: experiment_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: runnable
+support_level: advanced
+time_band: 1d+
+cost_band: high
+difficulty: hard
+data_access: public
+primary_outputs:
+- mse
+- mae
+- forecast_checkpoints
+launch_profiles:
+- id: quick_eval
+  label: Quick Eval
+  description: Run one packaged long-term forecasting script on a prepared dataset
+    using default ETTh2 configuration (seq_len=96, pred_len=96/192/336/720).
+- id: full_train_eval
+  label: Full Train + Eval
+  description: Run the complete CALF training and forecasting evaluation workflow
+    across all supported datasets (long-term and short-term forecasting).
+- id: few_shot_forecasting
+  label: Few-Shot Forecasting
+  description: Run few-shot forecasting evaluation to assess LLM generalization with
+    limited training data.
+- id: zero_shot_forecasting
+  label: Zero-Shot Forecasting
+  description: Run zero-shot forecasting evaluation to assess transfer capabilities.
+dataset_download:
+  primary_method: mixed
+  sources:
+  - name: Long-term datasets (Autoformer)
+    url: https://drive.google.com/drive/folders/1ZOYpTUa82_jCcxIdTmyr0LXQfvaM9vIy
+    type: google_drive
+    files:
+    - ETT-small/ETTh1.csv
+    - ETT-small/ETTh2.csv
+    - ETT-small/ETTm1.csv
+    - ETT-small/ETTm2.csv
+    - electricity/electricity.csv
+    - traffic/traffic.csv
+    - weather/weather.csv
+  - name: Short-term M4 dataset
+    url: https://drive.google.com/drive/folders/15zio96o3NK4XOoR5L88oaWcJDVOiqQo9
+    type: google_drive
+    files:
+    - m4/
+  notes:
+  - Long-term datasets must be organized under ./datasets/{dataset_name}/{filename}.csv
+  - M4 dataset should be placed under ./datasets/m4/
+  - Word token embeddings require extraction via pca.py before training (saves to
+    ./wte_pca_500.pt)
+credential_requirements:
+  mode: none
+  items: []
+  notes:
+  - Google Drive links for datasets require no authentication for public access
+  - External LLM weights (GPT-2 based) downloaded automatically via transformers library
+resources:
+  minimum:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 100
+    gpu_count: 1
+    gpu_vram_gb: 24
+    notes: Single GPU sufficient for seq_len=96, pred_len up to 720; batch_size may
+      need reduction on lower VRAM
+  recommended:
+    cpu_cores: 32
+    ram_gb: 128
+    disk_gb: 200
+    gpu_count: 1
+    gpu_vram_gb: 48
+    notes: 48GB VRAM allows batch_size=256 with all prediction lengths; higher VRAM
+      enables faster training
+environment:
+  python: '3.9'
+  cuda: '11.8'
+  pytorch: 2.1.0
+  flash_attn: null
+  key_packages:
+  - transformers==4.30.1
+  - torch>=2.1.0
+  - numpy
+  - pandas
+  - scikit-learn
+  notes:
+  - Python 3.9 explicitly required by project README
+  - Exact PyTorch/CUDA versions delegated to user environment; 2.1.0+ with CUDA 11.8
+    recommended
+  - See bundled requirements.txt for full dependency set
+  - LLM backbone based on GPT-2 architecture (downloaded via transformers)
+risk_flags:
+- high_compute_cost
+- multi_hour_training
+- large_download
+risk_notes:
+- Full evaluation across all datasets and prediction lengths requires extended training
+  time
+- Dataset downloads from Google Drive total several GB
+- GPU memory requirements scale with batch_size and prediction length
+- Training uses fixed seed (2021) for reproducibility
+recommended_when: 'Use this benchmark when evaluating cross-modal time-series alignment,
+  LLM fine-tuning for temporal data, long-horizon forecasting performance, or few-shot/zero-shot
+  generalization capabilities of language model backbones on time series tasks.
+
+  '
+not_recommended_when: 'Do not use this benchmark if you need a lightweight CPU-first
+  time-series benchmark, cannot provision GPUs with 24GB+ VRAM, or require inference-only
+  evaluation without training infrastructure.
+
+  '
+paper:
+  title: 'CALF: Aligning LLMs for Time Series Forecasting via Cross-modal Fine-Tuning'
+  authors:
+  - Peiyuan Liu
+  - Hang Guo
+  - Tao Dai
+  - Naiqi Li
+  - Jigang Bao
+  - Xudong Ren
+  - Yong Jiang
+  - Shu-Tao Xia
+  venue: arXiv preprint
+  year: 2024
+  arxiv: '2403.07300'
+  url: https://arxiv.org/abs/2403.07300
+  code_url: https://github.com/Hank0626/CALF
+  affiliations:
+  - Tsinghua Shenzhen International Graduate School
+  - Shenzhen University
+  - Pengcheng Laboratory
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.039_calf.zip
+  archive_type: zip
+  local_dir_name: paper-39-CALF
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.039_calf.zip
+  sha256: bf50089c65f0004d0e49a6d82c6225df9b7baacceead8814a715540189a49082
+  size_bytes: 3132751
+display:
+  palette_seed: teal-amber-signal
+  art_style: forecasting-lab
+  accent_priority: high
+  tags:
+  - large_language_models
+  - cross_modal_learning
+  - time_series_forecasting
+  - multivariate_forecasting
+  - llm_finetuning
+  - transformers
+image_path: ../image/039_aisb.t3.039_calf.jpg
+execution_anchors:
+  entry_point: run.py
+  main_script: scripts/long_term_forecasting/ETTh2.sh
+  key_modules:
+  - exp/exp_long_term_forecasting.py
+  - exp/exp_short_term_forecasting.py
+  - utils/metrics.py
+  - utils/cmLoss.py
+  - data_provider/data_factory.py
+  default_args:
+    task_name: long_term_forecast
+    model: CALF
+    seq_len: 96
+    pred_len: 96
+    batch_size: 256
+    learning_rate: 0.0005
+    train_epochs: 100
+    d_model: 768
+    n_heads: 4
+    gpt_layers: 6
+  metric_implementations:
+    mse:
+      source: utils/metrics.py::MSE
+      code_backed: true
+    mae:
+      source: utils/metrics.py::MAE
+      code_backed: true
+    rmse:
+      source: utils/metrics.py::RMSE
+      code_backed: true
+    mape:
+      source: utils/metrics.py::MAPE
+      code_backed: true
+    mspe:
+      source: utils/metrics.py::MSPE
+      code_backed: true
+output_artifacts:
+  checkpoints:
+    path: ./checkpoints
+    format: pytorch_model
+  results:
+    path: ./results
+    format: npy
+  summaries:
+    path: ./results_{task_name}.txt
+    format: text
+recovery_notes: 'Post-training artifacts include checkpoints in ./checkpoints, numerical
+  results in .npy format under ./results, and performance summaries in ./results_{task_name}.txt.
+  Early stopping with patience=5 is implemented. Training uses fixed seed (2021) for
+  reproducibility.
+
+  '

package/AISB/catalog/aisb.t3.039_calf.zh.yaml

@@ -0,0 +1,194 @@
+id: aisb.t3.039_calf
+name: 'CALF：通过跨模态微调实现大语言模型时间序列预测对齐'
+version: 0.1.0
+one_line: 跨模态微调框架，通过大语言模型文本表示对齐时间序列分布，实现最先进的多元预测。
+task_description: |-
+  本打包基准测试涵盖 CALF（跨模态大语言模型微调），这是一个弥合时间数据与 LLM 文本性质之间分布差异的框架。该任务涉及通过三种跨模态微调技术将类语言模型架构适配于时间序列输入：（1）跨模态匹配模块，通过主词嵌入提取和交叉注意力实现输入分布对齐；（2）特征正则化损失，用于中间层对齐和梯度引导；（3）输出一致性损失，用于解决表示空间差异。支持长期（ETT-small、electricity、traffic、weather）和短期（M4）预测任务，配置 seq_len/pred_len。
+
+task_mode: experiment_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: runnable
+support_level: advanced
+time_band: 1d+
+cost_band: high
+difficulty: hard
+data_access: public
+primary_outputs:
+- mse
+- mae
+- forecast_checkpoints
+launch_profiles:
+- id: quick_eval
+  label: 快速评估
+  description: 使用默认 ETTh2 配置（seq_len=96，pred_len=96/192/336/720）在准备好的数据集上运行一个打包的长期预测脚本。
+- id: full_train_eval
+  label: 完整训练 + 评估
+  description: 在所有支持的数据集（长期和短期预测）上运行完整的 CALF 训练和预测评估工作流程。
+- id: few_shot_forecasting
+  label: 小样本预测
+  description: 运行小样本预测评估以评估 LLM 在有限训练数据下的泛化能力。
+- id: zero_shot_forecasting
+  label: 零样本预测
+  description: 运行零样本预测评估以评估迁移能力。
+dataset_download:
+  primary_method: mixed
+  sources:
+  - name: 长期数据集（Autoformer）
+    url: https://drive.google.com/drive/folders/1ZOYpTUa82_jCcxIdTmyr0LXQfvaM9vIy
+    type: google_drive
+    files:
+    - ETT-small/ETTh1.csv
+    - ETT-small/ETTh2.csv
+    - ETT-small/ETTm1.csv
+    - ETT-small/ETTm2.csv
+    - electricity/electricity.csv
+    - traffic/traffic.csv
+    - weather/weather.csv
+  - name: 短期 M4 数据集
+    url: https://drive.google.com/drive/folders/15zio96o3NK4XOoR5L88oaWcJDVOiqQo9
+    type: google_drive
+    files:
+    - m4/
+  notes:
+  - 长期数据集必须按 ./datasets/{dataset_name}/{filename}.csv 组织
+  - M4 数据集应放置在 ./datasets/m4/ 下
+  - 词标记嵌入需要通过 pca.py 在训练前提取（保存至 ./wte_pca_500.pt）
+credential_requirements:
+  mode: none
+  items: []
+  notes:
+  - 数据集的 Google Drive 链接无需认证即可公开访问
+  - 外部 LLM 权重（基于 GPT-2）通过 transformers 库自动下载
+resources:
+  minimum:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 100
+    gpu_count: 1
+    gpu_vram_gb: 24
+    notes: 单 GPU 足以支持 seq_len=96，pred_len 最高 720；显存较低时可能需要减小 batch_size
+  recommended:
+    cpu_cores: 32
+    ram_gb: 128
+    disk_gb: 200
+    gpu_count: 1
+    gpu_vram_gb: 48
+    notes: 48GB 显存允许 batch_size=256 搭配所有预测长度；更高显存可加快训练速度
+environment:
+  python: '3.9'
+  cuda: '11.8'
+  pytorch: 2.1.0
+  flash_attn: null
+  key_packages:
+  - transformers==4.30.1
+  - torch>=2.1.0
+  - numpy
+  - pandas
+  - scikit-learn
+  notes:
+  - 项目 README 明确要求 Python 3.9
+  - 精确的 PyTorch/CUDA 版本由用户环境决定；推荐 2.1.0+ 搭配 CUDA 11.8
+  - 完整依赖项请参阅打包的 requirements.txt
+  - LLM 主干基于 GPT-2 架构（通过 transformers 下载）
+risk_flags:
+- high_compute_cost
+- multi_hour_training
+- large_download
+risk_notes:
+- 在所有数据集和预测长度上进行完整评估需要较长的训练时间
+- 从 Google Drive 下载数据集总计数 GB
+- GPU 显存需求随 batch_size 和预测长度增加而增加
+- 训练使用固定随机种子（2021）以确保可重复性
+recommended_when: 在评估跨模态时间序列对齐、时序数据 LLM 微调、长期预测性能，或语言模型主干在时间序列任务上的小样本/零样本泛化能力时使用此基准测试。
+not_recommended_when: 如果需要轻量级 CPU 优先的时间序列基准测试、无法提供 24GB+ 显存的 GPU，或需要无训练基础设施的纯推理评估，请勿使用此基准测试。
+paper:
+  title: 'CALF: Aligning LLMs for Time Series Forecasting via Cross-modal Fine-Tuning'
+  authors:
+  - Peiyuan Liu
+  - Hang Guo
+  - Tao Dai
+  - Naiqi Li
+  - Jigang Bao
+  - Xudong Ren
+  - Yong Jiang
+  - Shu-Tao Xia
+  venue: arXiv preprint
+  year: 2024
+  arxiv: '2403.07300'
+  url: https://arxiv.org/abs/2403.07300
+  code_url: https://github.com/Hank0626/CALF
+  affiliations:
+  - Tsinghua Shenzhen International Graduate School
+  - Shenzhen University
+  - Pengcheng Laboratory
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.039_calf.zip
+  archive_type: zip
+  local_dir_name: paper-39-CALF
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.039_calf.zip
+  sha256: bf50089c65f0004d0e49a6d82c6225df9b7baacceead8814a715540189a49082
+  size_bytes: 3132751
+display:
+  palette_seed: teal-amber-signal
+  art_style: forecasting-lab
+  accent_priority: high
+  tags:
+  - large_language_models
+  - cross_modal_learning
+  - time_series_forecasting
+  - multivariate_forecasting
+  - llm_finetuning
+  - transformers
+image_path: ../image/039_aisb.t3.039_calf.jpg
+execution_anchors:
+  entry_point: run.py
+  main_script: scripts/long_term_forecasting/ETTh2.sh
+  key_modules:
+  - exp/exp_long_term_forecasting.py
+  - exp/exp_short_term_forecasting.py
+  - utils/metrics.py
+  - utils/cmLoss.py
+  - data_provider/data_factory.py
+  default_args:
+    task_name: long_term_forecast
+    model: CALF
+    seq_len: 96
+    pred_len: 96
+    batch_size: 256
+    learning_rate: 0.0005
+    train_epochs: 100
+    d_model: 768
+    n_heads: 4
+    gpt_layers: 6
+  metric_implementations:
+    mse:
+      source: utils/metrics.py::MSE
+      code_backed: true
+    mae:
+      source: utils/metrics.py::MAE
+      code_backed: true
+    rmse:
+      source: utils/metrics.py::RMSE
+      code_backed: true
+    mape:
+      source: utils/metrics.py::MAPE
+      code_backed: true
+    mspe:
+      source: utils/metrics.py::MSPE
+      code_backed: true
+output_artifacts:
+  checkpoints:
+    path: ./checkpoints
+    format: pytorch_model
+  results:
+    path: ./results
+    format: npy
+  summaries:
+    path: ./results_{task_name}.txt
+    format: text
+recovery_notes: 训练后的产物包括 ./checkpoints 中的检查点、./results 下 .npy 格式的数值结果，以及 ./results_{task_name}.txt 中的性能摘要。已实现早停机制（patience=5）。训练使用固定随机种子（2021）以确保可重复性。

package/AISB/catalog/aisb.t3.040_graniteguardian.yaml

@@ -0,0 +1,199 @@
+id: aisb.t3.040_graniteguardian
+name: 'Granite Guardian: Comprehensive LLM Safeguarding'
+version: 0.1.0
+one_line: Multi-head LLM safeguard that detects jailbreaks, hallucinations, and other
+  risks across prompts and responses under configurable criteria.
+task_description: 'This packaged benchmark covers Granite Guardian style LLM safeguarding
+  across risk dimensions including jailbreaks, social bias, profanity, violence, sexual
+  content, unethical behavior, and RAG-specific hallucination risks (context relevance,
+  groundedness, answer relevance). The task involves evaluating or calibrating risk
+  detection heads so that safety detection metrics improve. The current snapshot bundles
+  cookbook-style usage routes and Jupyter notebooks demonstrating model inference
+  with vLLM, but the packaged XSTEST-style evaluator for faithful metric reproduction
+  is not yet included in the archive. External evaluation against the full metric
+  contract is required before treating this as a complete benchmark route.
+
+  '
+capability_tags:
+- llm_safety
+- guardrails
+- risk_detection
+- classification
+- evaluation
+- rag_quality
+- content_moderation
+- research_code_optimization
+aisb_direction: T3
+track_fit:
+- paper_track
+- benchmark_track
+task_mode: evaluation_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: external_eval_required
+support_level: recovery
+time_band: 2-6h
+cost_band: medium
+difficulty: medium
+data_access: public
+primary_outputs:
+- xstest_rh_auc
+- xstest_rh_f1
+- xstest_rr_auc
+- xstest_rr_f1
+- risk_scoring_examples
+- guardrail_auc
+- rag_hallucination_auc
+launch_profiles:
+- id: cookbook_demo
+  label: Cookbook Demo
+  description: 'Run the packaged Granite Guardian cookbooks (vLLM-based notebooks)
+    to verify model usage and risk scoring behavior for configured criteria. Covers
+    jailbreak detection, RAG groundedness checking, and bring-your-own-criteria workflows.
+
+    '
+- id: restore_eval
+  label: Restore Eval Route
+  description: 'Attach or restore the missing XSTEST-style evaluator before treating
+    this benchmark as a faithful metric reproduction route. Current snapshot lacks
+    executable anchors for XSTEST metrics.
+
+    '
+- id: external_benchmark_eval
+  label: External Benchmark Eval
+  description: 'Evaluate Granite Guardian models on external benchmarks including
+    XSTEST, LLM-AggreFact, and REVEAL to obtain reproducible AUC and F1 scores.
+
+    '
+dataset_download:
+  primary_method: huggingface
+  sources:
+  - name: Granite Guardian Models
+    url: https://huggingface.co/ibm-granite
+    notes:
+    - granite-guardian-3.3-8b
+    - granite-guardian-3.2-5b
+    - granite-guardian-3.2-3b-a800m
+    - granite-guardian-3.1-8b
+    - granite-guardian-3.1-2b
+  - name: Evaluation Datasets
+    url: https://huggingface.co/ibm
+    notes:
+    - SocialStigmaQA
+    - AttaQ
+    - ProvoQ
+    - WikiContradict
+credential_requirements:
+  mode: none
+  items: []
+  notes:
+  - All model weights and datasets are publicly accessible
+  - HF Spaces demo available at https://huggingface.co/spaces/ibm-granite/granite-guardian
+resources:
+  minimum:
+    cpu_cores: 8
+    ram_gb: 32
+    disk_gb: 80
+    gpu_count: 1
+    gpu_vram_gb: 24
+  recommended:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 150
+    gpu_count: 1
+    gpu_vram_gb: 48
+environment:
+  python: '3.10'
+  cuda: '11.8'
+  pytorch: 2.1.0
+  flash_attn: null
+  key_packages:
+  - torch
+  - transformers
+  - vllm
+  notes:
+  - See bundled README/requirements for the full dependency set
+  - vLLM recommended for efficient inference
+  - Content warning: examples may contain offensive language, stereotypes, or discriminatory
+      content
+risk_flags:
+- blocked_metrics
+- partial_snapshot
+risk_notes:
+- xstest_rh_auc metric has no executable code anchor in current snapshot
+- xstest_rh_f1 metric has no executable code anchor in current snapshot
+- xstest_rr_auc metric has no executable code anchor in current snapshot
+- xstest_rr_f1 metric has no executable code anchor in current snapshot
+- No benchmark execution was performed in this packaging pass
+- Treat as blocked until source restoration or richer local snapshot is recovered
+- Recovery rule: prefer existing code files over README-only claims
+recommended_when: 'Use this benchmark when you want a practical LLM-safeguard evaluation
+  task with configurable criteria, production-style monitoring use cases, or RAG pipeline
+  safety verification. Particularly suited for evaluating risk detection in retrieval-augmented
+  generation settings (groundedness, context relevance, answer relevance) and jailbreak
+  resistance testing.
+
+  '
+not_recommended_when: 'Do not use this if you need a fully executable benchmark with
+  XSTEST metric anchors in the current snapshot, or if your focus is on generative
+  quality rather than safety and verification. Not suitable when reproducible metric
+  reproduction is required without external evaluation setup.
+
+  '
+paper:
+  title: 'Granite Guardian: Comprehensive LLM Safeguarding'
+  authors: 'Inkit Padhi, Manish Nagireddy, Giandomenico Cornacchia, Subhajit Chaudhury,
+    Tejaswini Pedapati, Pierre Dognin, Keerthiram Murugesan, Erik Miehling, Martin
+    Santillan, Cooper Kieran Fraser, Giulio Zizzo, Muhammad Zaid Hameed, Mark Purcell,
+    Michael Desmond, Qian Pan, Zahra Ashktorab, Inge Vejsbjerg, Elizabeth Daly, Michael
+    Hind, Werner Geyer, Ambrish Rawat, Kush R. Varshney, Prasanna Sattigeri
+
+    '
+  venue: NAACL 2025 Industry Track
+  year: 2025
+  url: https://arxiv.org/abs/2412.07724
+  github: https://github.com/ibm-granite/granite-guardian
+  license: Apache 2.0
+  key_findings:
+  - AUC scores of 0.871 on harmful content benchmarks
+  - AUC scores of 0.854 on RAG-hallucination-related benchmarks
+  - Outperforms other open-source and closed-source models on deployment-focused metrics
+  - Trained on unique dataset combining human annotations and synthetic data
+  - Supports bring-your-own-criteria for custom risk detection
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.040_graniteguardian.zip
+  archive_type: zip
+  local_dir_name: paper-40-GraniteGuardian
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.040_graniteguardian.zip
+  sha256: c98d57a175b4dc64b7f4bb81e3c4c16adb38a98775335f077f780d491e92b4e2
+  size_bytes: 2545921
+display:
+  palette_seed: indigo-sand-shield
+  art_style: safety-console
+  accent_priority: high
+image_path: ../image/040_aisb.t3.040_graniteguardian.jpg
+code_anchors:
+  executable:
+  - cookbooks/granite-guardian-3.3/quickstart.ipynb
+  - cookbooks/granite-guardian-3.2/detailed_guide_vllm.ipynb
+  - cookbooks/granite-guardian-3.1/quick_start_vllm.ipynb
+  code_roots:
+  - cookbooks/
+  blocked_metrics:
+  - xstest_rh_auc
+  - xstest_rh_f1
+  - xstest_rr_auc
+  - xstest_rr_f1
+recovery_rule: 'Prefer existing code files over README-only claims when choosing origin_path,
+  source_ref, and evaluation_protocol.code_paths. Keep metrics_summary empty until
+  trusted execution outputs exist. Mark metrics as provisional/blocked if no executable
+  anchor exists in current snapshot.
+
+  '
+commercial:
+  annual_fee: null
+  notes: Models released as open-source under Apache 2.0 license