@researai/deepscientist 1.5.17 → 1.6.0

package/AISB/catalog/aisb.t3.034_ensemblewm.zh.yaml

@@ -0,0 +1,146 @@
+id: aisb.t3.034_ensemblewm
+name: 大语言模型集成水印技术
+version: 0.1.0
+one_line: 结合红绿水印、藏头诗和感觉运动规范的多特征水印检测，用于鲁棒的LLM输出验证。
+task_description: '该基准测试用于评估大语言模型输出的集成水印检测方法。集成方法结合三种不同的水印特征：红绿水印（令牌级logit操作）、藏头诗嵌入（句子首字母编码）和感觉运动规范（感知/动作类别选择）。任务涉及通过logit修改生成带水印文本，并检测清洁和改写后的输出中的水印。评估涵盖检测率、水印分数分布，以及使用基于T5的攻击管道对改写攻击的鲁棒性。统一检测函数可在所有集成配置下应用，无需修改。
+
+'
+task_mode: evaluation_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: runnable
+support_level: advanced
+time_band: 6-24h
+cost_band: medium
+difficulty: hard
+data_access: public
+primary_outputs:
+- detection_rate
+- watermark_scores
+- benchmark_report
+launch_profiles:
+- id: quick_eval
+  label: 快速评估
+  description: 在生成的输出上运行一个打包的水印检测评估路线。使用单一模型配置和预定义测试提示来测量基线检测率，无需攻击模拟。
+- id: full_eval
+  label: 完整评估
+  description: 执行完整的多特征水印检测工作流程，包括生成、软改写攻击（基于T5的可配置百分比单词替换）以及所有特征组合的评估。生成检测率矩阵和水印分数分布。
+dataset_download:
+  primary_method: mixed
+  sources:
+  - url: https://deepscientist.cc/AISB/034_ensemblewm
+    type: archive
+    format: zip
+  - url: https://huggingface.co/datasets/know-center/Lancaster_sensorimotor_norms
+    type: external
+    format: csv
+    description: Lancaster感觉运动规范数据集，包含39,707个词，覆盖11个感知维度和5个动作维度
+  notes:
+  - Lancaster感觉运动规范CSV已打包在压缩包中
+  - 测试提示来源于标准LLM评估数据集
+credential_requirements:
+  mode: none
+  items: []
+  notes:
+  - 无需外部API密钥
+  - 本地LLM权重需另行获取（例如通过HuggingFace获取Llama-3.1-8B）
+resources:
+  minimum:
+    cpu_cores: 8
+    ram_gb: 32
+    disk_gb: 80
+    gpu_count: 1
+    gpu_vram_gb: 24
+    notes: 支持量化模型（GPTQ 4位）以降低显存需求
+  recommended:
+    cpu_cores: 16
+    ram_gb: 64
+    disk_gb: 150
+    gpu_count: 1
+    gpu_vram_gb: 48
+    notes: 生成实验建议使用全精度；内存受限场景可使用4位量化
+environment:
+  python: '3.10'
+  cuda: '11.8'
+  pytorch: 2.1.0
+  flash_attn: 2.x
+  key_packages:
+  - transformers
+  - torch
+  - numpy
+  - pandas
+  - spacy
+  - datasets
+  - tqdm
+  - bitsandbytes
+  - auto-gptq
+  notes:
+  - 需要spacy en_core_web_sm模型：python -m spacy download en_core_web_sm
+  - T5改写攻击模型通过transformers加载（T5-small或T5-base）
+  - Llama模型通过AutoModelForCausalLM加载，可选BitsAndBytesConfig实现4位量化
+  - 完整依赖规范请参阅打包的requirements.txt
+risk_flags:
+- high_vram
+- extended_runtime
+risk_notes:
+- 生成实验需要大量GPU显存用于LLM推理
+- 包含攻击模拟的完整评估可能需要数小时，具体取决于数据集大小
+- 软攻击（改写）的batch_size参数会影响显存使用
+recommended_when: '当您需要一个专注于水印检测质量的LLM安全评估管道、想要评估集成检测对改写的鲁棒性，或需要比较单特征与多特征水印方案时，使用此基准测试。适用于评估纯检测方法，无需模型微调。
+
+'
+not_recommended_when: '如果您无法在本地托管开放LLM检查点、缺乏足够显存的GPU资源，或需要一个不包含生成和检测循环的基准测试，请勿使用此基准测试。不适用于评估文本分类或困惑度-based检测方法。
+
+'
+paper:
+  title: Ensemble Watermarks for Large Language Models
+  authors:
+  - Georg Niess
+  - Roman Kern
+  venue: arXiv preprint
+  year: 2024
+  url: https://arxiv.org/abs/2411.19563
+  code_url: https://github.com/
+  abstract: '随着大语言模型达到类似人类的流畅度，可靠地区分AI生成的文本与人类撰写的内容变得越来越困难。我们提出了一种多特征水印生成方法，将藏头诗和感觉运动规范与已建立的红绿水印相结合，实现了98%的检测率。经过改写攻击后，性能保持在95%的检测率，而红绿水印单独仅有49%。
+
+'
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.034_ensemblewm.zip
+  archive_type: zip
+  local_dir_name: aisb-t3-034-ensemblewm
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.034_ensemblewm.zip
+  sha256: 8eae0196937e9b32feade1727d417158061f0741b7b0b80a9f154bdd6aaba079
+  size_bytes: 25404700
+display:
+  palette_seed: aqua-ink-watermark
+  art_style: verification-grid
+  accent_priority: high
+  tags:
+  - watermarking
+  - detection
+  - llm-safety
+  - robustness
+  - ensemble-methods
+image_path: ../image/034_aisb.t3.034_ensemblewm.jpg
+metric_contract:
+  primary_metric: detection_rate
+  origin_path: detection_notebook.ipynb
+  source_ref: detection_rate
+  evaluation_protocol:
+    code_paths:
+    - batch_run/run_experiments_soft.py
+    - batch_run/run_attack_soft.py
+    - detection_notebook.ipynb
+    - modules/text_generation.py
+  metrics_summary: []
+  execution_status: pending
+  execution_notes: '静态代码审计确认所有分阶段指标的代码可执行锚点。本打包过程中未执行基准测试。指标值应视为暂定，需等待可信运行时输出。
+
+'
+executive_summary: '该基准测试为LLM输出中的集成水印检测提供了一个全面的评估框架。该方法结合令牌级（红绿）、句子级（藏头诗）和语义级（感觉运动）水印特征，以在改写攻击后仍能实现鲁棒检测。统一检测函数可在所有特征组合上运行，支持不同安全-鲁棒性权衡的灵活配置。
+
+'

package/AISB/catalog/aisb.t3.035_moralvalueswa.yaml

@@ -0,0 +1,207 @@
+schema_version: 1
+id: aisb.t3.035_moralvalueswa
+name: Comparing Moral Values in Western English-speaking societies and LLMs with Word
+  Associations
+version: 0.1.0
+one_line: Graph-based moral-value propagation benchmark comparing human and LLM word-association
+  networks against Moral Foundation Theory using Spearman correlations on five moral
+  dimensions.
+task_description: 'This benchmark evaluates moral-value alignment between humans and
+  LLMs by constructing word-association graphs and propagating moral scores from Moral
+  Foundation Theory (MFT) seed words through them via a random-walk algorithm. The
+  primary task is to optimize the propagation parameters and graph construction to
+  maximize Spearman correlation of predicted moral scores against the extended Moral
+  Foundations Dictionary (eMFD) across five dimensions: care, sanctity, fairness,
+  authority, and loyalty. The benchmark ships pre-computed LLaMA 3.1 word-association
+  data (llama_2.1_association.json, llama_2.1_moral.json) so the expensive LLM generation
+  step (100 prompts × 12k cues on GPU) can be skipped. The evaluation route (eval.py)
+  loads pre-computed moral scores, computes correlations against eMFD, and reports
+  per-dimension results. Two external data files must be obtained before running:
+  SWOW-EN.R100.csv (human word associations, requires registration at smallworldofwords.org)
+  and emfd_scoring.csv (auto-downloaded from GitHub if missing). The experiments.py
+  script provides additional analyses including alpha tuning, MAG comparison, and
+  cross-graph experiments from the paper.
+
+  '
+capability_tags:
+- research_code_optimization
+- moral_reasoning
+- graph_analysis
+- llm_evaluation
+- nlp
+aisb_direction: T3
+track_fit:
+- paper_track
+- benchmark_track
+task_mode: evaluation_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: runnable
+support_level: turnkey
+cost_band: low
+time_band: 2-6h
+difficulty: medium
+data_access: public
+primary_outputs:
+- correlation_care
+- correlation_sanctity
+- correlation_fairness
+- correlation_authority
+- correlation_loyalty
+- moral_value_graphs
+- analysis_report
+launch_profiles:
+- id: graph_eval
+  label: Graph Eval (Quick)
+  description: 'Run eval.py with pre-computed moral scores to reproduce Table 1 GMN-L
+    correlation results. Requires only emfd_scoring.csv (auto-downloaded). CPU-only,
+    completes in minutes.
+
+    '
+- id: full_experiments
+  label: Full Experiments
+  description: 'Run experiments.py to reproduce alpha tuning, MAG comparison, varying-alpha
+    sweeps, and cross-graph analyses from the paper. Requires SWOW-EN.R100.csv for
+    human association graph construction. CPU-feasible but benefits from GPU for getMoralValue
+    propagation.
+
+    '
+- id: llm_generation
+  label: LLM Association Generation
+  description: 'Re-run produce.py to regenerate LLaMA word associations from scratch.
+    Requires a HuggingFace token with access to meta-llama/Meta-Llama-3.1-8B-Instruct,
+    a GPU with ≥24 GB VRAM, and significant compute time (100 iterations × 12k cues).
+    Not required for evaluation; pre-computed outputs are bundled.
+
+    '
+dataset_download:
+  primary_method: mixed
+  sources:
+  - kind: bundled
+    url: null
+    access: public
+    note: 'Pre-computed LLaMA word associations (llama_2.1.json, llama_2.1_association.json,
+      llama_2.1_moral.json), MFD2 dictionary, cue words, and mag_words.json are included
+      in the snapshot under data/.
+
+      '
+  - kind: external_download
+    url: https://raw.githubusercontent.com/medianeuroscience/emfd/master/dictionaries/emfd_scoring.csv
+    access: public
+    note: 'eMFD scoring CSV. Auto-downloaded by eval.py if missing. ~200 KB.
+
+      '
+  - kind: external_registration
+    url: https://smallworldofwords.org/en/project/research
+    access: public
+    note: 'SWOW-EN.R100.csv (human word association data). Requires free registration
+      at smallworldofwords.org. Needed only for full_experiments and human graph construction.
+
+      '
+  - kind: external_download
+    url: https://provalisresearch.com/products/content-analysis-software/wordstat-dictionary/moral-foundations-dictionary/
+    access: public
+    note: 'MFD 2.0 dictionary (mfd2.txt). Already bundled in data/ but original source
+      listed for reference.
+
+      '
+  notes:
+  - Total disk footprint including all external files is well under 1 GB.
+  - The graph_eval launch profile needs only the bundled files plus emfd_scoring.csv.
+credential_requirements:
+  mode: optional
+  items:
+  - HuggingFace token (only if re-running LLM association generation via produce.py)
+  notes:
+  - produce.py requires HF_TOKEN with access to meta-llama/Meta-Llama-3.1-8B-Instruct
+  - Evaluation and experiments routes do not require any credentials
+resources:
+  minimum:
+    cpu_cores: 8
+    ram_gb: 16
+    disk_gb: 20
+    gpu_count: 0
+    gpu_vram_gb: 0
+  recommended:
+    cpu_cores: 16
+    ram_gb: 32
+    disk_gb: 80
+    gpu_count: 1
+    gpu_vram_gb: 24
+environment:
+  python: null
+  cuda: null
+  pytorch: null
+  flash_attn: null
+  key_packages:
+  - numpy
+  - pandas
+  - scipy
+  - gensim
+  - matplotlib
+  - tqdm
+  - torch
+  - transformers
+  - huggingface_hub
+  notes:
+  - CPU-only execution is sufficient for eval.py and most of experiments.py.
+  - GPU and transformers/torch are needed only for produce.py (LLM association generation).
+  - getMoralValue.py imports torch.cuda but the propagation algorithm itself is array-based.
+  - See bundled requirements.txt for the full dependency set.
+risk_flags:
+- external_data_registration
+- optional_credential
+risk_notes:
+- SWOW-EN.R100.csv requires free registration at smallworldofwords.org; without it
+  the human association graph cannot be rebuilt and some experiments.py functions
+  will fail.
+- emfd_scoring.csv is auto-fetched from GitHub but network access is needed on first
+  run.
+- Re-generating LLM associations via produce.py is computationally expensive (GPU
+  hours) and requires HuggingFace model access; this is optional since pre-computed
+  outputs are bundled.
+- No benchmark execution was performed during the packaging pass; metric values are
+  from paper Table 1.
+recommended_when: 'Use this benchmark when you want a lightweight graph-based NLP
+  evaluation task focused on moral reasoning that can reuse bundled word-association
+  data without retraining or prompting a language model. Good for testing optimization
+  of graph propagation algorithms, exploring moral foundation analysis, or comparing
+  human vs. LLM conceptual organization.
+
+  '
+not_recommended_when: 'Do not use this if you need a large-scale end-to-end LLM fine-tuning
+  benchmark, if you require vision or multimodal capabilities, or if you cannot obtain
+  the SWOW dataset registration and need the full experiments pipeline including human
+  association graph construction.
+
+  '
+paper:
+  title: Comparing Moral Values in Western English-speaking societies and LLMs with
+    Word Associations
+  authors:
+  - Chaoyi Xiang
+  - Chunhua Liu
+  - Simon De Deyne
+  - Lea Frermann
+  venue: ACL 2025 Main
+  year: 2025
+  url: https://aclanthology.org/2025.acl-long.177/
+  doi: 10.18653/v1/2025.acl-long.177
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.035_moralvalueswa.zip
+  archive_type: zip
+  local_dir_name: paper-35-MoralValuesWA
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.035_moralvalueswa.zip
+  sha256: 49fa6c7fa381c4e8059933d6e2aabda5b93a2d0d1eff27940bfd404a356a4b72
+  size_bytes: 48717
+commercial:
+  annual_fee: null
+display:
+  palette_seed: sepia-emerald-values
+  art_style: social-science
+  accent_priority: medium
+image_path: ../image/035_aisb.t3.035_moralvalueswa.jpg

package/AISB/catalog/aisb.t3.035_moralvalueswa.zh.yaml

@@ -0,0 +1,165 @@
+schema_version: 1
+id: aisb.t3.035_moralvalueswa
+name: 通过词汇联想比较西方英语社会与LLM的道德价值观
+version: 0.1.0
+one_line: 基于图的道德价值观传播基准，通过Spearman相关分析在五个道德维度上比较人类与LLM词汇联想网络与道德基础理论。
+task_description: '本基准通过构建词汇联想图并使用随机游走算法将道德基础理论（MFT）种子词的道德分数传播到图中，来评估人类与LLM之间的道德价值观一致性。主要任务是优化传播参数和图构建，以最大化预测道德分数与扩展道德基础词典（eMFD）在五个维度上的Spearman相关性：关怀、圣洁、公平、权威和忠诚。基准提供了预计算的LLaMA 3.1词汇联想数据（llama_2.1_association.json、llama_2.1_moral.json），因此可以跳过昂贵的LLM生成步骤（100个提示词×12k线索在GPU上运行）。评估路由（eval.py）加载预计算的道德分数，计算与eMFD的相关性，并报告每个维度的结果。运行前必须获取两个外部数据文件：SWOW-EN.R100.csv（人类词汇联想数据，需在smallworldofwords.org注册）和emfd_scoring.csv（缺失时自动从GitHub下载）。experiments.py脚本提供额外分析，包括alpha调优、MAG比较和论文中的跨图实验。
+
+  '
+capability_tags:
+- research_code_optimization
+- moral_reasoning
+- graph_analysis
+- llm_evaluation
+- nlp
+aisb_direction: T3
+track_fit:
+- paper_track
+- benchmark_track
+task_mode: evaluation_driven
+requires_execution: true
+requires_paper: true
+integrity_level: cas_plus_canary
+snapshot_status: runnable
+support_level: turnkey
+cost_band: low
+time_band: 2-6h
+difficulty: medium
+data_access: public
+primary_outputs:
+- correlation_care
+- correlation_sanctity
+- correlation_fairness
+- correlation_authority
+- correlation_loyalty
+- moral_value_graphs
+- analysis_report
+launch_profiles:
+- id: graph_eval
+  label: 图评估（快速）
+  description: '运行eval.py，使用预计算的道德分数复现表1中的GMN-L相关性结果。只需emfd_scoring.csv（自动下载）。仅需CPU，几分钟内完成。
+
+    '
+- id: full_experiments
+  label: 完整实验
+  description: '运行experiments.py复现论文中的alpha调优、MAG比较、变alpha扫描和跨图分析。需SWOW-EN.R100.csv用于人类联想图构建。CPU可执行，但getMoralValue传播受益于GPU。
+
+    '
+- id: llm_generation
+  label: LLM联想生成
+  description: '重新运行produce.py从头生成LLaMA词汇联想。需HuggingFace令牌访问meta-llama/Meta-Llama-3.1-8B-Instruct，需≥24GB显存的GPU，以及大量计算时间（100次迭代×12k线索）。评估不需要此步骤，预计算输出已捆绑。
+
+    '
+dataset_download:
+  primary_method: mixed
+  sources:
+  - kind: bundled
+    url: null
+    access: public
+    note: '预计算的LLaMA词汇联想数据（llama_2.1.json、llama_2.1_association.json、llama_2.1_moral.json）、MFD2词典、线索词和mag_words.json已包含在快照的data/目录下。
+
+      '
+  - kind: external_download
+    url: https://raw.githubusercontent.com/medianeuroscience/emfd/master/dictionaries/emfd_scoring.csv
+    access: public
+    note: 'eMFD评分CSV。eval.py缺失时自动下载。约200KB。
+
+      '
+  - kind: external_registration
+    url: https://smallworldofwords.org/en/project/research
+    access: public
+    note: 'SWOW-EN.R100.csv（人类词汇联想数据）。需在smallworldofwords.org免费注册。仅在full_experiments和人类图构建时需要。
+
+      '
+  - kind: external_download
+    url: https://provalisresearch.com/products/content-analysis-software/wordstat-dictionary/moral-foundations-dictionary/
+    access: public
+    note: 'MFD 2.0词典（mfd2.txt）。已捆绑在data/中，列为原始参考来源。
+
+      '
+  notes:
+  - 包含所有外部文件的总磁盘占用远低于1GB。
+  - graph_eval启动配置只需捆绑文件加上emfd_scoring.csv。
+credential_requirements:
+  mode: optional
+  items:
+  - HuggingFace令牌（仅在重新运行produce.py生成LLM联想时需要）
+  notes:
+  - produce.py需要具有meta-llama/Meta-Llama-3.1-8B-Instruct访问权限的HF_TOKEN
+  - 评估和实验路由不需要任何凭据
+resources:
+  minimum:
+    cpu_cores: 8
+    ram_gb: 16
+    disk_gb: 20
+    gpu_count: 0
+    gpu_vram_gb: 0
+  recommended:
+    cpu_cores: 16
+    ram_gb: 32
+    disk_gb: 80
+    gpu_count: 1
+    gpu_vram_gb: 24
+environment:
+  python: null
+  cuda: null
+  pytorch: null
+  flash_attn: null
+  key_packages:
+  - numpy
+  - pandas
+  - scipy
+  - gensim
+  - matplotlib
+  - tqdm
+  - torch
+  - transformers
+  - huggingface_hub
+  notes:
+  - 仅CPU执行足以运行eval.py和experiments.py的大部分内容。
+  - GPU和transformers/torch仅在produce.py（LLM联想生成）时需要。
+  - getMoralValue.py导入torch.cuda，但传播算法本身是基于数组的。
+  - 参见捆绑的requirements.txt获取完整依赖集。
+risk_flags:
+- external_data_registration
+- optional_credential
+risk_notes:
+- SWOW-EN.R100.csv需要在smallworldofwords.org免费注册；没有它将无法重建人类联想图，部分experiments.py函数会失败。
+- emfd_scoring.csv从GitHub自动获取，但首次运行需要网络访问。
+- 通过produce.py重新生成LLM联想计算成本高昂（GPU小时级），且需要HuggingFace模型访问权限；这是可选的，因为预计算输出已捆绑。
+- 打包过程中未执行基准测试运行；指标值来自论文表1。
+recommended_when: '当您需要一个轻量级的基于图的NLP评估任务来研究道德推理、且可以重用捆绑的词汇联想数据而无需重新训练或提示语言模型时，可以使用此基准。适合测试图传播算法的优化、探索道德基础分析，或比较人类与LLM的概念组织方式。
+
+  '
+not_recommended_when: '如果需要大规模端到端LLM微调基准、需要视觉或多模态能力，或无法获取SWOW数据集注册且需要包含人类联想图构建的完整实验管道时，请勿使用。
+
+  '
+paper:
+  title: Comparing Moral Values in Western English-speaking societies and LLMs with
+    Word Associations
+  authors:
+  - Chaoyi Xiang
+  - Chunhua Liu
+  - Simon De Deyne
+  - Lea Frermann
+  venue: ACL 2025 Main
+  year: 2025
+  url: https://aclanthology.org/2025.acl-long.177/
+  doi: 10.18653/v1/2025.acl-long.177
+download:
+  url: https://github.com/ResearAI/DeepScientist/releases/download/aisb-v0.0.1/aisb.t3.035_moralvalueswa.zip
+  archive_type: zip
+  local_dir_name: paper-35-MoralValuesWA
+  provider: github_release
+  repo: ResearAI/DeepScientist
+  tag: aisb-v0.0.1
+  asset_name: aisb.t3.035_moralvalueswa.zip
+  sha256: 49fa6c7fa381c4e8059933d6e2aabda5b93a2d0d1eff27940bfd404a356a4b72
+  size_bytes: 48717
+commercial:
+  annual_fee: null
+display:
+  palette_seed: sepia-emerald-values
+  art_style: social-science
+  accent_priority: medium
+image_path: ../image/035_aisb.t3.035_moralvalueswa.jpg