dsh-aris-panel 0.1.0

package/tools/semantic_scholar_fetch.py

@@ -0,0 +1,438 @@
+#!/usr/bin/env python3
+"""CLI helper for fetching Semantic Scholar papers.
+
+Designed to complement arxiv_fetch.py: arXiv handles preprints, this tool
+handles **published venue papers** (IEEE, ACM, Springer, etc.) with rich
+metadata (citations, venue, fieldsOfStudy, TLDR).
+
+Commands
+--------
+search       Relevance search for papers (offset pagination, max 100).
+search-bulk  Bulk search with token-based pagination (max 1000).
+paper        Fetch one paper by Semantic Scholar paper ID, DOI, CorpusId, ArXiv ID, etc.
+
+Filter flags (shared by search and search-bulk)
+-----------------------------------------------
+--fields-of-study   e.g. "Computer Science,Engineering"
+--publication-types  e.g. "JournalArticle", "Conference", "Review"
+--min-citations      e.g. 10
+--year               e.g. "2020-", "2020-2024"
+--venue              exact venue name, e.g. "IEEE Transactions on Signal Processing"
+--open-access        only papers with a public PDF
+
+Examples
+--------
+# Search for journal articles with >= 10 citations (best combo for quality filtering)
+python3 tools/semantic_scholar_fetch.py search "semantic communication" --max 10 \
+  --publication-types JournalArticle --min-citations 10
+
+# CS/Engineering papers from 2022 onward
+python3 tools/semantic_scholar_fetch.py search "semantic communication" --max 10 \
+  --fields-of-study "Computer Science,Engineering" --year "2022-"
+
+# Bulk search sorted by citation count, CS only
+python3 tools/semantic_scholar_fetch.py search-bulk "semantic communication" --max 50 \
+  --sort citationCount:desc --fields-of-study "Computer Science" --year "2020-"
+
+# Fetch a single paper by DOI or arXiv ID
+python3 tools/semantic_scholar_fetch.py paper "10.1109/JSAC.2021.3126077"
+python3 tools/semantic_scholar_fetch.py paper "ARXIV:2006.10685"
+
+# NOTE: --venue requires exact venue name (e.g. "IEEE Transactions on Signal Processing"),
+# not partial match like "IEEE". Prefer --publication-types + --fields-of-study instead.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from typing import Any
+
+_API_BASE = "https://api.semanticscholar.org/graph/v1"
+_USER_AGENT = "s2-fetch/1.1"
+_DEFAULT_TIMEOUT = 30
+
+# Good default for relevance search / single-paper fetch
+_DEFAULT_FIELDS = (
+    "paperId,title,abstract,year,venue,publicationVenue,publicationTypes,"
+    "publicationDate,url,openAccessPdf,authors,externalIds,citationCount,"
+    "referenceCount,fieldsOfStudy,s2FieldsOfStudy,tldr"
+)
+
+# Bulk search is intended for basic paper data; keep defaults conservative
+_DEFAULT_BULK_FIELDS = (
+    "paperId,title,abstract,year,venue,publicationDate,url,authors,"
+    "externalIds,citationCount,referenceCount,fieldsOfStudy"
+)
+
+
+def _headers() -> dict[str, str]:
+    headers = {
+        "User-Agent": _USER_AGENT,
+        "Accept": "application/json",
+    }
+    api_key = os.getenv("SEMANTIC_SCHOLAR_API_KEY", "").strip()
+    if api_key:
+        headers["x-api-key"] = api_key
+    return headers
+
+
+def _request_json(url: str, *, retries: int = 2, timeout: int = _DEFAULT_TIMEOUT) -> dict[str, Any]:
+    req = urllib.request.Request(url, headers=_headers())
+    last_err: Exception | None = None
+
+    for attempt in range(retries + 1):
+        try:
+            with urllib.request.urlopen(req, timeout=timeout) as resp:
+                raw = resp.read().decode("utf-8")
+            return json.loads(raw)
+        except urllib.error.HTTPError as exc:
+            body = ""
+            try:
+                body = exc.read().decode("utf-8", errors="replace")
+            except Exception:
+                pass
+
+            if exc.code in (429, 500, 502, 503, 504) and attempt < retries:
+                time.sleep(1.5 * (attempt + 1))
+                last_err = exc
+                continue
+
+            message = f"HTTP {exc.code}"
+            if body:
+                message += f": {body}"
+            raise RuntimeError(message) from exc
+        except urllib.error.URLError as exc:
+            if attempt < retries:
+                time.sleep(1.5 * (attempt + 1))
+                last_err = exc
+                continue
+            raise RuntimeError(f"Network error: {exc}") from exc
+        except json.JSONDecodeError as exc:
+            raise RuntimeError("Failed to parse JSON response from Semantic Scholar API") from exc
+
+    raise RuntimeError(f"Request failed after retries: {last_err}")
+
+
+def _clean_text(value: Any) -> str | None:
+    if value is None:
+        return None
+    text = str(value).strip().replace("\n", " ")
+    return text or None
+
+
+def _parse_author(author: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "authorId": author.get("authorId"),
+        "name": _clean_text(author.get("name")),
+    }
+
+
+def _parse_publication_venue(pub_venue: dict[str, Any] | None) -> dict[str, Any] | None:
+    if not pub_venue:
+        return None
+    return {
+        "id": pub_venue.get("id"),
+        "name": _clean_text(pub_venue.get("name")),
+        "type": _clean_text(pub_venue.get("type")),
+        "issn": _clean_text(pub_venue.get("issn")),
+        "url": _clean_text(pub_venue.get("url")),
+    }
+
+
+def _parse_paper(paper: dict[str, Any]) -> dict[str, Any]:
+    authors = paper.get("authors") or []
+    return {
+        "paperId": paper.get("paperId"),
+        "title": _clean_text(paper.get("title")),
+        "abstract": _clean_text(paper.get("abstract")),
+        "year": paper.get("year"),
+        "venue": _clean_text(paper.get("venue")),
+        "publicationVenue": _parse_publication_venue(paper.get("publicationVenue")),
+        "publicationTypes": paper.get("publicationTypes"),
+        "publicationDate": _clean_text(paper.get("publicationDate")),
+        "url": _clean_text(paper.get("url")),
+        "openAccessPdf": paper.get("openAccessPdf"),
+        "authors": [_parse_author(a) for a in authors],
+        "externalIds": paper.get("externalIds"),
+        "citationCount": paper.get("citationCount"),
+        "referenceCount": paper.get("referenceCount"),
+        "fieldsOfStudy": paper.get("fieldsOfStudy"),
+        "s2FieldsOfStudy": paper.get("s2FieldsOfStudy"),
+        "tldr": paper.get("tldr"),
+    }
+
+
+def search(
+    query: str,
+    max_results: int = 10,
+    offset: int = 0,
+    fields: str = _DEFAULT_FIELDS,
+    fields_of_study: str | None = None,
+    venue: str | None = None,
+    year: str | None = None,
+    min_citation_count: int | None = None,
+    publication_types: str | None = None,
+    open_access_pdf: bool = False,
+) -> dict[str, Any]:
+    params: dict[str, Any] = {
+        "query": query,
+        "limit": max_results,
+        "offset": offset,
+        "fields": fields,
+    }
+    if fields_of_study:
+        params["fieldsOfStudy"] = fields_of_study
+    if venue:
+        params["venue"] = venue
+    if year:
+        params["year"] = year
+    if min_citation_count is not None:
+        params["minCitationCount"] = min_citation_count
+    if publication_types:
+        params["publicationTypes"] = publication_types
+    if open_access_pdf:
+        params["openAccessPdf"] = ""
+    url = f"{_API_BASE}/paper/search?{urllib.parse.urlencode(params)}"
+    payload = _request_json(url)
+
+    data = payload.get("data") or []
+    return {
+        "mode": "search",
+        "total": payload.get("total"),
+        "offset": offset,
+        "next_offset": offset + len(data),
+        "data": [_parse_paper(item) for item in data],
+    }
+
+
+def search_bulk(
+    query: str,
+    max_results: int = 100,
+    token: str | None = None,
+    fields: str = _DEFAULT_BULK_FIELDS,
+    sort: str | None = None,
+    fields_of_study: str | None = None,
+    venue: str | None = None,
+    year: str | None = None,
+    min_citation_count: int | None = None,
+    publication_types: str | None = None,
+    open_access_pdf: bool = False,
+) -> dict[str, Any]:
+    params: dict[str, Any] = {
+        "query": query,
+        "limit": max_results,
+        "fields": fields,
+    }
+    if token:
+        params["token"] = token
+    if sort:
+        params["sort"] = sort
+    if fields_of_study:
+        params["fieldsOfStudy"] = fields_of_study
+    if venue:
+        params["venue"] = venue
+    if year:
+        params["year"] = year
+    if min_citation_count is not None:
+        params["minCitationCount"] = min_citation_count
+    if publication_types:
+        params["publicationTypes"] = publication_types
+    if open_access_pdf:
+        params["openAccessPdf"] = ""
+
+    url = f"{_API_BASE}/paper/search/bulk?{urllib.parse.urlencode(params)}"
+    payload = _request_json(url)
+
+    data = payload.get("data") or []
+    return {
+        "mode": "search-bulk",
+        "token": payload.get("token"),
+        "returned": len(data),
+        "sort": sort,
+        "data": [_parse_paper(item) for item in data],
+    }
+
+
+def get_paper(paper_id: str, fields: str = _DEFAULT_FIELDS) -> dict[str, Any]:
+    encoded_id = urllib.parse.quote(paper_id, safe="")
+    params = {"fields": fields}
+    url = f"{_API_BASE}/paper/{encoded_id}?{urllib.parse.urlencode(params)}"
+    payload = _request_json(url)
+    return _parse_paper(payload)
+
+
+def _add_filter_args(parser: argparse.ArgumentParser) -> None:
+    """Add shared filtering arguments to a search sub-parser."""
+    parser.add_argument(
+        "--fields-of-study",
+        default=None,
+        help="Comma-separated fields of study filter, e.g. 'Computer Science,Engineering'.",
+    )
+    parser.add_argument(
+        "--venue",
+        default=None,
+        help="Comma-separated venue filter, e.g. 'IEEE,ACM' or 'Nature'.",
+    )
+    parser.add_argument(
+        "--year",
+        default=None,
+        help="Year or range, e.g. '2023', '2020-2024', '2020-', '-2023'.",
+    )
+    parser.add_argument(
+        "--min-citations",
+        type=int,
+        default=None,
+        metavar="N",
+        help="Minimum citation count filter.",
+    )
+    parser.add_argument(
+        "--publication-types",
+        default=None,
+        help="Comma-separated types: JournalArticle,Conference,Review,etc.",
+    )
+    parser.add_argument(
+        "--open-access",
+        action="store_true",
+        default=False,
+        help="Only return papers with a public PDF.",
+    )
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="Search and fetch papers from Semantic Scholar.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    search_parser = subparsers.add_parser("search", help="Relevance search for papers")
+    search_parser.add_argument("query", help="Keyword query")
+    search_parser.add_argument(
+        "--max",
+        type=int,
+        default=10,
+        metavar="N",
+        help="Maximum number of results to return (default: 10).",
+    )
+    search_parser.add_argument(
+        "--offset",
+        type=int,
+        default=0,
+        help="Offset for pagination (default: 0).",
+    )
+    search_parser.add_argument(
+        "--fields",
+        default=_DEFAULT_FIELDS,
+        help="Comma-separated response fields to request.",
+    )
+    _add_filter_args(search_parser)
+
+    bulk_parser = subparsers.add_parser(
+        "search-bulk",
+        help="Bulk search for papers with token-based pagination",
+    )
+    bulk_parser.add_argument("query", help="Keyword query")
+    bulk_parser.add_argument(
+        "--max",
+        type=int,
+        default=100,
+        metavar="N",
+        help="Maximum number of results to return in this page (default: 100).",
+    )
+    bulk_parser.add_argument(
+        "--token",
+        default=None,
+        help="Continuation token returned by a previous bulk search page.",
+    )
+    bulk_parser.add_argument(
+        "--sort",
+        default=None,
+        help="Optional sort for bulk search, e.g. publicationDate:desc or citationCount:desc",
+    )
+    bulk_parser.add_argument(
+        "--fields",
+        default=_DEFAULT_BULK_FIELDS,
+        help="Comma-separated response fields to request.",
+    )
+    _add_filter_args(bulk_parser)
+
+    paper_parser = subparsers.add_parser("paper", help="Fetch one paper by ID")
+    paper_parser.add_argument(
+        "id",
+        help=(
+            "Semantic Scholar paper ID, DOI, CorpusId:..., ARXIV:..., PMID:..., MAG:..., ACL:..., etc."
+        ),
+    )
+    paper_parser.add_argument(
+        "--fields",
+        default=_DEFAULT_FIELDS,
+        help="Comma-separated response fields to request.",
+    )
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _build_parser().parse_args(argv)
+
+    try:
+        if args.command == "search":
+            result = search(
+                query=args.query,
+                max_results=args.max,
+                offset=args.offset,
+                fields=args.fields,
+                fields_of_study=args.fields_of_study,
+                venue=args.venue,
+                year=args.year,
+                min_citation_count=args.min_citations,
+                publication_types=args.publication_types,
+                open_access_pdf=args.open_access,
+            )
+            print(json.dumps(result, ensure_ascii=False, indent=2))
+            return 0
+
+        if args.command == "search-bulk":
+            result = search_bulk(
+                query=args.query,
+                max_results=args.max,
+                token=args.token,
+                fields=args.fields,
+                sort=args.sort,
+                fields_of_study=args.fields_of_study,
+                venue=args.venue,
+                year=args.year,
+                min_citation_count=args.min_citations,
+                publication_types=args.publication_types,
+                open_access_pdf=args.open_access,
+            )
+            print(json.dumps(result, ensure_ascii=False, indent=2))
+            return 0
+
+        if args.command == "paper":
+            result = get_paper(
+                paper_id=args.id,
+                fields=args.fields,
+            )
+            print(json.dumps(result, ensure_ascii=False, indent=2))
+            return 0
+
+        raise ValueError(f"Unsupported command: {args.command}")
+
+    except KeyboardInterrupt:
+        print("Interrupted.", file=sys.stderr)
+        return 130
+    except Exception as exc:
+        print(f"Error: {exc}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/tools/skill-groups.tsv

@@ -0,0 +1,116 @@
+# ARIS skill catalog — single source of truth for selective install.
+# Consumed by: install_aris.sh / install_aris_codex.sh / install_aris_copilot.sh /
+#              install_aris.ps1 / smart_update*.sh|ps1 / tools/skill_picker.py /
+#              tests/test_skill_groups.py
+#
+# Record types (tab-separated):
+#   group	<group-id>	<display-name>	<description>
+#   skill	<skill-name>	<group-id>	<requires>	<short-description>
+#
+# <requires> = skills the entry actually exercises in a DEFAULT run:
+#   (a) unconditional pipeline phases ("Invoke /x" with no gate);
+#   (b) default-on invocations — blocking (submission-assurance mandatory
+#       audits) or documented-silent-skip (auto-render, W&B health checks):
+#       a silent skip is silent feature loss, so it still belongs here;
+#   (c) hard artifact prerequisites the skill errors out demanding
+#       (wiki-enrich -> "run /research-wiki init first", meta-apply ->
+#       "run /meta-optimize first").
+# NOT requires: opt-in flags/branches, "use X instead" alternatives,
+# input-producers and downstream consumers, comparisons, doc links.
+# "-" = none. Every edge was verified against the SKILL.md invocation site
+# (#366 review pass); tests/test_skill_groups.py guards each edge textually.
+# <short-description> = one concise line shown in the interactive picker /
+# --list-groups. Every skills/<name>/SKILL.md must appear exactly once;
+# tests/test_skill_groups.py enforces catalog completeness.
+group	lit-search	文献检索	arXiv/OpenAlex/Semantic Scholar 等文献搜索、抓取与阅读
+group	ideation	选题与方向	找 idea、查新、方向打磨、research wiki 与总编排 pipeline
+group	review-loop	自动评审闭环	跨模型自动 review 循环与对抗性评审
+group	theory	公式与证明	公式推导、证明撰写与证明审查
+group	experiments	实验与算力	实验计划/执行/监控/分析与 GPU 集群、无服务器算力
+group	paper-core	论文写作主线	从大纲到 PDF 的写作、编译、数字/引用/诚信审计
+group	paper-visuals	配图与展示	插图、示意图、海报、幻灯片与演讲
+group	submission	投稿与基金	返修 rebuttal、重投 pipeline、基金申请书
+group	patent	专利	发明构建、权利要求、实施例到各国格式的专利流水线
+group	meta-utils	元工具与杂项	skill 语料自优化、通知、面试准备等
+skill	arxiv	lit-search	-	arXiv 搜索/下载/入库
+skill	alphaxiv	lit-search	-	单篇论文快速解读(AlphaXiv 摘要)
+skill	deepxiv	lit-search	-	DeepXiv 渐进式论文阅读
+skill	semantic-scholar	lit-search	-	正式发表论文检索(引用数/venue)
+skill	openalex	lit-search	-	OpenAlex 引用图谱与机构元数据
+skill	exa-search	lit-search	-	Exa AI 网页搜索+内容抽取
+skill	web-debug-search	lit-search	-	多来源调试检索（GitHub/Stack Exchange/中文社区/General Web）与版本兼容性
+skill	gemini-search	lit-search	-	Gemini 驱动的文献发现
+skill	research-lit	lit-search	-	文献综述与相关工作分析
+skill	comm-lit-review	lit-search	-	通信领域专用文献综述
+skill	idea-creator	ideation	-	给定方向生成并排序研究 idea
+skill	idea-discovery	ideation	idea-creator,novelty-check,research-lit,research-review,research-refine-pipeline,research-refine,render-html	W1:从方向到验证过的 idea 全流程
+skill	idea-discovery-robot	ideation	idea-creator,novelty-check,research-lit,research-review	W1 机器人/具身智能特化版
+skill	novelty-check	ideation	-	idea 查新(有没有人做过)
+skill	research-refine	ideation	-	粗糙方向打磨成聚焦提案
+skill	research-refine-pipeline	ideation	research-refine,experiment-plan	refine+实验计划一条龙
+skill	research-wiki	ideation	-	跨生命周期研究知识库
+skill	wiki-enrich	ideation	research-wiki	补全 wiki 论文页 TODO
+skill	research-pipeline	ideation	idea-discovery,experiment-bridge,auto-review-loop,render-html	端到端总 pipeline:找 idea→实验→论文
+skill	research-review	review-loop	-	研究方案/结果批判性评审
+skill	auto-review-loop	review-loop	result-to-claim,training-check,render-html	外部模型多轮 review→修复循环
+skill	auto-review-loop-llm	review-loop	-	用任意 OpenAI 兼容 API 做 review
+skill	auto-review-loop-minimax	review-loop	-	用 MiniMax API 做 review
+skill	kill-argument	review-loop	render-html	对抗性双线程评审(最强拒稿论证)
+skill	formula-derivation	theory	-	公式推导与理论线组织
+skill	proof-writer	theory	-	撰写严格数学证明
+skill	proof-checker	theory	render-html	证明缺口审查与修复
+skill	proof-orchestrator	theory	-	本地优先的证明编排、人工 GPT Pro 交接与显式 DeepSeek 复审
+skill	experiment-plan	experiments	-	claim 驱动的实验路线图
+skill	experiment-bridge	experiments	run-experiment,experiment-queue,monitor-experiment,training-check,ablation-planner	W1.5:从实验计划到跑通初步结果
+skill	experiment-queue	experiments	-	SSH 批量实验队列(OOM 重试)
+skill	run-experiment	experiments	-	部署运行实验(本地/远程/云 GPU)
+skill	monitor-experiment	experiments	-	监控实验进度收集结果
+skill	analyze-results	experiments	-	结果统计分析与对比表
+skill	training-check	experiments	-	训练健康度检查
+skill	system-profile	experiments	-	GPU 环境画像(跨机器复用)
+skill	dse-loop	experiments	-	设计空间探索自动调参循环
+skill	qzcli	experiments	-	启智平台 qzcli 作业管理
+skill	vast-gpu	experiments	-	Vast.ai 租卡跑实验
+skill	serverless-modal	experiments	-	Modal 无服务器 GPU
+skill	experiment-audit	experiments	-	实验诚实度审计(防假结果)
+skill	result-to-claim	experiments	-	结果能支撑什么 claim 的裁决
+skill	ablation-planner	experiments	-	消融实验规划
+skill	paper-writing	paper-core	paper-plan,paper-write,paper-compile,paper-figure,figure-spec,proof-checker,paper-claim-audit,citation-audit,auto-paper-improvement-loop,integrity-forensics,kill-argument	W3:从报告到投稿级 PDF 全流程
+skill	paper-plan	paper-core	-	论文大纲生成
+skill	paper-write	paper-core	-	按大纲逐节写 LaTeX
+skill	paper-compile	paper-core	-	LaTeX 编译修错出 PDF
+skill	paper-figure	paper-core	-	从实验结果出论文图表
+skill	citation-audit	paper-core	render-html	引用真实性/正确性核查
+skill	paper-claim-audit	paper-core	render-html	论文数字与原始结果核对
+skill	integrity-forensics	paper-core	-	投稿前诚信取证扫描
+skill	auto-paper-improvement-loop	paper-core	proof-checker,paper-claim-audit,kill-argument	论文 review→润色→重编译循环
+skill	writing-systems-papers	paper-core	-	系统方向论文写作要领
+skill	specification-writing	paper-core	embodiment-description,figure-description	技术规格文档撰写
+skill	overleaf-sync	paper-core	-	本地↔Overleaf 双向同步
+skill	render-html	paper-core	-	MD/JSON 工件渲染成 HTML 报告
+skill	paper-illustration	paper-visuals	-	AI 生成论文插图(Gemini)
+skill	paper-illustration-image2	paper-visuals	-	AI 插图备选(Codex 原生画图)
+skill	figure-spec	paper-visuals	-	确定性矢量架构/流程图(SVG)
+skill	mermaid-diagram	paper-visuals	-	Mermaid 流程/时序/类图
+skill	pixel-art	paper-visuals	-	像素画生成
+skill	paper-poster	paper-visuals	paper-poster-html	(已弃用)重定向到 poster-html
+skill	paper-poster-html	paper-visuals	-	学术海报生成(HTML 版)
+skill	paper-slides	paper-visuals	-	会议幻灯(beamer→PDF+PPTX)
+skill	paper-talk	paper-visuals	paper-slides,slides-polish	演讲稿+幻灯一体
+skill	slides-polish	paper-visuals	-	幻灯片打磨
+skill	rebuttal	submission	render-html	W4:审稿意见解析与安全回复
+skill	resubmit-pipeline	submission	auto-paper-improvement-loop,proof-checker,paper-claim-audit,citation-audit,kill-argument,paper-compile,integrity-forensics	被拒后重投全流程
+skill	grant-proposal	submission	research-lit,novelty-check,research-review	基金申请书(NSFC/NSF/科研費等)
+skill	patent-pipeline	patent	prior-art-search,patent-novelty-check,invention-structuring,claims-drafting,specification-writing,patent-review,jurisdiction-format	专利撰写全流程
+skill	invention-structuring	patent	-	粗 idea 结构化成发明披露
+skill	claims-drafting	patent	-	撰写权利要求书
+skill	embodiment-description	patent	-	实施例详细描述
+skill	figure-description	patent	-	专利附图说明
+skill	jurisdiction-format	patent	-	CN/US/EP 申请格式编译
+skill	patent-review	patent	-	专利申请质量审查
+skill	prior-art-search	patent	-	现有技术检索
+skill	patent-novelty-check	patent	-	专利新颖性核查
+skill	meta-optimize	meta-utils	-	skill 语料自优化提案
+skill	meta-apply	meta-utils	meta-optimize	落地已批准的语料补丁
+skill	feishu-notify	meta-utils	-	飞书通知
+skill	interview-cheatsheet	meta-utils	-	ML/LLM 面试速查手册生成