clearai-dsh 0.1.0

package/presets/clearai/template/skills/literature-review/scripts/search_databases.py

@@ -0,0 +1,303 @@
+#!/usr/bin/env python3
+"""
+Literature Database Search Script
+Searches multiple literature databases and aggregates results.
+"""
+
+import json
+import sys
+from typing import Dict, List
+from datetime import datetime
+
+def format_search_results(results: List[Dict], output_format: str = 'json') -> str:
+    """
+    Format search results for output.
+
+    Args:
+        results: List of search results
+        output_format: Format (json, markdown, or bibtex)
+
+    Returns:
+        Formatted string
+    """
+    if output_format == 'json':
+        return json.dumps(results, indent=2)
+
+    elif output_format == 'markdown':
+        md = f"# Literature Search Results\n\n"
+        md += f"**Search Date**: {datetime.now().strftime('%Y-%m-%d %H:%M')}\n"
+        md += f"**Total Results**: {len(results)}\n\n"
+
+        for i, result in enumerate(results, 1):
+            md += f"## {i}. {result.get('title', 'Untitled')}\n\n"
+            md += f"**Authors**: {result.get('authors', 'Unknown')}\n\n"
+            md += f"**Year**: {result.get('year', 'N/A')}\n\n"
+            md += f"**Source**: {result.get('source', 'Unknown')}\n\n"
+
+            if result.get('abstract'):
+                md += f"**Abstract**: {result['abstract']}\n\n"
+
+            if result.get('doi'):
+                md += f"**DOI**: [{result['doi']}](https://doi.org/{result['doi']})\n\n"
+
+            if result.get('url'):
+                md += f"**URL**: {result['url']}\n\n"
+
+            if result.get('citations'):
+                md += f"**Citations**: {result['citations']}\n\n"
+
+            md += "---\n\n"
+
+        return md
+
+    elif output_format == 'bibtex':
+        bibtex = ""
+        for i, result in enumerate(results, 1):
+            entry_type = result.get('type', 'article')
+            cite_key = f"{result.get('first_author', 'unknown')}{result.get('year', '0000')}"
+
+            bibtex += f"@{entry_type}{{{cite_key},\n"
+            bibtex += f"  title = {{{result.get('title', '')}}},\n"
+            bibtex += f"  author = {{{result.get('authors', '')}}},\n"
+            bibtex += f"  year = {{{result.get('year', '')}}},\n"
+
+            if result.get('journal'):
+                bibtex += f"  journal = {{{result['journal']}}},\n"
+
+            if result.get('volume'):
+                bibtex += f"  volume = {{{result['volume']}}},\n"
+
+            if result.get('pages'):
+                bibtex += f"  pages = {{{result['pages']}}},\n"
+
+            if result.get('doi'):
+                bibtex += f"  doi = {{{result['doi']}}},\n"
+
+            bibtex += "}\n\n"
+
+        return bibtex
+
+    else:
+        raise ValueError(f"Unknown format: {output_format}")
+
+def deduplicate_results(results: List[Dict]) -> List[Dict]:
+    """
+    Remove duplicate results based on DOI or title.
+
+    Args:
+        results: List of search results
+
+    Returns:
+        Deduplicated list
+    """
+    seen_dois = set()
+    seen_titles = set()
+    unique_results = []
+
+    for result in results:
+        doi = result.get('doi', '').lower().strip()
+        title = result.get('title', '').lower().strip()
+
+        # Check DOI first (more reliable)
+        if doi and doi in seen_dois:
+            continue
+
+        # Check title as fallback
+        if not doi and title in seen_titles:
+            continue
+
+        # Add to results
+        if doi:
+            seen_dois.add(doi)
+        if title:
+            seen_titles.add(title)
+
+        unique_results.append(result)
+
+    return unique_results
+
+def rank_results(results: List[Dict], criteria: str = 'citations') -> List[Dict]:
+    """
+    Rank results by specified criteria.
+
+    Args:
+        results: List of search results
+        criteria: Ranking criteria (citations, year, relevance)
+
+    Returns:
+        Ranked list
+    """
+    if criteria == 'citations':
+        return sorted(results, key=lambda x: x.get('citations', 0), reverse=True)
+    elif criteria == 'year':
+        return sorted(results, key=lambda x: x.get('year', '0'), reverse=True)
+    elif criteria == 'relevance':
+        return sorted(results, key=lambda x: x.get('relevance_score', 0), reverse=True)
+    else:
+        return results
+
+def filter_by_year(results: List[Dict], start_year: int = None, end_year: int = None) -> List[Dict]:
+    """
+    Filter results by publication year range.
+
+    Args:
+        results: List of search results
+        start_year: Minimum year (inclusive)
+        end_year: Maximum year (inclusive)
+
+    Returns:
+        Filtered list
+    """
+    filtered = []
+
+    for result in results:
+        try:
+            year = int(result.get('year', 0))
+            if start_year and year < start_year:
+                continue
+            if end_year and year > end_year:
+                continue
+            filtered.append(result)
+        except (ValueError, TypeError):
+            # Include if year parsing fails
+            filtered.append(result)
+
+    return filtered
+
+def generate_search_summary(results: List[Dict]) -> Dict:
+    """
+    Generate summary statistics for search results.
+
+    Args:
+        results: List of search results
+
+    Returns:
+        Summary dictionary
+    """
+    summary = {
+        'total_results': len(results),
+        'sources': {},
+        'year_distribution': {},
+        'avg_citations': 0,
+        'total_citations': 0
+    }
+
+    citations = []
+
+    for result in results:
+        # Count by source
+        source = result.get('source', 'Unknown')
+        summary['sources'][source] = summary['sources'].get(source, 0) + 1
+
+        # Count by year
+        year = result.get('year', 'Unknown')
+        summary['year_distribution'][year] = summary['year_distribution'].get(year, 0) + 1
+
+        # Collect citations
+        if result.get('citations'):
+            try:
+                citations.append(int(result['citations']))
+            except (ValueError, TypeError):
+                pass
+
+    if citations:
+        summary['avg_citations'] = sum(citations) / len(citations)
+        summary['total_citations'] = sum(citations)
+
+    return summary
+
+def main():
+    """Command-line interface for search result processing."""
+    if len(sys.argv) < 2:
+        print("Usage: python search_databases.py <results.json> [options]")
+        print("\nOptions:")
+        print("  --format FORMAT          Output format (json, markdown, bibtex)")
+        print("  --output FILE            Output file (default: stdout)")
+        print("  --rank CRITERIA          Rank by (citations, year, relevance)")
+        print("  --year-start YEAR        Filter by start year")
+        print("  --year-end YEAR          Filter by end year")
+        print("  --deduplicate            Remove duplicates")
+        print("  --summary                Show summary statistics")
+        sys.exit(1)
+
+    # Load results
+    results_file = sys.argv[1]
+    try:
+        with open(results_file, 'r', encoding='utf-8') as f:
+            results = json.load(f)
+    except Exception as e:
+        print(f"Error loading results: {e}")
+        sys.exit(1)
+
+    # Parse options
+    output_format = 'markdown'
+    output_file = None
+    rank_criteria = None
+    year_start = None
+    year_end = None
+    do_dedup = False
+    show_summary = False
+
+    i = 2
+    while i < len(sys.argv):
+        arg = sys.argv[i]
+
+        if arg == '--format' and i + 1 < len(sys.argv):
+            output_format = sys.argv[i + 1]
+            i += 2
+        elif arg == '--output' and i + 1 < len(sys.argv):
+            output_file = sys.argv[i + 1]
+            i += 2
+        elif arg == '--rank' and i + 1 < len(sys.argv):
+            rank_criteria = sys.argv[i + 1]
+            i += 2
+        elif arg == '--year-start' and i + 1 < len(sys.argv):
+            year_start = int(sys.argv[i + 1])
+            i += 2
+        elif arg == '--year-end' and i + 1 < len(sys.argv):
+            year_end = int(sys.argv[i + 1])
+            i += 2
+        elif arg == '--deduplicate':
+            do_dedup = True
+            i += 1
+        elif arg == '--summary':
+            show_summary = True
+            i += 1
+        else:
+            i += 1
+
+    # Process results
+    if do_dedup:
+        results = deduplicate_results(results)
+        print(f"After deduplication: {len(results)} results")
+
+    if year_start or year_end:
+        results = filter_by_year(results, year_start, year_end)
+        print(f"After year filter: {len(results)} results")
+
+    if rank_criteria:
+        results = rank_results(results, rank_criteria)
+        print(f"Ranked by: {rank_criteria}")
+
+    # Show summary
+    if show_summary:
+        summary = generate_search_summary(results)
+        print("\n" + "="*60)
+        print("SEARCH SUMMARY")
+        print("="*60)
+        print(json.dumps(summary, indent=2))
+        print()
+
+    # Format output
+    output = format_search_results(results, output_format)
+
+    # Write output
+    if output_file:
+        with open(output_file, 'w', encoding='utf-8') as f:
+            f.write(output)
+        print(f"✓ Results saved to: {output_file}")
+    else:
+        print(output)
+
+if __name__ == "__main__":
+    main()

package/presets/clearai/template/skills/literature-review/scripts/verify_citations.py

@@ -0,0 +1,221 @@
+#!/usr/bin/env python3
+"""
+Citation Verification Script
+Verifies DOIs, URLs, and citation metadata for accuracy.
+"""
+
+import re
+import requests
+import json
+from typing import Dict, List, Tuple
+import time
+
+class CitationVerifier:
+    def __init__(self):
+        self.session = requests.Session()
+        self.session.headers.update({
+            'User-Agent': 'CitationVerifier/1.0 (Literature Review Tool)'
+        })
+
+    def extract_dois(self, text: str) -> List[str]:
+        """Extract all DOIs from text."""
+        doi_pattern = r'10\.\d{4,}/[^\s\]\)"]+'
+        return re.findall(doi_pattern, text)
+
+    def verify_doi(self, doi: str) -> Tuple[bool, Dict]:
+        """
+        Verify a DOI and retrieve metadata.
+        Returns (is_valid, metadata)
+        """
+        try:
+            url = f"https://doi.org/api/handles/{doi}"
+            response = self.session.get(url, timeout=10)
+
+            if response.status_code == 200:
+                # DOI exists, now get metadata from CrossRef
+                metadata = self._get_crossref_metadata(doi)
+                return True, metadata
+            else:
+                return False, {}
+        except Exception as e:
+            return False, {"error": str(e)}
+
+    def _get_crossref_metadata(self, doi: str) -> Dict:
+        """Get metadata from CrossRef API."""
+        try:
+            url = f"https://api.crossref.org/works/{doi}"
+            response = self.session.get(url, timeout=10)
+
+            if response.status_code == 200:
+                data = response.json()
+                message = data.get('message', {})
+
+                # Extract key metadata
+                metadata = {
+                    'title': message.get('title', [''])[0],
+                    'authors': self._format_authors(message.get('author', [])),
+                    'year': self._extract_year(message),
+                    'journal': message.get('container-title', [''])[0],
+                    'volume': message.get('volume', ''),
+                    'pages': message.get('page', ''),
+                    'doi': doi
+                }
+                return metadata
+            return {}
+        except Exception as e:
+            return {"error": str(e)}
+
+    def _format_authors(self, authors: List[Dict]) -> str:
+        """Format author list."""
+        if not authors:
+            return ""
+
+        formatted = []
+        for author in authors[:3]:  # First 3 authors
+            given = author.get('given', '')
+            family = author.get('family', '')
+            if family:
+                formatted.append(f"{family}, {given[0]}." if given else family)
+
+        if len(authors) > 3:
+            formatted.append("et al.")
+
+        return ", ".join(formatted)
+
+    def _extract_year(self, message: Dict) -> str:
+        """Extract publication year."""
+        date_parts = message.get('published-print', {}).get('date-parts', [[]])
+        if not date_parts or not date_parts[0]:
+            date_parts = message.get('published-online', {}).get('date-parts', [[]])
+
+        if date_parts and date_parts[0]:
+            return str(date_parts[0][0])
+        return ""
+
+    def verify_url(self, url: str) -> Tuple[bool, int]:
+        """
+        Verify a URL is accessible.
+        Returns (is_accessible, status_code)
+        """
+        try:
+            response = self.session.head(url, timeout=10, allow_redirects=True)
+            is_accessible = response.status_code < 400
+            return is_accessible, response.status_code
+        except Exception:
+            return False, 0
+
+    def verify_citations_in_file(self, filepath: str) -> Dict:
+        """
+        Verify all citations in a markdown file.
+        Returns a report of verification results.
+        """
+        with open(filepath, 'r', encoding='utf-8') as f:
+            content = f.read()
+
+        dois = self.extract_dois(content)
+
+        report = {
+            'total_dois': len(dois),
+            'verified': [],
+            'failed': [],
+            'metadata': {}
+        }
+
+        for doi in dois:
+            print(f"Verifying DOI: {doi}")
+            is_valid, metadata = self.verify_doi(doi)
+
+            if is_valid:
+                report['verified'].append(doi)
+                report['metadata'][doi] = metadata
+            else:
+                report['failed'].append(doi)
+
+            time.sleep(0.5)  # Rate limiting
+
+        return report
+
+    def format_citation_apa(self, metadata: Dict) -> str:
+        """Format citation in APA style."""
+        authors = metadata.get('authors', '')
+        year = metadata.get('year', 'n.d.')
+        title = metadata.get('title', '')
+        journal = metadata.get('journal', '')
+        volume = metadata.get('volume', '')
+        pages = metadata.get('pages', '')
+        doi = metadata.get('doi', '')
+
+        citation = f"{authors} ({year}). {title}. "
+        if journal:
+            citation += f"*{journal}*"
+        if volume:
+            citation += f", *{volume}*"
+        if pages:
+            citation += f", {pages}"
+        if doi:
+            citation += f". https://doi.org/{doi}"
+
+        return citation
+
+    def format_citation_nature(self, metadata: Dict) -> str:
+        """Format citation in Nature style."""
+        authors = metadata.get('authors', '')
+        title = metadata.get('title', '')
+        journal = metadata.get('journal', '')
+        volume = metadata.get('volume', '')
+        pages = metadata.get('pages', '')
+        year = metadata.get('year', '')
+
+        citation = f"{authors} {title}. "
+        if journal:
+            citation += f"*{journal}* "
+        if volume:
+            citation += f"**{volume}**, "
+        if pages:
+            citation += f"{pages} "
+        if year:
+            citation += f"({year})"
+
+        return citation
+
+def main():
+    """Example usage."""
+    import sys
+
+    if len(sys.argv) < 2:
+        print("Usage: python verify_citations.py <markdown_file>")
+        sys.exit(1)
+
+    filepath = sys.argv[1]
+    verifier = CitationVerifier()
+
+    print(f"Verifying citations in: {filepath}")
+    report = verifier.verify_citations_in_file(filepath)
+
+    print("\n" + "="*60)
+    print("CITATION VERIFICATION REPORT")
+    print("="*60)
+    print(f"\nTotal DOIs found: {report['total_dois']}")
+    print(f"Verified: {len(report['verified'])}")
+    print(f"Failed: {len(report['failed'])}")
+
+    if report['failed']:
+        print("\nFailed DOIs:")
+        for doi in report['failed']:
+            print(f"  - {doi}")
+
+    if report['metadata']:
+        print("\n\nVerified Citations (APA format):")
+        for doi, metadata in report['metadata'].items():
+            citation = verifier.format_citation_apa(metadata)
+            print(f"\n{citation}")
+
+    # Save detailed report
+    output_file = filepath.replace('.md', '_citation_report.json')
+    with open(output_file, 'w', encoding='utf-8') as f:
+        json.dump(report, f, indent=2)
+
+    print(f"\n\nDetailed report saved to: {output_file}")
+
+if __name__ == "__main__":
+    main()

package/presets/clearai/template/skills/paper-lookup/SKILL.md

@@ -0,0 +1,59 @@
+---
+name: paper-lookup
+description: |
+  【论文检索·学术 API】通过 REST API 查 PubMed/arXiv/OpenAlex/Crossref 等学术库。适用：DOI/PMID 解析、论文元数据、引用关系、开放获取链接。不适用：行业快研与商业信息（用 web_search + domain-presearch）；完整综述流程（用 literature-review）。
+license: MIT license
+metadata:
+  version: 1.0-clearai
+  skill-author: K-Dense Inc. (adapted for ClearAI)
+  tier: system
+  origin: template
+  created_at: '2026-06-12T02:47:05.409595+00:00'
+---
+
+# Paper Lookup Skill（ClearAI 版）
+
+## 使用边界
+
+- **适用**：需要学术论文元数据、DOI 互转、预印本、引用图；补充 `web_search` 的学术精度。
+- **不适用**：替代 `web_search` 做行业新闻/企业情报；不单独产出完整综述。
+
+## ClearAI 工具与路径映射
+
+- API 调用 → `bash`（`curl` / Python `requests`），脚本可放 `lab/scripts/`
+- 结果落盘 → `lab/knowledge/paper_lookup_results.json`
+- 与综述衔接 → 供 `literature-review` / `citation-management` 消费
+- 经验回写 → `clear/memory/paper_lookup_lessons.md`
+
+## 核心工作流
+
+1. **理解查询**：主题检索 / 特定 DOI / 作者 / OA PDF？
+2. **选库**：见下表；详 endpoint 见 `references/` 各库文件
+3. **并行查询**：独立库可用同轮多个 `bash`
+4. **返回**：原始 JSON + 已查库列表 + 无结果须显式说明
+
+## 数据库选择（摘要）
+
+| 意图 | 主库 | 补充 |
+|------|------|------|
+| 生物医学主题 | PubMed | Semantic Scholar, OpenAlex |
+| 物理/数学/CS 预印本 | arXiv | OpenAlex |
+| 跨学科 | OpenAlex | Crossref, Semantic Scholar |
+| DOI 元数据 | Crossref | Unpaywall |
+| 开放获取链接 | Unpaywall | PMC, CORE |
+| 引用关系 | Semantic Scholar | OpenAlex |
+
+## 标识符
+
+| 类型 | 示例 |
+|------|------|
+| DOI | `10.1038/nature12373` |
+| PMID | `34567890` |
+| arXiv | `2103.15348` |
+
+## 与 web_search 分工
+
+- 行业案例、企业动态、中文工艺资讯 → `web_search`
+- 论文题名、摘要、DOI、引用链 → 本 Skill
+
+详细 API 示例见 `references/pubmed.md`、`references/crossref.md` 等（已从上游复制）。