clearai-dsh 0.3.0 → 0.4.0

package/presets/clearai/template/skills/citation-management/scripts/search_google_scholar.py

@@ -1,282 +0,0 @@
-#!/usr/bin/env python3
-"""
-Google Scholar Search Tool
-Search Google Scholar and export results.
-
-Note: This script requires the 'scholarly' library.
-Install with: pip install scholarly
-"""
-
-import sys
-import argparse
-import json
-import time
-import random
-from typing import List, Dict, Optional
-
-try:
-    from scholarly import scholarly, ProxyGenerator
-    SCHOLARLY_AVAILABLE = True
-except ImportError:
-    SCHOLARLY_AVAILABLE = False
-    print('Warning: scholarly library not installed. Install with: pip install scholarly', file=sys.stderr)
-
-class GoogleScholarSearcher:
-    """Search Google Scholar using scholarly library."""
-    
-    def __init__(self, use_proxy: bool = False):
-        """
-        Initialize searcher.
-        
-        Args:
-            use_proxy: Use free proxy (helps avoid rate limiting)
-        """
-        if not SCHOLARLY_AVAILABLE:
-            raise ImportError('scholarly library required. Install with: pip install scholarly')
-        
-        # Setup proxy if requested
-        if use_proxy:
-            try:
-                pg = ProxyGenerator()
-                pg.FreeProxies()
-                scholarly.use_proxy(pg)
-                print('Using free proxy', file=sys.stderr)
-            except Exception as e:
-                print(f'Warning: Could not setup proxy: {e}', file=sys.stderr)
-    
-    def search(self, query: str, max_results: int = 50,
-               year_start: Optional[int] = None, year_end: Optional[int] = None,
-               sort_by: str = 'relevance') -> List[Dict]:
-        """
-        Search Google Scholar.
-        
-        Args:
-            query: Search query
-            max_results: Maximum number of results
-            year_start: Start year filter
-            year_end: End year filter
-            sort_by: Sort order ('relevance' or 'citations')
-            
-        Returns:
-            List of result dictionaries
-        """
-        if not SCHOLARLY_AVAILABLE:
-            print('Error: scholarly library not installed', file=sys.stderr)
-            return []
-        
-        print(f'Searching Google Scholar: {query}', file=sys.stderr)
-        print(f'Max results: {max_results}', file=sys.stderr)
-        
-        results = []
-        
-        try:
-            # Perform search
-            search_query = scholarly.search_pubs(query)
-            
-            for i, result in enumerate(search_query):
-                if i >= max_results:
-                    break
-                
-                print(f'Retrieved {i+1}/{max_results}', file=sys.stderr)
-                
-                # Extract metadata
-                metadata = {
-                    'title': result.get('bib', {}).get('title', ''),
-                    'authors': ', '.join(result.get('bib', {}).get('author', [])),
-                    'year': result.get('bib', {}).get('pub_year', ''),
-                    'venue': result.get('bib', {}).get('venue', ''),
-                    'abstract': result.get('bib', {}).get('abstract', ''),
-                    'citations': result.get('num_citations', 0),
-                    'url': result.get('pub_url', ''),
-                    'eprint_url': result.get('eprint_url', ''),
-                }
-                
-                # Filter by year
-                if year_start or year_end:
-                    try:
-                        pub_year = int(metadata['year']) if metadata['year'] else 0
-                        if year_start and pub_year < year_start:
-                            continue
-                        if year_end and pub_year > year_end:
-                            continue
-                    except ValueError:
-                        pass
-                
-                results.append(metadata)
-                
-                # Rate limiting to avoid blocking
-                time.sleep(random.uniform(2, 5))
-            
-        except Exception as e:
-            print(f'Error during search: {e}', file=sys.stderr)
-        
-        # Sort if requested
-        if sort_by == 'citations' and results:
-            results.sort(key=lambda x: x.get('citations', 0), reverse=True)
-        
-        return results
-    
-    def metadata_to_bibtex(self, metadata: Dict) -> str:
-        """Convert metadata to BibTeX format."""
-        # Generate citation key
-        if metadata.get('authors'):
-            first_author = metadata['authors'].split(',')[0].strip()
-            last_name = first_author.split()[-1] if first_author else 'Unknown'
-        else:
-            last_name = 'Unknown'
-        
-        year = metadata.get('year', 'XXXX')
-        
-        # Get keyword from title
-        import re
-        title = metadata.get('title', '')
-        words = re.findall(r'\b[a-zA-Z]{4,}\b', title)
-        keyword = words[0].lower() if words else 'paper'
-        
-        citation_key = f'{last_name}{year}{keyword}'
-        
-        # Determine entry type (guess based on venue)
-        venue = metadata.get('venue', '').lower()
-        if 'proceedings' in venue or 'conference' in venue:
-            entry_type = 'inproceedings'
-            venue_field = 'booktitle'
-        else:
-            entry_type = 'article'
-            venue_field = 'journal'
-        
-        # Build BibTeX
-        lines = [f'@{entry_type}{{{citation_key},']
-        
-        # Convert authors format
-        if metadata.get('authors'):
-            authors = metadata['authors'].replace(',', ' and')
-            lines.append(f'  author  = {{{authors}}},')
-        
-        if metadata.get('title'):
-            lines.append(f'  title   = {{{metadata["title"]}}},')
-        
-        if metadata.get('venue'):
-            lines.append(f'  {venue_field} = {{{metadata["venue"]}}},')
-        
-        if metadata.get('year'):
-            lines.append(f'  year    = {{{metadata["year"]}}},')
-        
-        if metadata.get('url'):
-            lines.append(f'  url     = {{{metadata["url"]}}},')
-        
-        if metadata.get('citations'):
-            lines.append(f'  note    = {{Cited by: {metadata["citations"]}}},')
-        
-        # Remove trailing comma
-        if lines[-1].endswith(','):
-            lines[-1] = lines[-1][:-1]
-        
-        lines.append('}')
-        
-        return '\n'.join(lines)
-
-
-def main():
-    """Command-line interface."""
-    parser = argparse.ArgumentParser(
-        description='Search Google Scholar (requires scholarly library)',
-        epilog='Example: python search_google_scholar.py "machine learning" --limit 50'
-    )
-    
-    parser.add_argument(
-        'query',
-        help='Search query'
-    )
-    
-    parser.add_argument(
-        '--limit',
-        type=int,
-        default=50,
-        help='Maximum number of results (default: 50)'
-    )
-    
-    parser.add_argument(
-        '--year-start',
-        type=int,
-        help='Start year for filtering'
-    )
-    
-    parser.add_argument(
-        '--year-end',
-        type=int,
-        help='End year for filtering'
-    )
-    
-    parser.add_argument(
-        '--sort-by',
-        choices=['relevance', 'citations'],
-        default='relevance',
-        help='Sort order (default: relevance)'
-    )
-    
-    parser.add_argument(
-        '--use-proxy',
-        action='store_true',
-        help='Use free proxy to avoid rate limiting'
-    )
-    
-    parser.add_argument(
-        '-o', '--output',
-        help='Output file (default: stdout)'
-    )
-    
-    parser.add_argument(
-        '--format',
-        choices=['json', 'bibtex'],
-        default='json',
-        help='Output format (default: json)'
-    )
-    
-    args = parser.parse_args()
-    
-    if not SCHOLARLY_AVAILABLE:
-        print('\nError: scholarly library not installed', file=sys.stderr)
-        print('Install with: pip install scholarly', file=sys.stderr)
-        print('\nAlternatively, use PubMed search for biomedical literature:', file=sys.stderr)
-        print('  python search_pubmed.py "your query"', file=sys.stderr)
-        sys.exit(1)
-    
-    # Search
-    searcher = GoogleScholarSearcher(use_proxy=args.use_proxy)
-    results = searcher.search(
-        args.query,
-        max_results=args.limit,
-        year_start=args.year_start,
-        year_end=args.year_end,
-        sort_by=args.sort_by
-    )
-    
-    if not results:
-        print('No results found', file=sys.stderr)
-        sys.exit(1)
-    
-    # Format output
-    if args.format == 'json':
-        output = json.dumps({
-            'query': args.query,
-            'count': len(results),
-            'results': results
-        }, indent=2)
-    else:  # bibtex
-        bibtex_entries = [searcher.metadata_to_bibtex(r) for r in results]
-        output = '\n\n'.join(bibtex_entries) + '\n'
-    
-    # Write output
-    if args.output:
-        with open(args.output, 'w', encoding='utf-8') as f:
-            f.write(output)
-        print(f'Wrote {len(results)} results to {args.output}', file=sys.stderr)
-    else:
-        print(output)
-    
-    print(f'\nRetrieved {len(results)} results', file=sys.stderr)
-
-
-if __name__ == '__main__':
-    main()
-

package/presets/clearai/template/skills/citation-management/scripts/search_pubmed.py

@@ -1,398 +0,0 @@
-#!/usr/bin/env python3
-"""
-PubMed Search Tool
-Search PubMed using E-utilities API and export results.
-"""
-
-import sys
-import os
-import requests
-import argparse
-import json
-import time
-import xml.etree.ElementTree as ET
-from typing import List, Dict, Optional
-from datetime import datetime
-
-class PubMedSearcher:
-    """Search PubMed using NCBI E-utilities API."""
-    
-    def __init__(self, api_key: Optional[str] = None, email: Optional[str] = None):
-        """
-        Initialize searcher.
-        
-        Args:
-            api_key: NCBI API key (optional but recommended)
-            email: Email for Entrez (optional but recommended)
-        """
-        self.api_key = api_key or os.getenv('NCBI_API_KEY', '')
-        self.email = email or os.getenv('NCBI_EMAIL', '')
-        self.base_url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/'
-        self.session = requests.Session()
-        
-        # Rate limiting
-        self.delay = 0.11 if self.api_key else 0.34  # 10/sec with key, 3/sec without
-    
-    def search(self, query: str, max_results: int = 100,
-               date_start: Optional[str] = None, date_end: Optional[str] = None,
-               publication_types: Optional[List[str]] = None) -> List[str]:
-        """
-        Search PubMed and return PMIDs.
-        
-        Args:
-            query: Search query
-            max_results: Maximum number of results
-            date_start: Start date (YYYY/MM/DD or YYYY)
-            date_end: End date (YYYY/MM/DD or YYYY)
-            publication_types: List of publication types to filter
-            
-        Returns:
-            List of PMIDs
-        """
-        # Build query with filters
-        full_query = query
-        
-        # Add date range
-        if date_start or date_end:
-            start = date_start or '1900'
-            end = date_end or datetime.now().strftime('%Y')
-            full_query += f' AND {start}:{end}[Publication Date]'
-        
-        # Add publication types
-        if publication_types:
-            pub_type_query = ' OR '.join([f'"{pt}"[Publication Type]' for pt in publication_types])
-            full_query += f' AND ({pub_type_query})'
-        
-        print(f'Searching PubMed: {full_query}', file=sys.stderr)
-        
-        # ESearch to get PMIDs
-        esearch_url = self.base_url + 'esearch.fcgi'
-        params = {
-            'db': 'pubmed',
-            'term': full_query,
-            'retmax': max_results,
-            'retmode': 'json'
-        }
-        
-        if self.email:
-            params['email'] = self.email
-        if self.api_key:
-            params['api_key'] = self.api_key
-        
-        try:
-            response = self.session.get(esearch_url, params=params, timeout=30)
-            response.raise_for_status()
-            
-            data = response.json()
-            pmids = data['esearchresult']['idlist']
-            count = int(data['esearchresult']['count'])
-            
-            print(f'Found {count} results, retrieving {len(pmids)}', file=sys.stderr)
-            
-            return pmids
-            
-        except Exception as e:
-            print(f'Error searching PubMed: {e}', file=sys.stderr)
-            return []
-    
-    def fetch_metadata(self, pmids: List[str]) -> List[Dict]:
-        """
-        Fetch metadata for PMIDs.
-        
-        Args:
-            pmids: List of PubMed IDs
-            
-        Returns:
-            List of metadata dictionaries
-        """
-        if not pmids:
-            return []
-        
-        metadata_list = []
-        
-        # Fetch in batches of 200
-        batch_size = 200
-        for i in range(0, len(pmids), batch_size):
-            batch = pmids[i:i+batch_size]
-            print(f'Fetching metadata for PMIDs {i+1}-{min(i+batch_size, len(pmids))}...', file=sys.stderr)
-            
-            efetch_url = self.base_url + 'efetch.fcgi'
-            params = {
-                'db': 'pubmed',
-                'id': ','.join(batch),
-                'retmode': 'xml',
-                'rettype': 'abstract'
-            }
-            
-            if self.email:
-                params['email'] = self.email
-            if self.api_key:
-                params['api_key'] = self.api_key
-            
-            try:
-                response = self.session.get(efetch_url, params=params, timeout=60)
-                response.raise_for_status()
-                
-                # Parse XML
-                root = ET.fromstring(response.content)
-                articles = root.findall('.//PubmedArticle')
-                
-                for article in articles:
-                    metadata = self._extract_metadata_from_xml(article)
-                    if metadata:
-                        metadata_list.append(metadata)
-                
-                # Rate limiting
-                time.sleep(self.delay)
-                
-            except Exception as e:
-                print(f'Error fetching metadata for batch: {e}', file=sys.stderr)
-                continue
-        
-        return metadata_list
-    
-    def _extract_metadata_from_xml(self, article: ET.Element) -> Optional[Dict]:
-        """Extract metadata from PubmedArticle XML element."""
-        try:
-            medline_citation = article.find('.//MedlineCitation')
-            article_elem = medline_citation.find('.//Article')
-            journal = article_elem.find('.//Journal')
-            
-            # Get PMID
-            pmid = medline_citation.findtext('.//PMID', '')
-            
-            # Get DOI
-            doi = None
-            article_ids = article.findall('.//ArticleId')
-            for article_id in article_ids:
-                if article_id.get('IdType') == 'doi':
-                    doi = article_id.text
-                    break
-            
-            # Get authors
-            authors = []
-            author_list = article_elem.find('.//AuthorList')
-            if author_list is not None:
-                for author in author_list.findall('.//Author'):
-                    last_name = author.findtext('.//LastName', '')
-                    fore_name = author.findtext('.//ForeName', '')
-                    if last_name:
-                        if fore_name:
-                            authors.append(f'{last_name}, {fore_name}')
-                        else:
-                            authors.append(last_name)
-            
-            # Get year
-            year = article_elem.findtext('.//Journal/JournalIssue/PubDate/Year', '')
-            if not year:
-                medline_date = article_elem.findtext('.//Journal/JournalIssue/PubDate/MedlineDate', '')
-                if medline_date:
-                    import re
-                    year_match = re.search(r'\d{4}', medline_date)
-                    if year_match:
-                        year = year_match.group()
-            
-            metadata = {
-                'pmid': pmid,
-                'doi': doi,
-                'title': article_elem.findtext('.//ArticleTitle', ''),
-                'authors': ' and '.join(authors),
-                'journal': journal.findtext('.//Title', ''),
-                'year': year,
-                'volume': journal.findtext('.//JournalIssue/Volume', ''),
-                'issue': journal.findtext('.//JournalIssue/Issue', ''),
-                'pages': article_elem.findtext('.//Pagination/MedlinePgn', ''),
-                'abstract': article_elem.findtext('.//Abstract/AbstractText', '')
-            }
-            
-            return metadata
-            
-        except Exception as e:
-            print(f'Error extracting metadata: {e}', file=sys.stderr)
-            return None
-    
-    def metadata_to_bibtex(self, metadata: Dict) -> str:
-        """Convert metadata to BibTeX format."""
-        # Generate citation key
-        if metadata.get('authors'):
-            first_author = metadata['authors'].split(' and ')[0]
-            if ',' in first_author:
-                last_name = first_author.split(',')[0].strip()
-            else:
-                last_name = first_author.split()[0]
-        else:
-            last_name = 'Unknown'
-        
-        year = metadata.get('year', 'XXXX')
-        citation_key = f'{last_name}{year}pmid{metadata.get("pmid", "")}'
-        
-        # Build BibTeX entry
-        lines = [f'@article{{{citation_key},']
-        
-        if metadata.get('authors'):
-            lines.append(f'  author  = {{{metadata["authors"]}}},')
-        
-        if metadata.get('title'):
-            lines.append(f'  title   = {{{metadata["title"]}}},')
-        
-        if metadata.get('journal'):
-            lines.append(f'  journal = {{{metadata["journal"]}}},')
-        
-        if metadata.get('year'):
-            lines.append(f'  year    = {{{metadata["year"]}}},')
-        
-        if metadata.get('volume'):
-            lines.append(f'  volume  = {{{metadata["volume"]}}},')
-        
-        if metadata.get('issue'):
-            lines.append(f'  number  = {{{metadata["issue"]}}},')
-        
-        if metadata.get('pages'):
-            pages = metadata['pages'].replace('-', '--')
-            lines.append(f'  pages   = {{{pages}}},')
-        
-        if metadata.get('doi'):
-            lines.append(f'  doi     = {{{metadata["doi"]}}},')
-        
-        if metadata.get('pmid'):
-            lines.append(f'  note    = {{PMID: {metadata["pmid"]}}},')
-        
-        # Remove trailing comma
-        if lines[-1].endswith(','):
-            lines[-1] = lines[-1][:-1]
-        
-        lines.append('}')
-        
-        return '\n'.join(lines)
-
-
-def main():
-    """Command-line interface."""
-    parser = argparse.ArgumentParser(
-        description='Search PubMed using E-utilities API',
-        epilog='Example: python search_pubmed.py "CRISPR gene editing" --limit 100'
-    )
-    
-    parser.add_argument(
-        'query',
-        nargs='?',
-        help='Search query (PubMed syntax)'
-    )
-    
-    parser.add_argument(
-        '--query',
-        dest='query_arg',
-        help='Search query (alternative to positional argument)'
-    )
-    
-    parser.add_argument(
-        '--query-file',
-        help='File containing search query'
-    )
-    
-    parser.add_argument(
-        '--limit',
-        type=int,
-        default=100,
-        help='Maximum number of results (default: 100)'
-    )
-    
-    parser.add_argument(
-        '--date-start',
-        help='Start date (YYYY/MM/DD or YYYY)'
-    )
-    
-    parser.add_argument(
-        '--date-end',
-        help='End date (YYYY/MM/DD or YYYY)'
-    )
-    
-    parser.add_argument(
-        '--publication-types',
-        help='Comma-separated publication types (e.g., "Review,Clinical Trial")'
-    )
-    
-    parser.add_argument(
-        '-o', '--output',
-        help='Output file (default: stdout)'
-    )
-    
-    parser.add_argument(
-        '--format',
-        choices=['json', 'bibtex'],
-        default='json',
-        help='Output format (default: json)'
-    )
-    
-    parser.add_argument(
-        '--api-key',
-        help='NCBI API key (or set NCBI_API_KEY env var)'
-    )
-    
-    parser.add_argument(
-        '--email',
-        help='Email for Entrez (or set NCBI_EMAIL env var)'
-    )
-    
-    args = parser.parse_args()
-    
-    # Get query
-    query = args.query or args.query_arg
-    
-    if args.query_file:
-        try:
-            with open(args.query_file, 'r', encoding='utf-8') as f:
-                query = f.read().strip()
-        except Exception as e:
-            print(f'Error reading query file: {e}', file=sys.stderr)
-            sys.exit(1)
-    
-    if not query:
-        parser.print_help()
-        sys.exit(1)
-    
-    # Parse publication types
-    pub_types = None
-    if args.publication_types:
-        pub_types = [pt.strip() for pt in args.publication_types.split(',')]
-    
-    # Search PubMed
-    searcher = PubMedSearcher(api_key=args.api_key, email=args.email)
-    pmids = searcher.search(
-        query,
-        max_results=args.limit,
-        date_start=args.date_start,
-        date_end=args.date_end,
-        publication_types=pub_types
-    )
-    
-    if not pmids:
-        print('No results found', file=sys.stderr)
-        sys.exit(1)
-    
-    # Fetch metadata
-    metadata_list = searcher.fetch_metadata(pmids)
-    
-    # Format output
-    if args.format == 'json':
-        output = json.dumps({
-            'query': query,
-            'count': len(metadata_list),
-            'results': metadata_list
-        }, indent=2)
-    else:  # bibtex
-        bibtex_entries = [searcher.metadata_to_bibtex(m) for m in metadata_list]
-        output = '\n\n'.join(bibtex_entries) + '\n'
-    
-    # Write output
-    if args.output:
-        with open(args.output, 'w', encoding='utf-8') as f:
-            f.write(output)
-        print(f'Wrote {len(metadata_list)} results to {args.output}', file=sys.stderr)
-    else:
-        print(output)
-
-
-if __name__ == '__main__':
-    main()
-