@synsci/cli-darwin-x64-baseline 1.1.77 → 1.1.78

package/bin/skills/openalex-database/references/common_queries.md

@@ -0,0 +1,381 @@
+# Common OpenAlex Query Examples
+
+This document provides practical examples for common research queries using OpenAlex.
+
+## Finding Papers by Author
+
+**User query**: "Find papers by Albert Einstein"
+
+**Approach**: Two-step pattern
+1. Search for author to get ID
+2. Filter works by author ID
+
+**Python example**:
+```python
+from scripts.openalex_client import OpenAlexClient
+from scripts.query_helpers import find_author_works
+
+client = OpenAlexClient(email="your-email@example.edu")
+works = find_author_works("Albert Einstein", client, limit=100)
+
+for work in works:
+    print(f"{work['title']} ({work['publication_year']})")
+```
+
+## Finding Papers from an Institution
+
+**User query**: "What papers has MIT published in the last year?"
+
+**Approach**: Two-step pattern with date filter
+1. Search for institution to get ID
+2. Filter works by institution ID and year
+
+**Python example**:
+```python
+from scripts.query_helpers import find_institution_works
+
+works = find_institution_works("MIT", client, limit=200)
+
+# Filter for recent papers
+import datetime
+current_year = datetime.datetime.now().year
+recent_works = [w for w in works if w['publication_year'] == current_year]
+```
+
+## Highly Cited Papers on a Topic
+
+**User query**: "Find the most cited papers on CRISPR from the last 5 years"
+
+**Approach**: Search + filter + sort
+
+**Python example**:
+```python
+works = client.search_works(
+    search="CRISPR",
+    filter_params={
+        "publication_year": ">2019"
+    },
+    sort="cited_by_count:desc",
+    per_page=100
+)
+
+for work in works['results']:
+    title = work['title']
+    citations = work['cited_by_count']
+    year = work['publication_year']
+    print(f"{title} ({year}): {citations} citations")
+```
+
+## Open Access Papers on a Topic
+
+**User query**: "Find open access papers about climate change"
+
+**Approach**: Search + OA filter
+
+**Python example**:
+```python
+from scripts.query_helpers import get_open_access_papers
+
+papers = get_open_access_papers(
+    search_term="climate change",
+    client=client,
+    oa_status="any",  # or "gold", "green", "hybrid", "bronze"
+    limit=200
+)
+
+for paper in papers:
+    print(f"{paper['title']}")
+    print(f"  OA Status: {paper['open_access']['oa_status']}")
+    print(f"  URL: {paper['open_access']['oa_url']}")
+```
+
+## Publication Trends Analysis
+
+**User query**: "Show me publication trends for machine learning over the years"
+
+**Approach**: Use group_by to aggregate by year
+
+**Python example**:
+```python
+from scripts.query_helpers import get_publication_trends
+
+trends = get_publication_trends(
+    search_term="machine learning",
+    client=client
+)
+
+# Sort by year
+trends_sorted = sorted(trends, key=lambda x: x['key'])
+
+for trend in trends_sorted[-10:]:  # Last 10 years
+    year = trend['key']
+    count = trend['count']
+    print(f"{year}: {count} publications")
+```
+
+## Analyzing Research Output
+
+**User query**: "Analyze the research output of Stanford University from 2020-2024"
+
+**Approach**: Multiple aggregations for comprehensive analysis
+
+**Python example**:
+```python
+from scripts.query_helpers import analyze_research_output
+
+analysis = analyze_research_output(
+    entity_type='institution',
+    entity_name='Stanford University',
+    client=client,
+    years='2020-2024'
+)
+
+print(f"Institution: {analysis['entity_name']}")
+print(f"Total works: {analysis['total_works']}")
+print(f"Open access: {analysis['open_access_percentage']}%")
+print("\nTop topics:")
+for topic in analysis['top_topics'][:5]:
+    print(f"  - {topic['key_display_name']}: {topic['count']} works")
+```
+
+## Finding Papers by DOI (Batch)
+
+**User query**: "Get information for these 10 DOIs: ..."
+
+**Approach**: Batch lookup with pipe separator
+
+**Python example**:
+```python
+dois = [
+    "https://doi.org/10.1371/journal.pone.0266781",
+    "https://doi.org/10.1371/journal.pone.0267149",
+    "https://doi.org/10.1038/s41586-021-03819-2",
+    # ... up to 50 DOIs
+]
+
+works = client.batch_lookup(
+    entity_type='works',
+    ids=dois,
+    id_field='doi'
+)
+
+for work in works:
+    print(f"{work['title']} - {work['publication_year']}")
+```
+
+## Random Sample of Papers
+
+**User query**: "Give me 50 random papers from 2023"
+
+**Approach**: Use sample parameter with seed for reproducibility
+
+**Python example**:
+```python
+works = client.sample_works(
+    sample_size=50,
+    seed=42,  # For reproducibility
+    filter_params={
+        "publication_year": "2023",
+        "is_oa": "true"
+    }
+)
+
+print(f"Got {len(works)} random papers from 2023")
+```
+
+## Papers from Multiple Institutions
+
+**User query**: "Find papers with authors from both MIT and Stanford"
+
+**Approach**: Use + operator for AND within same attribute
+
+**Python example**:
+```python
+# First, get institution IDs
+mit_response = client._make_request(
+    '/institutions',
+    params={'search': 'MIT', 'per-page': 1}
+)
+mit_id = mit_response['results'][0]['id'].split('/')[-1]
+
+stanford_response = client._make_request(
+    '/institutions',
+    params={'search': 'Stanford', 'per-page': 1}
+)
+stanford_id = stanford_response['results'][0]['id'].split('/')[-1]
+
+# Find works with authors from both institutions
+works = client.search_works(
+    filter_params={
+        "authorships.institutions.id": f"{mit_id}+{stanford_id}"
+    },
+    per_page=100
+)
+
+print(f"Found {works['meta']['count']} collaborative papers")
+```
+
+## Papers in a Specific Journal
+
+**User query**: "Get all papers from Nature published in 2023"
+
+**Approach**: Two-step - find journal ID, then filter works
+
+**Python example**:
+```python
+# Step 1: Find journal source ID
+source_response = client._make_request(
+    '/sources',
+    params={'search': 'Nature', 'per-page': 1}
+)
+source = source_response['results'][0]
+source_id = source['id'].split('/')[-1]
+
+print(f"Found journal: {source['display_name']} (ID: {source_id})")
+
+# Step 2: Get works from that source
+works = client.search_works(
+    filter_params={
+        "primary_location.source.id": source_id,
+        "publication_year": "2023"
+    },
+    per_page=200
+)
+
+print(f"Found {works['meta']['count']} papers from Nature in 2023")
+```
+
+## Topic Analysis by Institution
+
+**User query**: "What topics does MIT research most?"
+
+**Approach**: Filter by institution, group by topics
+
+**Python example**:
+```python
+# Get MIT ID
+inst_response = client._make_request(
+    '/institutions',
+    params={'search': 'MIT', 'per-page': 1}
+)
+mit_id = inst_response['results'][0]['id'].split('/')[-1]
+
+# Group by topics
+topics = client.group_by(
+    entity_type='works',
+    group_field='topics.id',
+    filter_params={
+        "authorships.institutions.id": mit_id,
+        "publication_year": ">2020"
+    }
+)
+
+print("Top research topics at MIT (2020+):")
+for i, topic in enumerate(topics[:10], 1):
+    print(f"{i}. {topic['key_display_name']}: {topic['count']} works")
+```
+
+## Citation Analysis
+
+**User query**: "Find papers that cite this specific DOI"
+
+**Approach**: Get work by DOI, then use cited_by_api_url
+
+**Python example**:
+```python
+# Get the work
+doi = "https://doi.org/10.1038/s41586-021-03819-2"
+work = client.get_entity('works', doi)
+
+# Get papers that cite it
+cited_by_url = work['cited_by_api_url']
+
+# Extract just the query part and use it
+import requests
+response = requests.get(cited_by_url, params={'mailto': client.email})
+citing_works = response.json()
+
+print(f"{work['title']}")
+print(f"Total citations: {work['cited_by_count']}")
+print(f"\nRecent citing papers:")
+for citing_work in citing_works['results'][:5]:
+    print(f"  - {citing_work['title']} ({citing_work['publication_year']})")
+```
+
+## Large-Scale Data Extraction
+
+**User query**: "Get all papers on quantum computing from the last 3 years"
+
+**Approach**: Paginate through all results
+
+**Python example**:
+```python
+all_papers = client.paginate_all(
+    endpoint='/works',
+    params={
+        'search': 'quantum computing',
+        'filter': 'publication_year:2022-2024'
+    },
+    max_results=10000  # Limit to prevent excessive API calls
+)
+
+print(f"Retrieved {len(all_papers)} papers")
+
+# Save to CSV
+import csv
+with open('quantum_papers.csv', 'w', newline='') as f:
+    writer = csv.writer(f)
+    writer.writerow(['Title', 'Year', 'Citations', 'DOI', 'OA Status'])
+
+    for paper in all_papers:
+        writer.writerow([
+            paper['title'],
+            paper['publication_year'],
+            paper['cited_by_count'],
+            paper.get('doi', 'N/A'),
+            paper['open_access']['oa_status']
+        ])
+```
+
+## Complex Multi-Filter Query
+
+**User query**: "Find recent, highly-cited, open access papers on AI from top institutions"
+
+**Approach**: Combine multiple filters
+
+**Python example**:
+```python
+# Get IDs for top institutions
+top_institutions = ['MIT', 'Stanford', 'Oxford']
+inst_ids = []
+
+for inst_name in top_institutions:
+    response = client._make_request(
+        '/institutions',
+        params={'search': inst_name, 'per-page': 1}
+    )
+    if response['results']:
+        inst_id = response['results'][0]['id'].split('/')[-1]
+        inst_ids.append(inst_id)
+
+# Combine with pipe for OR
+inst_filter = '|'.join(inst_ids)
+
+# Complex query
+works = client.search_works(
+    search="artificial intelligence",
+    filter_params={
+        "publication_year": ">2022",
+        "cited_by_count": ">50",
+        "is_oa": "true",
+        "authorships.institutions.id": inst_filter
+    },
+    sort="cited_by_count:desc",
+    per_page=200
+)
+
+print(f"Found {works['meta']['count']} papers matching criteria")
+for work in works['results'][:10]:
+    print(f"{work['title']}")
+    print(f"  Citations: {work['cited_by_count']}, Year: {work['publication_year']}")
+```

package/bin/skills/openalex-database/scripts/openalex_client.py

@@ -0,0 +1,337 @@
+#!/usr/bin/env python3
+"""
+OpenAlex API Client with rate limiting and error handling.
+
+Provides a robust client for interacting with the OpenAlex API with:
+- Automatic rate limiting (polite pool: 10 req/sec)
+- Exponential backoff retry logic
+- Pagination support
+- Batch operations support
+"""
+
+import time
+import requests
+from typing import Dict, List, Optional, Any
+from urllib.parse import urljoin
+
+
+class OpenAlexClient:
+    """Client for OpenAlex API with rate limiting and error handling."""
+
+    BASE_URL = "https://api.openalex.org"
+
+    def __init__(self, email: Optional[str] = None, requests_per_second: int = 10):
+        """
+        Initialize OpenAlex client.
+
+        Args:
+            email: Email for polite pool (10x rate limit boost)
+            requests_per_second: Max requests per second (default: 10 for polite pool)
+        """
+        self.email = email
+        self.requests_per_second = requests_per_second
+        self.min_delay = 1.0 / requests_per_second
+        self.last_request_time = 0
+
+    def _rate_limit(self):
+        """Ensure requests don't exceed rate limit."""
+        current_time = time.time()
+        time_since_last = current_time - self.last_request_time
+        if time_since_last < self.min_delay:
+            time.sleep(self.min_delay - time_since_last)
+        self.last_request_time = time.time()
+
+    def _make_request(
+        self,
+        endpoint: str,
+        params: Optional[Dict] = None,
+        max_retries: int = 5
+    ) -> Dict[str, Any]:
+        """
+        Make API request with retry logic.
+
+        Args:
+            endpoint: API endpoint (e.g., '/works', '/authors')
+            params: Query parameters
+            max_retries: Maximum number of retry attempts
+
+        Returns:
+            JSON response as dictionary
+        """
+        if params is None:
+            params = {}
+
+        # Add email to params for polite pool
+        if self.email:
+            params['mailto'] = self.email
+
+        url = urljoin(self.BASE_URL, endpoint)
+
+        for attempt in range(max_retries):
+            try:
+                self._rate_limit()
+                response = requests.get(url, params=params, timeout=30)
+
+                if response.status_code == 200:
+                    return response.json()
+                elif response.status_code == 403:
+                    # Rate limited
+                    wait_time = 2 ** attempt
+                    print(f"Rate limited. Waiting {wait_time}s before retry...")
+                    time.sleep(wait_time)
+                elif response.status_code >= 500:
+                    # Server error
+                    wait_time = 2 ** attempt
+                    print(f"Server error. Waiting {wait_time}s before retry...")
+                    time.sleep(wait_time)
+                else:
+                    # Other error - don't retry
+                    response.raise_for_status()
+
+            except requests.exceptions.Timeout:
+                if attempt < max_retries - 1:
+                    wait_time = 2 ** attempt
+                    print(f"Request timeout. Waiting {wait_time}s before retry...")
+                    time.sleep(wait_time)
+                else:
+                    raise
+
+        raise Exception(f"Failed after {max_retries} retries")
+
+    def search_works(
+        self,
+        search: Optional[str] = None,
+        filter_params: Optional[Dict] = None,
+        per_page: int = 200,
+        page: int = 1,
+        sort: Optional[str] = None,
+        select: Optional[List[str]] = None
+    ) -> Dict[str, Any]:
+        """
+        Search works with filters.
+
+        Args:
+            search: Full-text search query
+            filter_params: Dictionary of filter parameters
+            per_page: Results per page (max: 200)
+            page: Page number
+            sort: Sort parameter (e.g., 'cited_by_count:desc')
+            select: List of fields to return
+
+        Returns:
+            API response with meta and results
+        """
+        params = {
+            'per-page': min(per_page, 200),
+            'page': page
+        }
+
+        if search:
+            params['search'] = search
+
+        if filter_params:
+            filter_str = ','.join([f"{k}:{v}" for k, v in filter_params.items()])
+            params['filter'] = filter_str
+
+        if sort:
+            params['sort'] = sort
+
+        if select:
+            params['select'] = ','.join(select)
+
+        return self._make_request('/works', params)
+
+    def get_entity(self, entity_type: str, entity_id: str) -> Dict[str, Any]:
+        """
+        Get single entity by ID.
+
+        Args:
+            entity_type: Type of entity ('works', 'authors', 'institutions', etc.)
+            entity_id: OpenAlex ID or external ID (DOI, ORCID, etc.)
+
+        Returns:
+            Entity object
+        """
+        endpoint = f"/{entity_type}/{entity_id}"
+        return self._make_request(endpoint)
+
+    def batch_lookup(
+        self,
+        entity_type: str,
+        ids: List[str],
+        id_field: str = 'openalex_id'
+    ) -> List[Dict[str, Any]]:
+        """
+        Look up multiple entities by ID efficiently.
+
+        Args:
+            entity_type: Type of entity ('works', 'authors', etc.)
+            ids: List of IDs (up to 50 per batch)
+            id_field: ID field name ('openalex_id', 'doi', 'orcid', etc.)
+
+        Returns:
+            List of entity objects
+        """
+        all_results = []
+
+        # Process in batches of 50
+        for i in range(0, len(ids), 50):
+            batch = ids[i:i+50]
+            filter_value = '|'.join(batch)
+
+            params = {
+                'filter': f"{id_field}:{filter_value}",
+                'per-page': 50
+            }
+
+            response = self._make_request(f"/{entity_type}", params)
+            all_results.extend(response.get('results', []))
+
+        return all_results
+
+    def paginate_all(
+        self,
+        endpoint: str,
+        params: Optional[Dict] = None,
+        max_results: Optional[int] = None
+    ) -> List[Dict[str, Any]]:
+        """
+        Paginate through all results.
+
+        Args:
+            endpoint: API endpoint
+            params: Query parameters
+            max_results: Maximum number of results to retrieve (None for all)
+
+        Returns:
+            List of all results
+        """
+        if params is None:
+            params = {}
+
+        params['per-page'] = 200  # Use maximum page size
+        params['page'] = 1
+
+        all_results = []
+
+        while True:
+            response = self._make_request(endpoint, params)
+            results = response.get('results', [])
+            all_results.extend(results)
+
+            # Check if we've hit max_results
+            if max_results and len(all_results) >= max_results:
+                return all_results[:max_results]
+
+            # Check if there are more pages
+            meta = response.get('meta', {})
+            total_count = meta.get('count', 0)
+            current_count = len(all_results)
+
+            if current_count >= total_count:
+                break
+
+            params['page'] += 1
+
+        return all_results
+
+    def sample_works(
+        self,
+        sample_size: int,
+        seed: Optional[int] = None,
+        filter_params: Optional[Dict] = None
+    ) -> List[Dict[str, Any]]:
+        """
+        Get random sample of works.
+
+        Args:
+            sample_size: Number of samples to retrieve
+            seed: Random seed for reproducibility
+            filter_params: Optional filters to apply
+
+        Returns:
+            List of sampled works
+        """
+        params = {
+            'sample': min(sample_size, 10000),  # API limit per request
+            'per-page': 200
+        }
+
+        if seed is not None:
+            params['seed'] = seed
+
+        if filter_params:
+            filter_str = ','.join([f"{k}:{v}" for k, v in filter_params.items()])
+            params['filter'] = filter_str
+
+        # For large samples, need multiple requests with different seeds
+        if sample_size > 10000:
+            all_samples = []
+            seen_ids = set()
+
+            for i in range((sample_size // 10000) + 1):
+                current_seed = seed + i if seed else i
+                params['seed'] = current_seed
+                params['sample'] = min(10000, sample_size - len(all_samples))
+
+                response = self._make_request('/works', params)
+                results = response.get('results', [])
+
+                # Deduplicate
+                for result in results:
+                    work_id = result.get('id')
+                    if work_id not in seen_ids:
+                        seen_ids.add(work_id)
+                        all_samples.append(result)
+
+                if len(all_samples) >= sample_size:
+                    break
+
+            return all_samples[:sample_size]
+        else:
+            response = self._make_request('/works', params)
+            return response.get('results', [])
+
+    def group_by(
+        self,
+        entity_type: str,
+        group_field: str,
+        filter_params: Optional[Dict] = None
+    ) -> List[Dict[str, Any]]:
+        """
+        Aggregate results by field.
+
+        Args:
+            entity_type: Type of entity ('works', 'authors', etc.)
+            group_field: Field to group by
+            filter_params: Optional filters
+
+        Returns:
+            List of grouped results with counts
+        """
+        params = {
+            'group_by': group_field
+        }
+
+        if filter_params:
+            filter_str = ','.join([f"{k}:{v}" for k, v in filter_params.items()])
+            params['filter'] = filter_str
+
+        response = self._make_request(f"/{entity_type}", params)
+        return response.get('group_by', [])
+
+
+if __name__ == "__main__":
+    # Example usage
+    client = OpenAlexClient(email="your-email@example.com")
+
+    # Search for works about machine learning
+    results = client.search_works(
+        search="machine learning",
+        filter_params={"publication_year": "2023"},
+        per_page=10
+    )
+
+    print(f"Found {results['meta']['count']} works")
+    for work in results['results']:
+        print(f"- {work['title']}")