@theihtisham/dev-pulse 1.0.0 → 1.1.0

package/devpulse/core/github_client.py

@@ -1,281 +1,281 @@
-"""GitHub API client with rate limiting and caching."""
-
-import time
-import logging
-from datetime import datetime, timedelta
-from typing import Any, Optional
-
-import requests
-
-from devpulse.core.config import get_settings
-
-logger = logging.getLogger("devpulse.github")
-
-# Suppress requests logging of URLs (may contain tokens)
-logging.getLogger("urllib3").setLevel(logging.WARNING)
-
-
-class GitHubClient:
-    """Authenticated GitHub API client with rate-limit awareness."""
-
-    BASE = "https://api.github.com"
-
-    def __init__(
-        self,
-        token: Optional[str] = None,
-        username: Optional[str] = None,
-        org: Optional[str] = None,
-    ) -> None:
-        settings = get_settings()
-        self.token = token or settings.github_token
-        self.username = username or settings.github_username
-        self.org = org or settings.github_org
-        self.base_url = settings.github_api_url
-        self.cache_ttl = settings.github_cache_ttl_seconds
-        self.rate_limit_rpm = settings.github_rate_limit_rpm
-
-        self._session = requests.Session()
-        if self.token:
-            self._session.headers.update(
-                {
-                    "Authorization": f"token {self.token}",
-                    "Accept": "application/vnd.github+json",
-                }
-            )
-        self._session.headers.update({"User-Agent": "DevPulse/1.0"})
-
-        self._last_request_time: float = 0.0
-        self._min_interval = 60.0 / max(self.rate_limit_rpm, 1)
-        self._cache: dict[str, tuple[float, Any]] = {}
-
-    # ── Rate limiting ────────────────────────────────────────────────
-
-    def _throttle(self) -> None:
-        elapsed = time.time() - self._last_request_time
-        if elapsed < self._min_interval:
-            time.sleep(self._min_interval - elapsed)
-        self._last_request_time = time.time()
-
-    def _get(self, url: str, params: Optional[dict[str, Any]] = None) -> Any:
-        """GET with rate limiting and caching."""
-        cache_key = f"{url}:{sorted((params or {}).items())}"
-        now = time.time()
-        if cache_key in self._cache:
-            ts, data = self._cache[cache_key]
-            if now - ts < self.cache_ttl:
-                return data
-
-        self._throttle()
-        resp = self._session.get(url, params=params, timeout=30)
-        resp.raise_for_status()
-
-        data = resp.json()
-        self._cache[cache_key] = (now, data)
-
-        # Check remaining rate limit
-        remaining = resp.headers.get("X-RateLimit-Remaining")
-        if remaining and int(remaining) < 5:
-            reset_time = int(resp.headers.get("X-RateLimit-Reset", 0))
-            wait = max(reset_time - int(time.time()), 1)
-            logger.warning("Rate limit low (%s remaining), waiting %ds", remaining, wait)
-            time.sleep(wait)
-
-        return data
-
-    def _paginated(self, url: str, params: Optional[dict[str, Any]] = None, max_pages: int = 10) -> list[dict[str, Any]]:
-        """Fetch all pages of a paginated endpoint."""
-        results: list[dict[str, Any]] = []
-        params = dict(params or {})
-        params.setdefault("per_page", 100)
-        params["page"] = 1
-
-        for _ in range(max_pages):
-            data = self._get(url, params)
-            if not isinstance(data, list) or not data:
-                break
-            results.extend(data)
-            if len(data) < params["per_page"]:
-                break
-            params["page"] += 1
-
-        return results
-
-    # ── Repository listing ───────────────────────────────────────────
-
-    def get_repos(self) -> list[str]:
-        """Get list of repository full names."""
-        if self.org:
-            repos = self._paginated(f"{self.base_url}/orgs/{self.org}/repos", {"type": "all"})
-        elif self.username:
-            repos = self._paginated(f"{self.base_url}/users/{self.username}/repos", {"type": "all"})
-        else:
-            return []
-        return [r["full_name"] for r in repos if isinstance(r, dict)]
-
-    # ── Commits ──────────────────────────────────────────────────────
-
-    def get_commits(
-        self, repo: str, since: Optional[str] = None, until: Optional[str] = None, author: Optional[str] = None
-    ) -> list[dict[str, Any]]:
-        """Fetch commits for a repository."""
-        params: dict[str, Any] = {"per_page": 100}
-        if since:
-            params["since"] = since
-        if until:
-            params["until"] = until
-        if author:
-            params["author"] = author
-
-        raw = self._paginated(f"{self.base_url}/repos/{repo}/commits", params, max_pages=5)
-        commits: list[dict[str, Any]] = []
-        for c in raw:
-            commit_data = c.get("commit", c)
-            author_info = commit_data.get("author", {})
-            commits.append(
-                {
-                    "sha": c.get("sha", ""),
-                    "repo": repo,
-                    "author": author_info.get("name", ""),
-                    "author_date": author_info.get("date", ""),
-                    "message": commit_data.get("message", "").split("\n")[0][:200],
-                    "additions": 0,
-                    "deletions": 0,
-                    "url": c.get("html_url", ""),
-                }
-            )
-        return commits
-
-    def get_commit_detail(self, repo: str, sha: str) -> dict[str, Any]:
-        """Get detailed stats for a single commit."""
-        data = self._get(f"{self.base_url}/repos/{repo}/commits/{sha}")
-        stats = data.get("stats", {})
-        return {"additions": stats.get("additions", 0), "deletions": stats.get("deletions", 0)}
-
-    # ── Pull Requests ────────────────────────────────────────────────
-
-    def get_pull_requests(
-        self, repo: str, state: str = "all", since: Optional[str] = None
-    ) -> list[dict[str, Any]]:
-        """Fetch pull requests for a repository."""
-        params: dict[str, Any] = {"state": state, "per_page": 100}
-        raw = self._paginated(f"{self.base_url}/repos/{repo}/pulls", params, max_pages=5)
-
-        prs: list[dict[str, Any]] = []
-        for p in raw:
-            if since and p.get("created_at", "") < since:
-                continue
-            prs.append(
-                {
-                    "number": p["number"],
-                    "repo": repo,
-                    "title": p.get("title", ""),
-                    "author": p.get("user", {}).get("login", ""),
-                    "state": "merged" if p.get("merged_at") else p.get("state", "open"),
-                    "created_at": p.get("created_at", ""),
-                    "merged_at": p.get("merged_at"),
-                    "closed_at": p.get("closed_at"),
-                    "additions": p.get("additions", 0),
-                    "deletions": p.get("deletions", 0),
-                    "changed_files": p.get("changed_files", 0),
-                    "review_comments": p.get("comments", 0) + p.get("review_comments", 0),
-                    "url": p.get("html_url", ""),
-                }
-            )
-        return prs
-
-    # ── Issues ───────────────────────────────────────────────────────
-
-    def get_issues(
-        self, repo: str, state: str = "all", since: Optional[str] = None
-    ) -> list[dict[str, Any]]:
-        """Fetch issues (excluding PRs) for a repository."""
-        params: dict[str, Any] = {"state": state, "per_page": 100}
-        if since:
-            params["since"] = since
-        raw = self._paginated(f"{self.base_url}/repos/{repo}/issues", params, max_pages=5)
-
-        issues: list[dict[str, Any]] = []
-        for i in raw:
-            if "pull_request" in i:
-                continue  # skip PRs returned by issues endpoint
-            issues.append(
-                {
-                    "number": i["number"],
-                    "repo": repo,
-                    "title": i.get("title", ""),
-                    "author": i.get("user", {}).get("login", ""),
-                    "state": i.get("state", "open"),
-                    "labels": [lbl.get("name", "") for lbl in i.get("labels", [])],
-                    "created_at": i.get("created_at", ""),
-                    "closed_at": i.get("closed_at"),
-                    "url": i.get("html_url", ""),
-                }
-            )
-        return issues
-
-    # ── Reviews ──────────────────────────────────────────────────────
-
-    def get_reviews(self, repo: str, pr_number: int) -> list[dict[str, Any]]:
-        """Fetch reviews for a pull request."""
-        raw = self._paginated(
-            f"{self.base_url}/repos/{repo}/pulls/{pr_number}/reviews", {"per_page": 100}
-        )
-        reviews: list[dict[str, Any]] = []
-        for r in raw:
-            reviews.append(
-                {
-                    "id": r["id"],
-                    "repo": repo,
-                    "pr_number": pr_number,
-                    "author": r.get("user", {}).get("login", ""),
-                    "state": r.get("state", ""),
-                    "submitted_at": r.get("submitted_at", ""),
-                    "body": r.get("body", ""),
-                }
-            )
-        return reviews
-
-    # ── Sync ─────────────────────────────────────────────────────────
-
-    def sync_all(
-        self,
-        repos: Optional[list[str]] = None,
-        since: Optional[str] = None,
-        db: Optional[Any] = None,
-    ) -> dict[str, int]:
-        """Sync commits, PRs, issues, and reviews from GitHub."""
-        if db is None:
-            from devpulse.core.database import Database
-            db = Database()
-
-        if repos is None:
-            repos = self.get_repos()
-
-        if not since:
-            since = (datetime.utcnow() - timedelta(days=30)).isoformat()
-
-        counts: dict[str, int] = {"commits": 0, "prs": 0, "issues": 0, "reviews": 0}
-
-        for repo in repos:
-            logger.info("Syncing %s ...", repo)
-            try:
-                commits = self.get_commits(repo, since=since)
-                counts["commits"] += db.upsert_commits(commits)
-
-                prs = self.get_pull_requests(repo, since=since)
-                counts["prs"] += db.upsert_pull_requests(prs)
-
-                # Fetch reviews for each PR
-                for pr in prs:
-                    try:
-                        reviews = self.get_reviews(repo, pr["number"])
-                        counts["reviews"] += db.upsert_reviews(reviews)
-                    except Exception:
-                        pass
-
-                issues = self.get_issues(repo, since=since)
-                counts["issues"] += db.upsert_issues(issues)
-            except Exception as exc:
-                logger.error("Error syncing %s: %s", repo, exc)
-
-        return counts
+"""GitHub API client with rate limiting and caching."""
+
+import time
+import logging
+from datetime import datetime, timedelta
+from typing import Any, Optional
+
+import requests
+
+from devpulse.core.config import get_settings
+
+logger = logging.getLogger("devpulse.github")
+
+# Suppress requests logging of URLs (may contain tokens)
+logging.getLogger("urllib3").setLevel(logging.WARNING)
+
+
+class GitHubClient:
+    """Authenticated GitHub API client with rate-limit awareness."""
+
+    BASE = "https://api.github.com"
+
+    def __init__(
+        self,
+        token: Optional[str] = None,
+        username: Optional[str] = None,
+        org: Optional[str] = None,
+    ) -> None:
+        settings = get_settings()
+        self.token = token or settings.github_token
+        self.username = username or settings.github_username
+        self.org = org or settings.github_org
+        self.base_url = settings.github_api_url
+        self.cache_ttl = settings.github_cache_ttl_seconds
+        self.rate_limit_rpm = settings.github_rate_limit_rpm
+
+        self._session = requests.Session()
+        if self.token:
+            self._session.headers.update(
+                {
+                    "Authorization": f"token {self.token}",
+                    "Accept": "application/vnd.github+json",
+                }
+            )
+        self._session.headers.update({"User-Agent": "DevPulse/1.0"})
+
+        self._last_request_time: float = 0.0
+        self._min_interval = 60.0 / max(self.rate_limit_rpm, 1)
+        self._cache: dict[str, tuple[float, Any]] = {}
+
+    # ── Rate limiting ────────────────────────────────────────────────
+
+    def _throttle(self) -> None:
+        elapsed = time.time() - self._last_request_time
+        if elapsed < self._min_interval:
+            time.sleep(self._min_interval - elapsed)
+        self._last_request_time = time.time()
+
+    def _get(self, url: str, params: Optional[dict[str, Any]] = None) -> Any:
+        """GET with rate limiting and caching."""
+        cache_key = f"{url}:{sorted((params or {}).items())}"
+        now = time.time()
+        if cache_key in self._cache:
+            ts, data = self._cache[cache_key]
+            if now - ts < self.cache_ttl:
+                return data
+
+        self._throttle()
+        resp = self._session.get(url, params=params, timeout=30)
+        resp.raise_for_status()
+
+        data = resp.json()
+        self._cache[cache_key] = (now, data)
+
+        # Check remaining rate limit
+        remaining = resp.headers.get("X-RateLimit-Remaining")
+        if remaining and int(remaining) < 5:
+            reset_time = int(resp.headers.get("X-RateLimit-Reset", 0))
+            wait = max(reset_time - int(time.time()), 1)
+            logger.warning("Rate limit low (%s remaining), waiting %ds", remaining, wait)
+            time.sleep(wait)
+
+        return data
+
+    def _paginated(self, url: str, params: Optional[dict[str, Any]] = None, max_pages: int = 10) -> list[dict[str, Any]]:
+        """Fetch all pages of a paginated endpoint."""
+        results: list[dict[str, Any]] = []
+        params = dict(params or {})
+        params.setdefault("per_page", 100)
+        params["page"] = 1
+
+        for _ in range(max_pages):
+            data = self._get(url, params)
+            if not isinstance(data, list) or not data:
+                break
+            results.extend(data)
+            if len(data) < params["per_page"]:
+                break
+            params["page"] += 1
+
+        return results
+
+    # ── Repository listing ───────────────────────────────────────────
+
+    def get_repos(self) -> list[str]:
+        """Get list of repository full names."""
+        if self.org:
+            repos = self._paginated(f"{self.base_url}/orgs/{self.org}/repos", {"type": "all"})
+        elif self.username:
+            repos = self._paginated(f"{self.base_url}/users/{self.username}/repos", {"type": "all"})
+        else:
+            return []
+        return [r["full_name"] for r in repos if isinstance(r, dict)]
+
+    # ── Commits ──────────────────────────────────────────────────────
+
+    def get_commits(
+        self, repo: str, since: Optional[str] = None, until: Optional[str] = None, author: Optional[str] = None
+    ) -> list[dict[str, Any]]:
+        """Fetch commits for a repository."""
+        params: dict[str, Any] = {"per_page": 100}
+        if since:
+            params["since"] = since
+        if until:
+            params["until"] = until
+        if author:
+            params["author"] = author
+
+        raw = self._paginated(f"{self.base_url}/repos/{repo}/commits", params, max_pages=5)
+        commits: list[dict[str, Any]] = []
+        for c in raw:
+            commit_data = c.get("commit", c)
+            author_info = commit_data.get("author", {})
+            commits.append(
+                {
+                    "sha": c.get("sha", ""),
+                    "repo": repo,
+                    "author": author_info.get("name", ""),
+                    "author_date": author_info.get("date", ""),
+                    "message": commit_data.get("message", "").split("\n")[0][:200],
+                    "additions": 0,
+                    "deletions": 0,
+                    "url": c.get("html_url", ""),
+                }
+            )
+        return commits
+
+    def get_commit_detail(self, repo: str, sha: str) -> dict[str, Any]:
+        """Get detailed stats for a single commit."""
+        data = self._get(f"{self.base_url}/repos/{repo}/commits/{sha}")
+        stats = data.get("stats", {})
+        return {"additions": stats.get("additions", 0), "deletions": stats.get("deletions", 0)}
+
+    # ── Pull Requests ────────────────────────────────────────────────
+
+    def get_pull_requests(
+        self, repo: str, state: str = "all", since: Optional[str] = None
+    ) -> list[dict[str, Any]]:
+        """Fetch pull requests for a repository."""
+        params: dict[str, Any] = {"state": state, "per_page": 100}
+        raw = self._paginated(f"{self.base_url}/repos/{repo}/pulls", params, max_pages=5)
+
+        prs: list[dict[str, Any]] = []
+        for p in raw:
+            if since and p.get("created_at", "") < since:
+                continue
+            prs.append(
+                {
+                    "number": p["number"],
+                    "repo": repo,
+                    "title": p.get("title", ""),
+                    "author": p.get("user", {}).get("login", ""),
+                    "state": "merged" if p.get("merged_at") else p.get("state", "open"),
+                    "created_at": p.get("created_at", ""),
+                    "merged_at": p.get("merged_at"),
+                    "closed_at": p.get("closed_at"),
+                    "additions": p.get("additions", 0),
+                    "deletions": p.get("deletions", 0),
+                    "changed_files": p.get("changed_files", 0),
+                    "review_comments": p.get("comments", 0) + p.get("review_comments", 0),
+                    "url": p.get("html_url", ""),
+                }
+            )
+        return prs
+
+    # ── Issues ───────────────────────────────────────────────────────
+
+    def get_issues(
+        self, repo: str, state: str = "all", since: Optional[str] = None
+    ) -> list[dict[str, Any]]:
+        """Fetch issues (excluding PRs) for a repository."""
+        params: dict[str, Any] = {"state": state, "per_page": 100}
+        if since:
+            params["since"] = since
+        raw = self._paginated(f"{self.base_url}/repos/{repo}/issues", params, max_pages=5)
+
+        issues: list[dict[str, Any]] = []
+        for i in raw:
+            if "pull_request" in i:
+                continue  # skip PRs returned by issues endpoint
+            issues.append(
+                {
+                    "number": i["number"],
+                    "repo": repo,
+                    "title": i.get("title", ""),
+                    "author": i.get("user", {}).get("login", ""),
+                    "state": i.get("state", "open"),
+                    "labels": [lbl.get("name", "") for lbl in i.get("labels", [])],
+                    "created_at": i.get("created_at", ""),
+                    "closed_at": i.get("closed_at"),
+                    "url": i.get("html_url", ""),
+                }
+            )
+        return issues
+
+    # ── Reviews ──────────────────────────────────────────────────────
+
+    def get_reviews(self, repo: str, pr_number: int) -> list[dict[str, Any]]:
+        """Fetch reviews for a pull request."""
+        raw = self._paginated(
+            f"{self.base_url}/repos/{repo}/pulls/{pr_number}/reviews", {"per_page": 100}
+        )
+        reviews: list[dict[str, Any]] = []
+        for r in raw:
+            reviews.append(
+                {
+                    "id": r["id"],
+                    "repo": repo,
+                    "pr_number": pr_number,
+                    "author": r.get("user", {}).get("login", ""),
+                    "state": r.get("state", ""),
+                    "submitted_at": r.get("submitted_at", ""),
+                    "body": r.get("body", ""),
+                }
+            )
+        return reviews
+
+    # ── Sync ─────────────────────────────────────────────────────────
+
+    def sync_all(
+        self,
+        repos: Optional[list[str]] = None,
+        since: Optional[str] = None,
+        db: Optional[Any] = None,
+    ) -> dict[str, int]:
+        """Sync commits, PRs, issues, and reviews from GitHub."""
+        if db is None:
+            from devpulse.core.database import Database
+            db = Database()
+
+        if repos is None:
+            repos = self.get_repos()
+
+        if not since:
+            since = (datetime.utcnow() - timedelta(days=30)).isoformat()
+
+        counts: dict[str, int] = {"commits": 0, "prs": 0, "issues": 0, "reviews": 0}
+
+        for repo in repos:
+            logger.info("Syncing %s ...", repo)
+            try:
+                commits = self.get_commits(repo, since=since)
+                counts["commits"] += db.upsert_commits(commits)
+
+                prs = self.get_pull_requests(repo, since=since)
+                counts["prs"] += db.upsert_pull_requests(prs)
+
+                # Fetch reviews for each PR
+                for pr in prs:
+                    try:
+                        reviews = self.get_reviews(repo, pr["number"])
+                        counts["reviews"] += db.upsert_reviews(reviews)
+                    except Exception:
+                        pass
+
+                issues = self.get_issues(repo, since=since)
+                counts["issues"] += db.upsert_issues(issues)
+            except Exception as exc:
+                logger.error("Error syncing %s: %s", repo, exc)
+
+        return counts