@theihtisham/dev-pulse 1.0.0

package/devpulse/core/github_client.py

@@ -0,0 +1,281 @@
+"""GitHub API client with rate limiting and caching."""
+
+import time
+import logging
+from datetime import datetime, timedelta
+from typing import Any, Optional
+
+import requests
+
+from devpulse.core.config import get_settings
+
+logger = logging.getLogger("devpulse.github")
+
+# Suppress requests logging of URLs (may contain tokens)
+logging.getLogger("urllib3").setLevel(logging.WARNING)
+
+
+class GitHubClient:
+    """Authenticated GitHub API client with rate-limit awareness."""
+
+    BASE = "https://api.github.com"
+
+    def __init__(
+        self,
+        token: Optional[str] = None,
+        username: Optional[str] = None,
+        org: Optional[str] = None,
+    ) -> None:
+        settings = get_settings()
+        self.token = token or settings.github_token
+        self.username = username or settings.github_username
+        self.org = org or settings.github_org
+        self.base_url = settings.github_api_url
+        self.cache_ttl = settings.github_cache_ttl_seconds
+        self.rate_limit_rpm = settings.github_rate_limit_rpm
+
+        self._session = requests.Session()
+        if self.token:
+            self._session.headers.update(
+                {
+                    "Authorization": f"token {self.token}",
+                    "Accept": "application/vnd.github+json",
+                }
+            )
+        self._session.headers.update({"User-Agent": "DevPulse/1.0"})
+
+        self._last_request_time: float = 0.0
+        self._min_interval = 60.0 / max(self.rate_limit_rpm, 1)
+        self._cache: dict[str, tuple[float, Any]] = {}
+
+    # ── Rate limiting ────────────────────────────────────────────────
+
+    def _throttle(self) -> None:
+        elapsed = time.time() - self._last_request_time
+        if elapsed < self._min_interval:
+            time.sleep(self._min_interval - elapsed)
+        self._last_request_time = time.time()
+
+    def _get(self, url: str, params: Optional[dict[str, Any]] = None) -> Any:
+        """GET with rate limiting and caching."""
+        cache_key = f"{url}:{sorted((params or {}).items())}"
+        now = time.time()
+        if cache_key in self._cache:
+            ts, data = self._cache[cache_key]
+            if now - ts < self.cache_ttl:
+                return data
+
+        self._throttle()
+        resp = self._session.get(url, params=params, timeout=30)
+        resp.raise_for_status()
+
+        data = resp.json()
+        self._cache[cache_key] = (now, data)
+
+        # Check remaining rate limit
+        remaining = resp.headers.get("X-RateLimit-Remaining")
+        if remaining and int(remaining) < 5:
+            reset_time = int(resp.headers.get("X-RateLimit-Reset", 0))
+            wait = max(reset_time - int(time.time()), 1)
+            logger.warning("Rate limit low (%s remaining), waiting %ds", remaining, wait)
+            time.sleep(wait)
+
+        return data
+
+    def _paginated(self, url: str, params: Optional[dict[str, Any]] = None, max_pages: int = 10) -> list[dict[str, Any]]:
+        """Fetch all pages of a paginated endpoint."""
+        results: list[dict[str, Any]] = []
+        params = dict(params or {})
+        params.setdefault("per_page", 100)
+        params["page"] = 1
+
+        for _ in range(max_pages):
+            data = self._get(url, params)
+            if not isinstance(data, list) or not data:
+                break
+            results.extend(data)
+            if len(data) < params["per_page"]:
+                break
+            params["page"] += 1
+
+        return results
+
+    # ── Repository listing ───────────────────────────────────────────
+
+    def get_repos(self) -> list[str]:
+        """Get list of repository full names."""
+        if self.org:
+            repos = self._paginated(f"{self.base_url}/orgs/{self.org}/repos", {"type": "all"})
+        elif self.username:
+            repos = self._paginated(f"{self.base_url}/users/{self.username}/repos", {"type": "all"})
+        else:
+            return []
+        return [r["full_name"] for r in repos if isinstance(r, dict)]
+
+    # ── Commits ──────────────────────────────────────────────────────
+
+    def get_commits(
+        self, repo: str, since: Optional[str] = None, until: Optional[str] = None, author: Optional[str] = None
+    ) -> list[dict[str, Any]]:
+        """Fetch commits for a repository."""
+        params: dict[str, Any] = {"per_page": 100}
+        if since:
+            params["since"] = since
+        if until:
+            params["until"] = until
+        if author:
+            params["author"] = author
+
+        raw = self._paginated(f"{self.base_url}/repos/{repo}/commits", params, max_pages=5)
+        commits: list[dict[str, Any]] = []
+        for c in raw:
+            commit_data = c.get("commit", c)
+            author_info = commit_data.get("author", {})
+            commits.append(
+                {
+                    "sha": c.get("sha", ""),
+                    "repo": repo,
+                    "author": author_info.get("name", ""),
+                    "author_date": author_info.get("date", ""),
+                    "message": commit_data.get("message", "").split("\n")[0][:200],
+                    "additions": 0,
+                    "deletions": 0,
+                    "url": c.get("html_url", ""),
+                }
+            )
+        return commits
+
+    def get_commit_detail(self, repo: str, sha: str) -> dict[str, Any]:
+        """Get detailed stats for a single commit."""
+        data = self._get(f"{self.base_url}/repos/{repo}/commits/{sha}")
+        stats = data.get("stats", {})
+        return {"additions": stats.get("additions", 0), "deletions": stats.get("deletions", 0)}
+
+    # ── Pull Requests ────────────────────────────────────────────────
+
+    def get_pull_requests(
+        self, repo: str, state: str = "all", since: Optional[str] = None
+    ) -> list[dict[str, Any]]:
+        """Fetch pull requests for a repository."""
+        params: dict[str, Any] = {"state": state, "per_page": 100}
+        raw = self._paginated(f"{self.base_url}/repos/{repo}/pulls", params, max_pages=5)
+
+        prs: list[dict[str, Any]] = []
+        for p in raw:
+            if since and p.get("created_at", "") < since:
+                continue
+            prs.append(
+                {
+                    "number": p["number"],
+                    "repo": repo,
+                    "title": p.get("title", ""),
+                    "author": p.get("user", {}).get("login", ""),
+                    "state": "merged" if p.get("merged_at") else p.get("state", "open"),
+                    "created_at": p.get("created_at", ""),
+                    "merged_at": p.get("merged_at"),
+                    "closed_at": p.get("closed_at"),
+                    "additions": p.get("additions", 0),
+                    "deletions": p.get("deletions", 0),
+                    "changed_files": p.get("changed_files", 0),
+                    "review_comments": p.get("comments", 0) + p.get("review_comments", 0),
+                    "url": p.get("html_url", ""),
+                }
+            )
+        return prs
+
+    # ── Issues ───────────────────────────────────────────────────────
+
+    def get_issues(
+        self, repo: str, state: str = "all", since: Optional[str] = None
+    ) -> list[dict[str, Any]]:
+        """Fetch issues (excluding PRs) for a repository."""
+        params: dict[str, Any] = {"state": state, "per_page": 100}
+        if since:
+            params["since"] = since
+        raw = self._paginated(f"{self.base_url}/repos/{repo}/issues", params, max_pages=5)
+
+        issues: list[dict[str, Any]] = []
+        for i in raw:
+            if "pull_request" in i:
+                continue  # skip PRs returned by issues endpoint
+            issues.append(
+                {
+                    "number": i["number"],
+                    "repo": repo,
+                    "title": i.get("title", ""),
+                    "author": i.get("user", {}).get("login", ""),
+                    "state": i.get("state", "open"),
+                    "labels": [lbl.get("name", "") for lbl in i.get("labels", [])],
+                    "created_at": i.get("created_at", ""),
+                    "closed_at": i.get("closed_at"),
+                    "url": i.get("html_url", ""),
+                }
+            )
+        return issues
+
+    # ── Reviews ──────────────────────────────────────────────────────
+
+    def get_reviews(self, repo: str, pr_number: int) -> list[dict[str, Any]]:
+        """Fetch reviews for a pull request."""
+        raw = self._paginated(
+            f"{self.base_url}/repos/{repo}/pulls/{pr_number}/reviews", {"per_page": 100}
+        )
+        reviews: list[dict[str, Any]] = []
+        for r in raw:
+            reviews.append(
+                {
+                    "id": r["id"],
+                    "repo": repo,
+                    "pr_number": pr_number,
+                    "author": r.get("user", {}).get("login", ""),
+                    "state": r.get("state", ""),
+                    "submitted_at": r.get("submitted_at", ""),
+                    "body": r.get("body", ""),
+                }
+            )
+        return reviews
+
+    # ── Sync ─────────────────────────────────────────────────────────
+
+    def sync_all(
+        self,
+        repos: Optional[list[str]] = None,
+        since: Optional[str] = None,
+        db: Optional[Any] = None,
+    ) -> dict[str, int]:
+        """Sync commits, PRs, issues, and reviews from GitHub."""
+        if db is None:
+            from devpulse.core.database import Database
+            db = Database()
+
+        if repos is None:
+            repos = self.get_repos()
+
+        if not since:
+            since = (datetime.utcnow() - timedelta(days=30)).isoformat()
+
+        counts: dict[str, int] = {"commits": 0, "prs": 0, "issues": 0, "reviews": 0}
+
+        for repo in repos:
+            logger.info("Syncing %s ...", repo)
+            try:
+                commits = self.get_commits(repo, since=since)
+                counts["commits"] += db.upsert_commits(commits)
+
+                prs = self.get_pull_requests(repo, since=since)
+                counts["prs"] += db.upsert_pull_requests(prs)
+
+                # Fetch reviews for each PR
+                for pr in prs:
+                    try:
+                        reviews = self.get_reviews(repo, pr["number"])
+                        counts["reviews"] += db.upsert_reviews(reviews)
+                    except Exception:
+                        pass
+
+                issues = self.get_issues(repo, since=since)
+                counts["issues"] += db.upsert_issues(issues)
+            except Exception as exc:
+                logger.error("Error syncing %s: %s", repo, exc)
+
+        return counts

package/devpulse/core/models.py

@@ -0,0 +1,142 @@
+"""Pydantic models for DevPulse data structures."""
+
+from datetime import datetime, date
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class Commit(BaseModel):
+    sha: str
+    repo: str
+    author: str
+    author_date: str
+    message: str = ""
+    additions: int = 0
+    deletions: int = 0
+    url: str = ""
+
+
+class PullRequest(BaseModel):
+    number: int
+    repo: str
+    title: str = ""
+    author: str = ""
+    state: str = "open"
+    created_at: str = ""
+    merged_at: Optional[str] = None
+    closed_at: Optional[str] = None
+    additions: int = 0
+    deletions: int = 0
+    changed_files: int = 0
+    review_comments: int = 0
+    url: str = ""
+
+
+class Issue(BaseModel):
+    number: int
+    repo: str
+    title: str = ""
+    author: str = ""
+    state: str = "open"
+    labels: list[str] = Field(default_factory=list)
+    created_at: str = ""
+    closed_at: Optional[str] = None
+    url: str = ""
+
+
+class Review(BaseModel):
+    id: int
+    repo: str
+    pr_number: int
+    author: str = ""
+    state: str = ""
+    submitted_at: str = ""
+    body: str = ""
+
+
+class DeveloperMetrics(BaseModel):
+    author: str
+    commits_count: int = 0
+    prs_created: int = 0
+    prs_merged: int = 0
+    issues_opened: int = 0
+    issues_closed: int = 0
+    reviews_given: int = 0
+    avg_pr_merge_time_hours: float = 0.0
+    avg_review_turnaround_hours: float = 0.0
+    lines_added: int = 0
+    lines_removed: int = 0
+    commits_per_day: float = 0.0
+    active_days: int = 0
+    period_days: int = 30
+
+
+class SprintData(BaseModel):
+    name: str
+    total_points: float = 0
+    completed_points: float = 0
+    remaining_points: float = 0
+    added_points: float = 0
+    start_date: str = ""
+    end_date: str = ""
+    velocity: float = 0
+    scope_creep_pct: float = 0
+
+
+class TeamHealth(BaseModel):
+    team_name: str = "team"
+    overall_score: float = 0.0
+    workload_balance: float = 0.0
+    burnout_risk: dict[str, float] = Field(default_factory=dict)
+    collaboration_score: float = 0.0
+    velocity_trend: str = "stable"
+    recommendations: list[str] = Field(default_factory=list)
+
+
+class CodeQuality(BaseModel):
+    repo: str
+    date: str = ""
+    test_coverage: float = 0.0
+    open_bugs: int = 0
+    tech_debt_score: float = 0.0
+    lines_added: int = 0
+    lines_removed: int = 0
+    files_changed: int = 0
+
+
+class Goal(BaseModel):
+    id: Optional[int] = None
+    title: str
+    description: str = ""
+    target_value: float
+    current_value: float = 0
+    metric: str
+    deadline: Optional[str] = None
+    status: str = "active"
+
+
+class DailyReport(BaseModel):
+    date: str
+    author: str
+    commits: int = 0
+    prs_opened: int = 0
+    prs_merged: int = 0
+    issues_opened: int = 0
+    issues_closed: int = 0
+    reviews_given: int = 0
+    lines_changed: int = 0
+    summary: str = ""
+
+
+class WeeklyReport(BaseModel):
+    week_start: str
+    week_end: str
+    authors: list[str] = Field(default_factory=list)
+    total_commits: int = 0
+    total_prs: int = 0
+    total_issues_closed: int = 0
+    avg_merge_time_hours: float = 0.0
+    top_contributors: list[dict[str, int]] = Field(default_factory=list)
+    insights: list[str] = Field(default_factory=list)
+    recommendations: list[str] = Field(default_factory=list)