feed-the-machine 1.1.0 → 1.3.0

package/ftm-inbox/backend/adapters/base.py

@@ -0,0 +1,230 @@
+"""
+BaseAdapter — abstract interface all poller adapters must implement.
+
+Every adapter handles one external system (Jira, Freshservice, Slack, etc.)
+and is responsible for fetching, normalizing, deduplicating, and storing
+items into the ftm-inbox database.
+
+Deduplication uses a SHA-256 hash of (source + source_id) so the same
+external item is never stored twice regardless of content changes.
+To track updates, adapters should update the existing row rather than
+insert a duplicate.
+"""
+
+import hashlib
+import json
+from abc import ABC, abstractmethod
+from datetime import datetime, timezone
+from typing import Any
+
+
+class NormalizedItem:
+    """Canonical representation of an inbox item before storage."""
+
+    __slots__ = (
+        "source",
+        "source_id",
+        "title",
+        "body",
+        "status",
+        "priority",
+        "assignee",
+        "requester",
+        "created_at",
+        "updated_at",
+        "tags",
+        "custom_fields",
+        "raw_payload",
+        "source_url",
+        "content_hash",
+    )
+
+    def __init__(
+        self,
+        source: str,
+        source_id: str,
+        title: str,
+        body: str = "",
+        status: str = "open",
+        priority: str = "medium",
+        assignee: str | None = None,
+        requester: str | None = None,
+        created_at: str | None = None,
+        updated_at: str | None = None,
+        tags: list[str] | None = None,
+        custom_fields: dict[str, Any] | None = None,
+        raw_payload: dict[str, Any] | None = None,
+        source_url: str | None = None,
+    ) -> None:
+        self.source = source
+        self.source_id = source_id
+        self.title = title
+        self.body = body
+        self.status = status
+        self.priority = priority
+        self.assignee = assignee
+        self.requester = requester
+        self.created_at = created_at
+        self.updated_at = updated_at
+        self.tags = tags or []
+        self.custom_fields = custom_fields or {}
+        self.raw_payload = raw_payload or {}
+        self.source_url = source_url
+        self.content_hash = _compute_hash(source, source_id)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "source": self.source,
+            "source_id": self.source_id,
+            "title": self.title,
+            "body": self.body,
+            "status": self.status,
+            "priority": self.priority,
+            "assignee": self.assignee,
+            "requester": self.requester,
+            "created_at": self.created_at,
+            "updated_at": self.updated_at,
+            "tags": json.dumps(self.tags),
+            "custom_fields": json.dumps(self.custom_fields),
+            "raw_payload": json.dumps(self.raw_payload),
+            "source_url": self.source_url,
+            "content_hash": self.content_hash,
+        }
+
+
+def _compute_hash(source: str, source_id: str) -> str:
+    """SHA-256 of 'source:source_id' — stable dedup key."""
+    raw = f"{source}:{source_id}".encode("utf-8")
+    return hashlib.sha256(raw).hexdigest()
+
+
+class BaseAdapter(ABC):
+    """
+    Abstract base class for all ftm-inbox poller adapters.
+
+    Subclasses must implement poll(), normalize(), and optionally override
+    deduplicate() and store() if the defaults don't suit the source.
+
+    Lifecycle per poll cycle:
+        raw_items  = self.poll()
+        normalized = [self.normalize(item) for item in raw_items]
+        new_items  = self.deduplicate(normalized, conn)
+        self.store(new_items, conn)
+    """
+
+    #: Unique identifier for this adapter, e.g. "jira", "freshservice"
+    source_name: str = ""
+
+    def __init__(self, credentials: dict[str, Any], config: dict[str, Any]) -> None:
+        """
+        Args:
+            credentials: Secrets from config.yml (API keys, tokens, etc.)
+            config:      Non-secret settings (base_url, project_key, etc.)
+        """
+        self.credentials = credentials
+        self.config = config
+
+    @abstractmethod
+    def poll(self) -> list[dict[str, Any]]:
+        """
+        Fetch raw items from the external system.
+
+        Returns:
+            List of raw dicts as returned by the source API.
+        """
+        ...
+
+    @abstractmethod
+    def normalize(self, raw_item: dict[str, Any]) -> NormalizedItem:
+        """
+        Convert a raw API item into a NormalizedItem.
+
+        Args:
+            raw_item: One element from the list returned by poll().
+
+        Returns:
+            NormalizedItem ready for deduplication and storage.
+        """
+        ...
+
+    def deduplicate(
+        self, items: list[NormalizedItem], conn
+    ) -> list[NormalizedItem]:
+        """
+        Filter out items already present in the database by content_hash.
+
+        Uses the UNIQUE constraint on inbox.content_hash. Returns only
+        items that do not yet exist.
+
+        Args:
+            items: Normalized items from this poll cycle.
+            conn:  Active sqlite3 connection.
+
+        Returns:
+            Subset of items not yet in the database.
+        """
+        if not items:
+            return []
+
+        hashes = [item.content_hash for item in items]
+        placeholders = ",".join("?" * len(hashes))
+        rows = conn.execute(
+            f"SELECT content_hash FROM inbox WHERE content_hash IN ({placeholders})",
+            hashes,
+        ).fetchall()
+        existing = {row["content_hash"] for row in rows}
+        return [item for item in items if item.content_hash not in existing]
+
+    def store(self, items: list[NormalizedItem], conn) -> int:
+        """
+        Insert new items into the inbox table.
+
+        Skips items that already exist (content_hash collision) rather than
+        raising — adapters that want upsert behavior should override this.
+
+        Args:
+            items: Deduplicated NormalizedItems to insert.
+            conn:  Active sqlite3 connection.
+
+        Returns:
+            Number of rows actually inserted.
+        """
+        if not items:
+            return 0
+
+        inserted = 0
+        for item in items:
+            data = item.to_dict()
+            try:
+                conn.execute(
+                    """
+                    INSERT INTO inbox
+                        (source, source_id, title, body, status, priority,
+                         assignee, requester, created_at, updated_at,
+                         tags, custom_fields, raw_payload, source_url, content_hash)
+                    VALUES
+                        (:source, :source_id, :title, :body, :status, :priority,
+                         :assignee, :requester, :created_at, :updated_at,
+                         :tags, :custom_fields, :raw_payload, :source_url, :content_hash)
+                    """,
+                    data,
+                )
+                inserted += 1
+            except Exception:
+                # Swallow duplicate key errors; log others in production
+                pass
+
+        conn.commit()
+        return inserted
+
+    def run_cycle(self, conn) -> int:
+        """
+        Execute a full poll-normalize-deduplicate-store cycle.
+
+        Returns:
+            Number of new items stored.
+        """
+        raw_items = self.poll()
+        normalized = [self.normalize(item) for item in raw_items]
+        new_items = self.deduplicate(normalized, conn)
+        return self.store(new_items, conn)

package/ftm-inbox/backend/adapters/freshservice.py

@@ -0,0 +1,104 @@
+"""
+FreshserviceAdapter — polls Freshservice tickets via the REST API v2.
+
+Required credentials:
+    api_key     Freshservice API key (Settings > API in Freshservice admin)
+
+Config keys:
+    domain      e.g. "yourorg.freshservice.com"
+    per_page    Number of tickets per page, default 100
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import requests
+from requests.auth import HTTPBasicAuth
+
+from backend.adapters._retry import retry
+from backend.adapters.base import BaseAdapter, NormalizedItem
+
+logger = logging.getLogger(__name__)
+
+_STATUS_MAP = {2: "open", 3: "pending", 4: "resolved", 5: "closed"}
+_PRIORITY_MAP = {1: "low", 2: "medium", 3: "high", 4: "urgent"}
+
+
+class FreshserviceAdapter(BaseAdapter):
+    """Polls Freshservice tickets via REST API v2."""
+
+    source_name = "freshservice"
+    required_credentials = ["api_key"]
+
+    def __init__(self, credentials: dict, config: dict) -> None:
+        super().__init__(credentials, config)
+        self._domain = config.get("domain", "")
+        if not self._domain:
+            raise ValueError("FreshserviceAdapter requires config['domain']")
+        self._auth = HTTPBasicAuth(credentials["api_key"], "X")
+        self._per_page = int(config.get("per_page", 100))
+
+    @retry(max_attempts=3, base_delay=1.0, exceptions=(requests.RequestException,))
+    def poll(self) -> list[dict]:
+        """Fetch tickets from Freshservice with pagination."""
+        url = f"https://{self._domain}/api/v2/tickets"
+        params: dict[str, Any] = {
+            "per_page": self._per_page,
+            "order_by": "updated_at",
+            "order_type": "desc",
+        }
+        response = requests.get(url, auth=self._auth, params=params, timeout=30)
+        response.raise_for_status()
+        data = response.json()
+        return data.get("tickets", [])
+
+    def normalize(self, raw_item: dict) -> NormalizedItem:
+        """Map a Freshservice ticket dict to NormalizedItem."""
+        ticket_id = str(raw_item.get("id", ""))
+
+        title = raw_item.get("subject") or "(no subject)"
+        body = raw_item.get("description_text") or raw_item.get("description") or ""
+
+        status_int = raw_item.get("status", 2)
+        status = _STATUS_MAP.get(status_int, "open")
+
+        priority_int = raw_item.get("priority", 2)
+        priority = _PRIORITY_MAP.get(priority_int, "medium")
+
+        requester_id = raw_item.get("requester_id")
+        requester = str(requester_id) if requester_id else None
+
+        responder_id = raw_item.get("responder_id")
+        assignee = str(responder_id) if responder_id else None
+
+        created_at = raw_item.get("created_at")
+        updated_at = raw_item.get("updated_at")
+
+        tags: list[str] = raw_item.get("tags") or []
+
+        custom_fields: dict[str, Any] = {}
+        raw_cf = raw_item.get("custom_fields") or {}
+        for key, value in raw_cf.items():
+            if value is not None:
+                custom_fields[key] = value
+
+        source_url = f"https://{self._domain}/a/tickets/{ticket_id}"
+
+        return NormalizedItem(
+            source=self.source_name,
+            source_id=ticket_id,
+            title=title,
+            body=body,
+            status=status,
+            priority=priority,
+            assignee=assignee,
+            requester=requester,
+            created_at=created_at,
+            updated_at=updated_at,
+            tags=tags,
+            custom_fields=custom_fields,
+            raw_payload=raw_item,
+            source_url=source_url,
+        )

package/ftm-inbox/backend/adapters/gmail.py

@@ -0,0 +1,125 @@
+"""
+GmailAdapter — polls Gmail for recent emails matching a label filter.
+
+Required credentials:
+    credentials_json_path   Path to Google OAuth credentials JSON
+    token_path              Path to stored OAuth token
+
+Config keys:
+    label_filter    Gmail label to filter by, default "INBOX"
+    max_results     Max emails per poll cycle, default 50
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+import requests
+
+from backend.adapters._retry import retry
+from backend.adapters.base import BaseAdapter, NormalizedItem
+
+logger = logging.getLogger(__name__)
+
+
+class GmailAdapter(BaseAdapter):
+    """Polls Gmail for recent emails using the Gmail REST API."""
+
+    source_name = "gmail"
+    required_credentials = ["credentials_json_path"]
+
+    def __init__(self, credentials: dict, config: dict) -> None:
+        super().__init__(credentials, config)
+        self._credentials_path = Path(
+            credentials.get("credentials_json_path", "")
+        ).expanduser()
+        self._token_path = Path(
+            credentials.get("token_path", "~/.config/ftm-inbox/gmail-token.json")
+        ).expanduser()
+        self._label_filter = config.get("label_filter", "INBOX")
+        self._max_results = int(config.get("max_results", 50))
+        self._access_token: str | None = None
+
+    def _get_access_token(self) -> str:
+        """Load OAuth access token from token file."""
+        if self._access_token:
+            return self._access_token
+
+        if not self._token_path.exists():
+            raise RuntimeError(
+                f"Gmail token file not found at {self._token_path}. "
+                "Run the setup wizard first."
+            )
+
+        token_data = json.loads(self._token_path.read_text())
+        self._access_token = token_data.get("access_token", "")
+        if not self._access_token:
+            raise RuntimeError("Gmail token file has no access_token field.")
+        return self._access_token
+
+    @retry(max_attempts=3, base_delay=1.0, exceptions=(requests.RequestException,))
+    def poll(self) -> list[dict]:
+        """Fetch recent emails from Gmail matching the label filter."""
+        token = self._get_access_token()
+        headers = {"Authorization": f"Bearer {token}"}
+
+        url = "https://gmail.googleapis.com/gmail/v1/users/me/messages"
+        params: dict[str, Any] = {
+            "maxResults": self._max_results,
+            "labelIds": self._label_filter,
+        }
+        response = requests.get(url, headers=headers, params=params, timeout=30)
+        response.raise_for_status()
+        data = response.json()
+        message_ids = [m["id"] for m in data.get("messages", [])]
+
+        messages: list[dict] = []
+        for msg_id in message_ids:
+            msg_url = f"{url}/{msg_id}"
+            msg_params = {"format": "metadata", "metadataHeaders": "Subject,From,Date"}
+            msg_response = requests.get(
+                msg_url, headers=headers, params=msg_params, timeout=30
+            )
+            if msg_response.ok:
+                messages.append(msg_response.json())
+
+        return messages
+
+    def normalize(self, raw_item: dict) -> NormalizedItem:
+        """Map a Gmail message dict to NormalizedItem."""
+        msg_id = raw_item.get("id", "")
+        headers = raw_item.get("payload", {}).get("headers", [])
+
+        header_map: dict[str, str] = {}
+        for h in headers:
+            header_map[h.get("name", "").lower()] = h.get("value", "")
+
+        title = header_map.get("subject") or "(no subject)"
+        requester = header_map.get("from")
+        created_at = header_map.get("date")
+
+        snippet = raw_item.get("snippet", "")
+
+        source_url = f"https://mail.google.com/mail/u/0/#inbox/{msg_id}"
+
+        label_ids: list[str] = raw_item.get("labelIds", [])
+
+        return NormalizedItem(
+            source=self.source_name,
+            source_id=msg_id,
+            title=title,
+            body=snippet,
+            status="open",
+            priority="medium",
+            assignee=None,
+            requester=requester,
+            created_at=created_at,
+            updated_at=None,
+            tags=label_ids,
+            custom_fields={},
+            raw_payload=raw_item,
+            source_url=source_url,
+        )

package/ftm-inbox/backend/adapters/jira.py

@@ -0,0 +1,136 @@
+"""
+JiraAdapter — polls a Jira project via the REST API v3 and normalizes issues.
+
+Required credentials:
+    api_token   Jira API token (generated at id.atlassian.net)
+    email       Atlassian account email associated with the token
+
+Config keys:
+    base_url    e.g. "https://myorg.atlassian.net"
+    jql         JQL filter string, default "assignee = currentUser() ORDER BY updated DESC"
+    max_results Number of issues to fetch per poll cycle, default 50
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import requests
+from requests.auth import HTTPBasicAuth
+
+from backend.adapters._retry import retry
+from backend.adapters.base import BaseAdapter, NormalizedItem
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_JQL = "assignee = currentUser() ORDER BY updated DESC"
+_DEFAULT_MAX_RESULTS = 50
+
+
+class JiraAdapter(BaseAdapter):
+    """Polls Jira issues using the REST API v3."""
+
+    source_name = "jira"
+    required_credentials = ["api_token", "email"]
+
+    def __init__(self, credentials: dict, config: dict) -> None:
+        super().__init__(credentials, config)
+        self._base_url = config.get("base_url", "").rstrip("/")
+        if not self._base_url:
+            raise ValueError("JiraAdapter requires config['base_url']")
+        self._auth = HTTPBasicAuth(
+            credentials["email"], credentials["api_token"]
+        )
+        self._jql = config.get("jql", _DEFAULT_JQL)
+        self._max_results = int(config.get("max_results", _DEFAULT_MAX_RESULTS))
+
+    @retry(max_attempts=3, base_delay=1.0, exceptions=(requests.RequestException,))
+    def poll(self) -> list[dict]:
+        """Fetch Jira issues matching the configured JQL query."""
+        url = f"{self._base_url}/rest/api/3/search"
+        params: dict[str, Any] = {
+            "jql": self._jql,
+            "maxResults": self._max_results,
+            "fields": (
+                "summary,description,status,priority,"
+                "assignee,reporter,created,updated,labels,issuetype"
+            ),
+        }
+        response = requests.get(url, auth=self._auth, params=params, timeout=30)
+        response.raise_for_status()
+        data = response.json()
+        return data.get("issues", [])
+
+    def normalize(self, raw_item: dict) -> NormalizedItem:
+        """Map a Jira issue dict to NormalizedItem."""
+        fields = raw_item.get("fields") or {}
+        key = raw_item.get("key", "")
+        issue_id = raw_item.get("id", key)
+
+        title = fields.get("summary") or key or "(no summary)"
+        body = _extract_jira_text(fields.get("description"))
+
+        status_obj = fields.get("status") or {}
+        status = (status_obj.get("name") or "open").lower()
+
+        priority_obj = fields.get("priority") or {}
+        priority = (priority_obj.get("name") or "medium").lower()
+
+        assignee_obj = fields.get("assignee") or {}
+        assignee = assignee_obj.get("displayName") or assignee_obj.get("emailAddress")
+
+        reporter_obj = fields.get("reporter") or {}
+        requester = reporter_obj.get("displayName") or reporter_obj.get("emailAddress")
+
+        created_at = fields.get("created")
+        updated_at = fields.get("updated")
+
+        tags: list[str] = fields.get("labels") or []
+
+        issuetype_obj = fields.get("issuetype") or {}
+        custom_fields: dict[str, Any] = {}
+        if issuetype_obj.get("name"):
+            custom_fields["issue_type"] = issuetype_obj["name"]
+
+        source_url = f"{self._base_url}/browse/{key}" if key else None
+
+        return NormalizedItem(
+            source=self.source_name,
+            source_id=str(issue_id),
+            title=title,
+            body=body,
+            status=status,
+            priority=priority,
+            assignee=assignee,
+            requester=requester,
+            created_at=created_at,
+            updated_at=updated_at,
+            tags=tags,
+            custom_fields=custom_fields,
+            raw_payload=raw_item,
+            source_url=source_url,
+        )
+
+
+def _extract_jira_text(adf: Any) -> str:
+    """Recursively pull plain text from an Atlassian Document Format node."""
+    if adf is None:
+        return ""
+    if isinstance(adf, str):
+        return adf
+    if not isinstance(adf, dict):
+        return ""
+
+    node_type = adf.get("type", "")
+    if node_type == "text":
+        return adf.get("text", "")
+
+    parts: list[str] = []
+    for child in adf.get("content") or []:
+        part = _extract_jira_text(child)
+        if part:
+            parts.append(part)
+
+    separator = "\n" if node_type in ("paragraph", "heading", "bulletList", "orderedList") else " "
+    return separator.join(parts).strip()