feed-the-machine 1.6.0 → 1.7.0

package/ftm-inbox/backend/adapters/base.py

@@ -1,230 +1,230 @@
-"""
-BaseAdapter — abstract interface all poller adapters must implement.
-
-Every adapter handles one external system (Jira, Freshservice, Slack, etc.)
-and is responsible for fetching, normalizing, deduplicating, and storing
-items into the ftm-inbox database.
-
-Deduplication uses a SHA-256 hash of (source + source_id) so the same
-external item is never stored twice regardless of content changes.
-To track updates, adapters should update the existing row rather than
-insert a duplicate.
-"""
-
-import hashlib
-import json
-from abc import ABC, abstractmethod
-from datetime import datetime, timezone
-from typing import Any
-
-
-class NormalizedItem:
-    """Canonical representation of an inbox item before storage."""
-
-    __slots__ = (
-        "source",
-        "source_id",
-        "title",
-        "body",
-        "status",
-        "priority",
-        "assignee",
-        "requester",
-        "created_at",
-        "updated_at",
-        "tags",
-        "custom_fields",
-        "raw_payload",
-        "source_url",
-        "content_hash",
-    )
-
-    def __init__(
-        self,
-        source: str,
-        source_id: str,
-        title: str,
-        body: str = "",
-        status: str = "open",
-        priority: str = "medium",
-        assignee: str | None = None,
-        requester: str | None = None,
-        created_at: str | None = None,
-        updated_at: str | None = None,
-        tags: list[str] | None = None,
-        custom_fields: dict[str, Any] | None = None,
-        raw_payload: dict[str, Any] | None = None,
-        source_url: str | None = None,
-    ) -> None:
-        self.source = source
-        self.source_id = source_id
-        self.title = title
-        self.body = body
-        self.status = status
-        self.priority = priority
-        self.assignee = assignee
-        self.requester = requester
-        self.created_at = created_at
-        self.updated_at = updated_at
-        self.tags = tags or []
-        self.custom_fields = custom_fields or {}
-        self.raw_payload = raw_payload or {}
-        self.source_url = source_url
-        self.content_hash = _compute_hash(source, source_id)
-
-    def to_dict(self) -> dict[str, Any]:
-        return {
-            "source": self.source,
-            "source_id": self.source_id,
-            "title": self.title,
-            "body": self.body,
-            "status": self.status,
-            "priority": self.priority,
-            "assignee": self.assignee,
-            "requester": self.requester,
-            "created_at": self.created_at,
-            "updated_at": self.updated_at,
-            "tags": json.dumps(self.tags),
-            "custom_fields": json.dumps(self.custom_fields),
-            "raw_payload": json.dumps(self.raw_payload),
-            "source_url": self.source_url,
-            "content_hash": self.content_hash,
-        }
-
-
-def _compute_hash(source: str, source_id: str) -> str:
-    """SHA-256 of 'source:source_id' — stable dedup key."""
-    raw = f"{source}:{source_id}".encode("utf-8")
-    return hashlib.sha256(raw).hexdigest()
-
-
-class BaseAdapter(ABC):
-    """
-    Abstract base class for all ftm-inbox poller adapters.
-
-    Subclasses must implement poll(), normalize(), and optionally override
-    deduplicate() and store() if the defaults don't suit the source.
-
-    Lifecycle per poll cycle:
-        raw_items  = self.poll()
-        normalized = [self.normalize(item) for item in raw_items]
-        new_items  = self.deduplicate(normalized, conn)
-        self.store(new_items, conn)
-    """
-
-    #: Unique identifier for this adapter, e.g. "jira", "freshservice"
-    source_name: str = ""
-
-    def __init__(self, credentials: dict[str, Any], config: dict[str, Any]) -> None:
-        """
-        Args:
-            credentials: Secrets from config.yml (API keys, tokens, etc.)
-            config:      Non-secret settings (base_url, project_key, etc.)
-        """
-        self.credentials = credentials
-        self.config = config
-
-    @abstractmethod
-    def poll(self) -> list[dict[str, Any]]:
-        """
-        Fetch raw items from the external system.
-
-        Returns:
-            List of raw dicts as returned by the source API.
-        """
-        ...
-
-    @abstractmethod
-    def normalize(self, raw_item: dict[str, Any]) -> NormalizedItem:
-        """
-        Convert a raw API item into a NormalizedItem.
-
-        Args:
-            raw_item: One element from the list returned by poll().
-
-        Returns:
-            NormalizedItem ready for deduplication and storage.
-        """
-        ...
-
-    def deduplicate(
-        self, items: list[NormalizedItem], conn
-    ) -> list[NormalizedItem]:
-        """
-        Filter out items already present in the database by content_hash.
-
-        Uses the UNIQUE constraint on inbox.content_hash. Returns only
-        items that do not yet exist.
-
-        Args:
-            items: Normalized items from this poll cycle.
-            conn:  Active sqlite3 connection.
-
-        Returns:
-            Subset of items not yet in the database.
-        """
-        if not items:
-            return []
-
-        hashes = [item.content_hash for item in items]
-        placeholders = ",".join("?" * len(hashes))
-        rows = conn.execute(
-            f"SELECT content_hash FROM inbox WHERE content_hash IN ({placeholders})",
-            hashes,
-        ).fetchall()
-        existing = {row["content_hash"] for row in rows}
-        return [item for item in items if item.content_hash not in existing]
-
-    def store(self, items: list[NormalizedItem], conn) -> int:
-        """
-        Insert new items into the inbox table.
-
-        Skips items that already exist (content_hash collision) rather than
-        raising — adapters that want upsert behavior should override this.
-
-        Args:
-            items: Deduplicated NormalizedItems to insert.
-            conn:  Active sqlite3 connection.
-
-        Returns:
-            Number of rows actually inserted.
-        """
-        if not items:
-            return 0
-
-        inserted = 0
-        for item in items:
-            data = item.to_dict()
-            try:
-                conn.execute(
-                    """
-                    INSERT INTO inbox
-                        (source, source_id, title, body, status, priority,
-                         assignee, requester, created_at, updated_at,
-                         tags, custom_fields, raw_payload, source_url, content_hash)
-                    VALUES
-                        (:source, :source_id, :title, :body, :status, :priority,
-                         :assignee, :requester, :created_at, :updated_at,
-                         :tags, :custom_fields, :raw_payload, :source_url, :content_hash)
-                    """,
-                    data,
-                )
-                inserted += 1
-            except Exception:
-                # Swallow duplicate key errors; log others in production
-                pass
-
-        conn.commit()
-        return inserted
-
-    def run_cycle(self, conn) -> int:
-        """
-        Execute a full poll-normalize-deduplicate-store cycle.
-
-        Returns:
-            Number of new items stored.
-        """
-        raw_items = self.poll()
-        normalized = [self.normalize(item) for item in raw_items]
-        new_items = self.deduplicate(normalized, conn)
-        return self.store(new_items, conn)
+"""
+BaseAdapter — abstract interface all poller adapters must implement.
+
+Every adapter handles one external system (Jira, Freshservice, Slack, etc.)
+and is responsible for fetching, normalizing, deduplicating, and storing
+items into the ftm-inbox database.
+
+Deduplication uses a SHA-256 hash of (source + source_id) so the same
+external item is never stored twice regardless of content changes.
+To track updates, adapters should update the existing row rather than
+insert a duplicate.
+"""
+
+import hashlib
+import json
+from abc import ABC, abstractmethod
+from datetime import datetime, timezone
+from typing import Any
+
+
+class NormalizedItem:
+    """Canonical representation of an inbox item before storage."""
+
+    __slots__ = (
+        "source",
+        "source_id",
+        "title",
+        "body",
+        "status",
+        "priority",
+        "assignee",
+        "requester",
+        "created_at",
+        "updated_at",
+        "tags",
+        "custom_fields",
+        "raw_payload",
+        "source_url",
+        "content_hash",
+    )
+
+    def __init__(
+        self,
+        source: str,
+        source_id: str,
+        title: str,
+        body: str = "",
+        status: str = "open",
+        priority: str = "medium",
+        assignee: str | None = None,
+        requester: str | None = None,
+        created_at: str | None = None,
+        updated_at: str | None = None,
+        tags: list[str] | None = None,
+        custom_fields: dict[str, Any] | None = None,
+        raw_payload: dict[str, Any] | None = None,
+        source_url: str | None = None,
+    ) -> None:
+        self.source = source
+        self.source_id = source_id
+        self.title = title
+        self.body = body
+        self.status = status
+        self.priority = priority
+        self.assignee = assignee
+        self.requester = requester
+        self.created_at = created_at
+        self.updated_at = updated_at
+        self.tags = tags or []
+        self.custom_fields = custom_fields or {}
+        self.raw_payload = raw_payload or {}
+        self.source_url = source_url
+        self.content_hash = _compute_hash(source, source_id)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "source": self.source,
+            "source_id": self.source_id,
+            "title": self.title,
+            "body": self.body,
+            "status": self.status,
+            "priority": self.priority,
+            "assignee": self.assignee,
+            "requester": self.requester,
+            "created_at": self.created_at,
+            "updated_at": self.updated_at,
+            "tags": json.dumps(self.tags),
+            "custom_fields": json.dumps(self.custom_fields),
+            "raw_payload": json.dumps(self.raw_payload),
+            "source_url": self.source_url,
+            "content_hash": self.content_hash,
+        }
+
+
+def _compute_hash(source: str, source_id: str) -> str:
+    """SHA-256 of 'source:source_id' — stable dedup key."""
+    raw = f"{source}:{source_id}".encode("utf-8")
+    return hashlib.sha256(raw).hexdigest()
+
+
+class BaseAdapter(ABC):
+    """
+    Abstract base class for all ftm-inbox poller adapters.
+
+    Subclasses must implement poll(), normalize(), and optionally override
+    deduplicate() and store() if the defaults don't suit the source.
+
+    Lifecycle per poll cycle:
+        raw_items  = self.poll()
+        normalized = [self.normalize(item) for item in raw_items]
+        new_items  = self.deduplicate(normalized, conn)
+        self.store(new_items, conn)
+    """
+
+    #: Unique identifier for this adapter, e.g. "jira", "freshservice"
+    source_name: str = ""
+
+    def __init__(self, credentials: dict[str, Any], config: dict[str, Any]) -> None:
+        """
+        Args:
+            credentials: Secrets from config.yml (API keys, tokens, etc.)
+            config:      Non-secret settings (base_url, project_key, etc.)
+        """
+        self.credentials = credentials
+        self.config = config
+
+    @abstractmethod
+    def poll(self) -> list[dict[str, Any]]:
+        """
+        Fetch raw items from the external system.
+
+        Returns:
+            List of raw dicts as returned by the source API.
+        """
+        ...
+
+    @abstractmethod
+    def normalize(self, raw_item: dict[str, Any]) -> NormalizedItem:
+        """
+        Convert a raw API item into a NormalizedItem.
+
+        Args:
+            raw_item: One element from the list returned by poll().
+
+        Returns:
+            NormalizedItem ready for deduplication and storage.
+        """
+        ...
+
+    def deduplicate(
+        self, items: list[NormalizedItem], conn
+    ) -> list[NormalizedItem]:
+        """
+        Filter out items already present in the database by content_hash.
+
+        Uses the UNIQUE constraint on inbox.content_hash. Returns only
+        items that do not yet exist.
+
+        Args:
+            items: Normalized items from this poll cycle.
+            conn:  Active sqlite3 connection.
+
+        Returns:
+            Subset of items not yet in the database.
+        """
+        if not items:
+            return []
+
+        hashes = [item.content_hash for item in items]
+        placeholders = ",".join("?" * len(hashes))
+        rows = conn.execute(
+            f"SELECT content_hash FROM inbox WHERE content_hash IN ({placeholders})",
+            hashes,
+        ).fetchall()
+        existing = {row["content_hash"] for row in rows}
+        return [item for item in items if item.content_hash not in existing]
+
+    def store(self, items: list[NormalizedItem], conn) -> int:
+        """
+        Insert new items into the inbox table.
+
+        Skips items that already exist (content_hash collision) rather than
+        raising — adapters that want upsert behavior should override this.
+
+        Args:
+            items: Deduplicated NormalizedItems to insert.
+            conn:  Active sqlite3 connection.
+
+        Returns:
+            Number of rows actually inserted.
+        """
+        if not items:
+            return 0
+
+        inserted = 0
+        for item in items:
+            data = item.to_dict()
+            try:
+                conn.execute(
+                    """
+                    INSERT INTO inbox
+                        (source, source_id, title, body, status, priority,
+                         assignee, requester, created_at, updated_at,
+                         tags, custom_fields, raw_payload, source_url, content_hash)
+                    VALUES
+                        (:source, :source_id, :title, :body, :status, :priority,
+                         :assignee, :requester, :created_at, :updated_at,
+                         :tags, :custom_fields, :raw_payload, :source_url, :content_hash)
+                    """,
+                    data,
+                )
+                inserted += 1
+            except Exception:
+                # Swallow duplicate key errors; log others in production
+                pass
+
+        conn.commit()
+        return inserted
+
+    def run_cycle(self, conn) -> int:
+        """
+        Execute a full poll-normalize-deduplicate-store cycle.
+
+        Returns:
+            Number of new items stored.
+        """
+        raw_items = self.poll()
+        normalized = [self.normalize(item) for item in raw_items]
+        new_items = self.deduplicate(normalized, conn)
+        return self.store(new_items, conn)

package/ftm-inbox/backend/adapters/freshservice.py

@@ -1,104 +1,104 @@
-"""
-FreshserviceAdapter — polls Freshservice tickets via the REST API v2.
-
-Required credentials:
-    api_key     Freshservice API key (Settings > API in Freshservice admin)
-
-Config keys:
-    domain      e.g. "yourorg.freshservice.com"
-    per_page    Number of tickets per page, default 100
-"""
-
-from __future__ import annotations
-
-import logging
-from typing import Any
-
-import requests
-from requests.auth import HTTPBasicAuth
-
-from backend.adapters._retry import retry
-from backend.adapters.base import BaseAdapter, NormalizedItem
-
-logger = logging.getLogger(__name__)
-
-_STATUS_MAP = {2: "open", 3: "pending", 4: "resolved", 5: "closed"}
-_PRIORITY_MAP = {1: "low", 2: "medium", 3: "high", 4: "urgent"}
-
-
-class FreshserviceAdapter(BaseAdapter):
-    """Polls Freshservice tickets via REST API v2."""
-
-    source_name = "freshservice"
-    required_credentials = ["api_key"]
-
-    def __init__(self, credentials: dict, config: dict) -> None:
-        super().__init__(credentials, config)
-        self._domain = config.get("domain", "")
-        if not self._domain:
-            raise ValueError("FreshserviceAdapter requires config['domain']")
-        self._auth = HTTPBasicAuth(credentials["api_key"], "X")
-        self._per_page = int(config.get("per_page", 100))
-
-    @retry(max_attempts=3, base_delay=1.0, exceptions=(requests.RequestException,))
-    def poll(self) -> list[dict]:
-        """Fetch tickets from Freshservice with pagination."""
-        url = f"https://{self._domain}/api/v2/tickets"
-        params: dict[str, Any] = {
-            "per_page": self._per_page,
-            "order_by": "updated_at",
-            "order_type": "desc",
-        }
-        response = requests.get(url, auth=self._auth, params=params, timeout=30)
-        response.raise_for_status()
-        data = response.json()
-        return data.get("tickets", [])
-
-    def normalize(self, raw_item: dict) -> NormalizedItem:
-        """Map a Freshservice ticket dict to NormalizedItem."""
-        ticket_id = str(raw_item.get("id", ""))
-
-        title = raw_item.get("subject") or "(no subject)"
-        body = raw_item.get("description_text") or raw_item.get("description") or ""
-
-        status_int = raw_item.get("status", 2)
-        status = _STATUS_MAP.get(status_int, "open")
-
-        priority_int = raw_item.get("priority", 2)
-        priority = _PRIORITY_MAP.get(priority_int, "medium")
-
-        requester_id = raw_item.get("requester_id")
-        requester = str(requester_id) if requester_id else None
-
-        responder_id = raw_item.get("responder_id")
-        assignee = str(responder_id) if responder_id else None
-
-        created_at = raw_item.get("created_at")
-        updated_at = raw_item.get("updated_at")
-
-        tags: list[str] = raw_item.get("tags") or []
-
-        custom_fields: dict[str, Any] = {}
-        raw_cf = raw_item.get("custom_fields") or {}
-        for key, value in raw_cf.items():
-            if value is not None:
-                custom_fields[key] = value
-
-        source_url = f"https://{self._domain}/a/tickets/{ticket_id}"
-
-        return NormalizedItem(
-            source=self.source_name,
-            source_id=ticket_id,
-            title=title,
-            body=body,
-            status=status,
-            priority=priority,
-            assignee=assignee,
-            requester=requester,
-            created_at=created_at,
-            updated_at=updated_at,
-            tags=tags,
-            custom_fields=custom_fields,
-            raw_payload=raw_item,
-            source_url=source_url,
-        )
+"""
+FreshserviceAdapter — polls Freshservice tickets via the REST API v2.
+
+Required credentials:
+    api_key     Freshservice API key (Settings > API in Freshservice admin)
+
+Config keys:
+    domain      e.g. "yourorg.freshservice.com"
+    per_page    Number of tickets per page, default 100
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import requests
+from requests.auth import HTTPBasicAuth
+
+from backend.adapters._retry import retry
+from backend.adapters.base import BaseAdapter, NormalizedItem
+
+logger = logging.getLogger(__name__)
+
+_STATUS_MAP = {2: "open", 3: "pending", 4: "resolved", 5: "closed"}
+_PRIORITY_MAP = {1: "low", 2: "medium", 3: "high", 4: "urgent"}
+
+
+class FreshserviceAdapter(BaseAdapter):
+    """Polls Freshservice tickets via REST API v2."""
+
+    source_name = "freshservice"
+    required_credentials = ["api_key"]
+
+    def __init__(self, credentials: dict, config: dict) -> None:
+        super().__init__(credentials, config)
+        self._domain = config.get("domain", "")
+        if not self._domain:
+            raise ValueError("FreshserviceAdapter requires config['domain']")
+        self._auth = HTTPBasicAuth(credentials["api_key"], "X")
+        self._per_page = int(config.get("per_page", 100))
+
+    @retry(max_attempts=3, base_delay=1.0, exceptions=(requests.RequestException,))
+    def poll(self) -> list[dict]:
+        """Fetch tickets from Freshservice with pagination."""
+        url = f"https://{self._domain}/api/v2/tickets"
+        params: dict[str, Any] = {
+            "per_page": self._per_page,
+            "order_by": "updated_at",
+            "order_type": "desc",
+        }
+        response = requests.get(url, auth=self._auth, params=params, timeout=30)
+        response.raise_for_status()
+        data = response.json()
+        return data.get("tickets", [])
+
+    def normalize(self, raw_item: dict) -> NormalizedItem:
+        """Map a Freshservice ticket dict to NormalizedItem."""
+        ticket_id = str(raw_item.get("id", ""))
+
+        title = raw_item.get("subject") or "(no subject)"
+        body = raw_item.get("description_text") or raw_item.get("description") or ""
+
+        status_int = raw_item.get("status", 2)
+        status = _STATUS_MAP.get(status_int, "open")
+
+        priority_int = raw_item.get("priority", 2)
+        priority = _PRIORITY_MAP.get(priority_int, "medium")
+
+        requester_id = raw_item.get("requester_id")
+        requester = str(requester_id) if requester_id else None
+
+        responder_id = raw_item.get("responder_id")
+        assignee = str(responder_id) if responder_id else None
+
+        created_at = raw_item.get("created_at")
+        updated_at = raw_item.get("updated_at")
+
+        tags: list[str] = raw_item.get("tags") or []
+
+        custom_fields: dict[str, Any] = {}
+        raw_cf = raw_item.get("custom_fields") or {}
+        for key, value in raw_cf.items():
+            if value is not None:
+                custom_fields[key] = value
+
+        source_url = f"https://{self._domain}/a/tickets/{ticket_id}"
+
+        return NormalizedItem(
+            source=self.source_name,
+            source_id=ticket_id,
+            title=title,
+            body=body,
+            status=status,
+            priority=priority,
+            assignee=assignee,
+            requester=requester,
+            created_at=created_at,
+            updated_at=updated_at,
+            tags=tags,
+            custom_fields=custom_fields,
+            raw_payload=raw_item,
+            source_url=source_url,
+        )