@cohortapp/agent-sdk 2.3.0

package/scripts/secondary-inbox-poller.py

@@ -0,0 +1,437 @@
+#!/usr/bin/env python3
+"""Alex Gmail Inbox Poller — checks alex@example.com for unseen emails via IMAP.
+
+Designed to run as part of Robin's polling infrastructure. Writes new inbox items
+to state/inbox/gmail/ for the inbox processor to triage using the 5-category protocol.
+
+Can be run standalone: python3 scripts/secondary-inbox-poller.py
+Or called from the main poller via scripts/poller/index.mjs
+
+Exit codes:
+  0 — success (even if no new mail)
+  1 — fatal error (missing credentials)
+"""
+import sys
+import os
+import re
+import imaplib
+import email
+import json
+import hashlib
+import traceback
+from email.header import decode_header
+from email.utils import parsedate_to_datetime, parseaddr
+from datetime import datetime, timezone
+
+# ── Configuration ──
+
+PRINCIPAL_EMAIL = "alex@example.com"
+MAX_EMAILS_PER_CYCLE = 20
+SNIPPET_LENGTH = 500
+
+# Paths (relative to repo root)
+REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+INBOX_DIR = os.path.join(REPO_ROOT, "state", "inbox", "gmail")
+ATTACHMENTS_DIR = os.path.join(REPO_ROOT, "state", "inbox", "attachments")
+CURSOR_FILE = os.path.join(REPO_ROOT, "state", "polling", "alex-gmail-cursor.yaml")
+
+
+# ── Credential Loading ──
+
+def load_password():
+    """Load MEHRAN_GMAIL_APP_PASSWORD from environment or .env file."""
+    password = os.environ.get("MEHRAN_GMAIL_APP_PASSWORD", "")
+    if password:
+        return password
+
+    env_path = os.path.join(REPO_ROOT, ".env")
+    if os.path.exists(env_path):
+        with open(env_path) as f:
+            for line in f:
+                line = line.strip()
+                if line.startswith("#") or "=" not in line:
+                    continue
+                if line.startswith("MEHRAN_GMAIL_APP_PASSWORD="):
+                    val = line.split("=", 1)[1].strip()
+                    # Strip surrounding quotes if present
+                    if (val.startswith('"') and val.endswith('"')) or \
+                       (val.startswith("'") and val.endswith("'")):
+                        val = val[1:-1]
+                    return val
+    return ""
+
+
+# ── Header Decoding ──
+
+def decode_header_value(value):
+    """Decode RFC 2047 encoded header value."""
+    if not value:
+        return ""
+    decoded_parts = decode_header(value)
+    result = []
+    for part, encoding in decoded_parts:
+        if isinstance(part, bytes):
+            result.append(part.decode(encoding or "utf-8", errors="replace"))
+        else:
+            result.append(str(part))
+    return " ".join(result)
+
+
+# ── Body Extraction ──
+
+def strip_html(html):
+    """Crude HTML-to-text conversion using stdlib only (no external deps)."""
+    # Remove style and script blocks
+    text = re.sub(r"<(style|script)[^>]*>.*?</\1>", "", html, flags=re.DOTALL | re.IGNORECASE)
+    # Replace <br>, <p>, <div> with newlines
+    text = re.sub(r"<br\s*/?>", "\n", text, flags=re.IGNORECASE)
+    text = re.sub(r"</(p|div|tr|li)>", "\n", text, flags=re.IGNORECASE)
+    # Strip all remaining tags
+    text = re.sub(r"<[^>]+>", "", text)
+    # Decode common HTML entities
+    text = text.replace("&nbsp;", " ").replace("&amp;", "&")
+    text = text.replace("&lt;", "<").replace("&gt;", ">")
+    text = text.replace("&quot;", '"').replace("&#39;", "'")
+    text = re.sub(r"&\w+;", "", text)  # remove remaining entities
+    # Collapse whitespace
+    text = re.sub(r"[ \t]+", " ", text)
+    text = re.sub(r"\n{3,}", "\n\n", text)
+    return text.strip()
+
+
+def extract_attachments(msg, mid_hash):
+    """Extract and save email attachments to local storage.
+
+    Returns a list of dicts with attachment metadata and local file paths.
+    Skips inline images and signature files (< 1KB images).
+    """
+    attachments = []
+    os.makedirs(ATTACHMENTS_DIR, exist_ok=True)
+
+    if not msg.is_multipart():
+        return attachments
+
+    for part in msg.walk():
+        content_disposition = str(part.get("Content-Disposition", ""))
+        filename = part.get_filename()
+
+        # Only process actual attachments (not inline body parts)
+        if "attachment" not in content_disposition and not filename:
+            continue
+
+        if not filename:
+            continue
+
+        try:
+            payload = part.get_payload(decode=True)
+            if not payload:
+                continue
+
+            # Skip tiny inline images (likely signatures)
+            content_type = part.get_content_type() or ""
+            if content_type.startswith("image/") and len(payload) < 1024:
+                continue
+
+            # Sanitise filename
+            safe_name = re.sub(r"[^a-zA-Z0-9._-]", "_", filename)
+            local_name = f"{mid_hash}-{safe_name}"
+            local_path = os.path.join(ATTACHMENTS_DIR, local_name)
+
+            # Write attachment to disk
+            with open(local_path, "wb") as f:
+                f.write(payload)
+
+            attachments.append({
+                "name": filename,
+                "mimetype": content_type,
+                "size": len(payload),
+                "local_path": local_path,
+            })
+            print(f"[alex-gmail]   Attachment saved: {filename} ({len(payload)} bytes)")
+
+        except Exception as e:
+            print(f"[alex-gmail]   Attachment error for {filename}: {e}")
+            continue
+
+    return attachments
+
+
+def get_body_snippet(msg, max_length=SNIPPET_LENGTH):
+    """Extract text body from email, truncated to max_length chars.
+
+    Prefers text/plain; falls back to text/html with tag stripping.
+    """
+    plain_body = ""
+    html_body = ""
+
+    if msg.is_multipart():
+        for part in msg.walk():
+            content_type = part.get_content_type()
+            content_disposition = str(part.get("Content-Disposition", ""))
+            if "attachment" in content_disposition:
+                continue
+            try:
+                payload = part.get_payload(decode=True)
+                if not payload:
+                    continue
+                charset = part.get_content_charset() or "utf-8"
+                decoded = payload.decode(charset, errors="replace")
+                if content_type == "text/plain" and not plain_body:
+                    plain_body = decoded
+                elif content_type == "text/html" and not html_body:
+                    html_body = decoded
+            except Exception:
+                pass
+    else:
+        try:
+            payload = msg.get_payload(decode=True)
+            if payload:
+                charset = msg.get_content_charset() or "utf-8"
+                decoded = payload.decode(charset, errors="replace")
+                if msg.get_content_type() == "text/html":
+                    html_body = decoded
+                else:
+                    plain_body = decoded
+        except Exception:
+            pass
+
+    # Prefer plain text; fall back to stripped HTML
+    body = plain_body.strip()
+    if not body and html_body:
+        body = strip_html(html_body)
+
+    if len(body) > max_length:
+        return body[:max_length] + "..."
+    return body
+
+
+# ── Cursor Management ──
+
+def read_cursor():
+    """Read the last poll state from cursor file."""
+    if not os.path.exists(CURSOR_FILE):
+        return {"last_poll": None, "last_message_id": None, "messages_processed": 0}
+
+    cursor = {}
+    try:
+        with open(CURSOR_FILE) as f:
+            for line in f:
+                line = line.strip()
+                if ":" not in line:
+                    continue
+                key, val = line.split(":", 1)
+                key = key.strip()
+                val = val.strip().strip('"')
+                if val == "null" or val == "":
+                    val = None
+                elif key == "messages_processed":
+                    try:
+                        val = int(val)
+                    except ValueError:
+                        val = 0
+                cursor[key] = val
+    except Exception:
+        return {"last_poll": None, "last_message_id": None, "messages_processed": 0}
+
+    return {
+        "last_poll": cursor.get("last_poll"),
+        "last_message_id": cursor.get("last_message_id"),
+        "messages_processed": cursor.get("messages_processed", 0),
+    }
+
+
+def write_cursor(last_poll, last_message_id, messages_processed):
+    """Write the poll state to cursor file."""
+    os.makedirs(os.path.dirname(CURSOR_FILE), exist_ok=True)
+
+    # Quote values properly, handling None
+    lp = f'"{last_poll}"' if last_poll else "null"
+    lm = f'"{last_message_id}"' if last_message_id else "null"
+
+    content = f"""last_poll: {lp}
+last_message_id: {lm}
+messages_processed: {messages_processed}
+"""
+    with open(CURSOR_FILE, "w") as f:
+        f.write(content)
+
+
+# ── Dedup ──
+
+def message_id_hash(message_id, from_addr="", subject="", date=""):
+    """Generate a short hash for deduplication. Uses message-id if available, else fallback."""
+    key = message_id or f"{from_addr}{subject}{date}"
+    return hashlib.md5(key.encode("utf-8", errors="replace")).hexdigest()[:16]
+
+
+def is_already_processed(mid_hash):
+    """Check if this message has already been written to inbox (processed or pending)."""
+    base_name = f"{mid_hash}-alex-email.json"
+    pending = os.path.join(INBOX_DIR, base_name)
+    processed = os.path.join(INBOX_DIR, base_name + ".processed")
+    return os.path.exists(pending) or os.path.exists(processed)
+
+
+# ── Main Poll ──
+
+def poll_alex_inbox():
+    """Poll Alex's Gmail inbox for unseen emails and write to inbox queue."""
+    password = load_password()
+    if not password:
+        print("[alex-gmail] MEHRAN_GMAIL_APP_PASSWORD not set, skipping")
+        return 0
+
+    os.makedirs(INBOX_DIR, exist_ok=True)
+
+    cursor = read_cursor()
+    now = datetime.now(timezone.utc).isoformat()
+    new_count = 0
+    skipped = 0
+    last_mid = cursor.get("last_message_id")
+    total_processed = cursor.get("messages_processed", 0) or 0
+
+    mail = None
+    try:
+        # Connect to Gmail IMAP
+        mail = imaplib.IMAP4_SSL("imap.gmail.com")
+        mail.login(PRINCIPAL_EMAIL, password)
+        mail.select("INBOX")
+
+        # Search for unseen messages
+        status, data = mail.search(None, "UNSEEN")
+        if status != "OK" or not data[0]:
+            print("[alex-gmail] No unseen messages")
+            write_cursor(now, last_mid, total_processed)
+            mail.logout()
+            return 0
+
+        msg_ids = data[0].split()
+        total_unseen = len(msg_ids)
+        print(f"[alex-gmail] Found {total_unseen} unseen messages")
+
+        # Rate limit: process at most MAX_EMAILS_PER_CYCLE
+        if len(msg_ids) > MAX_EMAILS_PER_CYCLE:
+            msg_ids = msg_ids[-MAX_EMAILS_PER_CYCLE:]
+            print(f"[alex-gmail] Rate limited to last {MAX_EMAILS_PER_CYCLE}")
+
+        for mid in msg_ids:
+            try:
+                # Fetch headers first (PEEK so we don't mark as read)
+                status, header_data = mail.fetch(
+                    mid,
+                    "(BODY.PEEK[HEADER.FIELDS (FROM TO SUBJECT DATE MESSAGE-ID)])"
+                )
+                if status != "OK" or not header_data[0]:
+                    continue
+
+                raw_header = header_data[0][1]
+                msg_header = email.message_from_bytes(raw_header)
+
+                from_raw = msg_header.get("From", "")
+                to_raw = msg_header.get("To", "")
+                subject_raw = msg_header.get("Subject", "")
+                date_raw = msg_header.get("Date", "")
+                message_id_raw = msg_header.get("Message-ID", "")
+
+                # Decode headers
+                from_decoded = decode_header_value(from_raw)
+                subject_decoded = decode_header_value(subject_raw)
+
+                # Parse from address into name + email
+                from_name, from_addr = parseaddr(from_decoded)
+                if not from_name:
+                    from_name = from_addr
+
+                # Generate hash for dedup
+                mid_hash = message_id_hash(
+                    message_id_raw, from_addr, subject_decoded, date_raw
+                )
+
+                # Skip if already processed
+                if is_already_processed(mid_hash):
+                    skipped += 1
+                    continue
+
+                # Fetch full message for reliable body extraction (PEEK to not mark as read)
+                snippet = ""
+                email_attachments = []
+                try:
+                    status2, full_data = mail.fetch(mid, "(BODY.PEEK[])")
+                    if status2 == "OK" and full_data[0]:
+                        full_msg = email.message_from_bytes(full_data[0][1])
+                        snippet = get_body_snippet(full_msg)
+                        email_attachments = extract_attachments(full_msg, mid_hash)
+                except Exception:
+                    snippet = ""
+
+                # Parse date to ISO-8601
+                try:
+                    parsed_date = parsedate_to_datetime(date_raw)
+                    iso_date = parsed_date.isoformat()
+                except Exception:
+                    iso_date = date_raw
+
+                # Build inbox record
+                record = {
+                    "received_at": now,
+                    "event_type": "alex_email",
+                    "source": "alex_gmail_poll",
+                    "email": {
+                        "from": from_addr,
+                        "from_name": from_name,
+                        "to": PRINCIPAL_EMAIL,
+                        "subject": subject_decoded,
+                        "date": iso_date,
+                        "message_id": message_id_raw,
+                        "snippet": snippet,
+                    },
+                    "attachments": email_attachments if email_attachments else None,
+                    "triage_hint": None,
+                }
+
+                # Write inbox item
+                filepath = os.path.join(INBOX_DIR, f"{mid_hash}-alex-email.json")
+                with open(filepath, "w") as f:
+                    json.dump(record, f, indent=2, ensure_ascii=False, default=str)
+
+                new_count += 1
+                last_mid = message_id_raw
+                print(f"[alex-gmail] New: {from_name} <{from_addr}> — {subject_decoded[:60]}")
+
+            except Exception as e:
+                print(f"[alex-gmail] Error processing message: {e}")
+                continue
+
+        total_processed += new_count
+        write_cursor(now, last_mid, total_processed)
+        mail.logout()
+
+        print(f"[alex-gmail] Poll complete: {new_count} new, {skipped} skipped, {total_unseen} unseen total")
+        return new_count
+
+    except imaplib.IMAP4.error as e:
+        print(f"[alex-gmail] IMAP error: {e}")
+        if mail:
+            try:
+                mail.logout()
+            except Exception:
+                pass
+        # Write cursor even on error so we don't lose state
+        write_cursor(now, last_mid, total_processed)
+        return 0
+
+    except Exception as e:
+        print(f"[alex-gmail] Connection error: {e}")
+        traceback.print_exc(file=sys.stdout)
+        if mail:
+            try:
+                mail.logout()
+            except Exception:
+                pass
+        write_cursor(now, last_mid, total_processed)
+        return 0
+
+
+if __name__ == "__main__":
+    count = poll_alex_inbox()
+    print(f"[alex-gmail] Finished — {count} new items written to inbox")