heathrow 0.7.0

data/lib/heathrow/sources/messenger.rb

@@ -0,0 +1,212 @@
+require 'digest'
+require 'shellwords'
+require 'json'
+require 'time'
+require_relative 'base'
+
+module Heathrow
+  module Sources
+    class Messenger < Base
+      COOKIE_DIR = File.join(Dir.home, '.heathrow', 'cookies')
+      COOKIE_FILE = File.join(COOKIE_DIR, 'messenger.json')
+      FETCH_SCRIPT = File.join(__dir__, 'messenger_fetch_marionette.py')
+
+      # Required cookies for authentication
+      REQUIRED_COOKIES = %w[c_user xs]
+      OPTIONAL_COOKIES = %w[datr fr]
+
+      def initialize(name, config, db)
+        super
+        @cookies = config['cookies'] || load_cookies
+      end
+
+      def sync(source_id)
+        return 0 unless valid_cookies?
+
+        begin
+          data = fetch_via_playwright
+          return 0 unless data
+          threads = data['threads'] || []
+          return 0 if threads.empty?
+
+          count = 0
+          threads.each do |thread|
+            count += process_thread(source_id, thread)
+          end
+          count
+        rescue => e
+          STDERR.puts "Messenger error: #{e.message}" if ENV['DEBUG']
+          0
+        end
+      end
+
+      def fetch
+        return [] unless enabled?
+        source = @db.get_source_by_name(@name)
+        return [] unless source
+        sync(source['id'])
+        update_last_fetch
+        []
+      end
+
+      def valid_cookies?
+        return false unless @cookies.is_a?(Hash)
+        REQUIRED_COOKIES.all? { |k| @cookies[k] && !@cookies[k].empty? }
+      end
+
+      def self.extract_firefox_cookies
+        profile_dirs = Dir.glob(File.join(Dir.home, '.mozilla', 'firefox', '*'))
+        cookies_db = profile_dirs.map { |d| File.join(d, 'cookies.sqlite') }
+                                  .find { |f| File.exist?(f) }
+        return nil unless cookies_db
+
+        tmp = "/tmp/heathrow_ff_cookies_#{$$}.sqlite"
+        system("cp #{Shellwords.escape(cookies_db)} #{Shellwords.escape(tmp)} 2>/dev/null")
+        return nil unless File.exist?(tmp)
+
+        cookies = {}
+        begin
+          require 'sqlite3'
+          db = SQLite3::Database.new(tmp)
+          db.results_as_hash = true
+
+          (REQUIRED_COOKIES + OPTIONAL_COOKIES).each do |name|
+            row = db.get_first_row(
+              "SELECT value FROM moz_cookies WHERE name = ? AND (host LIKE '%messenger.com' OR host LIKE '%facebook.com') ORDER BY expiry DESC LIMIT 1",
+              [name]
+            )
+            cookies[name] = row['value'] if row
+          end
+
+          db.close
+        ensure
+          File.delete(tmp) if File.exist?(tmp)
+        end
+
+        cookies
+      end
+
+      def save_cookies(cookies = nil)
+        cookies ||= @cookies
+        Dir.mkdir(COOKIE_DIR) unless Dir.exist?(COOKIE_DIR)
+        File.write(COOKIE_FILE, cookies.to_json)
+        File.chmod(0600, COOKIE_FILE)
+        @cookies = cookies
+      end
+
+      SEND_SCRIPT = File.join(__dir__, 'messenger_send.py')
+
+      def send_message(thread_id, _subject, body)
+        return { success: false, message: "No thread ID" } unless thread_id
+        return { success: false, message: "Empty message" } if body.nil? || body.strip.empty?
+
+        result = `python3 #{Shellwords.escape(SEND_SCRIPT)} #{Shellwords.escape(thread_id)} #{Shellwords.escape(body.strip)} 2>/dev/null`
+
+        data = JSON.parse(result) rescue nil
+        if data
+          data.transform_keys!(&:to_sym)
+          data
+        else
+          { success: false, message: "Messenger send: no response from script" }
+        end
+      rescue => e
+        { success: false, message: "Messenger send error: #{e.message}" }
+      end
+
+      private
+
+      def load_cookies
+        return {} unless File.exist?(COOKIE_FILE)
+        JSON.parse(File.read(COOKIE_FILE))
+      rescue
+        {}
+      end
+
+      def fetch_via_playwright
+        # Use Marionette (real Firefox tab) since Meta blocks headless browsers
+        result = `python3 #{Shellwords.escape(FETCH_SCRIPT)} 2>/dev/null`
+        return nil if result.nil? || result.strip.empty?
+
+        data = JSON.parse(result)
+        if data['error']
+          STDERR.puts "Messenger fetch error: #{data['error']}" if ENV['DEBUG']
+          save_session_error(data['error']) if data['error'] == 'login_required'
+          return nil
+        end
+        data
+      rescue JSON::ParserError => e
+        STDERR.puts "Messenger JSON parse error: #{e.message}" if ENV['DEBUG']
+        nil
+      end
+
+      def process_thread(source_id, thread)
+        return 0 unless thread['id'] && thread['id'].to_s.match?(/^\d+$/)
+
+        thread_name = thread['name'] || 'Unknown'
+        return 0 if thread_name.empty?
+
+        messages = thread['messages'] || []
+
+        # Fallback: old format with just a snippet
+        if messages.empty?
+          snippet = thread['snippet'] || ''
+          snippet = '' if snippet =~ /^Messages and calls are secured|^End-to-end encrypted/i
+          return 0 if snippet.empty? && !thread['unread']
+          messages = [{ 'id' => "last_#{thread['id']}", 'sender' => thread_name,
+                        'text' => snippet, 'timestamp' => Time.now.to_i }]
+        end
+
+        count = 0
+        messages.each do |msg|
+          text = msg['text'] || ''
+          text = '' if text =~ /^Messages and calls are secured|^End-to-end encrypted/i
+          next if text.empty?
+
+          msg_id = msg['id'] || "#{thread['id']}_#{msg['timestamp']}"
+          ext_id = "msng_#{thread['id']}_#{msg_id}"
+          sender = msg['sender'] || thread_name
+          timestamp = (msg['timestamp'] || Time.now.to_i).to_i
+
+          data = {
+            source_id: source_id,
+            external_id: ext_id,
+            thread_id: thread['id'].to_s,
+            sender: sender,
+            sender_name: sender,
+            recipients: [thread_name],
+            subject: thread_name,
+            content: text,
+            html_content: nil,
+            timestamp: timestamp,
+            received_at: Time.now.to_i,
+            read: msg == messages.first && thread['unread'] ? false : true,
+            starred: false,
+            archived: false,
+            labels: ['Messenger'],
+            attachments: nil,
+            metadata: {
+              thread_id: thread['id'],
+              message_id: msg_id,
+              platform: 'messenger'
+            },
+            raw_data: { thread_id: thread['id'], name: thread_name }
+          }
+
+          begin
+            @db.insert_message(data)
+            count += 1
+          rescue SQLite3::ConstraintException
+            # Already exists
+          end
+        end
+        count
+      end
+
+      def save_session_error(message)
+        error_file = File.join(COOKIE_DIR, 'messenger_error.txt')
+        Dir.mkdir(COOKIE_DIR) unless Dir.exist?(COOKIE_DIR)
+        File.write(error_file, "#{Time.now.iso8601}: #{message}\n", mode: 'a')
+      end
+    end
+  end
+end

data/lib/heathrow/sources/messenger_fetch.js

@@ -0,0 +1,297 @@
+#!/usr/bin/env node
+// Fetch Messenger inbox threads + message history using Playwright headless browser
+// Called by messenger.rb, outputs JSON to stdout
+// Facebook uses E2EE so we must scrape the rendered DOM, not GraphQL
+
+const { firefox } = require('playwright');
+const path = require('path');
+const fs = require('fs');
+const { execSync } = require('child_process');
+
+const COOKIE_FILE = path.join(process.env.HOME, '.heathrow', 'cookies', 'messenger.json');
+const TIMEOUT = 30000;
+const MAX_THREADS = 10;   // How many threads to fetch history for
+const MSG_PER_THREAD = 10; // Messages to extract per thread
+const DEBUG = process.argv.includes('--debug');
+
+// Extract fresh cookies from Firefox's cookies.sqlite
+function refreshCookiesFromFirefox() {
+  try {
+    const profilesDir = path.join(process.env.HOME, '.mozilla', 'firefox');
+    const profiles = fs.readdirSync(profilesDir);
+    let cookiesDb = null;
+    for (const p of profiles) {
+      const candidate = path.join(profilesDir, p, 'cookies.sqlite');
+      if (fs.existsSync(candidate)) { cookiesDb = candidate; break; }
+    }
+    if (!cookiesDb) return null;
+
+    const tmp = `/tmp/heathrow_msng_fetch_${process.pid}.sqlite`;
+    fs.copyFileSync(cookiesDb, tmp);
+
+    const names = ['c_user', 'xs', 'datr', 'fr', 'sb', 'wd'];
+    const cookies = {};
+
+    for (const name of names) {
+      try {
+        const result = execSync(
+          `sqlite3 "${tmp}" "SELECT value FROM moz_cookies WHERE name = '${name}' AND (host LIKE '%messenger.com' OR host LIKE '%facebook.com') ORDER BY expiry DESC LIMIT 1"`,
+          { encoding: 'utf8', timeout: 5000 }
+        ).trim();
+        if (result) cookies[name] = decodeURIComponent(result);
+      } catch (e) {}
+    }
+
+    try { fs.unlinkSync(tmp); } catch (e) {}
+
+    if (cookies.c_user && cookies.xs) {
+      const dir = path.dirname(COOKIE_FILE);
+      if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+      fs.writeFileSync(COOKIE_FILE, JSON.stringify(cookies));
+      fs.chmodSync(COOKIE_FILE, 0o600);
+      return cookies;
+    }
+    return null;
+  } catch (e) {
+    return null;
+  }
+}
+
+async function fetchMessengerThreads() {
+  let browser;
+  try {
+    // Always refresh cookies from Firefox
+    let cookies = refreshCookiesFromFirefox();
+    if (!cookies) {
+      if (fs.existsSync(COOKIE_FILE)) {
+        cookies = JSON.parse(fs.readFileSync(COOKIE_FILE, 'utf8'));
+      } else {
+        cookies = {};
+      }
+    }
+    if (!cookies.c_user || !cookies.xs) {
+      console.log(JSON.stringify({ error: 'missing_cookies', threads: [] }));
+      return;
+    }
+
+    const makeCookies = (domain) => Object.entries(cookies).map(([name, value]) => ({
+      name, value: String(value), domain, path: '/',
+      httpOnly: true, secure: true, sameSite: 'None'
+    }));
+
+    browser = await firefox.launch({ headless: true });
+    const context = await browser.newContext({
+      userAgent: 'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0',
+      viewport: { width: 1280, height: 720 }
+    });
+
+    await context.addCookies([...makeCookies('.messenger.com'), ...makeCookies('.facebook.com')]);
+    const page = await context.newPage();
+
+    await page.goto('https://www.messenger.com/t/', {
+      waitUntil: 'domcontentloaded', timeout: TIMEOUT
+    });
+
+    // Wait for thread list
+    try {
+      await page.waitForSelector('a[href*="/t/"]', { timeout: 15000 });
+    } catch (e) {
+      await page.waitForTimeout(5000);
+    }
+
+    // Dismiss PIN dialog
+    try {
+      const pinInput = await page.$('input[type="text"], input[type="tel"], input[aria-label*="PIN"]');
+      if (pinInput) {
+        for (const digit of '314159') {
+          await page.keyboard.press(digit);
+          await page.waitForTimeout(100);
+        }
+        await page.waitForTimeout(3000);
+      }
+    } catch (e) {}
+
+    // Dismiss modals
+    try {
+      const closeBtn = await page.$('[aria-label="Close"], [aria-label="Dismiss"]');
+      if (closeBtn) await closeBtn.click();
+      await page.waitForTimeout(1000);
+    } catch (e) {}
+
+    await page.waitForTimeout(2000);
+
+    if (page.url().includes('login') || page.url().includes('checkpoint')) {
+      console.log(JSON.stringify({ error: 'login_required', threads: [] }));
+      return;
+    }
+
+    // Get thread list from sidebar DOM
+    const threadList = await page.evaluate(() => {
+      const skipNames = /^(Media & files|Privacy & support|Marketplace|Message Requests|Archived Chats|Communities|Chats)$/i;
+      const skipSnippets = /^(Messages and calls are secured|End-to-end encrypted|Active \d|Active now|You're now connected|Say hi to your new)/i;
+      const statusPatterns = /^(Active now|Active \d+[hm] ago)$/i;
+      const results = [];
+      const links = document.querySelectorAll('a[href*="/t/"]');
+      const seen = new Set();
+
+      for (const link of links) {
+        const match = link.href.match(/\/t\/(\d+)/);
+        if (!match || seen.has(match[1])) continue;
+        seen.add(match[1]);
+
+        const row = link.closest('[role="row"], [role="listitem"]') || link.parentElement;
+        if (!row) continue;
+
+        const spans = Array.from(row.querySelectorAll('span'))
+          .map(s => s.textContent.trim())
+          .filter(t => t.length > 0 && t.length < 200);
+        if (spans.length === 0) continue;
+
+        let name = null;
+        for (const s of spans) {
+          if (statusPatterns.test(s) || skipNames.test(s) || s.includes(' unread')) continue;
+          name = s;
+          break;
+        }
+        if (!name || skipNames.test(name)) continue;
+
+        const unread = row.innerHTML.includes('Unread') ||
+                       row.querySelector('[aria-label*="unread"]') !== null;
+
+        results.push({ id: match[1], name, unread });
+      }
+      return results;
+    });
+
+    if (DEBUG) {
+      console.error(`Found ${threadList.length} threads in sidebar`);
+    }
+
+    // Visit each thread and extract messages from the rendered conversation
+    const threads = [];
+    const toVisit = threadList.slice(0, MAX_THREADS);
+
+    for (const thread of toVisit) {
+      try {
+        await page.goto(`https://www.messenger.com/t/${thread.id}`, {
+          waitUntil: 'domcontentloaded', timeout: 15000
+        });
+
+        // Wait for messages to render
+        try {
+          await page.waitForSelector('[role="row"]', { timeout: 8000 });
+        } catch (e) {
+          await page.waitForTimeout(3000);
+        }
+        await page.waitForTimeout(1500);
+
+        // Extract messages from the conversation view
+        const messages = await page.evaluate(({threadName, msgLimit}) => {
+          const msgs = [];
+          const skipText = /^(Messages and calls are secured|New messages and calls are secured|End-to-end encrypted|You're now connected|Say hi to|You (created|named) the group|Only people in this chat can)/i;
+          const datePattern = /^(January|February|March|April|May|June|July|August|September|October|November|December|\d{1,2}\/\d{1,2}|Yesterday|Today|Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday)/i;
+
+          // Messages are in [role="row"] inside [role="main"]
+          const main = document.querySelector('[role="main"]');
+          if (!main) return [];
+
+          const rows = main.querySelectorAll('[role="row"]');
+          let currentSender = threadName;
+
+          for (const row of rows) {
+            const fullText = row.textContent.trim();
+
+            // Skip date separator rows
+            if (datePattern.test(fullText) && fullText.length < 60) continue;
+            // Skip E2E banner
+            if (skipText.test(fullText)) continue;
+            // Skip empty or header rows
+            if (!fullText || fullText.length < 2) continue;
+
+            // Get all [dir="auto"] spans in this row for sender + message parsing
+            const spans = Array.from(row.querySelectorAll('[dir="auto"]'))
+              .map(el => el.textContent.trim())
+              .filter(t => t.length > 0);
+
+            if (spans.length === 0) continue;
+
+            // Detect sender: "You sent" or a short name before the message
+            let sender = currentSender;
+            let messageText = '';
+
+            if (spans[0] === 'You sent' || fullText.startsWith('You sent')) {
+              sender = 'You';
+              // Message text is in subsequent spans (skip "You sent" and "Enter")
+              messageText = spans.filter(s => s !== 'You sent' && s !== 'Enter')
+                .find(s => s.length > 1 && !datePattern.test(s) && !skipText.test(s)) || '';
+            } else {
+              // First span might be sender name (short, no spaces typically for first name)
+              // Message text is duplicated in spans (Messenger renders it twice)
+              const firstSpan = spans[0];
+              const restSpans = spans.slice(1).filter(s => s !== 'Enter');
+
+              if (firstSpan.length < 30 && restSpans.length > 0 && firstSpan !== restSpans[0]) {
+                // First span is likely sender name
+                sender = firstSpan;
+                currentSender = sender;
+                messageText = restSpans[0] || '';
+              } else if (restSpans.length > 0) {
+                messageText = firstSpan;
+              } else {
+                messageText = firstSpan;
+              }
+            }
+
+            // Clean up: Messenger often duplicates the message text
+            if (!messageText || messageText === 'Enter' || skipText.test(messageText)) continue;
+            if (datePattern.test(messageText) && messageText.length < 60) continue;
+
+            msgs.push({ text: messageText, sender });
+          }
+
+          // Newest last in DOM, reverse for newest-first
+          const seen = new Set();
+          return msgs.reverse().filter(m => {
+            if (seen.has(m.text)) return false;
+            seen.add(m.text);
+            return true;
+          }).slice(0, msgLimit);
+        }, {threadName: thread.name, msgLimit: MSG_PER_THREAD});
+
+        if (DEBUG) {
+          console.error(`Thread ${thread.name}: ${messages.length} messages`);
+        }
+
+        // Assign sequential timestamps (we don't have real ones from DOM)
+        const now = Date.now() / 1000;
+        const msgsWithTs = messages.map((m, i) => ({
+          id: `${thread.id}_${i}`,
+          sender: m.sender,
+          text: m.text,
+          timestamp: Math.floor(now - (messages.length - 1 - i) * 60)  // 1 min apart
+        }));
+
+        threads.push({
+          id: thread.id,
+          name: thread.name,
+          unread: thread.unread,
+          messages: msgsWithTs
+        });
+
+      } catch (e) {
+        if (DEBUG) console.error(`Failed thread ${thread.id}: ${e.message}`);
+        // Still include the thread with no messages
+        threads.push({ id: thread.id, name: thread.name, unread: thread.unread, messages: [] });
+      }
+    }
+
+    console.log(JSON.stringify({ threads, source: 'dom' }));
+
+  } catch (err) {
+    console.log(JSON.stringify({ error: err.message, threads: [] }));
+  } finally {
+    if (browser) await browser.close();
+  }
+}
+
+fetchMessengerThreads();

data/lib/heathrow/sources/messenger_fetch_marionette.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""Fetch Messenger DM threads and messages via Firefox Marionette.
+
+Connects to a running Firefox instance on Marionette port 2828,
+finds the Messenger tab, scrapes thread list and message history
+from the DOM. Outputs JSON to stdout, debug info to stderr.
+
+Requires: marionette_driver (pip install marionette_driver)
+"""
+
+import json
+import sys
+
+MAX_THREADS = 20
+DEBUG = '--debug' in sys.argv
+
+
+def debug(msg):
+    if DEBUG:
+        print(f"[marionette] {msg}", file=sys.stderr)
+
+
+def output_error(message):
+    print(json.dumps({"error": message, "threads": []}))
+    sys.exit(0)
+
+
+def find_messenger_tab(client):
+    """Find the tab with messenger.com open."""
+    handles = client.window_handles
+    debug(f"Found {len(handles)} tabs")
+
+    for handle in handles:
+        client.switch_to_window(handle)
+        url = client.get_url()
+        debug(f"  Tab: {url[:80]}")
+        if 'messenger.com' in url:
+            debug(f"Found Messenger tab: {url}")
+            return handle
+
+    return None
+
+
+SCRAPE_THREADS_JS = """
+const skipNames = /^(Media & files|Privacy & support|Marketplace|Message Requests|Archived Chats|Communities|Chats)$/i;
+const statusPatterns = /^(Active now|Active \\d+[hm] ago)$/i;
+const results = [];
+const links = document.querySelectorAll('a[href*="/t/"]');
+const seen = new Set();
+
+for (const link of links) {
+    const match = link.href.match(/\\/t\\/(\\d+)/);
+    if (!match || seen.has(match[1])) continue;
+    seen.add(match[1]);
+
+    const row = link.closest('[role="row"], [role="listitem"]') || link.parentElement;
+    if (!row) continue;
+
+    const spans = Array.from(row.querySelectorAll('span'))
+        .map(s => s.textContent.trim())
+        .filter(t => t.length > 0 && t.length < 200);
+    if (spans.length === 0) continue;
+
+    let name = null;
+    let snippet = '';
+    for (const s of spans) {
+        if (statusPatterns.test(s) || skipNames.test(s) || s.includes(' unread')) continue;
+        if (!name) { name = s; continue; }
+        if (!snippet && s !== name && s.length > 1) { snippet = s; }
+    }
+    if (!name || skipNames.test(name)) continue;
+
+    const unread = row.innerHTML.includes('Unread') ||
+                   row.querySelector('[aria-label*="unread"]') !== null;
+
+    results.push({id: match[1], name: name, unread: unread, snippet: snippet});
+}
+return results;
+"""
+
+
+
+def main():
+    try:
+        from marionette_driver.marionette import Marionette
+    except ImportError:
+        output_error("marionette_driver not installed (pip install marionette_driver)")
+
+    client = None
+    try:
+        client = Marionette(host='localhost', port=2828)
+        client.start_session()
+        debug("Connected to Marionette")
+
+        handle = find_messenger_tab(client)
+        if not handle:
+            output_error("No Messenger tab found in Firefox")
+
+        # Scrape thread list + snippets from the sidebar (no navigation needed)
+        debug("Scraping thread list from sidebar...")
+        thread_list = client.execute_script(SCRAPE_THREADS_JS)
+
+        if not thread_list:
+            debug("No threads found in sidebar")
+            print(json.dumps({"threads": [], "source": "marionette"}))
+            return
+
+        debug(f"Found {len(thread_list)} threads in sidebar")
+
+        threads = []
+        for thread in thread_list[:MAX_THREADS]:
+            snippet = thread.get('snippet', '')
+            debug(f"  {thread['name']}: snippet={snippet[:50] if snippet else '(none)'}, unread={thread.get('unread')}")
+            threads.append({
+                "id": thread['id'],
+                "name": thread['name'],
+                "unread": thread.get('unread', False),
+                "snippet": snippet,
+                "messages": []
+            })
+
+        print(json.dumps({"threads": threads, "source": "marionette"}))
+
+    except ConnectionRefusedError:
+        output_error("Cannot connect to Firefox Marionette on port 2828. Start Firefox with --marionette.")
+    except Exception as e:
+        output_error(str(e))
+    finally:
+        if client:
+            try:
+                client.delete_session()
+                debug("Session deleted")
+            except Exception:
+                pass
+
+
+if __name__ == '__main__':
+    main()

data/lib/heathrow/sources/messenger_send.js

@@ -0,0 +1,32 @@
+#!/usr/bin/env node
+// Send a Messenger DM by opening the thread in the user's browser.
+// Since Meta blocks all non-browser API access, this opens the DM thread
+// in the real browser with the message pre-copied to clipboard.
+//
+// Usage: messenger_send.js <thread_id> <message>
+// Outputs JSON: { success: true/false, message: "..." }
+
+const { execSync, spawn } = require('child_process');
+
+const threadId = process.argv[2];
+const message = process.argv[3];
+
+if (!threadId || !message) {
+  console.log(JSON.stringify({ success: false, message: 'Usage: messenger_send.js <thread_id> <message>' }));
+  process.exit(0);
+}
+
+const url = `https://www.messenger.com/t/${threadId}`;
+
+// Copy message to clipboard
+try {
+  execSync('xclip -selection clipboard', { input: message.trim(), timeout: 3000 });
+} catch (e) {}
+
+// Open in browser
+spawn('xdg-open', [url], { detached: true, stdio: 'ignore' }).unref();
+
+console.log(JSON.stringify({
+  success: true,
+  message: 'Opened Messenger in browser. Message copied to clipboard (Ctrl+V to paste).'
+}));