heathrow 0.7.9 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8023c45996b5eba44dfbb1e9718fe11d149ec7f14f35dd5569fe5172b53a018e
-  data.tar.gz: d1f94f8964b220fb7f831ca60298a3ca68f1dc4bdc4b9949282bb7add4c8d65e
+  metadata.gz: e278aa83b425434144c53f99442726b3f67e2d0bc06cd34817256b7b862da924
+  data.tar.gz: 5b33d7429d0d8022393c994106d959236ea42f137401588819b49512e713755a
 SHA512:
-  metadata.gz: 5b438204942e8ad23d166b36be5acc3dc9a0fa1c2456f9946f258c02d1597176bf0dafde75081e21c469839487938bc8181bf3151cff4fca30d7bd617ae53d18
-  data.tar.gz: 1944ede7efac79dac8ace2640dee9c2260754898f8dc24ed1b7793898ed2b3eb57b47cef9fc7102ab8b1da09b20be898aa23312872704301b5aa3fb96a501caf
+  metadata.gz: '0596e613d29bcfc00ebe98afb84bb9f5fc9db44edbd33f1ef78c2604e451d639d1d526e7152808c7002cd47c2e12699e861dc4d5e972cbb51dc7457a9f38036d'
+  data.tar.gz: a29bf091fd86b3f5c2a731f42e799276260e45bee61b2bff4dee1f6f9dbb69b831f01b2dc4321635d4907b63370970f9f5181227b02b494412a7fd26c066ec46

data/heathrow.gemspec

@@ -29,6 +29,6 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   # Runtime dependencies - keep it simple!
-  spec.add_runtime_dependency 'rcurses', '>= 5.0'
+  spec.add_runtime_dependency 'rcurses', '~> 7.0'
   spec.add_runtime_dependency 'sqlite3', '>= 1.4'
 end

data/lib/heathrow/message_composer.rb

@@ -163,7 +163,7 @@ module Heathrow
       template << "Date: #{format_date(original_date)}"
       template << "Subject: #{original_subject}"
       template << ""
-      template << original_content
+      template << original_content.gsub("\r", "")
 
       # Signature at the bottom
       sig = get_signature

data/lib/heathrow/sources/messenger.rb

@@ -10,6 +10,7 @@ module Heathrow
       COOKIE_DIR = File.join(Dir.home, '.heathrow', 'cookies')
       COOKIE_FILE = File.join(COOKIE_DIR, 'messenger.json')
       FETCH_SCRIPT = File.join(__dir__, 'messenger_fetch_marionette.py')
+      THREAD_SCRIPT = File.join(__dir__, 'messenger_fetch_thread.py')
 
       # Required cookies for authentication
       REQUIRED_COOKIES = %w[c_user xs]
@@ -21,25 +22,106 @@ module Heathrow
       end
 
       def sync(source_id)
-        return 0 unless valid_cookies?
+        unless valid_cookies?
+          @sync_error = "Messenger: invalid cookies"
+          return 0
+        end
 
         begin
           data = fetch_via_playwright
-          return 0 unless data
+          unless data
+            @sync_error = "Messenger: no data (is the tab open?)"
+            return 0
+          end
+
+          if data['error']
+            @sync_error = "Messenger: #{data['error']}"
+            return 0
+          end
+
           threads = data['threads'] || []
-          return 0 if threads.empty?
+          if threads.empty?
+            @sync_error = "Messenger: no threads found"
+            return 0
+          end
 
           count = 0
           threads.each do |thread|
             count += process_thread(source_id, thread)
           end
+          @sync_error = nil
           count
         rescue => e
-          STDERR.puts "Messenger error: #{e.message}" if ENV['DEBUG']
+          @sync_error = "Messenger: #{e.message}"
           0
         end
       end
 
+      attr_reader :sync_error
+
+      # Deep-fetch a single thread: navigate into it and scrape visible messages
+      def sync_thread(source_id, thread_id, thread_name)
+        result = `timeout 15 python3 #{Shellwords.escape(THREAD_SCRIPT)} #{Shellwords.escape(thread_id)} 2>/dev/null`
+        return 0 if result.nil? || result.strip.empty?
+
+        data = JSON.parse(result)
+        if data['error']
+          @sync_error = "Messenger thread: #{data['error']}"
+          return 0
+        end
+
+        messages = data['messages'] || []
+        return 0 if messages.empty?
+
+        count = 0
+        messages.each_with_index do |msg, i|
+          text = (msg['text'] || '').strip
+          next if text.empty? || text.length < 2
+          # Skip UI garbage
+          next if text =~ /^(Today|Yesterday) at \d/i
+          next if text =~ /^Enter, Message sent/i
+          next if text =~ /^You (sent|replied|reacted)/i
+
+          sender = msg['sender'] || ''
+          sender = thread_name if sender.empty?
+
+          ext_id = "msng_#{thread_id}_d#{Digest::MD5.hexdigest(text)[0..11]}"
+
+          msg_data = {
+            source_id: source_id,
+            external_id: ext_id,
+            thread_id: thread_id.to_s,
+            sender: sender,
+            sender_name: sender,
+            recipients: [thread_name],
+            subject: thread_name,
+            content: text,
+            html_content: nil,
+            timestamp: Time.now.to_i,
+            received_at: Time.now.to_i,
+            read: true,
+            starred: false,
+            archived: false,
+            labels: ['Messenger'],
+            attachments: nil,
+            metadata: { thread_id: thread_id, message_id: ext_id, platform: 'messenger' },
+            raw_data: { thread_id: thread_id, name: thread_name }
+          }
+
+          begin
+            @db.insert_message(msg_data)
+            count += 1
+          rescue SQLite3::ConstraintException
+            # Already exists
+          end
+        end
+        @sync_error = nil
+        count
+      rescue => e
+        @sync_error = "Messenger thread: #{e.message}"
+        0
+      end
+
       def fetch
         return [] unless enabled?
         source = @db.get_source_by_name(@name)
@@ -124,7 +206,7 @@ module Heathrow
 
       def fetch_via_playwright
         # Use Marionette (real Firefox tab) since Meta blocks headless browsers
-        result = `python3 #{Shellwords.escape(FETCH_SCRIPT)} 2>/dev/null`
+        result = `timeout 30 python3 #{Shellwords.escape(FETCH_SCRIPT)} 2>/dev/null`
         return nil if result.nil? || result.strip.empty?
 
         data = JSON.parse(result)

data/lib/heathrow/sources/messenger_fetch_marionette.py

@@ -78,8 +78,6 @@ for (const link of links) {
 return results;
 """
 
-
-
 def main():
     try:
         from marionette_driver.marionette import Marionette
@@ -109,8 +107,10 @@ def main():
 
         threads = []
         for thread in thread_list[:MAX_THREADS]:
+            tid = thread['id']
             snippet = thread.get('snippet', '')
             debug(f"  {thread['name']}: snippet={snippet[:50] if snippet else '(none)'}, unread={thread.get('unread')}")
+
             threads.append({
                 "id": thread['id'],
                 "name": thread['name'],

data/lib/heathrow/sources/messenger_fetch_thread.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+"""Fetch messages from a single Messenger conversation via Firefox Marionette.
+
+Usage: messenger_fetch_thread.py <thread_id>
+
+Connects to Firefox Marionette, navigates to the conversation,
+scrapes visible messages, returns JSON to stdout.
+"""
+
+import json
+import sys
+import time
+
+DEBUG = '--debug' in sys.argv
+THREAD_ID = None
+
+for arg in sys.argv[1:]:
+    if arg != '--debug' and arg.isdigit():
+        THREAD_ID = arg
+
+
+def debug(msg):
+    if DEBUG:
+        print(f"[thread-fetch] {msg}", file=sys.stderr)
+
+
+def main():
+    if not THREAD_ID:
+        print(json.dumps({"error": "No thread ID provided", "messages": []}))
+        return
+
+    try:
+        from marionette_driver.marionette import Marionette
+    except ImportError:
+        print(json.dumps({"error": "marionette_driver not installed", "messages": []}))
+        return
+
+    client = None
+    try:
+        client = Marionette(host='localhost', port=2828)
+        client.start_session()
+        debug("Connected")
+
+        # Find Messenger tab
+        for handle in client.window_handles:
+            client.switch_to_window(handle)
+            if 'messenger.com' in client.get_url():
+                break
+        else:
+            print(json.dumps({"error": "No Messenger tab found", "messages": []}))
+            return
+
+        # Navigate to the conversation
+        target_url = f"https://www.messenger.com/t/{THREAD_ID}"
+        current_url = client.get_url()
+        if f"/t/{THREAD_ID}" not in current_url:
+            debug(f"Navigating to {target_url}")
+            client.navigate(target_url)
+            time.sleep(2)
+        else:
+            debug("Already on target conversation")
+
+        # Scrape messages from the main content area
+        messages = client.execute_script("""
+const msgs = [];
+const mainArea = document.querySelector('[role="main"]');
+if (!mainArea) return msgs;
+
+// Find all message groups - each group has a sender
+const groups = mainArea.querySelectorAll('[role="row"]');
+
+for (const group of groups) {
+    // Get text content from dir="auto" spans (actual message text)
+    const textEls = Array.from(group.querySelectorAll('[dir="auto"]'));
+    if (textEls.length === 0) continue;
+
+    // Filter out UI chrome
+    const texts = textEls
+        .map(e => e.textContent.trim())
+        .filter(t => {
+            if (t.length < 1 || t.length > 5000) return false;
+            if (/^(Active now|Active \\d|Seen by|You sent|\\d+ (hour|minute|day|week)|Loading|Replying to|End-to-end encrypted|Messenger|Media & files|Privacy & support)/i.test(t)) return false;
+            if (/^(Today|Yesterday)$/i.test(t)) return false;
+            if (/^\\d{1,2}:\\d{2}\\s*(AM|PM)?$/i.test(t)) return false;
+            if (/^\\w{3} \\d{1,2}, \\d{4}$/i.test(t)) return false;
+            return true;
+        });
+    if (texts.length === 0) continue;
+
+    const text = texts.join(' ');
+    if (text.length < 1) continue;
+
+    // Try sender from img alt
+    let sender = '';
+    const img = group.querySelector('img[alt]');
+    if (img && img.alt && img.alt.length < 60 && !/^\\d/.test(img.alt)) {
+        sender = img.alt;
+    }
+
+    msgs.push({sender: sender, text: text});
+}
+return msgs;
+""") or []
+
+        debug(f"Found {len(messages)} raw messages")
+
+        # Deduplicate consecutive identical texts
+        deduped = []
+        prev_text = ''
+        for m in messages:
+            if m['text'] != prev_text:
+                deduped.append(m)
+                prev_text = m['text']
+
+        debug(f"After dedup: {len(deduped)} messages")
+        print(json.dumps({"messages": deduped}))
+
+    except ConnectionRefusedError:
+        print(json.dumps({"error": "Cannot connect to Marionette on port 2828", "messages": []}))
+    except Exception as e:
+        print(json.dumps({"error": str(e), "messages": []}))
+    finally:
+        if client:
+            try:
+                client.delete_session()
+            except Exception:
+                pass
+
+
+if __name__ == '__main__':
+    main()