heathrow 0.7.0

data/lib/heathrow/sources/rss.rb

@@ -0,0 +1,299 @@
+require 'rss'
+require 'time'
+require 'digest'
+require 'shellwords'
+require_relative 'base'
+
+module Heathrow
+  module Sources
+    class RSS < Base
+      def initialize(name, config, db)
+        super
+        # feeds can be simple URLs or hashes with {url:, title:, tags:}
+        @feeds = config['feeds'] || []
+      end
+
+      # Sync all feeds into the database. Called from application.rb on refresh.
+      def sync(source_id)
+        count = 0
+        changed = false
+        @feeds.each do |feed_entry|
+          url, title, tags = parse_feed_entry(feed_entry)
+          begin
+            n = sync_feed(source_id, url, title, tags)
+            count += n
+            if feed_entry.is_a?(Hash)
+              feed_entry['last_status'] = 'ok'
+              feed_entry['last_sync'] = Time.now.to_i
+              changed = true
+            end
+          rescue => e
+            STDERR.puts "RSS error #{url}: #{e.message}" if ENV['DEBUG']
+            if feed_entry.is_a?(Hash)
+              feed_entry['last_status'] = "error: #{e.message}"[0..120]
+              feed_entry['last_sync'] = Time.now.to_i
+              changed = true
+            end
+          end
+        end
+        if changed
+          @config['feeds'] = @feeds
+          save_config
+        end
+        count
+      end
+
+      # Legacy interface for source_manager
+      def fetch
+        return [] unless enabled?
+        source = @db.get_source_by_name(@name)
+        return [] unless source
+        sync(source['id'])
+        update_last_fetch
+        []
+      end
+
+      def add_feed(url, title: nil, tags: [])
+        entry = { 'url' => url, 'title' => title, 'tags' => tags }
+        @feeds << entry unless @feeds.any? { |f| feed_url(f) == url }
+        @config['feeds'] = @feeds
+        save_config
+      end
+
+      def remove_feed(url)
+        @feeds.reject! { |f| feed_url(f) == url }
+        @config['feeds'] = @feeds
+        save_config
+      end
+
+      def list_feeds
+        @feeds.map { |f| { url: feed_url(f), title: feed_title(f), tags: feed_tags(f) } }
+      end
+
+      def test_connection(&progress)
+        ok = 0
+        fail_names = []
+        fail_details = []
+        @feeds.each_with_index do |f, i|
+          url = feed_url(f)
+          title = feed_title(f) || url
+          progress.call("Testing #{i+1}/#{@feeds.size}: #{title}") if progress
+          begin
+            data = http_get(url)
+            if data && !data.empty?
+              ok += 1
+            else
+              fail_names << title
+              fail_details << title
+            end
+          rescue => e
+            fail_names << title
+            fail_details << "#{title}: #{e.message[0..50]}"
+          end
+        end
+        if fail_names.empty?
+          { success: true, message: "All #{ok} feeds OK" }
+        else
+          { success: false, message: "#{ok}/#{@feeds.size} OK. Failed: #{fail_details.join(', ')}", failed_feeds: fail_names }
+        end
+      end
+
+      private
+
+      def parse_feed_entry(entry)
+        if entry.is_a?(Hash)
+          [entry['url'], entry['title'], entry['tags'] || []]
+        else
+          [entry.to_s, nil, []]
+        end
+      end
+
+      def feed_url(entry)
+        entry.is_a?(Hash) ? entry['url'] : entry.to_s
+      end
+
+      def feed_title(entry)
+        entry.is_a?(Hash) ? entry['title'] : nil
+      end
+
+      def feed_tags(entry)
+        entry.is_a?(Hash) ? (entry['tags'] || []) : []
+      end
+
+      def sync_feed(source_id, url, custom_title, tags)
+        count = 0
+
+        raw = http_get(url)
+        raise "fetch failed (no response)" unless raw && !raw.empty?
+
+        feed = ::RSS::Parser.parse(raw, false)
+        raise "parse failed (not a valid feed)" unless feed && feed.items
+
+        is_atom = feed.is_a?(::RSS::Atom::Feed)
+        feed_title = custom_title || (is_atom ? atom_text(feed.title) : feed.channel&.title) || url
+
+        feed.items.each do |item|
+          link = item_link(item, is_atom)
+          ext_id = link || item_id(item, is_atom) || Digest::MD5.hexdigest(item_title(item, is_atom) + url)
+
+          title = strip_html(item_title(item, is_atom)).gsub(/\s+/, ' ').strip
+          # Keep HTML for rich rendering; also store plain text
+          html_content = item_content(item, is_atom)
+          plain_content = strip_html(html_content)
+
+          timestamp = extract_timestamp(item)
+          author = strip_html(extract_author(item) || '')
+          categories = extract_categories(item)
+
+          labels = [feed_title] + tags + categories
+          labels.uniq!
+
+          data = {
+            source_id: source_id,
+            external_id: "rss_#{Digest::MD5.hexdigest(ext_id)}",
+            sender: (author.nil? || author.empty?) ? feed_title : author,
+            sender_name: (author.nil? || author.empty?) ? feed_title : author,
+            recipients: [feed_title],
+            subject: title,
+            content: plain_content,
+            html_content: html_content,
+            timestamp: timestamp.to_i,
+            received_at: Time.now.to_i,
+            read: false,
+            starred: false,
+            archived: false,
+            labels: labels,
+            metadata: {
+              link: link,
+              feed_url: url,
+              feed_title: feed_title,
+              author: author,
+              categories: categories,
+              tags: tags
+            },
+            raw_data: { link: link, feed_title: feed_title, author: author, categories: categories }
+          }
+
+          begin
+            @db.insert_message(data)
+            count += 1
+          rescue SQLite3::ConstraintException
+            # Already exists
+          end
+        end
+
+        count
+      end
+
+      def http_get(url, _redirects = 8)
+        # Use curl — Ruby's net-http gem has chunked encoding bugs
+        result = `curl -sL --max-time 15 --max-redirs 8 -A 'Heathrow/1.0 (RSS Reader)' #{Shellwords.escape(url)} 2>/dev/null`
+        $?.success? && !result.empty? ? result : nil
+      rescue => e
+        STDERR.puts "RSS fetch error #{url}: #{e.message}" if ENV['DEBUG']
+        nil
+      end
+
+      # Atom/RSS2 normalization helpers
+      def atom_text(obj)
+        return nil unless obj
+        obj.respond_to?(:content) ? obj.content : obj.to_s
+      end
+
+      def item_title(item, is_atom)
+        if is_atom
+          atom_text(item.title) || 'No title'
+        else
+          item.title || 'No title'
+        end
+      end
+
+      def item_link(item, is_atom)
+        if is_atom
+          # Atom links: find rel="alternate" or first link
+          if item.respond_to?(:links) && item.links
+            alt = item.links.find { |l| l.rel.nil? || l.rel == 'alternate' }
+            (alt || item.links.first)&.href
+          elsif item.link.respond_to?(:href)
+            item.link.href
+          else
+            item.link.to_s
+          end
+        else
+          item.link
+        end
+      end
+
+      def item_id(item, is_atom)
+        if is_atom
+          atom_text(item.id)
+        else
+          item.guid&.content
+        end
+      end
+
+      def item_content(item, is_atom)
+        if is_atom
+          # Prefer content over summary for Atom
+          c = atom_text(item.content) if item.respond_to?(:content) && item.content
+          c ||= atom_text(item.summary) if item.respond_to?(:summary) && item.summary
+          c || ''
+        else
+          item.description || (item.content_encoded rescue nil) || ''
+        end
+      end
+
+      def extract_timestamp(item)
+        if item.respond_to?(:pubDate) && item.pubDate
+          item.pubDate
+        elsif item.respond_to?(:dc_date) && item.dc_date
+          item.dc_date
+        elsif item.respond_to?(:updated) && item.updated
+          item.updated.content rescue Time.now
+        else
+          Time.now
+        end
+      end
+
+      def extract_author(item)
+        if item.respond_to?(:author) && item.author
+          # Atom author objects have a .name method; .to_s may give raw XML
+          author = if item.author.respond_to?(:name) && item.author.name
+                     atom_text(item.author.name)
+                   else
+                     item.author.to_s
+                   end
+          return author unless author.nil? || author.empty?
+        end
+        if item.respond_to?(:dc_creator) && item.dc_creator
+          item.dc_creator
+        else
+          nil
+        end
+      end
+
+      def extract_categories(item)
+        cats = []
+        if item.respond_to?(:categories) && item.categories
+          cats = item.categories.map { |c| c.content rescue c.to_s }.compact
+        end
+        cats
+      rescue
+        []
+      end
+
+      def strip_html(text)
+        return '' if text.nil? || text.empty?
+        text.gsub(/<[^>]*>/, '')
+            .gsub(/&nbsp;/, ' ')
+            .gsub(/&amp;/, '&')
+            .gsub(/&lt;/, '<')
+            .gsub(/&gt;/, '>')
+            .gsub(/&quot;/, '"')
+            .gsub(/&#39;/, "'")
+            .gsub(/\n\s*\n\s*\n/, "\n\n")
+            .strip
+      end
+    end
+  end
+end

data/lib/heathrow/sources/slack.rb

@@ -0,0 +1,375 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'net/http'
+require 'json'
+require 'uri'
+require 'time'
+
+module Heathrow
+  module Sources
+    # Slack source implementation
+    class Slack
+      def initialize(source)
+        @source = source
+        @base_url = 'https://slack.com/api'
+        
+        # Cache for user and channel names
+        @users_cache = {}
+        @channels_cache = {}
+        @last_fetch = {}
+      end
+      
+      def name
+        'slack'
+      end
+      
+      def test_connection
+        uri = URI("#{@base_url}/auth.test")
+        response = make_api_request(uri)
+        
+        if response && response['ok']
+          puts "Successfully connected to Slack workspace: #{response['team']}"
+          true
+        else
+          error = response ? response['error'] : 'Unknown error'
+          puts "Failed to connect: #{error}"
+          false
+        end
+      rescue => e
+        puts "Connection error: #{e.message}"
+        false
+      end
+      
+      def fetch_messages
+        messages = []
+        
+        channel_ids = @source.config['channel_ids'] || []
+        dm_user_ids = @source.config['dm_user_ids'] || []
+        
+        # Fetch channel messages
+        channel_ids.each do |channel_id|
+          channel_messages = fetch_channel_messages(channel_id)
+          messages.concat(channel_messages)
+        end
+        
+        # Fetch direct messages
+        dm_user_ids.each do |user_id|
+          dm_messages = fetch_direct_messages(user_id)
+          messages.concat(dm_messages)
+        end
+        
+        # Fetch all channels if no specific ones configured
+        if channel_ids.empty? && dm_user_ids.empty?
+          all_channels = fetch_all_conversations
+          all_channels.each do |channel|
+            # Debug logging
+            File.write('/tmp/slack_channels.log', "Channel: #{channel['name']} (#{channel['id']}), is_member: #{channel['is_member']}\n", mode: 'a') if ENV['DEBUG']
+            
+            # Only fetch from channels we're a member of
+            next unless channel['is_member']
+            
+            if channel['is_im']
+              # Direct message
+              dm_messages = fetch_direct_messages(channel['user'] || channel['id'])
+              messages.concat(dm_messages)
+            else
+              # Any other type of channel (public, private, group)
+              channel_messages = fetch_channel_messages(channel['id'])
+              messages.concat(channel_messages)
+            end
+          end
+        end
+        
+        messages
+      end
+      
+      def send_message(recipient, content, metadata = {})
+        channel = resolve_channel(recipient)
+        
+        uri = URI("#{@base_url}/chat.postMessage")
+        params = {
+          channel: channel,
+          text: content,
+          as_user: true
+        }
+        
+        # Handle thread replies
+        if metadata[:thread_ts]
+          params[:thread_ts] = metadata[:thread_ts]
+        end
+        
+        response = make_api_request(uri, params)
+        
+        if response && response['ok']
+          { success: true, message_id: response['ts'] }
+        else
+          error = response ? response['error'] : 'Unknown error'
+          { success: false, error: error }
+        end
+      rescue => e
+        { success: false, error: e.message }
+      end
+      
+      def mark_as_read(message_id, channel_id = nil)
+        return { success: false, error: 'Channel ID required for Slack' } unless channel_id
+        
+        # Determine the correct API endpoint based on channel type
+        channel_type = detect_channel_type(channel_id)
+        endpoint = case channel_type
+                   when 'channel' then 'channels.mark'
+                   when 'group' then 'groups.mark'
+                   when 'im' then 'im.mark'
+                   when 'mpim' then 'mpim.mark'
+                   else 'conversations.mark'
+                   end
+        
+        uri = URI("#{@base_url}/#{endpoint}")
+        params = {
+          channel: channel_id,
+          ts: message_id
+        }
+        
+        response = make_api_request(uri, params)
+        
+        if response && response['ok']
+          { success: true }
+        else
+          error = response ? response['error'] : 'Unknown error'
+          { success: false, error: error }
+        end
+      rescue => e
+        { success: false, error: e.message }
+      end
+      
+      private
+      
+      def make_api_request(uri, params = {}, method = :post)
+        api_token = @source.config['api_token'] || @source.config['token']
+        
+        if method == :get
+          uri.query = URI.encode_www_form(params) unless params.empty?
+          request = Net::HTTP::Get.new(uri)
+        else
+          request = Net::HTTP::Post.new(uri)
+          request['Content-Type'] = 'application/json'
+          request.body = params.to_json unless params.empty?
+        end
+        
+        request['Authorization'] = "Bearer #{api_token}"
+        
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true
+        http.read_timeout = 10  # 10 second timeout
+        http.open_timeout = 5   # 5 second connection timeout
+        
+        response = http.request(request)
+        JSON.parse(response.body) rescue nil
+      end
+      
+      def fetch_channel_messages(channel_id)
+        channel_name = get_channel_name(channel_id)
+        
+        uri = URI("#{@base_url}/conversations.history")
+        params = {
+          channel: channel_id,
+          limit: 50
+        }
+        
+        # Get messages since last fetch if available
+        if @last_fetch[channel_id]
+          params[:oldest] = @last_fetch[channel_id].to_f.to_s
+        end
+        
+        response = make_api_request(uri, params, :get)
+        
+        messages = []
+        if response && response['ok']
+          response['messages'].each do |msg|
+            next if msg['type'] != 'message' || msg['subtype'] == 'bot_message'
+            
+            user_name = get_user_name(msg['user'])
+            
+            messages << {
+              'source_id' => @source.id,
+              'source_type' => 'slack',
+              'external_id' => "slack_#{@source.id}_#{channel_id}_#{msg['ts']}",
+              'sender' => user_name,
+              'recipient' => "##{channel_name}",
+              'subject' => nil,
+              'content' => msg['text'],
+              'timestamp' => Time.at(msg['ts'].to_f).iso8601,
+              'is_read' => 0,
+              'metadata' => {
+                'channel_id' => channel_id,
+                'channel_name' => channel_name,
+                'thread_ts' => msg['thread_ts'],
+                'ts' => msg['ts'],
+                'workspace' => @source.config['workspace']
+              }.to_json,
+              'raw_data' => msg.to_json,
+              'attachments' => nil
+            }
+          end
+          
+          @last_fetch[channel_id] = Time.now
+        end
+        
+        messages
+      rescue => e
+        puts "Error fetching channel messages: #{e.message}"
+        []
+      end
+      
+      def fetch_direct_messages(user_id)
+        # First, open the DM channel
+        uri = URI("#{@base_url}/conversations.open")
+        params = { users: user_id }
+        response = make_api_request(uri, params)
+        
+        return [] unless response && response['ok']
+        
+        channel_id = response['channel']['id']
+        user_name = get_user_name(user_id)
+        
+        # Now fetch the messages
+        uri = URI("#{@base_url}/conversations.history")
+        params = {
+          channel: channel_id,
+          limit: 100
+        }
+        
+        if @last_fetch[channel_id]
+          params[:oldest] = @last_fetch[channel_id].to_f.to_s
+        end
+        
+        response = make_api_request(uri, params)
+        
+        messages = []
+        if response && response['ok']
+          response['messages'].each do |msg|
+            next if msg['type'] != 'message'
+            
+            sender_name = get_user_name(msg['user'])
+            
+            messages << {
+              'source_id' => @source.id,
+              'source_type' => 'slack',
+              'external_id' => "slack_dm_#{@source.id}_#{channel_id}_#{msg['ts']}",
+              'sender' => sender_name,
+              'recipient' => "DM with #{user_name}",
+              'subject' => nil,
+              'content' => msg['text'],
+              'timestamp' => Time.at(msg['ts'].to_f).iso8601,
+              'is_read' => 0,
+              'metadata' => {
+                'channel_id' => channel_id,
+                'user_id' => user_id,
+                'ts' => msg['ts'],
+                'is_dm' => true,
+                'workspace' => @source.config['workspace']
+              }.to_json,
+              'raw_data' => msg.to_json,
+              'attachments' => nil
+            }
+          end
+          
+          @last_fetch[channel_id] = Time.now
+        end
+        
+        messages
+      rescue => e
+        puts "Error fetching DMs: #{e.message}"
+        []
+      end
+      
+      def fetch_all_conversations
+        uri = URI("#{@base_url}/conversations.list")
+        params = {
+          types: 'public_channel,private_channel,mpim,im',
+          limit: 100
+        }
+        
+        response = make_api_request(uri, params, :get)
+        
+        if response && response['ok']
+          response['channels'] || []
+        else
+          []
+        end
+      rescue => e
+        puts "Error fetching conversations: #{e.message}"
+        []
+      end
+      
+      def get_user_name(user_id)
+        return @users_cache[user_id] if @users_cache[user_id]
+        
+        uri = URI("#{@base_url}/users.info")
+        params = { user: user_id }
+        response = make_api_request(uri, params, :get)
+        
+        if response && response['ok']
+          name = response['user']['profile']['display_name'] || 
+                 response['user']['profile']['real_name'] || 
+                 response['user']['name']
+          @users_cache[user_id] = name
+          name
+        else
+          user_id
+        end
+      rescue => e
+        user_id
+      end
+      
+      def get_channel_name(channel_id)
+        return @channels_cache[channel_id] if @channels_cache[channel_id]
+        
+        uri = URI("#{@base_url}/conversations.info")
+        params = { channel: channel_id }
+        response = make_api_request(uri, params, :get)
+        
+        if response && response['ok']
+          name = response['channel']['name']
+          @channels_cache[channel_id] = name
+          name
+        else
+          channel_id
+        end
+      rescue => e
+        channel_id
+      end
+      
+      def detect_channel_type(channel_id)
+        case channel_id[0]
+        when 'C' then 'channel'
+        when 'G' then 'group'
+        when 'D' then 'im'
+        when 'M' then 'mpim'
+        else 'channel'
+        end
+      end
+      
+      def resolve_channel(recipient)
+        # If it's already a channel ID, use it
+        return recipient if recipient =~ /^[CGDM]/
+        
+        # If it starts with #, look up the channel
+        if recipient.start_with?('#')
+          channel_name = recipient[1..-1]
+          # Would need to fetch channel list and find matching name
+          # For now, assume it's provided as ID
+          recipient
+        else
+          # Assume it's a user name for DM
+          # Would need to look up user ID
+          recipient
+        end
+      end
+      
+      def generate_message_id
+        "slack_#{@source.id}_#{Time.now.to_i}_#{rand(1000)}"
+      end
+    end
+  end
+end