feed2email 0.5.0 → 0.6.0

data/lib/feed2email/feed.rb

@@ -1,89 +1,173 @@
+require 'feedzirra'
+require 'forwardable'
+require 'net/http'
+require 'open-uri'
+require 'stringio'
+require 'uri'
+require 'zlib'
+require 'feed2email/core_ext'
+require 'feed2email/entry'
+require 'feed2email/feed_history'
+require 'feed2email/feed_meta'
+require 'feed2email/feeds'
+require 'feed2email/version'
+
 module Feed2Email
   class Feed
-    FEEDS_FILE = File.join(CONFIG_DIR, 'feeds.yml')
-    HISTORY_FILE = File.join(CONFIG_DIR, 'history.yml')
+    extend Forwardable
+
+    class << self
+      extend Forwardable
 
-    def self.log(*args)
-      Feed2Email::Logger.instance.log(*args)
+      def_delegators :Feed2Email, :config, :log
     end
 
     def self.process_all
-      Feed2Email::Config.instance.read!
-
-      log :debug, 'Loading feed subscriptions...'
-      feed_uris = YAML.load(open(FEEDS_FILE)) rescue nil
-
-      if !feed_uris.is_a? Array
-        $stderr.puts "Error: missing or invalid feeds file #{FEEDS_FILE}"
-        exit 4
+      feed_uris.each_with_index do |uri, i|
+        feed = new(uri)
+        feed.process
+        feed_uris[i] = feed.uri # persist possible permanent redirect
       end
 
-      log :info, "Subscribed to #{n = feed_uris.size} feed#{n == 1 ? '' : 's'}"
-
-      log :debug, 'Loading history...'
-      @@history = YAML.load(open(HISTORY_FILE)) rescue {}
+      feed_uris.sync
+    end
 
-      feed_uris.each do |uri|
-        log :info, "Found feed #{uri}"
-        Feed.new(uri).process
-      end
+    def self.feed_uris
+      return @feed_uris if @feed_uris
 
-      log :debug, 'Writing history...'
-      open(HISTORY_FILE, 'w') {|f| f.write(@@history.to_yaml) }
+      log :debug, 'Loading feed subscriptions...'
+      @feed_uris = Feeds.new(File.join(CONFIG_DIR, 'feeds.yml'))
+      log :info, "Subscribed to #{'feed'.pluralize(feed_uris.size)}"
+      @feed_uris
     end
 
+    attr_reader :uri
+
     def initialize(uri)
       @uri = uri
     end
 
     def process
-      if fetched?
-        log :debug, 'Feed is fetched'
+      log :info, "Processing feed #{uri} ..."
 
-        if entries.any?
-          log :info,
-            "Processing #{n = entries.size} entr#{n == 1 ? 'y' : 'ies'}..."
-          process_entries
-        else
-          log :warn, 'Feed does not have entries'
-        end
+      return unless fetch_and_parse_feed
+
+      if entries.any?
+        process_entries
+        history.sync
+        meta.sync
       else
-        log :error, 'Feed could not be fetched'
+        log :warn, 'Feed does not have entries'
       end
     end
 
     private
 
-    def config
-      Feed2Email::Config.instance.config
-    end
+    def fetch_feed
+      log :debug, 'Fetching feed...'
+
+      begin
+        handle_permanent_redirection
+
+        open(uri, fetch_feed_options) do |f|
+          if f.meta['last-modified'] || meta.has_key?(:last_modified)
+            meta[:last_modified] = f.meta['last-modified']
+          end
 
-    def data
-      if @data.nil?
-        log :debug, 'Fetching and parsing feed...'
+          if f.meta['etag'] || meta.has_key?(:etag)
+            meta[:etag] = f.meta['etag']
+          end
 
-        begin
-          @data = Feedzirra::Feed.fetch_and_parse(@uri,
-            :user_agent => "feed2email/#{VERSION}",
-            :compress   => true
-          )
-        rescue => e
-          log :error, "#{e.class}: #{e.message.strip}"
-          e.backtrace.each {|line| log :debug, line }
+          return decode_content(f.read, f.meta['content-encoding'])
         end
+      rescue OpenURI::HTTPError => e
+        if e.message == '304 Not Modified'
+          log :info, 'Feed not modified; skipping...'
+          return false
+        end
+
+        raise
+      rescue => e
+        log :error, 'Failed to fetch feed'
+        log_exception(e)
+        return false
       end
+    end
+
+    def handle_permanent_redirection
+      parsed_uri = URI.parse(uri)
+      http = Net::HTTP.new(parsed_uri.host, parsed_uri.port)
+      http.use_ssl = (parsed_uri.scheme == 'https')
+      response = http.head(parsed_uri.request_uri)
 
-      @data
+      if response.code == '301' && response['location'] =~ %r{\Ahttps?://}
+        self.uri = response['location']
+        log :warn, "Got permanently redirected! Updated feed location to #{uri}"
+      end
     end
 
-    def entries
-      @entries ||= data.entries[0..max_entries - 1].map {|entry_data|
-        Entry.new(entry_data, @uri, title)
+    def decode_content(data, content_encoding)
+      case content_encoding
+      when 'gzip'
+        gz = Zlib::GzipReader.new(StringIO.new(data))
+        xml = gz.read
+        gz.close
+      when 'deflate'
+        xml = Zlib::Inflate.inflate(data)
+      else
+        xml = data
+      end
+
+      xml
+    end
+
+    def fetch_feed_options
+      options = {
+        'User-Agent' => "feed2email/#{VERSION}",
+        'Accept-Encoding' => 'gzip, deflate',
       }
+
+      if meta[:last_modified]
+        options['If-Modified-Since'] = meta[:last_modified]
+      end
+
+      if meta[:etag]
+        options['If-None-Match'] = meta[:etag]
+      end
+
+      options
+    end
+
+    def parse_feed(xml_data)
+      log :debug, 'Parsing feed...'
+
+      begin
+        Feedzirra::Feed.parse(xml_data)
+      rescue => e
+        log :error, 'Failed to parse feed'
+        log_exception(e)
+        return false
+      end
+    end
+
+    def fetch_and_parse_feed
+      if xml_data = fetch_feed
+        @data = parse_feed(xml_data)
+      end
+
+      @data && @data.respond_to?(:entries)
+    end
+
+    def uri=(uri)
+      history.uri = uri
+      meta.uri = uri
+      @uri = uri
     end
 
-    def fetched?
-      data.respond_to?(:entries)
+    def entries
+      @entries ||= data.entries.first(max_entries).map {|entry_data|
+        Entry.new(entry_data, uri, title)
+      }
     end
 
     def log(*args)
@@ -91,50 +175,65 @@ module Feed2Email
     end
 
     def max_entries
-      (config['max_entries'] || 20).to_i
+      config['max_entries'].to_i
     end
 
     def process_entries
-      entries.each do |entry|
-        log :info, "Found entry #{entry.uri}"
-
-        if seen_before?
-          if seen_entries.include?(entry.uri)
-            log :debug, 'Skipping seen entry...'
-          else
-            log :debug, 'Processing new entry...'
-
-            begin
-              entry.process
-            rescue => e
-              log :error, "#{e.class}: #{e.message.strip}"
-              e.backtrace.each {|line| log :debug, line }
-            end
-
-            seen_entries << entry.uri if e.nil? # record in history if no errors
-            e = nil
-          end
+      log :info, "Processing #{'entry'.pluralize(entries.size, 'entries')}..."
+      entries.each {|entry| process_entry(entry) }
+    end
+
+    def process_entry(entry)
+      log :info, "Processing entry #{entry.uri} ..."
+
+      if history.any?
+        if history.include?(entry.uri)
+          log :debug, 'Skipping old entry...'
         else
-          log :debug, 'Skipping new entry...'
-          seen_entries << entry.uri # record in history
+          # Sleep between entry processing to avoid Net::SMTPServerBusy errors
+          if config['send_delay'] > 0
+            log :debug,
+              "Sleeping for #{'second'.pluralize(config['send_delay'])}"
+            sleep(config['send_delay'])
+          end
+
+          log :debug, 'Sending new entry...'
+
+          begin
+            entry.send_mail
+          rescue => e
+            log_exception(e)
+          end
+
+          if e.nil? # no errors
+            history << entry.uri
+          end
+
+          e = nil
         end
+      else
+        log :debug, 'Skipping new feed entry...'
+        history << entry.uri
       end
     end
 
-    def seen_before?
-      if @seen_before.nil?
-        @seen_before = !@@history[@uri].nil?
-      end
-
-      @seen_before
+    def history
+      @history ||= FeedHistory.new(uri)
     end
 
-    def seen_entries
-      @@history[@uri] ||= []
+    def meta
+      @meta ||= FeedMeta.new(uri)
     end
 
-    def title
-      data.title
+    def log_exception(error)
+      log :error, "#{error.class}: #{error.message.strip}"
+      error.backtrace.each {|line| log :debug, line }
     end
+
+    def_delegator :data, :title, :title
+
+    def_delegator :Feed2Email, :config, :config
+
+    def data; @data end
   end
 end

data/lib/feed2email/feed_data_file.rb

@@ -0,0 +1,65 @@
+require 'digest/md5'
+require 'yaml'
+
+module Feed2Email
+  class FeedDataFile
+    def initialize(uri)
+      @uri = uri
+      @dirty = false
+    end
+
+    def uri=(new_uri)
+      return if new_uri == uri
+
+      data # load data if not already loaded
+      remove_file
+      mark_dirty
+      @uri = new_uri
+    end
+
+    def sync
+      open(path, 'w') {|f| f.write(data.to_yaml) } if dirty
+    end
+
+    private
+
+    def load_data
+      begin
+        @data = YAML.load(open(path))
+      rescue Errno::ENOENT
+        @data = data_type.new
+      end
+    end
+
+    def path
+      File.join(CONFIG_DIR, filename)
+    end
+
+    def filename
+      "#{filename_prefix}-#{filename_suffix}.yml"
+    end
+
+    def filename_suffix
+      Digest::MD5.hexdigest(uri)
+    end
+
+    def data
+      @data ||= load_data
+    end
+
+    def mark_dirty
+      @dirty = true
+    end
+
+    def remove_file
+      begin
+        File.unlink(path)
+      rescue Errno::ENOENT
+      end
+    end
+
+    def dirty; @dirty end
+
+    def uri; @uri end
+  end
+end

data/lib/feed2email/feed_history.rb

@@ -0,0 +1,28 @@
+require 'feed2email/feed_data_file'
+
+module Feed2Email
+  class FeedHistory < FeedDataFile
+    def <<(entry_uri)
+      mark_dirty
+      data << entry_uri
+    end
+
+    def any?
+      @old_feed ||= File.exist?(path)
+    end
+
+    def include?(entry_uri)
+      data.include?(entry_uri) # delegate
+    end
+
+    private
+
+    def data_type
+      Array
+    end
+
+    def filename_prefix
+      'history'
+    end
+  end
+end

data/lib/feed2email/feed_meta.rb

@@ -0,0 +1,28 @@
+require 'feed2email/feed_data_file'
+
+module Feed2Email
+  class FeedMeta < FeedDataFile
+    def [](key)
+      data[key]
+    end
+
+    def []=(key, value)
+      mark_dirty if data[key] != value
+      data[key] = value
+    end
+
+    def has_key?(key)
+      data.has_key?(key)
+    end
+
+    private
+
+    def data_type
+      Hash
+    end
+
+    def filename_prefix
+      'meta'
+    end
+  end
+end

data/lib/feed2email/feeds.rb

@@ -0,0 +1,77 @@
+require 'forwardable'
+require 'yaml'
+
+module Feed2Email
+  class Feeds
+    extend Forwardable
+
+    class MissingFeedsError < StandardError; end
+    class InvalidFeedsSyntaxError < StandardError; end
+    class InvalidFeedsDataTypeError < StandardError; end
+
+    def initialize(path)
+      @path = path
+      @dirty = false
+      check
+    end
+
+    def_delegators :data, :size, :each_with_index
+
+    def []=(index, uri)
+      mark_dirty if data[index] != uri
+      data[index] = uri
+    end
+
+    def sync
+      open(path, 'w') {|f| f.write(data.to_yaml) } if dirty
+    end
+
+    private
+
+    def check
+      check_existence
+      check_syntax
+      check_data_type
+    end
+
+    def check_existence
+      if !File.exist?(path)
+        raise MissingFeedsError, "Missing feeds file #{path}"
+      end
+    end
+
+    def check_syntax
+      begin
+        load_yaml
+      rescue Psych::SyntaxError
+        raise InvalidFeedsSyntaxError,
+          "Invalid YAML syntax for feeds file #{path}"
+      end
+    end
+
+    def check_data_type
+      if !data.is_a?(Array)
+        raise InvalidFeedsDataTypeError,
+          "Invalid data type (not an Array) for feeds file #{path}"
+      end
+    end
+
+    def load_yaml
+      @data = YAML.load(read_file)
+    end
+
+    def read_file
+      File.read(path)
+    end
+
+    def mark_dirty
+      @dirty = true
+    end
+
+    def path; @path end
+
+    def data; @data end
+
+    def dirty; @dirty end
+  end
+end

data/lib/feed2email/logger.rb

@@ -1,41 +1,42 @@
+require 'logger'
+
 module Feed2Email
   class Logger
-    include Singleton
+    def initialize(log_path, log_level)
+      @log_path = log_path
+      @log_level = log_level
+    end
 
     def log(severity, message)
-      logger.add(::Logger.const_get(severity.upcase), message) if log?
+      logger.add(::Logger.const_get(severity.upcase), message)
     end
 
     private
 
-    def config
-      Feed2Email::Config.instance.config
-    end
-
-    def log?
-      log_path != false
+    def log_to
+      if log_path == true
+        $stdout
+      elsif log_path # truthy but not true (a path)
+        File.expand_path(log_path)
+      end
     end
 
-    def log_level
-      config['log_level'] || 'info'
-    end
+    def logger
+      return @logger if @logger
 
-    def log_path
-      config['log_path']
-    end
+      @logger = ::Logger.new(log_to)
 
-    def log_to
-      if log_path.nil? || log_path == true
-        STDOUT
+      if log_level
+        @logger.level = ::Logger.const_get(log_level.upcase)
       else
-        File.expand_path(log_path)
+        @logger.level = ::Logger::INFO
       end
-    end
 
-    def logger
-      @logger ||= ::Logger.new(log_to)
-      @logger.level = ::Logger.const_get(log_level.upcase)
       @logger
     end
+
+    def log_path; @log_path end
+
+    def log_level; @log_level end
   end
 end