feed2email 0.7.0 → 0.8.0

data/lib/feed2email/config.rb

@@ -14,7 +14,12 @@ module Feed2Email
     end
 
     def [](option)
-      merged_config[option] # delegate
+      config[option] # delegate
+    end
+
+    def smtp_configured?
+      config['smtp_host'] && config['smtp_port'] && config['smtp_user'] &&
+        config['smtp_pass']
     end
 
     private
@@ -81,8 +86,8 @@ module Feed2Email
       File.read(path)
     end
 
-    def merged_config
-      @merged_config ||= defaults.merge(data)
+    def config
+      @config ||= defaults.merge(data)
     end
 
     def defaults

data/lib/feed2email/configurable.rb

@@ -0,0 +1,7 @@
+module Feed2Email
+  module Configurable
+    def config
+      Feed2Email.config # delegate
+    end
+  end
+end

data/lib/feed2email/feed.rb

@@ -1,53 +1,24 @@
 require 'feedzirra'
-require 'forwardable'
-require 'net/http'
 require 'open-uri'
 require 'stringio'
-require 'uri'
 require 'zlib'
+require 'feed2email/configurable'
 require 'feed2email/core_ext'
 require 'feed2email/entry'
 require 'feed2email/feed_history'
-require 'feed2email/feed_meta'
-require 'feed2email/feeds'
+require 'feed2email/loggable'
+require 'feed2email/redirection_checker'
 require 'feed2email/version'
 
 module Feed2Email
   class Feed
-    extend Forwardable
+    include Configurable
+    include Loggable
 
-    def self.feed_uris; @feed_uris end
+    attr_reader :meta
 
-    def self.logger
-      Feed2Email.logger # delegate
-    end
-
-    def self.smtp_connection
-      Feed2Email.smtp_connection # delegate
-    end
-
-    logger.debug 'Loading feed subscriptions...'
-    @feed_uris = Feeds.new(File.join(CONFIG_DIR, 'feeds.yml'))
-    logger.info "Subscribed to #{'feed'.pluralize(feed_uris.size)}"
-
-    def self.process_all
-      begin
-        feed_uris.each_with_index do |uri, i|
-          feed = new(uri)
-          feed.process
-          feed_uris[i] = feed.uri # persist possible permanent redirect
-        end
-      ensure
-        smtp_connection.finalize
-      end
-
-      feed_uris.sync
-    end
-
-    attr_reader :uri
-
-    def initialize(uri)
-      @uri = uri
+    def initialize(meta)
+      @meta = meta
     end
 
     def process
@@ -56,23 +27,39 @@ module Feed2Email
       return unless fetch_and_parse_feed
 
       if entries.any?
-        process_entries
+        processed = process_entries
         history.sync
-        meta.sync
       else
+        processed = true
         logger.warn 'Feed does not have entries'
       end
+
+      processed
     end
 
     private
 
+    def apply_send_delay
+      return if config['send_delay'] == 0
+
+      return if last_email_sent_at.nil?
+
+      secs_since_last_email = Time.now - last_email_sent_at
+      secs_to_sleep = config['send_delay'] - secs_since_last_email
+
+      return if secs_to_sleep <= 0
+
+      logger.debug "Sleeping for #{secs_to_sleep} seconds..."
+      sleep(secs_to_sleep)
+    end
+
     def fetch_feed
       logger.debug 'Fetching feed...'
 
       begin
-        handle_permanent_redirection
+        cache_feed = !permanently_redirected?
 
-        open(uri, fetch_feed_options) do |f|
+        open(uri, fetch_feed_options(cache_feed)) do |f|
           if f.meta['last-modified'] || meta.has_key?(:last_modified)
             meta[:last_modified] = f.meta['last-modified']
           end
@@ -83,31 +70,28 @@ module Feed2Email
 
           return decode_content(f.read, f.meta['content-encoding'])
         end
-      rescue OpenURI::HTTPError => e
-        if e.message == '304 Not Modified'
+      rescue => e
+        if e.is_a?(OpenURI::HTTPError) && e.message == '304 Not Modified'
           logger.info 'Feed not modified; skipping...'
-          return false
+        else
+          logger.error 'Failed to fetch feed'
+          log_exception(e)
         end
 
-        raise
-      rescue => e
-        logger.error 'Failed to fetch feed'
-        log_exception(e)
         return false
       end
     end
 
-    def handle_permanent_redirection
-      parsed_uri = URI.parse(uri)
-      http = Net::HTTP.new(parsed_uri.host, parsed_uri.port)
-      http.use_ssl = (parsed_uri.scheme == 'https')
-      response = http.head(parsed_uri.request_uri)
+    def permanently_redirected?
+      checker = RedirectionChecker.new(uri)
 
-      if response.code == '301' && response['location'] =~ %r{\Ahttps?://}
-        self.uri = response['location']
-        logger.warn(
-          "Got permanently redirected! Updated feed location to #{uri}")
-      end
+      return false unless checker.permanently_redirected?
+
+      self.uri = checker.location
+      logger.warn 'Got permanently redirected!'
+      logger.warn "Updated feed location to #{checker.location}"
+
+      true
     end
 
     def decode_content(data, content_encoding)
@@ -125,18 +109,20 @@ module Feed2Email
       xml
     end
 
-    def fetch_feed_options
+    def fetch_feed_options(cache_feed)
       options = {
         'User-Agent' => "feed2email/#{VERSION}",
         'Accept-Encoding' => 'gzip, deflate',
       }
 
-      if meta[:last_modified]
-        options['If-Modified-Since'] = meta[:last_modified]
-      end
+      if cache_feed
+        if meta[:last_modified]
+          options['If-Modified-Since'] = meta[:last_modified]
+        end
 
-      if meta[:etag]
-        options['If-None-Match'] = meta[:etag]
+        if meta[:etag]
+          options['If-None-Match'] = meta[:etag]
+        end
       end
 
       options
@@ -162,10 +148,13 @@ module Feed2Email
       @data && @data.respond_to?(:entries)
     end
 
+    def uri
+      meta[:uri]
+    end
+
     def uri=(uri)
       history.uri = uri
-      meta.uri = uri
-      @uri = uri
+      meta[:uri] = uri
     end
 
     def entries
@@ -174,59 +163,58 @@ module Feed2Email
       }
     end
 
-    def logger
-      Feed2Email.logger # delegate
-    end
-
     def max_entries
       config['max_entries'].to_i
     end
 
     def process_entries
       logger.info "Processing #{'entry'.pluralize(entries.size, 'entries')}..."
-      entries.each {|entry| process_entry(entry) }
+      entries.all? {|e| process_entry(e) } # false if any entry fails
     end
 
     def process_entry(entry)
       logger.info "Processing entry #{entry.uri} ..."
 
-      if history.any?
-        if history.include?(entry.uri)
-          logger.debug 'Skipping old entry...'
-        else
-          # Sleep between entry processing to avoid Net::SMTPServerBusy errors
-          if config['send_delay'] > 0
-            logger.debug(
-              "Sleeping for #{'second'.pluralize(config['send_delay'])}")
-            sleep(config['send_delay'])
-          end
+      unless history.any?
+        logger.debug 'Skipping new feed entry...'
+        history << entry.uri
+        return true
+      end
 
-          logger.debug 'Sending new entry...'
+      if history.include?(entry.uri)
+        logger.debug 'Skipping old entry...'
+        return true
+      end
 
-          begin
-            entry.send_mail
-          rescue => e
-            log_exception(e)
-          end
+      apply_send_delay
 
-          if e.nil? # no errors
-            history << entry.uri
-          end
+      logger.debug 'Sending new entry...'
 
-          e = nil
-        end
-      else
-        logger.debug 'Skipping new feed entry...'
+      begin
+        mail_sent = entry.send_mail
+      rescue => e
+        log_exception(e)
+        return false
+      end
+
+      if mail_sent
+        self.last_email_sent_at = Time.now
         history << entry.uri
       end
+
+      mail_sent
     end
 
     def history
       @history ||= FeedHistory.new(uri)
     end
 
-    def meta
-      @meta ||= FeedMeta.new(uri)
+    def last_email_sent_at
+      @last_email_sent_at
+    end
+
+    def last_email_sent_at=(time)
+      @last_email_sent_at = time
     end
 
     def log_exception(error)
@@ -234,9 +222,9 @@ module Feed2Email
       error.backtrace.each {|line| logger.debug line }
     end
 
-    def_delegator :data, :title, :title
-
-    def_delegator :Feed2Email, :config, :config
+    def title
+      data.title # delegate
+    end
 
     def data; @data end
   end

data/lib/feed2email/feed_autodiscoverer.rb

@@ -0,0 +1,55 @@
+require 'nokogiri'
+require 'open-uri'
+require 'uri'
+
+module Feed2Email
+  class FeedAutodiscoverer
+    def initialize(uri)
+      @uri = uri
+    end
+
+    def content_type; @content_type end
+
+    def feeds
+      return @feeds if @feeds
+      fetch
+      @feeds = discoverable? ? discover : []
+    end
+
+    private
+
+    def data; @data end
+
+    def discover
+      head = Nokogiri::HTML(data).at_css('head')
+
+      if base = head.at_css('base[href]')
+        base_uri = base['href']
+      else
+        base_uri = uri
+      end
+
+      head.css('link[rel=alternate]').select {|link|
+        link['href'] && link['type'] =~ /\Aapplication\/(rss|atom)\+xml\z/
+      }.map do |link|
+        if link['href'] =~ %r{\Ahttps?://} # absolute
+          uri = link['href']
+        else
+          uri = URI.join(base_uri, link['href']).to_s # relative
+        end
+
+        { uri: uri, content_type: link['type'], title: link['title'] }
+      end
+    end
+
+    def discoverable?
+      content_type == 'text/html'
+    end
+
+    def fetch
+      @data, @content_type = open(uri) {|f| [f.read, f.content_type] }
+    end
+
+    def uri; @uri end
+  end
+end

data/lib/feed2email/feed_history.rb

@@ -1,28 +1,82 @@
-require 'feed2email/feed_data_file'
+require 'digest/md5'
+require 'yaml'
 
 module Feed2Email
-  class FeedHistory < FeedDataFile
-    def <<(entry_uri)
-      mark_dirty
-      data << entry_uri
+  class FeedHistory
+    def initialize(uri)
+      @uri = uri
+      @dirty = false
     end
 
     def any?
       @old_feed ||= File.exist?(path)
     end
 
+    def path
+      File.join(CONFIG_DIR, filename)
+    end
+
     def include?(entry_uri)
       data.include?(entry_uri) # delegate
     end
 
+    def sync
+      open(path, 'w') {|f| f.write(to_yaml) } if dirty
+    end
+
+    def uri=(new_uri)
+      return if new_uri == uri
+
+      data # load data if not already loaded
+      remove_file
+      mark_dirty
+      @uri = new_uri
+    end
+
+    def <<(entry_uri)
+      mark_dirty
+      data << entry_uri
+    end
+
     private
 
-    def data_type
-      Array
+    def data
+      @data ||= load_data
+    end
+
+    def dirty; @dirty end
+
+    def filename
+      "history-#{filename_suffix}.yml"
+    end
+
+    def filename_suffix
+      Digest::MD5.hexdigest(uri)
     end
 
-    def filename_prefix
-      'history'
+    def load_data
+      begin
+        @data = YAML.load(open(path))
+      rescue Errno::ENOENT
+        @data = []
+      end
     end
+
+    def mark_dirty
+      @dirty = true
+    end
+
+    def remove_file
+      begin
+        File.unlink(path)
+      rescue Errno::ENOENT
+      end
+    end
+
+    def to_yaml
+      data.to_yaml # delegate
+    end
+
+    def uri; @uri end
   end
 end