feed2email 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bf67c31096e64a3b8e790b968fbd7fecd5384c78
-  data.tar.gz: 0a7567b5e3b8aa17aa9977d9a5485fa5d6fec138
+  metadata.gz: d2edf473e3d4740e4593c3c15cf7e2bb3c106815
+  data.tar.gz: 063706f3316c3f7610fa9b361adeca02dc2ecfe3
 SHA512:
-  metadata.gz: e187b0d14d5687d410b9860b6ab5cbeed2fe5d38764d95da23f460a198f607547cc8b35e5faae8b65a74c2d0add7259ab37cfec13607ff9c3de0643d0264eda6
-  data.tar.gz: 4d735364a8a9f84d6a66de1c3a7f622056517ec72dd04f648c2a9fad9966ea188f44dea1c5222f9b1c3ff7f64ae64f81f66970e5b4733272ec8b5e0c10f65907
+  metadata.gz: a2780f0f6836d9071e8021bbe206c3c369a0a516d6c38f5678448dea229d00a332e206b4d8ce734569d5b933ed518b80edcb5ea2c9136e127ba33bd1fde75b5a
+  data.tar.gz: bcc5b42264fccb8ff5e54479120c6c50d67f188ddd343b1d894bb7b810e923e9b3c1f6a4693875ee6cae387250d465f0965f4d2be5ed5cb4e5e1b7c6c662fe92

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+### 0.4.0
+
+* Major rewrite to keep history of processed (seen) entries
+* Handle feed fetching/parsing errors
+* Limit the number of entries to process per feed
+* Fix prepending of feed URI to path entry permalinks
+
 ### 0.3.0
 
 * Add logging

data/Gemfile.lock

@@ -10,7 +10,7 @@ GEM
   specs:
     activesupport (3.1.12)
       multi_json (~> 1.0)
-    builder (3.2.0)
+    builder (3.2.2)
     curb (0.7.18)
     feedzirra (0.1.3)
       activesupport (~> 3.1.1)
@@ -22,24 +22,25 @@ GEM
       rake (>= 0.8.7)
       rdoc (~> 3.8)
       sax-machine (~> 0.1.0)
-    i18n (0.6.4)
-    json (1.7.7)
+    i18n (0.6.5)
+    json (1.8.0)
     loofah (1.2.1)
       nokogiri (>= 1.4.4)
-    mail (2.5.3)
-      i18n (>= 0.4.0)
+    mail (2.5.4)
       mime-types (~> 1.16)
       treetop (~> 1.4.8)
-    mime-types (1.22)
-    multi_json (1.7.2)
-    nokogiri (1.5.9)
+    mime-types (1.24)
+    mini_portile (0.5.1)
+    multi_json (1.7.9)
+    nokogiri (1.6.0)
+      mini_portile (~> 0.5.0)
     polyglot (0.3.3)
-    rake (10.0.4)
+    rake (10.1.0)
     rdoc (3.12.2)
       json (~> 1.4)
     sax-machine (0.1.0)
       nokogiri (> 0.0.0)
-    treetop (1.4.12)
+    treetop (1.4.15)
       polyglot
       polyglot (>= 0.3.1)
 

data/README.md

@@ -45,6 +45,8 @@ pair is separated with a colon: `foo: bar`
   avoid SMTP server throttling errors (default is `10`; use `0` to disable)
 * `log_path` (optional) is the _absolute_ path to the log file (default is
   `true` which logs to standard output; use `false` to disable)
+* `max_entries` (optional) is the maximum number of entries to process per feed
+  (default is `20`; use `0` for unlimited)
 
 ### SMTP
 
@@ -99,13 +101,12 @@ immediately. During dry run mode:
 
 * No feeds are fetched and, thus, no email is sent (existing feed entries are
   considered already seen)
-* `~/.feed2email/state.yml` is created containing the timestamp of when each
-  feed was last fetched
+* `~/.feed2email/history.yml` is created containing processed (seen) entries per
+  feed
 
 If you want to receive existing entries from a specific feed, you can manually
-alter the timestamp for that feed in `state.yml` to a value in the past. Next
-time you run feed2email, all entries published past that timestamp will be sent
-with email.
+delete them from `history.yml`. Next time feed2email runs, they will be
+processed (sent as email).
 
 You can use [cron][] to run feed2email automatically e.g. once every hour.
 

data/lib/feed2email/entry.rb

@@ -1,14 +1,9 @@
 module Feed2Email
   class Entry
-    attr_reader :feed
-
-    def self.process(data, feed)
-      Entry.new(data, feed).process
-    end
-
-    def initialize(data, feed)
+    def initialize(data, feed_uri, feed_title)
       @data = data
-      @feed = feed
+      @feed_uri = feed_uri
+      @feed_title = feed_title
     end
 
     def author
@@ -20,14 +15,7 @@ module Feed2Email
     end
 
     def process
-      log :debug, "Processing entry #{uri} ..."
-
-      if send?
-        log :debug, 'Sending email...'
-        to_mail.send
-      else
-        log :debug, 'Entry should not be sent; skipping...'
-      end
+      Mail.new(self, @feed_title).send
     end
 
     def title
@@ -36,49 +24,12 @@ module Feed2Email
 
     def uri
       @uri ||= begin
-        if @data.url[0] == '/'
-          @feed.uri.chomp('/') + @data.url
+        if @data.url[0] == '/' # invalid entry URL is a path
+          @feed_uri[%r{https?://[^/]+}] + @data.url # prepend feed URI
         else
           @data.url
         end
       end
     end
-
-    private
-
-    def log(*args)
-      Feed2Email::Logger.instance.log(*args)
-    end
-
-    def published_at
-      @data.published
-    end
-
-    def send?
-      if published_at
-        log :debug, 'Entry has publication timestamp'
-
-        if published_at.past? # respect entries published in the future
-          log :debug, 'Entry published in the past'
-
-          if published_at > @feed.fetch_time
-            log :debug, 'Entry not seen before'
-            return true
-          else
-            log :debug, 'Entry seen before'
-          end
-        else
-          log :warn, "Entry #{uri} published in the future"
-        end
-      else
-        log :warn, "Entry #{uri} does not have publication timestamp"
-      end
-
-      false
-    end
-
-    def to_mail
-      Mail.new(self)
-    end
   end
 end

data/lib/feed2email/feed.rb

@@ -1,20 +1,12 @@
 module Feed2Email
   class Feed
     FEEDS_FILE = File.join(CONFIG_DIR, 'feeds.yml')
-    STATE_FILE = File.join(CONFIG_DIR, 'state.yml')
+    HISTORY_FILE = File.join(CONFIG_DIR, 'history.yml')
 
     def self.log(*args)
       Feed2Email::Logger.instance.log(*args)
     end
 
-    def self.pluralize(n, singular, plural)
-      "#{n} #{n == 1 ? singular : plural}"
-    end
-
-    def self.process(uri)
-      Feed.new(uri).process
-    end
-
     def self.process_all
       Feed2Email::Config.instance.read!
 
@@ -26,111 +18,121 @@ module Feed2Email
         exit 4
       end
 
-      log :info, "Subscribed to #{pluralize(feed_uris.size, 'feed', 'feeds')}"
+      log :info, "Subscribed to #{n = feed_uris.size} feed#{n == 1 ? '' : 's'}"
 
-      log :debug, 'Loading fetch times...'
-      @@fetch_times = YAML.load(open(STATE_FILE)) rescue {}
+      log :debug, 'Loading history...'
+      @@history = YAML.load(open(HISTORY_FILE)) rescue {}
 
-      feed_uris.each {|uri| Feed.process(uri) }
+      feed_uris.each do |uri|
+        log :info, "Found feed #{uri}"
+        Feed.new(uri).process
+      end
 
-      log :debug, 'Writing fetch times...'
-      open(STATE_FILE, 'w') {|f| f.write(@@fetch_times.to_yaml) }
+      log :debug, 'Writing history...'
+      open(HISTORY_FILE, 'w') {|f| f.write(@@history.to_yaml) }
     end
 
-    attr_reader :uri
-
     def initialize(uri)
       @uri = uri
     end
 
-    def fetch_time
-      @@fetch_times[@uri]
-    end
-
-    def pluralize(*args)
-      Feed2Email::Feed.pluralize(*args) # delegate
-    end
-
     def process
-      log :info, "Processing feed #{@uri} ..."
-
-      if seen_before?
-        log :debug, 'Feed seen before'
-
-        if fetched?
-          log :debug, 'Feed is fetched'
-
-          if have_entries?
-            log :info, "Processing #{pluralize(entries.size, 'entry', 'entries')}..."
+      if fetched?
+        log :debug, 'Feed is fetched'
 
-            begin
-              process_entries
-            rescue => e
-              log :error, "#{e.class}: #{e.message.strip}"
-            end
-          else
-            log :warn, 'Feed does not have entries'
-          end
+        if entries.any?
+          log :info,
+            "Processing #{n = entries.size} entr#{n == 1 ? 'y' : 'ies'}..."
+          process_entries
         else
-          log :error, 'Feed could not be fetched'
+          log :warn, 'Feed does not have entries'
         end
       else
-        log :info, 'Feed not seen before; skipping...'
+        log :error, 'Feed could not be fetched'
       end
-
-      if e.nil? && (!seen_before? || fetched?)
-        log :debug, 'Syncing fetch time...'
-        sync_fetch_time
-      end
-    end
-
-    def title
-      data.title
     end
 
     private
 
+    def config
+      Feed2Email::Config.instance.config
+    end
+
     def data
       if @data.nil?
         log :debug, 'Fetching and parsing feed...'
-        @data = Feedzirra::Feed.fetch_and_parse(@uri,
-          :user_agent => "feed2email/#{VERSION}",
-          :compress   => true
-        )
-        @fetched_at = Time.now
+
+        begin
+          @data = Feedzirra::Feed.fetch_and_parse(@uri,
+            :user_agent => "feed2email/#{VERSION}",
+            :compress   => true
+          )
+        rescue => e
+          log :error, "#{e.class}: #{e.message.strip}"
+        end
       end
 
       @data
     end
 
     def entries
-      data.entries
+      @entries ||= data.entries[0..max_entries - 1].map {|entry_data|
+        Entry.new(entry_data, @uri, title)
+      }
     end
 
     def fetched?
       data.respond_to?(:entries)
     end
 
-    def have_entries?
-      entries.any?
-    end
-
     def log(*args)
       Feed2Email::Feed.log(*args) # delegate
     end
 
+    def max_entries
+      (config['max_entries'] || 20).to_i
+    end
+
     def process_entries
-      entries.each do |entry_data|
-        Entry.process(entry_data, self)
+      entries.each do |entry|
+        log :info, "Found entry #{entry.uri}"
+
+        if seen_before?
+          if seen_entries.include?(entry.uri)
+            log :debug, 'Skipping seen entry...'
+          else
+            log :debug, 'Processing new entry...'
+
+            begin
+              entry.process
+            rescue => e
+              log :error, "#{e.class}: #{e.message.strip}"
+            end
+
+            seen_entries << entry.uri if e.nil? # record in history if no errors
+            e = nil
+          end
+        else
+          log :debug, 'Skipping new entry...'
+          seen_entries << entry.uri # record in history
+        end
       end
     end
 
     def seen_before?
-      fetch_time.is_a?(Time)
+      if @seen_before.nil?
+        @seen_before = !@@history[@uri].nil?
+      end
+
+      @seen_before
+    end
+
+    def seen_entries
+      @@history[@uri] ||= []
     end
 
-    def sync_fetch_time
-      @@fetch_times[@uri] = @fetched_at || Time.now
+    def title
+      data.title
     end
   end
 end

data/lib/feed2email/logger.rb

@@ -24,7 +24,7 @@ module Feed2Email
       if log_path.nil? || log_path == true
         STDOUT
       else
-        log_path
+        File.expand_path(log_path)
       end
     end
 

data/lib/feed2email/mail.rb

@@ -1,7 +1,8 @@
 module Feed2Email
   class Mail
-    def initialize(entry)
+    def initialize(entry, feed_title)
       @entry = entry
+      @feed_title = feed_title
     end
 
     def send
@@ -42,7 +43,7 @@ module Feed2Email
 
     def from
       from_data = {
-        :name  => @entry.feed.title,
+        :name  => @feed_title,
         :email => from_address,
       }
       '"%{name}" <%{email}>' % from_data

data/lib/feed2email/version.rb

@@ -1,3 +1,3 @@
 module Feed2Email
-  VERSION = '0.3.0'
+  VERSION = '0.4.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: feed2email
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Aggelos Orfanakos
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-25 00:00:00.000000000 Z
+date: 2013-09-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: feedzirra