RubyGems - feed2email - Versions diffs - 0.8.0 → 0.9.0 - Mend

feed2email 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (32) hide show

checksums.yaml +4 -4
data/CHANGELOG.md +21 -10
data/README.md +138 -86
data/bin/f2e +6 -4
data/bin/feed2email +6 -4
data/bin/feed2email-migrate +23 -0
data/lib/feed2email.rb +6 -9
data/lib/feed2email/cli.rb +112 -75
data/lib/feed2email/config.rb +72 -52
data/lib/feed2email/core_ext.rb +10 -0
data/lib/feed2email/database.rb +75 -0
data/lib/feed2email/entry.rb +149 -15
data/lib/feed2email/feed.rb +111 -130
data/lib/feed2email/feed_autodiscoverer.rb +8 -10
data/lib/feed2email/migrate/convert_feeds_migration.rb +29 -0
data/lib/feed2email/migrate/feeds_import_migration.rb +42 -0
data/lib/feed2email/migrate/history_import_migration.rb +42 -0
data/lib/feed2email/migrate/migration.rb +42 -0
data/lib/feed2email/migrate/split_history_migration.rb +32 -0
data/lib/feed2email/open-uri.rb +7 -0
data/lib/feed2email/opml_exporter.rb +109 -0
data/lib/feed2email/opml_importer.rb +52 -0
data/lib/feed2email/redirection_checker.rb +2 -2
data/lib/feed2email/smtp_connection.rb +59 -0
data/lib/feed2email/version.rb +1 -1
metadata +55 -30
data/bin/feed2email-migrate-feedlist +0 -36
data/bin/feed2email-migrate-history +0 -29
data/lib/feed2email/feed_history.rb +0 -82
data/lib/feed2email/feed_list.rb +0 -147
data/lib/feed2email/lazy_smtp_connection.rb +0 -35
data/lib/feed2email/mail.rb +0 -84

data/lib/feed2email/database.rb ADDED Viewed

@@ -0,0 +1,75 @@
+require 'sequel'
+require 'feed2email'
+module Feed2Email
+  def self.database
+    @database
+  end
+  def self.database=(database)
+    @database = database
+  end
+  def self.database_path
+    root.join('feed2email.db').to_s
+  end
+  class Database
+    def self.setup
+      return if Feed2Email.database
+      Feed2Email.database = new(
+        adapter:       'sqlite',
+        database:      Feed2Email.database_path,
+        loggers:       [Feed2Email.logger],
+        sql_log_level: :debug
+      )
+      Feed2Email.database.setup
+    end
+    def initialize(connect_options)
+      @connect_options = connect_options
+    end
+    def setup
+      unless connection
+        setup_connection
+        setup_schema
+      end
+    end
+    private
+    def connect_options; @connect_options end
+    def connection; @connection end
+    def path; connect_options[:database] end
+    def setup_connection
+      @connection = Sequel::Model.db = Sequel.connect(connect_options)
+    end
+    def setup_schema
+      connection.create_table? :feeds do
+        primary_key :id
+        String :uri, null: false, unique: true
+        TrueClass :enabled, null: false, default: true
+        String :etag
+        String :last_modified
+        Time :last_processed_at
+        Time :created_at
+        Time :updated_at
+      end
+      connection.create_table? :entries do
+        primary_key :id
+        foreign_key :feed_id, :feeds, null: false, index: true,
+                                      on_delete: :cascade
+        String :uri, null: false, unique: true
+        Time :created_at
+        Time :updated_at
+      end
+    end
+  end
+end

data/lib/feed2email/entry.rb CHANGED Viewed

@@ -1,41 +1,175 @@
-require 'feed2email/mail'
+require 'mail'
+require 'sequel'
+require 'uri'
+require 'feed2email'
+require 'feed2email/configurable'
+require 'feed2email/core_ext'
+require 'feed2email/database'
+require 'feed2email/loggable'
+require 'feed2email/version'
 module Feed2Email
-  class Entry
-    def initialize(data, feed_uri, feed_title)
-      @data = data
-      @feed_uri = feed_uri
-      @feed_title = feed_title
+  Database.setup
+  if config['send_method'] == 'smtp'
+    require 'feed2email/smtp_connection'
+    SMTPConnection.setup
+  end
+  class Entry < Sequel::Model(:entries)
+    plugin :timestamps
+    many_to_one :feed
+    class << self
+      attr_accessor :last_email_sent_at
+    end
+    include Configurable
+    include Loggable
+    attr_accessor :data
+    attr_accessor :feed_data
+    attr_accessor :feed_uri
+    def process
+      unless feed.old?
+        logger.debug 'Skipping new feed entry...'
+        save # record as seen
+        return true
+      end
+      if old?
+        logger.debug 'Skipping old entry...'
+        return true
+      end
+      return send_mail
+    end
+    private
+    def apply_send_delay
+      return if config['send_delay'] == 0 || config['send_method'] == 'file'
+      return if last_email_sent_at.nil?
+      secs_since_last_email = Time.now - last_email_sent_at
+      secs_to_sleep = config['send_delay'] - secs_since_last_email
+      return if secs_to_sleep <= 0
+      logger.debug "Sleeping for #{secs_to_sleep} seconds..."
+      sleep(secs_to_sleep)
+    end
+    def author; data.author end
+    def body_html
+      %{
+        <html>
+        <body>
+        <h1><a href="%{uri}">%{title}</a></h1>
+        %{content}
+        <p>%{published}</p>
+        <p><a href="%{uri}">%{uri}</a></p>
+        <p>--<br>
+        Sent by <a href="https://github.com/agorf/feed2email">feed2email
+        #{VERSION}</a> at #{Time.now}</p>
+        </body>
+        </html>
+      }.gsub(/^\s+/, '') % {
+        content:   content,
+        published: published_line,
+        title:     title.strip_html,
+        uri:       uri.escape_html,
+      }
+    end
+    def body_text
+      body_html.to_markdown
     end
-    def author
-      @data.author
+    def build_mail
+      Mail.new.tap do |m|
+        m.from      = %{"#{feed_title}" <#{config['sender']}>}
+        m.to        = config['recipient']
+        m.subject   = title.strip_html
+        m.html_part = build_mail_part('text/html', body_html)
+        m.text_part = build_mail_part('text/plain', body_text)
+        m.delivery_method(*delivery_method_params)
+      end
+    end
+    def build_mail_part(content_type, body)
+      part = Mail::Part.new
+      part.content_type = "#{content_type}; charset=UTF-8"
+      part.body = body
+      part
     end
     def content
-      @data.content || @data.summary
+      data.content || data.summary
     end
-    def published
-      @data.published
+    def delivery_method_params
+      case config['send_method']
+      when 'file'
+        [:file, location: config['mail_path']]
+      when 'sendmail'
+        [:sendmail, location: config['sendmail_path']]
+      when 'smtp'
+        [:smtp_connection, connection: Feed2Email.smtp_connection]
+      end
+    end
+    def feed_title; feed_data.title end
+    def last_email_sent_at; Entry.last_email_sent_at end
+    def last_email_sent_at=(time)
+      Entry.last_email_sent_at = time
+    end
+    def old?
+      feed.entries_dataset.where(uri: uri).any?
+    end
+    def published; data.published end
+    def published_line
+      return nil unless author || published
+      text = 'Published'
+      text << " by #{author}" if author
+      text << " at #{published}" if published
+      text
     end
     def send_mail
-      Mail.new(self, @feed_title).send
+      apply_send_delay
+      logger.debug 'Sending new entry...'
+      if build_mail.deliver!
+        self.last_email_sent_at = Time.now
+        save # record as seen
+        return true
+      end
     end
     def title
-      @data.title.strip
+      data.title.strip
     end
     def uri
       return @uri if @uri
-      @uri = @data.url
+      @uri = data.url
       # Make relative entry URL absolute by prepending feed URL
       if @uri && @uri.start_with?('/')
-        @uri = @feed_uri[%r{https?://[^/]+}] + @uri
+        @uri = URI.join(feed_uri[%r{https?://[^/]+}], @uri)
       end
       @uri

data/lib/feed2email/feed.rb CHANGED Viewed

@@ -1,72 +1,102 @@
 require 'feedzirra'
-require 'open-uri'
+require 'sequel'
 require 'stringio'
 require 'zlib'
+require 'feed2email'
+require 'feed2email/config'
 require 'feed2email/configurable'
 require 'feed2email/core_ext'
+require 'feed2email/database'
 require 'feed2email/entry'
-require 'feed2email/feed_history'
 require 'feed2email/loggable'
+require 'feed2email/open-uri'
 require 'feed2email/redirection_checker'
 require 'feed2email/version'
 module Feed2Email
-  class Feed
+  Database.setup
+  class Feed < Sequel::Model(:feeds)
+    plugin :dirty
+    plugin :timestamps
+    one_to_many :entries
+    subset(:enabled, enabled: true)
+    def_dataset_method(:by_smallest_id) { order(:id) }
     include Configurable
     include Loggable
-    attr_reader :meta
-    def initialize(meta)
-      @meta = meta
-    end
+    def old?; last_processed_at end
     def process
       logger.info "Processing feed #{uri} ..."
-      return unless fetch_and_parse_feed
+      return false unless fetch_and_parse
+      if processable?
+        # Reset feed caching parameters unless all entries were processed. This
+        # makes sure the feed will be fetched on next processing.
+        unless process_entries
+          self.last_modified = initial_value(:last_modified)
+          self.etag = initial_value(:etag)
+        end
-      if entries.any?
-        processed = process_entries
-        history.sync
+        self.last_processed_at = Time.now
+        save(changed: true)
       else
-        processed = true
         logger.warn 'Feed does not have entries'
       end
+    end
-      processed
+    def to_s
+      parts = [id.to_s.rjust(3)] # align right 1-999
+      parts << "\e[31mDISABLED\e[0m" unless enabled
+      parts << uri
+      parts.join(' ')
     end
-    private
+    def toggle
+      update(enabled: !enabled)
+    end
-    def apply_send_delay
-      return if config['send_delay'] == 0
+    def uncache
+      !cached? || update(last_modified: nil, etag: nil)
+    end
-      return if last_email_sent_at.nil?
+    private
-      secs_since_last_email = Time.now - last_email_sent_at
-      secs_to_sleep = config['send_delay'] - secs_since_last_email
+    def cached?
+      last_modified || etag
+    end
-      return if secs_to_sleep <= 0
+    def decode_content(data, content_encoding)
+      case content_encoding
+      when 'gzip'
+        gz = Zlib::GzipReader.new(StringIO.new(data))
+        xml = gz.read
+        gz.close
+      when 'deflate'
+        xml = Zlib::Inflate.inflate(data)
+      else
+        xml = data
+      end
-      logger.debug "Sleeping for #{secs_to_sleep} seconds..."
-      sleep(secs_to_sleep)
+      xml
     end
-    def fetch_feed
+    def fetch
       logger.debug 'Fetching feed...'
       begin
-        cache_feed = !permanently_redirected?
-        open(uri, fetch_feed_options(cache_feed)) do |f|
-          if f.meta['last-modified'] || meta.has_key?(:last_modified)
-            meta[:last_modified] = f.meta['last-modified']
-          end
+        open(uri, fetch_options) do |f|
+          handle_redirection if uri != f.base_uri.to_s
-          if f.meta['etag'] || meta.has_key?(:etag)
-            meta[:etag] = f.meta['etag']
-          end
+          self.last_modified = f.meta['last-modified']
+          self.etag = f.meta['etag']
           return decode_content(f.read, f.meta['content-encoding'])
         end
@@ -82,53 +112,48 @@ module Feed2Email
       end
     end
-    def permanently_redirected?
-      checker = RedirectionChecker.new(uri)
-      return false unless checker.permanently_redirected?
-      self.uri = checker.location
-      logger.warn 'Got permanently redirected!'
-      logger.warn "Updated feed location to #{checker.location}"
-      true
-    end
-    def decode_content(data, content_encoding)
-      case content_encoding
-      when 'gzip'
-        gz = Zlib::GzipReader.new(StringIO.new(data))
-        xml = gz.read
-        gz.close
-      when 'deflate'
-        xml = Zlib::Inflate.inflate(data)
-      else
-        xml = data
+    def fetch_and_parse
+      if xml_data = fetch
+        @parsed_feed = parse(xml_data)
+        @parsed_feed && @parsed_feed.respond_to?(:entries)
       end
-      xml
     end
-    def fetch_feed_options(cache_feed)
+    def fetch_options
       options = {
         'User-Agent' => "feed2email/#{VERSION}",
         'Accept-Encoding' => 'gzip, deflate',
       }
-      if cache_feed
-        if meta[:last_modified]
-          options['If-Modified-Since'] = meta[:last_modified]
+      unless permanently_redirected?
+        if last_modified
+          options['If-Modified-Since'] = last_modified
         end
-        if meta[:etag]
-          options['If-None-Match'] = meta[:etag]
+        if etag
+          options['If-None-Match'] = etag
         end
       end
       options
     end
-    def parse_feed(xml_data)
+    def handle_redirection
+      checker = RedirectionChecker.new(uri)
+      if checker.permanently_redirected?
+        logger.warn 'Got permanently redirected!'
+        self.uri = checker.location
+        logger.warn "Updated feed location to #{checker.location}"
+      end
+    end
+    def log_exception(error)
+      logger.error "#{error.class}: #{error.message.strip}"
+      error.backtrace.each {|line| logger.debug line }
+    end
+    def parse(xml_data)
       logger.debug 'Parsing feed...'
       begin
@@ -140,92 +165,48 @@ module Feed2Email
       end
     end
-    def fetch_and_parse_feed
-      if xml_data = fetch_feed
-        @data = parse_feed(xml_data)
-      end
-      @data && @data.respond_to?(:entries)
+    def parsed_entries
+      parsed_feed.entries
     end
-    def uri
-      meta[:uri]
-    end
-    def uri=(uri)
-      history.uri = uri
-      meta[:uri] = uri
-    end
+    def parsed_feed; @parsed_feed end
-    def entries
-      @entries ||= data.entries.first(max_entries).map {|entry_data|
-        Entry.new(entry_data, uri, title)
-      }
-    end
-    def max_entries
-      config['max_entries'].to_i
+    def permanently_redirected?
+      column_changed?(:uri)
     end
     def process_entries
-      logger.info "Processing #{'entry'.pluralize(entries.size, 'entries')}..."
-      entries.all? {|e| process_entry(e) } # false if any entry fails
-    end
-    def process_entry(entry)
-      logger.info "Processing entry #{entry.uri} ..."
-      unless history.any?
-        logger.debug 'Skipping new feed entry...'
-        history << entry.uri
-        return true
-      end
+      total = processable_entries.size
+      processed = true
-      if history.include?(entry.uri)
-        logger.debug 'Skipping old entry...'
-        return true
+      processable_entries.each_with_index do |parsed_entry, i|
+        logger.info "Processing entry #{i + 1}/#{total} #{parsed_entry.url} ..."
+        processed = false unless process_entry(parsed_entry)
       end
-      apply_send_delay
+      processed
+    end
-      logger.debug 'Sending new entry...'
+    def process_entry(parsed_entry)
+      entry = Entry.new(feed_id: id, uri: parsed_entry.url)
+      entry.data      = parsed_entry
+      entry.feed_data = parsed_feed
+      entry.feed_uri  = uri
       begin
-        mail_sent = entry.send_mail
+        return entry.process
       rescue => e
         log_exception(e)
         return false
       end
-      if mail_sent
-        self.last_email_sent_at = Time.now
-        history << entry.uri
-      end
-      mail_sent
     end
-    def history
-      @history ||= FeedHistory.new(uri)
+    def processable?
+      processable_entries.size > 0
     end
-    def last_email_sent_at
-      @last_email_sent_at
+    def processable_entries
+      parsed_entries.first(config['max_entries'])
     end
-    def last_email_sent_at=(time)
-      @last_email_sent_at = time
-    end
-    def log_exception(error)
-      logger.error "#{error.class}: #{error.message.strip}"
-      error.backtrace.each {|line| logger.debug line }
-    end
-    def title
-      data.title # delegate
-    end
-    def data; @data end
   end
 end