RubyGems - feed2email - Versions diffs - 0.8.0 → 0.9.0 - Mend

feed2email 0.8.0 → 0.9.0

Files changed (32) hide show

checksums.yaml +4 -4
data/CHANGELOG.md +21 -10
data/README.md +138 -86
data/bin/f2e +6 -4
data/bin/feed2email +6 -4
data/bin/feed2email-migrate +23 -0
data/lib/feed2email.rb +6 -9
data/lib/feed2email/cli.rb +112 -75
data/lib/feed2email/config.rb +72 -52
data/lib/feed2email/core_ext.rb +10 -0
data/lib/feed2email/database.rb +75 -0
data/lib/feed2email/entry.rb +149 -15
data/lib/feed2email/feed.rb +111 -130
data/lib/feed2email/feed_autodiscoverer.rb +8 -10
data/lib/feed2email/migrate/convert_feeds_migration.rb +29 -0
data/lib/feed2email/migrate/feeds_import_migration.rb +42 -0
data/lib/feed2email/migrate/history_import_migration.rb +42 -0
data/lib/feed2email/migrate/migration.rb +42 -0
data/lib/feed2email/migrate/split_history_migration.rb +32 -0
data/lib/feed2email/open-uri.rb +7 -0
data/lib/feed2email/opml_exporter.rb +109 -0
data/lib/feed2email/opml_importer.rb +52 -0
data/lib/feed2email/redirection_checker.rb +2 -2
data/lib/feed2email/smtp_connection.rb +59 -0
data/lib/feed2email/version.rb +1 -1
metadata +55 -30
data/bin/feed2email-migrate-feedlist +0 -36
data/bin/feed2email-migrate-history +0 -29
data/lib/feed2email/feed_history.rb +0 -82
data/lib/feed2email/feed_list.rb +0 -147
data/lib/feed2email/lazy_smtp_connection.rb +0 -35
data/lib/feed2email/mail.rb +0 -84

data/lib/feed2email/database.rb ADDED Viewed

@@ -0,0 +1,75 @@
+require 'sequel'
+require 'feed2email'
+module Feed2Email
+  def self.database
+    @database
+  end
+  def self.database=(database)
+    @database = database
+  end
+  def self.database_path
+    root.join('feed2email.db').to_s
+  end
+  class Database
+    def self.setup
+      return if Feed2Email.database
+      Feed2Email.database = new(
+        adapter:       'sqlite',
+        database:      Feed2Email.database_path,
+        loggers:       [Feed2Email.logger],
+        sql_log_level: :debug
+      )
+      Feed2Email.database.setup
+    end
+    def initialize(connect_options)
+      @connect_options = connect_options
+    end
+    def setup
+      unless connection
+        setup_connection
+        setup_schema
+      end
+    end
+    private
+    def connect_options; @connect_options end
+    def connection; @connection end
+    def path; connect_options[:database] end
+    def setup_connection
+      @connection = Sequel::Model.db = Sequel.connect(connect_options)
+    end
+    def setup_schema
+      connection.create_table? :feeds do
+        primary_key :id
+        String :uri, null: false, unique: true
+        TrueClass :enabled, null: false, default: true
+        String :etag
+        String :last_modified
+        Time :last_processed_at
+        Time :created_at
+        Time :updated_at
+      end
+      connection.create_table? :entries do
+        primary_key :id
+        foreign_key :feed_id, :feeds, null: false, index: true,
+                                      on_delete: :cascade
+        String :uri, null: false, unique: true
+        Time :created_at
+        Time :updated_at
+      end
+    end
+  end
+end

data/lib/feed2email/entry.rb CHANGED Viewed

@@ -1,41 +1,175 @@
-require 'feed2email/mail'
+require 'mail'
+require 'sequel'
+require 'uri'
+require 'feed2email'
+require 'feed2email/configurable'
+require 'feed2email/core_ext'
+require 'feed2email/database'
+require 'feed2email/loggable'
+require 'feed2email/version'
 module Feed2Email
-  class Entry
-    def initialize(data, feed_uri, feed_title)
-      @data = data
-      @feed_uri = feed_uri
-      @feed_title = feed_title
+  Database.setup
+  if config['send_method'] == 'smtp'
+    require 'feed2email/smtp_connection'
+    SMTPConnection.setup
+  end
+  class Entry < Sequel::Model(:entries)
+    plugin :timestamps
+    many_to_one :feed
+    class << self
+      attr_accessor :last_email_sent_at
+    end
+    include Configurable
+    include Loggable
+    attr_accessor :data
+    attr_accessor :feed_data
+    attr_accessor :feed_uri
+    def process
+      unless feed.old?
+        logger.debug 'Skipping new feed entry...'
+        save # record as seen
+        return true
+      end
+      if old?
+        logger.debug 'Skipping old entry...'
+        return true
+      end
+      return send_mail
+    end
+    private
+    def apply_send_delay
+      return if config['send_delay'] == 0 || config['send_method'] == 'file'
+      return if last_email_sent_at.nil?
+      secs_since_last_email = Time.now - last_email_sent_at
+      secs_to_sleep = config['send_delay'] - secs_since_last_email
+      return if secs_to_sleep <= 0
+      logger.debug "Sleeping for #{secs_to_sleep} seconds..."
+      sleep(secs_to_sleep)
+    end
+    def author; data.author end
+    def body_html
+      %{
+        <html>
+        <body>
+        <h1><a href="%{uri}">%{title}</a></h1>
+        %{content}
+        <p>%{published}</p>
+        <p><a href="%{uri}">%{uri}</a></p>
+        <p>--<br>
+        Sent by <a href="https://github.com/agorf/feed2email">feed2email
+        #{VERSION}</a> at #{Time.now}</p>
+        </body>
+        </html>
+      }.gsub(/^\s+/, '') % {
+        content:   content,
+        published: published_line,
+        title:     title.strip_html,
+        uri:       uri.escape_html,
+      }
+    end
+    def body_text
+      body_html.to_markdown
     end
-    def author
-      @data.author
+    def build_mail
+      Mail.new.tap do |m|
+        m.from      = %{"#{feed_title}" <#{config['sender']}>}
+        m.to        = config['recipient']
+        m.subject   = title.strip_html
+        m.html_part = build_mail_part('text/html', body_html)
+        m.text_part = build_mail_part('text/plain', body_text)
+        m.delivery_method(*delivery_method_params)
+      end
+    end
+    def build_mail_part(content_type, body)
+      part = Mail::Part.new
+      part.content_type = "#{content_type}; charset=UTF-8"
+      part.body = body
+      part
     end
     def content
-      @data.content || @data.summary
+      data.content || data.summary
     end
-    def published
-      @data.published
+    def delivery_method_params
+      case config['send_method']
+      when 'file'
+        [:file, location: config['mail_path']]
+      when 'sendmail'
+        [:sendmail, location: config['sendmail_path']]
+      when 'smtp'
+        [:smtp_connection, connection: Feed2Email.smtp_connection]
+      end
+    end
+    def feed_title; feed_data.title end
+    def last_email_sent_at; Entry.last_email_sent_at end
+    def last_email_sent_at=(time)
+      Entry.last_email_sent_at = time
+    end
+    def old?
+      feed.entries_dataset.where(uri: uri).any?
+    end
+    def published; data.published end
+    def published_line
+      return nil unless author || published
+      text = 'Published'
+      text << " by #{author}" if author
+      text << " at #{published}" if published
+      text
     end
     def send_mail
-      Mail.new(self, @feed_title).send
+      apply_send_delay
+      logger.debug 'Sending new entry...'
+      if build_mail.deliver!
+        self.last_email_sent_at = Time.now
+        save # record as seen
+        return true
+      end
     end
     def title
-      @data.title.strip
+      data.title.strip
     end
     def uri
       return @uri if @uri
-      @uri = @data.url
+      @uri = data.url
       # Make relative entry URL absolute by prepending feed URL
       if @uri && @uri.start_with?('/')
-        @uri = @feed_uri[%r{https?://[^/]+}] + @uri
+        @uri = URI.join(feed_uri[%r{https?://[^/]+}], @uri)
       end
       @uri

data/lib/feed2email/feed.rb CHANGED Viewed

@@ -1,72 +1,102 @@
 require 'feedzirra'
-require 'open-uri'
+require 'sequel'
 require 'stringio'
 require 'zlib'
+require 'feed2email'
+require 'feed2email/config'
 require 'feed2email/configurable'
 require 'feed2email/core_ext'
+require 'feed2email/database'
 require 'feed2email/entry'
-require 'feed2email/feed_history'
 require 'feed2email/loggable'
+require 'feed2email/open-uri'
 require 'feed2email/redirection_checker'
 require 'feed2email/version'
 module Feed2Email
-  class Feed
+  Database.setup
+  class Feed < Sequel::Model(:feeds)
+    plugin :dirty
+    plugin :timestamps
+    one_to_many :entries
+    subset(:enabled, enabled: true)
+    def_dataset_method(:by_smallest_id) { order(:id) }
     include Configurable
     include Loggable
-    attr_reader :meta
-    def initialize(meta)
-      @meta = meta
-    end
+    def old?; last_processed_at end
     def process
       logger.info "Processing feed #{uri} ..."
-      return unless fetch_and_parse_feed
+      return false unless fetch_and_parse
+      if processable?
+        # Reset feed caching parameters unless all entries were processed. This
+        # makes sure the feed will be fetched on next processing.
+        unless process_entries
+          self.last_modified = initial_value(:last_modified)
+          self.etag = initial_value(:etag)
+        end
-      if entries.any?
-        processed = process_entries
-        history.sync
+        self.last_processed_at = Time.now
+        save(changed: true)
       else
-        processed = true
         logger.warn 'Feed does not have entries'
       end
+    end
-      processed
+    def to_s
+      parts = [id.to_s.rjust(3)] # align right 1-999
+      parts << "\e[31mDISABLED\e[0m" unless enabled
+      parts << uri
+      parts.join(' ')
     end
-    private
+    def toggle
+      update(enabled: !enabled)
+    end
-    def apply_send_delay
-      return if config['send_delay'] == 0
+    def uncache
+      !cached? || update(last_modified: nil, etag: nil)
+    end
-      return if last_email_sent_at.nil?
+    private
-      secs_since_last_email = Time.now - last_email_sent_at
-      secs_to_sleep = config['send_delay'] - secs_since_last_email
+    def cached?
+      last_modified || etag
+    end
-      return if secs_to_sleep <= 0
+    def decode_content(data, content_encoding)
+      case content_encoding
+      when 'gzip'
+        gz = Zlib::GzipReader.new(StringIO.new(data))
+        xml = gz.read
+        gz.close
+      when 'deflate'
+        xml = Zlib::Inflate.inflate(data)
+      else
+        xml = data
+      end
-      logger.debug "Sleeping for #{secs_to_sleep} seconds..."
-      sleep(secs_to_sleep)
+      xml
     end
-    def fetch_feed
+    def fetch
       logger.debug 'Fetching feed...'
       begin
-        cache_feed = !permanently_redirected?
-        open(uri, fetch_feed_options(cache_feed)) do |f|
-          if f.meta['last-modified'] || meta.has_key?(:last_modified)
-            meta[:last_modified] = f.meta['last-modified']
-          end
+        open(uri, fetch_options) do |f|
+          handle_redirection if uri != f.base_uri.to_s
-          if f.meta['etag'] || meta.has_key?(:etag)
-            meta[:etag] = f.meta['etag']
-          end
+          self.last_modified = f.meta['last-modified']
+          self.etag = f.meta['etag']
           return decode_content(f.read, f.meta['content-encoding'])
         end
@@ -82,53 +112,48 @@ module Feed2Email
       end
     end
-    def permanently_redirected?
-      checker = RedirectionChecker.new(uri)
-      return false unless checker.permanently_redirected?
-      self.uri = checker.location
-      logger.warn 'Got permanently redirected!'
-      logger.warn "Updated feed location to #{checker.location}"
-      true
-    end
-    def decode_content(data, content_encoding)
-      case content_encoding
-      when 'gzip'
-        gz = Zlib::GzipReader.new(StringIO.new(data))
-        xml = gz.read
-        gz.close
-      when 'deflate'
-        xml = Zlib::Inflate.inflate(data)
-      else
-        xml = data
+    def fetch_and_parse
+      if xml_data = fetch
+        @parsed_feed = parse(xml_data)
+        @parsed_feed && @parsed_feed.respond_to?(:entries)
       end
-      xml
     end
-    def fetch_feed_options(cache_feed)
+    def fetch_options
       options = {
         'User-Agent' => "feed2email/#{VERSION}",
         'Accept-Encoding' => 'gzip, deflate',
       }
-      if cache_feed
-        if meta[:last_modified]
-          options['If-Modified-Since'] = meta[:last_modified]
+      unless permanently_redirected?
+        if last_modified
+          options['If-Modified-Since'] = last_modified
         end
-        if meta[:etag]
-          options['If-None-Match'] = meta[:etag]
+        if etag
+          options['If-None-Match'] = etag
         end
       end
       options
     end
-    def parse_feed(xml_data)
+    def handle_redirection
+      checker = RedirectionChecker.new(uri)
+      if checker.permanently_redirected?
+        logger.warn 'Got permanently redirected!'
+        self.uri = checker.location
+        logger.warn "Updated feed location to #{checker.location}"
+      end
+    end
+    def log_exception(error)
+      logger.error "#{error.class}: #{error.message.strip}"
+      error.backtrace.each {|line| logger.debug line }
+    end
+    def parse(xml_data)
       logger.debug 'Parsing feed...'
       begin
@@ -140,92 +165,48 @@ module Feed2Email
       end
     end
-    def fetch_and_parse_feed
-      if xml_data = fetch_feed
-        @data = parse_feed(xml_data)
-      end
-      @data && @data.respond_to?(:entries)
+    def parsed_entries
+      parsed_feed.entries
     end
-    def uri
-      meta[:uri]
-    end
-    def uri=(uri)
-      history.uri = uri
-      meta[:uri] = uri
-    end
+    def parsed_feed; @parsed_feed end
-    def entries
-      @entries ||= data.entries.first(max_entries).map {|entry_data|
-        Entry.new(entry_data, uri, title)
-      }
-    end
-    def max_entries
-      config['max_entries'].to_i
+    def permanently_redirected?
+      column_changed?(:uri)
     end
     def process_entries
-      logger.info "Processing #{'entry'.pluralize(entries.size, 'entries')}..."
-      entries.all? {|e| process_entry(e) } # false if any entry fails
-    end
-    def process_entry(entry)
-      logger.info "Processing entry #{entry.uri} ..."
-      unless history.any?
-        logger.debug 'Skipping new feed entry...'
-        history << entry.uri
-        return true
-      end
+      total = processable_entries.size
+      processed = true
-      if history.include?(entry.uri)
-        logger.debug 'Skipping old entry...'
-        return true
+      processable_entries.each_with_index do |parsed_entry, i|
+        logger.info "Processing entry #{i + 1}/#{total} #{parsed_entry.url} ..."
+        processed = false unless process_entry(parsed_entry)
       end
-      apply_send_delay
+      processed
+    end
-      logger.debug 'Sending new entry...'
+    def process_entry(parsed_entry)
+      entry = Entry.new(feed_id: id, uri: parsed_entry.url)
+      entry.data      = parsed_entry
+      entry.feed_data = parsed_feed
+      entry.feed_uri  = uri
       begin
-        mail_sent = entry.send_mail
+        return entry.process
       rescue => e
         log_exception(e)
         return false
       end
-      if mail_sent
-        self.last_email_sent_at = Time.now
-        history << entry.uri
-      end
-      mail_sent
     end
-    def history
-      @history ||= FeedHistory.new(uri)
+    def processable?
+      processable_entries.size > 0
     end
-    def last_email_sent_at
-      @last_email_sent_at
+    def processable_entries
+      parsed_entries.first(config['max_entries'])
     end
-    def last_email_sent_at=(time)
-      @last_email_sent_at = time
-    end
-    def log_exception(error)
-      logger.error "#{error.class}: #{error.message.strip}"
-      error.backtrace.each {|line| logger.debug line }
-    end
-    def title
-      data.title # delegate
-    end
-    def data; @data end
   end
 end