feedjira 2.0.0 → 2.2.0

data/lib/feedjira/core_ext/time.rb

@@ -1,6 +1,7 @@
-require "time"
-require "date"
+require 'time'
+require 'date'
 
+# rubocop:disable Style/DocumentationMethod
 class Time
   # Parse a time string and convert it to UTC without raising errors.
   # Parses a flattened 14-digit time (YYYYmmddHHMMMSS) as UTC.
@@ -10,22 +11,28 @@ class Time
   #
   # === Returns
   # A Time instance in UTC or nil if there were errors while parsing.
+  # rubocop:disable Metrics/MethodLength
   def self.parse_safely(dt)
-    if dt
-      case
-      when dt.is_a?(Time)
-        dt.utc
-      when dt.respond_to?(:empty?) && dt.empty?
-        nil
-      when dt.respond_to?(:to_datetime)
-        dt.to_datetime.utc
-      when dt.to_s =~ /\A\d{14}\z/
-        parse("#{dt.to_s}Z", true)
-      else
-        parse(dt.to_s, true).utc
-      end
+    if dt.is_a?(Time)
+      dt.utc
+    elsif dt.respond_to?(:to_datetime)
+      dt.to_datetime.utc
+    elsif dt.respond_to? :to_s
+      parse_string_safely dt.to_s
     end
-  rescue StandardError
+  rescue StandardError => e
+    Feedjira.logger.debug { "Failed to parse time #{dt}" }
+    Feedjira.logger.debug(e)
     nil
-  end unless method_defined?(:parse_safely)
+  end
+
+  def self.parse_string_safely(string)
+    return nil if string.empty?
+
+    if string =~ /\A\d{14}\z/
+      parse("#{string}Z", true)
+    else
+      parse(string).utc
+    end
+  end
 end

data/lib/feedjira/core_ext.rb

@@ -1,3 +1,3 @@
-require "feedjira/core_ext/time"
-require "feedjira/core_ext/date"
-require "feedjira/core_ext/string"
+require 'feedjira/core_ext/time'
+require 'feedjira/core_ext/date'
+require 'feedjira/core_ext/string'

data/lib/feedjira/date_time_utilities/date_time_epoch_parser.rb

@@ -0,0 +1,13 @@
+# rubocop:disable Style/Documentation
+# rubocop:disable Style/DocumentationMethod
+module Feedjira
+  module DateTimeUtilities
+    class DateTimeEpochParser
+      def self.parse(string)
+        epoch_time = string.to_i
+        return Time.at(epoch_time).to_datetime if epoch_time.to_s == string
+        raise "#{string} is not a valid epoch time"
+      end
+    end
+  end
+end

data/lib/feedjira/date_time_utilities/date_time_language_parser.rb

@@ -0,0 +1,24 @@
+# rubocop:disable Style/Documentation
+# rubocop:disable Style/DocumentationMethod
+module Feedjira
+  module DateTimeUtilities
+    class DateTimeLanguageParser
+      MONTHS_ENGLISH =
+        %w(Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec).freeze
+      MONTHS_SPANISH =
+        %w(Ene Feb Mar Abr May Jun Jul Ago Sep Oct Nov Dic).freeze
+
+      def self.parse(string)
+        DateTime.parse(translate(string))
+      end
+
+      def self.translate(string)
+        MONTHS_SPANISH.each_with_index do |m, i|
+          rgx = Regexp.new("\s#{m}\s", Regexp::IGNORECASE)
+          return string.gsub(rgx, MONTHS_ENGLISH[i]) if string =~ rgx
+        end
+        raise "No translation found for #{string}"
+      end
+    end
+  end
+end

data/lib/feedjira/date_time_utilities/date_time_pattern_parser.rb

@@ -0,0 +1,34 @@
+# rubocop:disable Style/Documentation
+# rubocop:disable Style/DocumentationMethod
+module Feedjira
+  module DateTimeUtilities
+    class DateTimePatternParser
+      # rubocop:disable Style/AsciiComments
+      # Japanese Symbols are required for strange Date Strings like
+      # '水, 31 8 2016 07:37:00 PDT'
+      JAPANESE_SYMBOLS = %w(日 月 火 水 木 金 土).freeze
+      PATTERNS = ['%m/%d/%Y %T %p', '%d %m %Y %T %Z'].freeze
+
+      # rubocop:disable Metrics/MethodLength
+      def self.parse(string)
+        PATTERNS.each do |p|
+          begin
+            datetime = DateTime.strptime(prepare(string), p)
+            return datetime
+          rescue StandardError => e
+            Feedjira.logger.debug("Failed to parse date #{string}")
+            Feedjira.logger.debug(e)
+            nil
+          end
+        end
+        raise "No pattern matched #{string}"
+      end
+
+      def self.prepare(string)
+        rgx = Regexp.new("^(#{JAPANESE_SYMBOLS.join('|')}),\s")
+        string.gsub(rgx, '')
+      end
+      private_class_method :prepare
+    end
+  end
+end

data/lib/feedjira/date_time_utilities.rb

@@ -0,0 +1,32 @@
+# rubocop:disable Style/Documentation
+module Feedjira
+  module DateTimeUtilities
+    # This is our date parsing heuristic.
+    # Date Parsers are attempted in order.
+    DATE_PARSERS = [
+      DateTimePatternParser,
+      DateTimeLanguageParser,
+      DateTimeEpochParser,
+      DateTime
+    ].freeze
+
+    # Parse the given string starting with the most common parser (default ruby)
+    # and going over all other available parsers
+    # rubocop:disable Metrics/MethodLength
+    def parse_datetime(string)
+      res = DATE_PARSERS.find do |parser|
+        begin
+          return parser.parse(string).feed_utils_to_gm_time
+        rescue StandardError => e
+          Feedjira.logger.debug { "Failed to parse date #{string}" }
+          Feedjira.logger.debug(e)
+          nil
+        end
+      end
+
+      Feedjira.logger.warn { "Failed to parse date #{string}" } if res.nil?
+
+      res
+    end
+  end
+end

data/lib/feedjira/feed.rb

@@ -1,85 +1,112 @@
+# rubocop:disable Style/Documentation
+# rubocop:disable Style/DocumentationMethod
 module Feedjira
   class Feed
-    def self.parse_with(parser, xml, &block)
-      parser.parse xml, &block
-    end
+    class << self
+      def parse_with(parser, xml, &block)
+        parser.parse xml, &block
+      end
 
-    def self.parse(xml, &block)
-      if parser = determine_feed_parser_for_xml(xml)
+      def parse(xml, &block)
+        parser = determine_feed_parser_for_xml(xml)
+        raise NoParserAvailable, 'No valid parser for XML.' unless parser
         parse_with parser, xml, &block
-      else
-        raise NoParserAvailable.new("No valid parser for XML.")
       end
-    end
 
-    def self.determine_feed_parser_for_xml(xml)
-      start_of_doc = xml.slice(0, 2000)
-      feed_classes.detect {|klass| klass.able_to_parse?(start_of_doc)}
-    end
+      def determine_feed_parser_for_xml(xml)
+        start_of_doc = xml.slice(0, 2000)
+        feed_classes.detect { |klass| klass.able_to_parse?(start_of_doc) }
+      end
 
-    def self.add_feed_class(klass)
-      feed_classes.unshift klass
-    end
+      def add_feed_class(klass)
+        feed_classes.unshift klass
+      end
 
-    def self.feed_classes
-      @feed_classes ||= [
-        Feedjira::Parser::RSSFeedBurner,
-        Feedjira::Parser::GoogleDocsAtom,
-        Feedjira::Parser::AtomFeedBurner,
-        Feedjira::Parser::Atom,
-        Feedjira::Parser::ITunesRSS,
-        Feedjira::Parser::RSS
-      ]
-    end
+      def feed_classes
+        @feed_classes ||= Feedjira.parsers
+      end
 
-    def self.add_common_feed_element(element_tag, options = {})
-      feed_classes.each do |k|
-        k.element element_tag, options
+      def reset_parsers!
+        @feed_classes = nil
       end
-    end
 
-    def self.add_common_feed_elements(element_tag, options = {})
-      feed_classes.each do |k|
-        k.elements element_tag, options
+      def add_common_feed_element(element_tag, options = {})
+        feed_classes.each do |k|
+          k.element element_tag, options
+        end
       end
-    end
 
-    def self.add_common_feed_entry_element(element_tag, options = {})
-      call_on_each_feed_entry :element, element_tag, options
-    end
+      def add_common_feed_elements(element_tag, options = {})
+        feed_classes.each do |k|
+          k.elements element_tag, options
+        end
+      end
 
-    def self.add_common_feed_entry_elements(element_tag, options = {})
-      call_on_each_feed_entry :elements, element_tag, options
-    end
+      def add_common_feed_entry_element(element_tag, options = {})
+        call_on_each_feed_entry :element, element_tag, options
+      end
 
-    def self.call_on_each_feed_entry(method, *parameters)
-      feed_classes.each do |k|
-        k.sax_config.collection_elements.each_value do |vl|
-          vl.find_all{|v| (v.accessor == 'entries') && (v.data_class.class == Class)}.each do |v|
-              v.data_class.send(method, *parameters)
+      def add_common_feed_entry_elements(element_tag, options = {})
+        call_on_each_feed_entry :elements, element_tag, options
+      end
+
+      def call_on_each_feed_entry(method, *parameters)
+        feed_classes.each do |klass|
+          klass.sax_config.collection_elements.each_value do |value|
+            collection_configs = value.select do |v|
+              v.accessor == 'entries' && v.data_class.class == Class
+            end
+
+            collection_configs.each do |config|
+              config.data_class.send(method, *parameters)
+            end
           end
         end
       end
-    end
 
-    def self.fetch_and_parse(url)
-      response = connection(url).get
-      raise FetchFailure.new("Fetch failed - #{response.status}") unless response.success?
-      xml = response.body
-      parser_klass = determine_feed_parser_for_xml xml
-      raise NoParserAvailable.new("No valid parser for XML.") unless parser_klass
-
-      feed = parse_with parser_klass, xml
-      feed.feed_url = url
-      feed.etag = response.headers['etag'].to_s.gsub(/"/, '')
-      feed.last_modified = response.headers['last-modified']
-      feed
-    end
+      def fetch_and_parse(url)
+        response = connection(url).get
+        unless response.success?
+          raise FetchFailure, "Fetch failed - #{response.status}"
+        end
+        feed = parse response.body
+        feed.feed_url = url
+        feed.etag = response.headers['etag'].to_s.delete '"'
+
+        feed.last_modified = parse_last_modified(response)
+        feed
+      end
+
+      # rubocop:disable LineLength
+      def connection(url)
+        Faraday.new(url: url, headers: headers, request: request_options) do |conn|
+          conn.use FaradayMiddleware::FollowRedirects, limit: Feedjira.follow_redirect_limit
+          conn.adapter(*Faraday.default_adapter)
+        end
+      end
+      # rubocop:enable LineLength
+
+      private
+
+      def headers
+        {
+          user_agent: Feedjira.user_agent
+        }
+      end
+
+      def request_options
+        {
+          timeout: Feedjira.request_timeout
+        }
+      end
 
-    def self.connection(url)
-      Faraday.new(url: url) do |conn|
-        conn.use FaradayMiddleware::FollowRedirects, limit: 3
-        conn.adapter :net_http
+      def parse_last_modified(response)
+        lm = response.headers['last-modified']
+        DateTime.parse(lm).to_time
+      rescue StandardError => e
+        Feedjira.logger.warn { "Failed to parse last modified '#{lm}'" }
+        Feedjira.logger.debug(e)
+        nil
       end
     end
   end

data/lib/feedjira/feed_entry_utilities.rb

@@ -1,23 +1,25 @@
+# rubocop:disable Style/Documentation
+# rubocop:disable Style/DocumentationMethod
 module Feedjira
   module FeedEntryUtilities
-
     include Enumerable
+    include DateTimeUtilities
 
     def published
       @published ||= @updated
     end
 
     def parse_datetime(string)
-      begin
-        DateTime.parse(string).feed_utils_to_gm_time
-      rescue
-        warn "Failed to parse date #{string.inspect}"
-        nil
-      end
+      DateTime.parse(string).feed_utils_to_gm_time
+    rescue StandardError => e
+      Feedjira.logger.warn { "Failed to parse date #{string.inspect}" }
+      Feedjira.logger.debug(e)
+      nil
     end
 
     ##
-    # Returns the id of the entry or its url if not id is present, as some formats don't support it
+    # Returns the id of the entry or its url if not id is present, as some
+    # formats don't support it
     def id
       @entry_id ||= @url
     end
@@ -26,41 +28,40 @@ module Feedjira
     # Writer for published. By default, we keep the "oldest" publish time found.
     def published=(val)
       parsed = parse_datetime(val)
-      @published = parsed if !@published || parsed < @published
+      @published = parsed if parsed && (!@published || parsed < @published)
     end
 
     ##
     # Writer for updated. By default, we keep the most recent update time found.
     def updated=(val)
       parsed = parse_datetime(val)
-      @updated = parsed if !@updated || parsed > @updated
+      @updated = parsed if parsed && (!@updated || parsed > @updated)
     end
 
     def sanitize!
-      %w[title author summary content image].each do |name|
-        if self.respond_to?(name) && self.send(name).respond_to?(:sanitize!)
-          self.send(name).send :sanitize!
+      %w(title author summary content image).each do |name|
+        if respond_to?(name) && send(name).respond_to?(:sanitize!)
+          send(name).send :sanitize!
         end
       end
     end
 
-    alias_method :last_modified, :published
+    alias last_modified published
 
     def each
-      @rss_fields ||= self.instance_variables
+      @rss_fields ||= instance_variables
 
       @rss_fields.each do |field|
-        yield(field.to_s.sub('@', ''), self.instance_variable_get(field))
+        yield(field.to_s.sub('@', ''), instance_variable_get(field))
       end
     end
 
     def [](field)
-      self.instance_variable_get("@#{field.to_s}")
+      instance_variable_get("@#{field}")
     end
 
     def []=(field, value)
-      self.instance_variable_set("@#{field.to_s}", value)
+      instance_variable_set("@#{field}", value)
     end
-
   end
 end

data/lib/feedjira/feed_utilities.rb

@@ -1,6 +1,8 @@
+# rubocop:disable Style/Documentation
+# rubocop:disable Style/DocumentationMethod
 module Feedjira
   module FeedUtilities
-    UPDATABLE_ATTRIBUTES = %w(title feed_url url last_modified etag)
+    UPDATABLE_ATTRIBUTES = %w(title feed_url url last_modified etag).freeze
 
     attr_writer   :new_entries, :updated, :last_modified
     attr_accessor :etag
@@ -11,7 +13,7 @@ module Feedjira
 
     module ClassMethods
       def parse(xml, &block)
-        xml = xml.lstrip
+        xml = strip_whitespace(xml)
         xml = preprocess(xml) if preprocess_xml
         super xml, &block
       end
@@ -28,11 +30,20 @@ module Feedjira
       def preprocess_xml
         @preprocess_xml
       end
+
+      def strip_whitespace(xml)
+        if Feedjira.strip_whitespace
+          xml.strip
+        else
+          xml.lstrip
+        end
+      end
     end
 
     def last_modified
       @last_modified ||= begin
-        entry = entries.reject {|e| e.published.nil? }.sort_by { |entry| entry.published if entry.published }.last
+        published = entries.reject { |e| e.published.nil? }
+        entry = published.sort_by { |e| e.published if e.published }.last
         entry ? entry.published : nil
       end
     end
@@ -45,13 +56,13 @@ module Feedjira
       @new_entries ||= []
     end
 
-    def has_new_entries?
-      new_entries.size > 0
+    def new_entries?
+      !new_entries.empty?
     end
 
     def update_from_feed(feed)
       self.new_entries += find_new_entries_for(feed)
-      self.entries.unshift(*self.new_entries)
+      entries.unshift(*self.new_entries)
 
       @updated = false
 
@@ -61,7 +72,8 @@ module Feedjira
     end
 
     def update_attribute(feed, name)
-      old_value, new_value = send(name), feed.send(name)
+      old_value = send(name)
+      new_value = feed.send(name)
 
       if old_value != new_value
         send("#{name}=", new_value)
@@ -72,33 +84,36 @@ module Feedjira
     end
 
     def sanitize_entries!
-      entries.each {|entry| entry.sanitize!}
+      entries.each(&:sanitize!)
     end
 
     private
 
+    # This implementation is a hack, which is why it's so ugly. It's to get
+    # around the fact that not all feeds have a published date. However,
+    # they're always ordered with the newest one first. So we go through the
+    # entries just parsed and insert each one as a new entry until we get to
+    # one that has the same id as the the newest for the feed.
     def find_new_entries_for(feed)
-      # this implementation is a hack, which is why it's so ugly.
-      # it's to get around the fact that not all feeds have a published date.
-      # however, they're always ordered with the newest one first.
-      # So we go through the entries just parsed and insert each one as a new entry
-      # until we get to one that has the same id as the the newest for the feed
-      return feed.entries if self.entries.length == 0
-      latest_entry = self.entries.first
+      return feed.entries if entries.length.zero?
+
+      latest_entry = entries.first
       found_new_entries = []
+
       feed.entries.each do |entry|
-        if entry.entry_id.nil? && latest_entry.entry_id.nil?
-          break if entry.url == latest_entry.url
-        else
-          break if entry.entry_id == latest_entry.entry_id || entry.url == latest_entry.url
-        end
+        break unless new_entry?(entry, latest_entry)
         found_new_entries << entry
       end
+
       found_new_entries
     end
 
-    def existing_entry?(test_entry)
-      entries.any? { |entry| entry.id == test_entry.id }
+    def new_entry?(entry, latest)
+      nil_ids = entry.entry_id.nil? && latest.entry_id.nil?
+      new_id = entry.entry_id != latest.entry_id
+      new_url = entry.url != latest.url
+
+      (nil_ids || new_id) && new_url
     end
   end
 end

data/lib/feedjira/parser/atom.rb

@@ -1,19 +1,21 @@
+# rubocop:disable Style/DocumentationMethod
 module Feedjira
   module Parser
     # Parser for dealing with Atom feeds.
     class Atom
       include SAXMachine
       include FeedUtilities
+
       element :title
-      element :subtitle, :as => :description
-      element :link, :as => :url, :value => :href, :with => {:type => "text/html"}
-      element :link, :as => :feed_url, :value => :href, :with => {:rel => "self"}
-      elements :link, :as => :links, :value => :href
-      elements :link, :as => :hubs, :value => :href, :with => {:rel => "hub"}
-      elements :entry, :as => :entries, :class => AtomEntry
+      element :subtitle, as: :description
+      element :link, as: :url, value: :href, with: { type: 'text/html' }
+      element :link, as: :feed_url, value: :href, with: { rel: 'self' }
+      elements :link, as: :links, value: :href
+      elements :link, as: :hubs, value: :href, with: { rel: 'hub' }
+      elements :entry, as: :entries, class: AtomEntry
 
-      def self.able_to_parse?(xml) #:nodoc:
-        /\<feed[^\>]+xmlns\s?=\s?[\"|\'](http:\/\/www\.w3\.org\/2005\/Atom|http:\/\/purl\.org\/atom\/ns\#)[\"|\'][^\>]*\>/ =~ xml
+      def self.able_to_parse?(xml)
+        %r{\<feed[^\>]+xmlns\s?=\s?[\"\'](http://www\.w3\.org/2005/Atom|http://purl\.org/atom/ns\#)[\"\'][^\>]*\>} =~ xml # rubocop:disable Metrics/LineLength
       end
 
       def url

data/lib/feedjira/parser/atom_entry.rb

@@ -1,5 +1,5 @@
+# rubocop:disable Style/DocumentationMethod
 module Feedjira
-
   module Parser
     # Parser for dealing with Atom feed entries.
     class AtomEntry
@@ -7,28 +7,26 @@ module Feedjira
       include FeedEntryUtilities
 
       element :title
-      element :link, :as => :url, :value => :href, :with => {:type => "text/html", :rel => "alternate"}
-      element :name, :as => :author
+      element :link, as: :url, value: :href, with: { type: 'text/html', rel: 'alternate' } # rubocop:disable Metrics/LineLength
+      element :name, as: :author
       element :content
       element :summary
 
-      element :"media:content", :as => :image, :value => :url
-      element :enclosure, :as => :image, :value => :href
+      element :"media:content", as: :image, value: :url
+      element :enclosure, as: :image, value: :href
 
       element :published
-      element :id, :as => :entry_id
-      element :created, :as => :published
-      element :issued, :as => :published
+      element :id, as: :entry_id
+      element :created, as: :published
+      element :issued, as: :published
       element :updated
-      element :modified, :as => :updated
-      elements :category, :as => :categories, :value => :term
-      elements :link, :as => :links, :value => :href
+      element :modified, as: :updated
+      elements :category, as: :categories, value: :term
+      elements :link, as: :links, value: :href
 
       def url
         @url ||= links.first
       end
     end
-
   end
-
 end