html2rss 0.32.4 → 0.33.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a53d8374b6b4ce7551c9f1bffd0e14960995f7df7380554f1292e0a70317cf58
-  data.tar.gz: 90e91d4275c03c120924cca66230a7aecb3606f2f50c6224d019c138cae244e2
+  metadata.gz: 5e2ebceddb0e3a74ddfcb848eb78f454061a4208b06dbc5443349b102d7edcd6
+  data.tar.gz: eeaaa250afd8e6b991bbc112470cd07da4285649f9a7df9c0c8d57b90d67ec8d
 SHA512:
-  metadata.gz: d2543d4431b2790c89c19ab207d0cdbbc7416493ea1184347746bc26e21068415d68f025e0cc253082224ca5191a3984163e3e48ef3e727cb3b78c19cddd3d64
-  data.tar.gz: ac588fece229b58ab2ea4de91bb467fdbfaf199f43febfd8be6b39d817fc9b9b2867c21cb472949e886ad4dd50b436d9c5c149c0976a87b8cd67d09a6a62f9f6
+  metadata.gz: c71a7060557c917c1022fb03ea02d73dfbf86a9f9db1edf4b8a46f5855801a721236382346326d34b91029a2c9b902e53d81c11b650de67910fcc21edfa99ece
+  data.tar.gz: 64511f26248a9a713cadefe2efae797b243b097063f6170b0e39b5751d38a4b2cda12ab90730e5e87e44129d308618dbd88ff62c704a4ca3e08bdd86d8053dac

data/lib/html2rss/article/enclosure.rb

@@ -16,12 +16,8 @@ module Html2rss
       def self.guess_content_type_from_url(url, default: 'application/octet-stream')
         return default unless url
 
-        path = url.path
-        ext = File.extname(path)
-        ext = ext[1..] if ext.start_with?('.')
-
-        content_type = MIME::Types.type_for(ext)
-        content_type.first&.to_s || 'application/octet-stream'
+        found = MIME::Types.type_for(url.path).first
+        found&.content_type || default
       end
 
       # @param url [Html2rss::Url] absolute enclosure URL

data/lib/html2rss/auto_source/cleanup.rb

@@ -234,8 +234,6 @@ module Html2rss
         end
 
         def excluded_destination?(url)
-          return false unless url
-
           facts = LinkDestination::DestinationFacts.build(url)
           return true if facts.high_confidence_junk_path || facts.high_confidence_utility_destination
 
@@ -286,7 +284,7 @@ module Html2rss
         def item_url(item)
           raw = item.url
           raw.nil? || raw.to_s.empty? ? nil : Html2rss::Url.from_absolute(raw.to_s)
-        rescue ArgumentError
+        rescue Html2rss::Url::InvalidUrlError
           nil
         end
 

data/lib/html2rss/auto_source/scraper/html.rb

@@ -34,8 +34,6 @@ module Html2rss
           return false unless parsed_body
 
           new(parsed_body, url: DETECTION_BASE_URL).extractable?
-        rescue ArgumentError
-          false
         end
 
         # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML (when +document:+ omitted)
@@ -68,8 +66,6 @@ module Html2rss
         # @return [Boolean]
         def extractable?
           ranked_segments.any?
-        rescue ArgumentError
-          false
         end
 
         private
@@ -83,8 +79,6 @@ module Html2rss
               materialize(ranked)
             end
           end
-        rescue ArgumentError
-          []
         end
 
         def ranked_segments
@@ -96,11 +90,11 @@ module Html2rss
         end
 
         def list_ranked
-          @list_ranked ||= rank_strategy(:list, permit_unanchored: false)
+          @list_ranked ||= document ? rank_strategy(:list, permit_unanchored: false) : []
         end
 
         def cluster_ranked
-          @cluster_ranked ||= rank_strategy(:cluster, permit_unanchored: true)
+          @cluster_ranked ||= document ? rank_strategy(:cluster, permit_unanchored: true) : []
         end
 
         def rank_strategy(strategy, permit_unanchored:)

data/lib/html2rss/auto_source/scraper/json_document.rb

@@ -47,7 +47,6 @@ module Html2rss
         # @return [Enumerator, void] article enumerator when no block is given
         def each(&block)
           return enum_for(:each) unless block
-          return unless extractable?
 
           if json_feed?
             each_json_feed_item(&block)
@@ -103,7 +102,7 @@ module Html2rss
           return unless str
 
           Url.from_relative(str, @url)
-        rescue ArgumentError
+        rescue Html2rss::Url::InvalidUrlError
           nil
         end
 
@@ -112,7 +111,7 @@ module Html2rss
           return unless str
 
           Url.from_absolute(str)
-        rescue ArgumentError
+        rescue Html2rss::Url::InvalidUrlError
           nil
         end
 

data/lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb

@@ -28,149 +28,149 @@ module Html2rss
           # Snake `_at`/`_on` or camel `At`/`On` suffix after a closed published-at miss.
           DATE_KEY_SUFFIX = /_at\z|_on\z|[a-z]At\z|[a-z]On\z/
 
-          module_function
-
-          # rubocop:disable Metrics/MethodLength
-          # @param entry [Hash] raw article entry candidate
-          # @param base_url [String, Html2rss::Url] base URL for relative link resolution
-          # @return [Hash{Symbol => Object, nil}] normalized article hash for downstream extraction
-          def normalise(entry, base_url:)
-            return unless entry.is_a?(Hash)
-
-            title = string(ValueFinder.fetch(entry, TITLE_KEYS))
-            description = string(ValueFinder.fetch(entry, DESCRIPTION_KEYS))
-            article_url = resolve_link(entry, keys: URL_KEYS, base_url:,
-                                              log_key: 'JsonState: invalid URL encountered')
-            return unless article_url
-            return if title.nil? && description.nil?
-
-            {
-              title:,
-              description:,
-              url: article_url,
-              image: resolve_link(entry, keys: IMAGE_KEYS, base_url:,
-                                         log_key: 'JsonState: invalid image URL encountered'),
-              published_at: published_at(entry),
-              categories: categories(entry),
-              id: identifier(entry, article_url)
-            }.compact
-          end
-          # rubocop:enable Metrics/MethodLength
-
-          # @param value [Object] candidate scalar, or Hash wrapping +rendered+/+value+/+text+
-          # @return [String, nil] normalized non-empty string (HTML preserved for descriptions)
-          def string(value)
-            value = unwrap_scalar(value)
-            return unless value.is_a?(String) || value.is_a?(Numeric)
-
-            trimmed = value.to_s.strip
-            trimmed unless trimmed.empty?
-          end
+          class << self
+            # rubocop:disable Metrics/MethodLength
+            # @param entry [Hash] raw article entry candidate
+            # @param base_url [String, Html2rss::Url] base URL for relative link resolution
+            # @return [Hash{Symbol => Object, nil}] normalized article hash for downstream extraction
+            def normalise(entry, base_url:)
+              return unless entry.is_a?(Hash)
+
+              title = string(ValueFinder.fetch(entry, TITLE_KEYS))
+              description = string(ValueFinder.fetch(entry, DESCRIPTION_KEYS))
+              article_url = resolve_link(entry, keys: URL_KEYS, base_url:,
+                                                log_key: 'JsonState: invalid URL encountered')
+              return unless article_url
+              return if title.nil? && description.nil?
+
+              {
+                title:,
+                description:,
+                url: article_url,
+                image: resolve_link(entry, keys: IMAGE_KEYS, base_url:,
+                                           log_key: 'JsonState: invalid image URL encountered'),
+                published_at: published_at(entry),
+                categories: categories(entry),
+                id: identifier(entry, article_url)
+              }.compact
+            end
+            # rubocop:enable Metrics/MethodLength
 
-          # @param value [Object] candidate scalar or nested Hash
-          # @return [Object, nil] String/Numeric after at most {MAX_URL_UNWRAP} Hash unwraps
-          def unwrap_scalar(value)
-            depth = 0
-            while value.is_a?(Hash) && depth < MAX_URL_UNWRAP
-              nested = SCALAR_KEYS.find { |key| value.key?(key) }
-              break unless nested
+            # @param value [Object] candidate scalar, or Hash wrapping +rendered+/+value+/+text+
+            # @return [String, nil] normalized non-empty string (HTML preserved for descriptions)
+            def string(value)
+              value = unwrap_scalar(value)
+              return unless value.is_a?(String) || value.is_a?(Numeric)
 
-              value = value[nested]
-              depth += 1
+              trimmed = value.to_s.strip
+              trimmed unless trimmed.empty?
             end
-            value unless value.is_a?(Hash) || value.is_a?(Array)
-          end
 
-          # @param entry [Hash] raw article entry candidate
-          # @param keys [Array<Symbol>] preferred link keys
-          # @param base_url [String, Html2rss::Url] base URL for relative link resolution
-          # @param log_key [String] structured log message key
-          # @return [Html2rss::Url, nil] resolved absolute URL
-          def resolve_link(entry, keys:, base_url:, log_key:)
-            value = unwrap_url(ValueFinder.fetch(entry, keys), keys) ||
-                    unwrap_url(ValueFinder.fetch(entry[:links], keys), keys)
-            string = string(value)
-            return unless string
-
-            Url.from_relative(string, base_url)
-          rescue ArgumentError
-            Log.debug("#{log_key}: url=#{string}")
-            nil
-          end
+            # @param value [Object] candidate scalar or nested Hash
+            # @return [Object, nil] String/Numeric after at most {MAX_URL_UNWRAP} Hash unwraps
+            def unwrap_scalar(value)
+              depth = 0
+              while value.is_a?(Hash) && depth < MAX_URL_UNWRAP
+                nested = SCALAR_KEYS.find { |key| value.key?(key) }
+                break unless nested
 
-          # @param value [Object] candidate URL value, possibly a nested Hash
-          # @param keys [Array<Symbol>] preferred link keys
-          # @return [Object, nil] scalar URL candidate after at most {MAX_URL_UNWRAP} Hash unwraps
-          def unwrap_url(value, keys)
-            depth = 0
-            while value.is_a?(Hash) && depth < MAX_URL_UNWRAP
-              value = ValueFinder.fetch(value, keys)
-              depth += 1
+                value = value[nested]
+                depth += 1
+              end
+              value unless value.is_a?(Hash) || value.is_a?(Array)
             end
-            value unless value.is_a?(Hash)
-          end
 
-          # @param entry [Hash] raw article entry candidate
-          # @return [String, nil] publication timestamp string or unix timestamp digits
-          def published_at(entry)
-            value = ValueFinder.fetch(entry, PUBLISHED_AT_KEYS)
-            string(value) || string(suffix_date_value(entry))
-          end
-
-          # @param hash [Hash, nil] entry or nested attributes/data hash
-          # @return [Object, nil] first string or numeric value whose key matches {DATE_KEY_SUFFIX}
-          def suffix_date_value(hash)
-            return unless hash.is_a?(Hash)
+            # @param entry [Hash] raw article entry candidate
+            # @param keys [Array<Symbol>] preferred link keys
+            # @param base_url [String, Html2rss::Url] base URL for relative link resolution
+            # @param log_key [String] structured log message key
+            # @return [Html2rss::Url, nil] resolved absolute URL
+            def resolve_link(entry, keys:, base_url:, log_key:)
+              value = unwrap_url(ValueFinder.fetch(entry, keys), keys) ||
+                      unwrap_url(ValueFinder.fetch(entry[:links], keys), keys)
+              string = string(value)
+              return unless string
+
+              Url.from_relative(string, base_url)
+            rescue Html2rss::Url::InvalidUrlError
+              Log.debug("#{log_key}: url=#{string}")
+              nil
+            end
 
-            hash.each do |key, value|
-              next unless date_suffix_key?(key)
-              next unless value.is_a?(String) || value.is_a?(Numeric)
+            # @param value [Object] candidate URL value, possibly a nested Hash
+            # @param keys [Array<Symbol>] preferred link keys
+            # @return [Object, nil] scalar URL candidate after at most {MAX_URL_UNWRAP} Hash unwraps
+            def unwrap_url(value, keys)
+              depth = 0
+              while value.is_a?(Hash) && depth < MAX_URL_UNWRAP
+                value = ValueFinder.fetch(value, keys)
+                depth += 1
+              end
+              value unless value.is_a?(Hash)
+            end
 
-              return value
+            # @param entry [Hash] raw article entry candidate
+            # @return [String, nil] publication timestamp string or unix timestamp digits
+            def published_at(entry)
+              value = ValueFinder.fetch(entry, PUBLISHED_AT_KEYS)
+              string(value) || string(suffix_date_value(entry))
             end
 
-            suffix_date_value(hash[:attributes]) || suffix_date_value(hash[:data])
-          end
+            # @param hash [Hash, nil] entry or nested attributes/data hash
+            # @return [Object, nil] first string or numeric value whose key matches {DATE_KEY_SUFFIX}
+            def suffix_date_value(hash)
+              return unless hash.is_a?(Hash)
 
-          # @param key [Symbol, String] candidate published-at key
-          # @return [Boolean] whether the key ends in a date suffix (_at/_on/At/On)
-          def date_suffix_key?(key)
-            key.to_s.match?(DATE_KEY_SUFFIX)
-          end
+              hash.each do |key, value|
+                next unless date_suffix_key?(key)
+                next unless value.is_a?(String) || value.is_a?(Numeric)
 
-          # rubocop:disable Metrics/MethodLength
-          # @param entry [Hash] raw article entry candidate
-          # @return [Array<String>, nil] normalized unique categories
-          def categories(entry)
-            raw = ValueFinder.fetch(entry, CATEGORY_KEYS)
-            names = case raw
-                    when Array then raw
-                    when Hash then raw.values
-                    when String then [raw]
-                    else []
-                    end
-
-            result = names.flat_map do |value|
-              case value
-              when Hash
-                string(ValueFinder.fetch(value, %i[name title label]))
-              else
-                string(value)
+                return value
               end
-            end.compact
 
-            result.uniq!
-            result unless result.empty?
-          end
-          # rubocop:enable Metrics/MethodLength
-
-          # @param entry [Hash] raw article entry candidate
-          # @param article_url [Html2rss::Url] resolved article URL
-          # @return [String] stable article identifier fallbacking to resolved URL
-          def identifier(entry, article_url)
-            value = ValueFinder.fetch(entry, ID_KEYS)
-            value = ValueFinder.fetch(value, ID_KEYS) if value.is_a?(Hash)
-            string(value) || article_url.to_s
+              suffix_date_value(hash[:attributes]) || suffix_date_value(hash[:data])
+            end
+
+            # @param key [Symbol, String] candidate published-at key
+            # @return [Boolean] whether the key ends in a date suffix (_at/_on/At/On)
+            def date_suffix_key?(key)
+              key.to_s.match?(DATE_KEY_SUFFIX)
+            end
+
+            # rubocop:disable Metrics/MethodLength
+            # @param entry [Hash] raw article entry candidate
+            # @return [Array<String>, nil] normalized unique categories
+            def categories(entry)
+              raw = ValueFinder.fetch(entry, CATEGORY_KEYS)
+              names = case raw
+                      when Array then raw
+                      when Hash then raw.values
+                      when String then [raw]
+                      else []
+                      end
+
+              result = names.flat_map do |value|
+                case value
+                when Hash
+                  string(ValueFinder.fetch(value, %i[name title label]))
+                else
+                  string(value)
+                end
+              end.compact
+
+              result.uniq!
+              result unless result.empty?
+            end
+            # rubocop:enable Metrics/MethodLength
+
+            # @param entry [Hash] raw article entry candidate
+            # @param article_url [Html2rss::Url] resolved article URL
+            # @return [String] stable article identifier fallbacking to resolved URL
+            def identifier(entry, article_url)
+              value = ValueFinder.fetch(entry, ID_KEYS)
+              value = ValueFinder.fetch(value, ID_KEYS) if value.is_a?(Hash)
+              string(value) || article_url.to_s
+            end
           end
         end
       end

data/lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb

@@ -6,51 +6,51 @@ module Html2rss
       class JsonState
         # Identifies arrays that look like collections of article hashes.
         module CandidateDetector
-          module_function
-
-          # @param document [Hash, Array, Object] candidate document node
-          # @return [Boolean] whether the node contains article-like arrays
-          def candidate_array?(document)
-            case document
-            when Array
-              return true if array_of_articles?(document)
-
-              document.any? { traversable_candidate?(_1) }
-            when Hash then document.each_value.any? { candidate_array?(_1) }
-            else false
+          class << self
+            # @param document [Hash, Array, Object] candidate document node
+            # @return [Boolean] whether the node contains article-like arrays
+            def candidate_array?(document)
+              case document
+              when Array
+                return true if array_of_articles?(document)
+
+                document.any? { traversable_candidate?(_1) }
+              when Hash then document.each_value.any? { candidate_array?(_1) }
+              else false
+              end
             end
-          end
 
-          # @param value [Hash, Array, Object] candidate nested value
-          # @return [Boolean] whether nested value should be traversed for article candidates
-          def traversable_candidate?(value)
-            case value
-            when Array, Hash then candidate_array?(value)
-            else false
+            # @param value [Hash, Array, Object] candidate nested value
+            # @return [Boolean] whether nested value should be traversed for article candidates
+            def traversable_candidate?(value)
+              case value
+              when Array, Hash then candidate_array?(value)
+              else false
+              end
             end
-          end
 
-          # @param array [Array<Object>] candidate list of entries
-          # @return [Boolean] whether array includes hash entries with title and URL fields
-          def array_of_articles?(array)
-            array.any? do |element|
-              next unless element.is_a?(Hash)
+            # @param array [Array<Object>] candidate list of entries
+            # @return [Boolean] whether array includes hash entries with title and URL fields
+            def array_of_articles?(array)
+              array.any? do |element|
+                next unless element.is_a?(Hash)
 
-              title_from(element) && url_from(element)
+                title_from(element) && url_from(element)
+              end
             end
-          end
 
-          # @param object [Hash] article candidate object
-          # @return [Object, nil] detected title-like value
-          def title_from(object)
-            ValueFinder.fetch(object, ArticleNormalizer::TITLE_KEYS)
-          end
+            # @param object [Hash] article candidate object
+            # @return [Object, nil] detected title-like value
+            def title_from(object)
+              ValueFinder.fetch(object, ArticleNormalizer::TITLE_KEYS)
+            end
 
-          # @param object [Hash] article candidate object
-          # @return [Object, nil] detected URL-like value (including links.self)
-          def url_from(object)
-            ValueFinder.fetch(object, ArticleNormalizer::URL_KEYS) ||
-              ValueFinder.fetch(object[:links], ArticleNormalizer::URL_KEYS)
+            # @param object [Hash] article candidate object
+            # @return [Object, nil] detected URL-like value (including links.self)
+            def url_from(object)
+              ValueFinder.fetch(object, ArticleNormalizer::URL_KEYS) ||
+                ValueFinder.fetch(object[:links], ArticleNormalizer::URL_KEYS)
+            end
           end
         end
       end