html2rss 0.32.4 → 0.33.0

data/lib/html2rss/status.rb

@@ -13,6 +13,10 @@ module Html2rss
   # Stable telemetry payload for cross-repo consumers (e.g. html2rss-web observability).
   # Tallies and counters are validated and frozen at construction (including Marshal load).
   class Status
+    # Keys kept in {#to_h} even when zero or empty.
+    ALWAYS_REPORTED = %i[version dedup_dropped].freeze
+    private_constant :ALWAYS_REPORTED
+
     class << self
       ##
       # Builds status from extracted articles and scrape telemetry.
@@ -71,9 +75,9 @@ module Html2rss
       version:, scraper_tallies:, dedup_dropped:, selected_strategy: nil, attempt_count: 0,
       strategy_attempts: [], admission_drops: {}, entry_url: nil, scrape_url: nil, entry_resolution: nil
     )
-      dedup = Integer(dedup_dropped)
-      attempts = Integer(attempt_count)
-      validate_counters!(dedup:, attempts:, selected_strategy:)
+      dedup = Option.integer(:dedup_dropped, dedup_dropped, min: 0)
+      attempts = Option.integer(:attempt_count, attempt_count, min: 0)
+      validate_selected_strategy!(selected_strategy, attempts)
 
       super(
         version: version.to_s.dup.freeze,
@@ -93,20 +97,8 @@ module Html2rss
     # Observability hash for web (+scraper_status+). Omits empty/absent optional keys.
     #
     # @return [Hash{Symbol => Object}]
-    # rubocop:disable-next Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity -- omit-empty optional keys stay explicit
     def to_h
-      {
-        version:,
-        dedup_dropped:,
-        **(scraper_tallies.any? ? { scraper_tallies: } : {}),
-        **(selected_strategy.nil? ? {} : { selected_strategy: }),
-        **(attempt_count.positive? ? { attempt_count: } : {}),
-        **(strategy_attempts.any? ? { strategy_attempts: } : {}),
-        **(admission_drops.any? ? { admission_drops: } : {}),
-        **(entry_url ? { entry_url: } : {}),
-        **(scrape_url ? { scrape_url: } : {}),
-        **(entry_resolution ? { entry_resolution: } : {})
-      }
+      super.reject { |key, value| !ALWAYS_REPORTED.include?(key) && omit?(value) }
     end
 
     ##
@@ -124,6 +116,14 @@ module Html2rss
 
     private
 
+    def omit?(value)
+      case value
+      when nil then true
+      when Integer then value.zero?
+      else value.empty?
+      end
+    end
+
     def marshal_dump
       [version, scraper_tallies, dedup_dropped, selected_strategy, attempt_count, strategy_attempts,
        admission_drops, entry_url, scrape_url, entry_resolution]
@@ -133,16 +133,16 @@ module Html2rss
       version, scraper_tallies, dedup_dropped, selected_strategy, attempt_count,
         strategy_attempts, admission_drops, entry_url, scrape_url, entry_resolution = payload
       initialize(version:, scraper_tallies:, dedup_dropped:, selected_strategy:, attempt_count:,
-                 strategy_attempts:, admission_drops: admission_drops || {}, entry_url:, scrape_url:,
+                 strategy_attempts:, admission_drops:, entry_url:, scrape_url:,
                  entry_resolution:)
     end
 
     def freeze_tallies(tallies)
-      tallies.to_h.transform_keys(&:to_s).transform_values { |count| Integer(count) }.freeze
+      tallies.to_h { |name, count| [name.to_s, count] }.freeze
     end
 
     def freeze_attempts(attempts)
-      Array(attempts).map { |attempt| attempt.to_h.freeze }.freeze
+      attempts.map { |attempt| attempt.dup.freeze }.freeze
     end
 
     def freeze_optional_string(value)
@@ -161,17 +161,10 @@ module Html2rss
       end.transform_keys(&:to_sym).freeze
     end
 
-    def validate_counters!(dedup:, attempts:, selected_strategy:)
-      raise ArgumentError, 'dedup_dropped must be >= 0' if dedup.negative?
-      raise ArgumentError, 'attempt_count must be >= 0' if attempts.negative?
-
-      unless selected_strategy.nil? || selected_strategy.is_a?(Symbol)
-        raise ArgumentError, 'selected_strategy must be a Symbol or nil'
-      end
-
-      return unless selected_strategy && attempts < 1
-
-      raise ArgumentError, 'attempt_count must be >= 1 when selected_strategy is set'
+    def validate_selected_strategy!(selected_strategy, attempts)
+      return if selected_strategy.nil?
+      raise ArgumentError, 'selected_strategy must be a Symbol or nil' unless selected_strategy.is_a?(Symbol)
+      raise ArgumentError, 'attempt_count must be >= 1 when selected_strategy is set' if attempts < 1
     end
   end
 end

data/lib/html2rss/syndication/alternate_links.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Syndication
+    ##
+    # Scans raw HTML for +<link rel="alternate">+ hints that point at RSS/Atom feeds.
+    module AlternateLinks
+      LINK_TAG_RE = /<link\b[^>]*>/i
+      HREF_RE = /\bhref\s*=\s*["']([^"']+)["']/i
+      TYPE_RE = /\btype\s*=\s*["']([^"']+)["']/i
+      REL_RE = /\brel\s*=\s*["']([^"']+)["']/i
+      private_constant :LINK_TAG_RE, :HREF_RE, :TYPE_RE, :REL_RE
+
+      class << self
+        ##
+        # @param html [String] raw HTML
+        # @return [Array<String>] hrefs of alternate links that look like feeds
+        def hrefs(html)
+          html.scan(LINK_TAG_RE).filter_map { |tag| feed_href(tag) }
+        end
+
+        private
+
+        def feed_href(tag)
+          return unless tag[REL_RE, 1].to_s.downcase.split(/\s+/).include?('alternate')
+
+          href = tag[HREF_RE, 1]
+          return if href.nil? || href.empty?
+
+          href.strip if syndication_type?(tag[TYPE_RE, 1]) || href.match?(/rss|atom|feed|\.xml/i)
+        end
+
+        def syndication_type?(type)
+          type.to_s.downcase.match?(/rss|atom/)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/syndication/discovery.rb

@@ -6,214 +6,144 @@ module Html2rss
     # Finds a same-origin RSS/Atom URL for a page via head alternates and path guesses.
     #
     # Ports configs +probe_rss+ path/alternate logic onto {RequestSession#follow_up}.
-    module Discovery # rubocop:disable Metrics/ModuleLength -- discovery + path probes stay co-located
+    module Discovery
       # Common feed path suffixes probed after head +rel=alternate+ hints.
       DEFAULT_PATHS = CandidateCatalog::FEED_PATHS
 
-      LINK_TAG_RE = /<link\b[^>]*>/i
-      HREF_RE = /\bhref\s*=\s*["']([^"']+)["']/i
-      TYPE_RE = /\btype\s*=\s*["']([^"']+)["']/i
-      REL_RE = /\brel\s*=\s*["']([^"']+)["']/i
-      private_constant :LINK_TAG_RE, :HREF_RE, :TYPE_RE, :REL_RE
-
-      module_function
-
-      ##
-      # Probes candidates lazily and returns the first feedish URL.
-      #
-      # @param page_url [String, Html2rss::Url] page or origin URL
-      # @param request_session [Html2rss::RequestSession] shared session for follow-ups
-      # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML when available
-      # @param html [String, nil] raw HTML when a document is unavailable
-      # @param extra_paths [Array<String>] additional path guesses
-      # @param max_probes [Integer, nil] max candidate GETs (nil = uncapped)
-      # @return [Html2rss::Url, nil]
-      # rubocop:disable-next Metrics/ParameterLists -- discovery kwargs stay co-located
-      def best_feed_url(page_url:, request_session:, parsed_body: nil, html: nil, extra_paths: [],
-                        max_probes: nil)
-        best_feed_response(
-          page_url:, request_session:, parsed_body:, html:, extra_paths:, max_probes:
-        )&.url
-      end
-
-      ##
-      # Like {#best_feed_url} but returns the validated syndication response for parsing.
-      #
-      # @param page_url [String, Html2rss::Url]
-      # @param request_session [Html2rss::RequestSession]
-      # @param parsed_body [Nokogiri::HTML::Document, nil]
-      # @param html [String, nil]
-      # @param extra_paths [Array<String>]
-      # @param max_probes [Integer, nil] max candidate GETs (nil = uncapped)
-      # @return [Html2rss::RequestService::Response, nil]
-      # rubocop:disable-next Metrics/ParameterLists, Metrics/MethodLength -- discovery kwargs stay co-located
-      def best_feed_response(page_url:, request_session:, parsed_body: nil, html: nil, extra_paths: [],
-                             max_probes: nil)
-        page = Html2rss::Url.from_absolute(page_url)
-        candidates = candidate_urls(page_url: page, parsed_body:, html:, extra_paths:)
-        Log.debug(
-          "Syndication::Discovery: host=#{page.host} candidate_count=#{candidates.size}"
-        )
-
-        first_feedish_response(
-          candidates, request_session:, origin_url: page, max_probes:
-        ).tap do |response|
-          next unless response
-
-          Log.info({
-            event_name: 'syndication.discovery.selected',
-            host: page.host,
-            message: "Syndication::Discovery: host=#{page.host} selected_feed_url=#{response.url}"
-          }.to_json)
+      class << self
+        ##
+        # Probes candidates lazily and returns the first feedish URL.
+        #
+        # @param page_url [String, Html2rss::Url] page or origin URL
+        # @param request_session [Html2rss::RequestSession] shared session for follow-ups
+        # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML when available
+        # @param html [String, nil] raw HTML when a document is unavailable
+        # @param extra_paths [Array<String>] additional path guesses
+        # @param max_probes [Integer, nil] max candidate GETs (nil = uncapped)
+        # @return [Html2rss::Url, nil]
+        # rubocop:disable-next Metrics/ParameterLists -- discovery kwargs stay co-located
+        def best_feed_url(page_url:, request_session:, parsed_body: nil, html: nil, extra_paths: [],
+                          max_probes: nil)
+          best_feed_response(
+            page_url:, request_session:, parsed_body:, html:, extra_paths:, max_probes:
+          )&.url
         end
-      end
-
-      ##
-      # Ordered candidate feed URLs (head alternates first, then path guesses).
-      #
-      # @param page_url [String, Html2rss::Url]
-      # @param parsed_body [Nokogiri::HTML::Document, nil]
-      # @param html [String, nil]
-      # @param extra_paths [Array<String>]
-      # @return [Array<Html2rss::Url>]
-      def candidate_urls(page_url:, parsed_body: nil, html: nil, extra_paths: [])
-        page = Html2rss::Url.from_absolute(page_url)
-        (alternate_feed_urls(page, parsed_body:, html:) +
-          path_guess_urls(page, extra_paths:)).uniq
-      end
 
-      ##
-      # @param response [Html2rss::RequestService::Response, nil]
-      # @return [Boolean]
-      def feedish?(response)
-        return false unless response
-        return false unless successful_status?(response)
-
-        response.feed_response?
-      end
-
-      def successful_status?(response)
-        status = response.status
-        status.nil? || status.between?(200, 299)
-      end
-      module_function :successful_status?
-      private_class_method :successful_status?
-
-      ##
-      # Directory-relative feed path guesses for a page URL.
-      #
-      # @param page_url [String, Html2rss::Url]
-      # @return [Array<String>] relative paths
-      def page_dir_paths(page_url)
-        path = Html2rss::Url.from_absolute(page_url).path
-        return [] if path.nil? || path.empty? || path == '/'
-
-        dir = path.sub(%r{/[^/]*$}, '/')
-        ["#{dir}feed", "#{dir}rss.xml", "#{dir}atom.xml"]
-      end
-
-      def first_feedish_response(urls, request_session:, origin_url:, max_probes: nil)
-        scoped = max_probes.nil? ? urls.lazy : urls.lazy.take(max_probes)
-        scoped.filter_map do |url|
-          response = probe(url, request_session:, origin_url:)
-          response if feedish?(response)
-        end.first
-      end
-      module_function :first_feedish_response
-      private_class_method :first_feedish_response
-
-      def first_feedish_url(urls, request_session:, origin_url:, max_probes: nil)
-        first_feedish_response(urls, request_session:, origin_url:, max_probes:)&.url
-      end
-      module_function :first_feedish_url
-      private_class_method :first_feedish_url
-
-      def probe(url, request_session:, origin_url:)
-        request_session.follow_up(url:, relation: :auto_source, origin_url:)
-      rescue Html2rss::Error => error
-        Log.debug(
-          "Syndication::Discovery: probe skipped host=#{origin_url.host} " \
-          "(#{error.class})"
-        )
-        nil
-      end
-      module_function :probe
-      private_class_method :probe
-
-      def alternate_feed_urls(page, parsed_body:, html:)
-        from_document = feed_links_from_document(parsed_body, page)
-        return from_document if from_document.any?
+        ##
+        # Like {#best_feed_url} but returns the validated syndication response for parsing.
+        #
+        # @param page_url [String, Html2rss::Url]
+        # @param request_session [Html2rss::RequestSession]
+        # @param parsed_body [Nokogiri::HTML::Document, nil]
+        # @param html [String, nil]
+        # @param extra_paths [Array<String>]
+        # @param max_probes [Integer, nil] max candidate GETs (nil = uncapped)
+        # @return [Html2rss::RequestService::Response, nil]
+        # rubocop:disable-next Metrics/ParameterLists, Metrics/MethodLength -- discovery kwargs stay co-located
+        def best_feed_response(page_url:, request_session:, parsed_body: nil, html: nil, extra_paths: [],
+                               max_probes: nil)
+          page = Html2rss::Url.from_absolute(page_url)
+          candidates = candidate_urls(page_url: page, parsed_body:, html:, extra_paths:)
+          Log.debug(
+            "Syndication::Discovery: host=#{page.host} candidate_count=#{candidates.size}"
+          )
+
+          first_feedish_response(
+            candidates, request_session:, origin_url: page, max_probes:
+          ).tap do |response|
+            next unless response
+
+            Log.info({
+              event_name: 'syndication.discovery.selected',
+              host: page.host,
+              message: "Syndication::Discovery: host=#{page.host} selected_feed_url=#{response.url}"
+            }.to_json)
+          end
+        end
 
-        alternate_feed_hrefs(html.to_s, page)
-      end
-      module_function :alternate_feed_urls
-      private_class_method :alternate_feed_urls
+        ##
+        # Ordered candidate feed URLs (head alternates first, then path guesses).
+        #
+        # @param page_url [String, Html2rss::Url]
+        # @param parsed_body [Nokogiri::HTML::Document, nil]
+        # @param html [String, nil]
+        # @param extra_paths [Array<String>]
+        # @return [Array<Html2rss::Url>]
+        def candidate_urls(page_url:, parsed_body: nil, html: nil, extra_paths: [])
+          page = Html2rss::Url.from_absolute(page_url)
+          (alternate_feed_urls(page, parsed_body:, html:) +
+            path_guess_urls(page, extra_paths:)).uniq
+        end
 
-      def feed_links_from_document(parsed_body, page)
-        return [] unless parsed_body.is_a?(Nokogiri::HTML::Document)
+        ##
+        # @param response [Html2rss::RequestService::Response, nil]
+        # @return [Boolean]
+        def feedish?(response)
+          return false unless response
+          return false unless successful_status?(response)
 
-        Html::FeedLink.from_document(parsed_body).filter_map do |link|
-          absolute_url(page, link.href)
+          response.feed_response?
         end
-      end
-      module_function :feed_links_from_document
-      private_class_method :feed_links_from_document
 
-      def path_guess_urls(page, extra_paths:)
-        paths = (page_dir_paths(page) + DEFAULT_PATHS + Array(extra_paths)).uniq
-        paths.filter_map { |path| absolute_url(page, path) }
-      end
-      module_function :path_guess_urls
-      private_class_method :path_guess_urls
+        ##
+        # Directory-relative feed path guesses for a page URL.
+        #
+        # @param page_url [String, Html2rss::Url]
+        # @return [Array<String>] relative paths
+        def page_dir_paths(page_url)
+          path = Html2rss::Url.from_absolute(page_url).path
+          return [] if path == '/'
+
+          dir = path.sub(%r{/[^/]*$}, '/')
+          ["#{dir}feed", "#{dir}rss.xml", "#{dir}atom.xml"]
+        end
 
-      def alternate_feed_hrefs(html, page)
-        return [] if html.nil? || html.empty?
+        private
 
-        html.scan(LINK_TAG_RE).filter_map { |tag| href_from_alternate_link(tag, page) }.uniq
-      end
-      module_function :alternate_feed_hrefs
-      private_class_method :alternate_feed_hrefs
+        def successful_status?(response)
+          status = response.status
+          status.nil? || status.between?(200, 299)
+        end
 
-      def href_from_alternate_link(tag, page)
-        return unless alternate_rel?(tag)
+        def first_feedish_response(urls, request_session:, origin_url:, max_probes: nil)
+          scoped = max_probes.nil? ? urls.lazy : urls.lazy.take(max_probes)
+          scoped.filter_map do |url|
+            response = probe(url, request_session:, origin_url:)
+            response if feedish?(response)
+          end.first
+        end
 
-        href = tag[HREF_RE, 1]
-        return if href.nil? || href.empty?
-        return unless syndication_type?(tag[TYPE_RE, 1]) || feed_like_href?(href)
+        def probe(url, request_session:, origin_url:)
+          request_session.follow_up(url:, relation: :auto_source, origin_url:)
+        rescue Html2rss::Error => error
+          Log.debug("Syndication::Discovery: probe skipped host=#{origin_url.host} (#{error.class})")
+          nil
+        end
 
-        absolute_url(page, href.strip)
-      end
-      module_function :href_from_alternate_link
-      private_class_method :href_from_alternate_link
+        def alternate_feed_urls(page, parsed_body:, html:)
+          from_document = feed_links_from_document(parsed_body, page)
+          return from_document if from_document.any?
 
-      def alternate_rel?(tag)
-        tag[REL_RE, 1].to_s.downcase.split(/\s+/).include?('alternate')
-      end
-      module_function :alternate_rel?
-      private_class_method :alternate_rel?
+          AlternateLinks.hrefs(html.to_s).filter_map { |href| absolute_url(page, href) }.uniq
+        end
 
-      def syndication_type?(type)
-        return false if type.nil? || type.empty?
+        def feed_links_from_document(parsed_body, page)
+          return [] unless parsed_body.is_a?(Nokogiri::HTML::Document)
 
-        t = type.downcase
-        t.include?('rss+xml') || t.include?('atom+xml') || t.include?('rss') || t.include?('atom')
-      end
-      module_function :syndication_type?
-      private_class_method :syndication_type?
+          Html::FeedLink.from_document(parsed_body).filter_map { |link| absolute_url(page, link.href) }
+        end
 
-      def feed_like_href?(href)
-        href.match?(/rss|atom|feed|\.xml/i)
-      end
-      module_function :feed_like_href?
-      private_class_method :feed_like_href?
+        def path_guess_urls(page, extra_paths:)
+          paths = (page_dir_paths(page) + DEFAULT_PATHS + Array(extra_paths)).uniq
+          paths.filter_map { |path| absolute_url(page, path) }
+        end
 
-      def absolute_url(base, path)
-        Html2rss::Url.from_relative(path, base)
-      rescue ArgumentError
-        nil
+        def absolute_url(base, path)
+          Html2rss::Url.from_relative(path, base)
+        rescue Html2rss::Url::InvalidUrlError
+          nil
+        end
       end
-      module_function :absolute_url
-      private_class_method :absolute_url
     end
   end
 end