html2rss 0.32.3 → 0.33.0

data/lib/html2rss/syndication/discovery.rb

@@ -6,214 +6,144 @@ module Html2rss
     # Finds a same-origin RSS/Atom URL for a page via head alternates and path guesses.
     #
     # Ports configs +probe_rss+ path/alternate logic onto {RequestSession#follow_up}.
-    module Discovery # rubocop:disable Metrics/ModuleLength -- discovery + path probes stay co-located
+    module Discovery
       # Common feed path suffixes probed after head +rel=alternate+ hints.
       DEFAULT_PATHS = CandidateCatalog::FEED_PATHS
 
-      LINK_TAG_RE = /<link\b[^>]*>/i
-      HREF_RE = /\bhref\s*=\s*["']([^"']+)["']/i
-      TYPE_RE = /\btype\s*=\s*["']([^"']+)["']/i
-      REL_RE = /\brel\s*=\s*["']([^"']+)["']/i
-      private_constant :LINK_TAG_RE, :HREF_RE, :TYPE_RE, :REL_RE
-
-      module_function
-
-      ##
-      # Probes candidates lazily and returns the first feedish URL.
-      #
-      # @param page_url [String, Html2rss::Url] page or origin URL
-      # @param request_session [Html2rss::RequestSession] shared session for follow-ups
-      # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML when available
-      # @param html [String, nil] raw HTML when a document is unavailable
-      # @param extra_paths [Array<String>] additional path guesses
-      # @param max_probes [Integer, nil] max candidate GETs (nil = uncapped)
-      # @return [Html2rss::Url, nil]
-      # rubocop:disable-next Metrics/ParameterLists -- discovery kwargs stay co-located
-      def best_feed_url(page_url:, request_session:, parsed_body: nil, html: nil, extra_paths: [],
-                        max_probes: nil)
-        best_feed_response(
-          page_url:, request_session:, parsed_body:, html:, extra_paths:, max_probes:
-        )&.url
-      end
-
-      ##
-      # Like {#best_feed_url} but returns the validated syndication response for parsing.
-      #
-      # @param page_url [String, Html2rss::Url]
-      # @param request_session [Html2rss::RequestSession]
-      # @param parsed_body [Nokogiri::HTML::Document, nil]
-      # @param html [String, nil]
-      # @param extra_paths [Array<String>]
-      # @param max_probes [Integer, nil] max candidate GETs (nil = uncapped)
-      # @return [Html2rss::RequestService::Response, nil]
-      # rubocop:disable-next Metrics/ParameterLists, Metrics/MethodLength -- discovery kwargs stay co-located
-      def best_feed_response(page_url:, request_session:, parsed_body: nil, html: nil, extra_paths: [],
-                             max_probes: nil)
-        page = Html2rss::Url.from_absolute(page_url)
-        candidates = candidate_urls(page_url: page, parsed_body:, html:, extra_paths:)
-        Log.debug(
-          "Syndication::Discovery: host=#{page.host} candidate_count=#{candidates.size}"
-        )
-
-        first_feedish_response(
-          candidates, request_session:, origin_url: page, max_probes:
-        ).tap do |response|
-          next unless response
-
-          Log.info({
-            event_name: 'syndication.discovery.selected',
-            host: page.host,
-            message: "Syndication::Discovery: host=#{page.host} selected_feed_url=#{response.url}"
-          }.to_json)
+      class << self
+        ##
+        # Probes candidates lazily and returns the first feedish URL.
+        #
+        # @param page_url [String, Html2rss::Url] page or origin URL
+        # @param request_session [Html2rss::RequestSession] shared session for follow-ups
+        # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML when available
+        # @param html [String, nil] raw HTML when a document is unavailable
+        # @param extra_paths [Array<String>] additional path guesses
+        # @param max_probes [Integer, nil] max candidate GETs (nil = uncapped)
+        # @return [Html2rss::Url, nil]
+        # rubocop:disable-next Metrics/ParameterLists -- discovery kwargs stay co-located
+        def best_feed_url(page_url:, request_session:, parsed_body: nil, html: nil, extra_paths: [],
+                          max_probes: nil)
+          best_feed_response(
+            page_url:, request_session:, parsed_body:, html:, extra_paths:, max_probes:
+          )&.url
         end
-      end
-
-      ##
-      # Ordered candidate feed URLs (head alternates first, then path guesses).
-      #
-      # @param page_url [String, Html2rss::Url]
-      # @param parsed_body [Nokogiri::HTML::Document, nil]
-      # @param html [String, nil]
-      # @param extra_paths [Array<String>]
-      # @return [Array<Html2rss::Url>]
-      def candidate_urls(page_url:, parsed_body: nil, html: nil, extra_paths: [])
-        page = Html2rss::Url.from_absolute(page_url)
-        (alternate_feed_urls(page, parsed_body:, html:) +
-          path_guess_urls(page, extra_paths:)).uniq
-      end
 
-      ##
-      # @param response [Html2rss::RequestService::Response, nil]
-      # @return [Boolean]
-      def feedish?(response)
-        return false unless response
-        return false unless successful_status?(response)
-
-        response.feed_response?
-      end
-
-      def successful_status?(response)
-        status = response.status
-        status.nil? || status.between?(200, 299)
-      end
-      module_function :successful_status?
-      private_class_method :successful_status?
-
-      ##
-      # Directory-relative feed path guesses for a page URL.
-      #
-      # @param page_url [String, Html2rss::Url]
-      # @return [Array<String>] relative paths
-      def page_dir_paths(page_url)
-        path = Html2rss::Url.from_absolute(page_url).path
-        return [] if path.nil? || path.empty? || path == '/'
-
-        dir = path.sub(%r{/[^/]*$}, '/')
-        ["#{dir}feed", "#{dir}rss.xml", "#{dir}atom.xml"]
-      end
-
-      def first_feedish_response(urls, request_session:, origin_url:, max_probes: nil)
-        scoped = max_probes.nil? ? urls.lazy : urls.lazy.take(max_probes)
-        scoped.filter_map do |url|
-          response = probe(url, request_session:, origin_url:)
-          response if feedish?(response)
-        end.first
-      end
-      module_function :first_feedish_response
-      private_class_method :first_feedish_response
-
-      def first_feedish_url(urls, request_session:, origin_url:, max_probes: nil)
-        first_feedish_response(urls, request_session:, origin_url:, max_probes:)&.url
-      end
-      module_function :first_feedish_url
-      private_class_method :first_feedish_url
-
-      def probe(url, request_session:, origin_url:)
-        request_session.follow_up(url:, relation: :auto_source, origin_url:)
-      rescue Html2rss::Error => error
-        Log.debug(
-          "Syndication::Discovery: probe skipped host=#{origin_url.host} " \
-          "(#{error.class})"
-        )
-        nil
-      end
-      module_function :probe
-      private_class_method :probe
-
-      def alternate_feed_urls(page, parsed_body:, html:)
-        from_document = feed_links_from_document(parsed_body, page)
-        return from_document if from_document.any?
+        ##
+        # Like {#best_feed_url} but returns the validated syndication response for parsing.
+        #
+        # @param page_url [String, Html2rss::Url]
+        # @param request_session [Html2rss::RequestSession]
+        # @param parsed_body [Nokogiri::HTML::Document, nil]
+        # @param html [String, nil]
+        # @param extra_paths [Array<String>]
+        # @param max_probes [Integer, nil] max candidate GETs (nil = uncapped)
+        # @return [Html2rss::RequestService::Response, nil]
+        # rubocop:disable-next Metrics/ParameterLists, Metrics/MethodLength -- discovery kwargs stay co-located
+        def best_feed_response(page_url:, request_session:, parsed_body: nil, html: nil, extra_paths: [],
+                               max_probes: nil)
+          page = Html2rss::Url.from_absolute(page_url)
+          candidates = candidate_urls(page_url: page, parsed_body:, html:, extra_paths:)
+          Log.debug(
+            "Syndication::Discovery: host=#{page.host} candidate_count=#{candidates.size}"
+          )
+
+          first_feedish_response(
+            candidates, request_session:, origin_url: page, max_probes:
+          ).tap do |response|
+            next unless response
+
+            Log.info({
+              event_name: 'syndication.discovery.selected',
+              host: page.host,
+              message: "Syndication::Discovery: host=#{page.host} selected_feed_url=#{response.url}"
+            }.to_json)
+          end
+        end
 
-        alternate_feed_hrefs(html.to_s, page)
-      end
-      module_function :alternate_feed_urls
-      private_class_method :alternate_feed_urls
+        ##
+        # Ordered candidate feed URLs (head alternates first, then path guesses).
+        #
+        # @param page_url [String, Html2rss::Url]
+        # @param parsed_body [Nokogiri::HTML::Document, nil]
+        # @param html [String, nil]
+        # @param extra_paths [Array<String>]
+        # @return [Array<Html2rss::Url>]
+        def candidate_urls(page_url:, parsed_body: nil, html: nil, extra_paths: [])
+          page = Html2rss::Url.from_absolute(page_url)
+          (alternate_feed_urls(page, parsed_body:, html:) +
+            path_guess_urls(page, extra_paths:)).uniq
+        end
 
-      def feed_links_from_document(parsed_body, page)
-        return [] unless parsed_body.is_a?(Nokogiri::HTML::Document)
+        ##
+        # @param response [Html2rss::RequestService::Response, nil]
+        # @return [Boolean]
+        def feedish?(response)
+          return false unless response
+          return false unless successful_status?(response)
 
-        Html::FeedLink.from_document(parsed_body).filter_map do |link|
-          absolute_url(page, link.href)
+          response.feed_response?
         end
-      end
-      module_function :feed_links_from_document
-      private_class_method :feed_links_from_document
 
-      def path_guess_urls(page, extra_paths:)
-        paths = (page_dir_paths(page) + DEFAULT_PATHS + Array(extra_paths)).uniq
-        paths.filter_map { |path| absolute_url(page, path) }
-      end
-      module_function :path_guess_urls
-      private_class_method :path_guess_urls
+        ##
+        # Directory-relative feed path guesses for a page URL.
+        #
+        # @param page_url [String, Html2rss::Url]
+        # @return [Array<String>] relative paths
+        def page_dir_paths(page_url)
+          path = Html2rss::Url.from_absolute(page_url).path
+          return [] if path == '/'
+
+          dir = path.sub(%r{/[^/]*$}, '/')
+          ["#{dir}feed", "#{dir}rss.xml", "#{dir}atom.xml"]
+        end
 
-      def alternate_feed_hrefs(html, page)
-        return [] if html.nil? || html.empty?
+        private
 
-        html.scan(LINK_TAG_RE).filter_map { |tag| href_from_alternate_link(tag, page) }.uniq
-      end
-      module_function :alternate_feed_hrefs
-      private_class_method :alternate_feed_hrefs
+        def successful_status?(response)
+          status = response.status
+          status.nil? || status.between?(200, 299)
+        end
 
-      def href_from_alternate_link(tag, page)
-        return unless alternate_rel?(tag)
+        def first_feedish_response(urls, request_session:, origin_url:, max_probes: nil)
+          scoped = max_probes.nil? ? urls.lazy : urls.lazy.take(max_probes)
+          scoped.filter_map do |url|
+            response = probe(url, request_session:, origin_url:)
+            response if feedish?(response)
+          end.first
+        end
 
-        href = tag[HREF_RE, 1]
-        return if href.nil? || href.empty?
-        return unless syndication_type?(tag[TYPE_RE, 1]) || feed_like_href?(href)
+        def probe(url, request_session:, origin_url:)
+          request_session.follow_up(url:, relation: :auto_source, origin_url:)
+        rescue Html2rss::Error => error
+          Log.debug("Syndication::Discovery: probe skipped host=#{origin_url.host} (#{error.class})")
+          nil
+        end
 
-        absolute_url(page, href.strip)
-      end
-      module_function :href_from_alternate_link
-      private_class_method :href_from_alternate_link
+        def alternate_feed_urls(page, parsed_body:, html:)
+          from_document = feed_links_from_document(parsed_body, page)
+          return from_document if from_document.any?
 
-      def alternate_rel?(tag)
-        tag[REL_RE, 1].to_s.downcase.split(/\s+/).include?('alternate')
-      end
-      module_function :alternate_rel?
-      private_class_method :alternate_rel?
+          AlternateLinks.hrefs(html.to_s).filter_map { |href| absolute_url(page, href) }.uniq
+        end
 
-      def syndication_type?(type)
-        return false if type.nil? || type.empty?
+        def feed_links_from_document(parsed_body, page)
+          return [] unless parsed_body.is_a?(Nokogiri::HTML::Document)
 
-        t = type.downcase
-        t.include?('rss+xml') || t.include?('atom+xml') || t.include?('rss') || t.include?('atom')
-      end
-      module_function :syndication_type?
-      private_class_method :syndication_type?
+          Html::FeedLink.from_document(parsed_body).filter_map { |link| absolute_url(page, link.href) }
+        end
 
-      def feed_like_href?(href)
-        href.match?(/rss|atom|feed|\.xml/i)
-      end
-      module_function :feed_like_href?
-      private_class_method :feed_like_href?
+        def path_guess_urls(page, extra_paths:)
+          paths = (page_dir_paths(page) + DEFAULT_PATHS + Array(extra_paths)).uniq
+          paths.filter_map { |path| absolute_url(page, path) }
+        end
 
-      def absolute_url(base, path)
-        Html2rss::Url.from_relative(path, base)
-      rescue ArgumentError
-        nil
+        def absolute_url(base, path)
+          Html2rss::Url.from_relative(path, base)
+        rescue Html2rss::Url::InvalidUrlError
+          nil
+        end
       end
-      module_function :absolute_url
-      private_class_method :absolute_url
     end
   end
 end

data/lib/html2rss/syndication/parser.rb

@@ -6,136 +6,113 @@ module Html2rss
   module Syndication
     ##
     # Parses RSS 2.0 and Atom documents into AutoSource article hashes.
-    module Parser # rubocop:disable Metrics/ModuleLength -- feed formats + structured parse warn
-      module_function
-
-      ##
-      # @param response [Html2rss::RequestService::Response]
-      # @return [Array<Hash{Symbol => Object}>] article hashes (may be empty)
-      # @raise [ArgumentError] when the response is not a syndication feed
-      def parse_response(response)
-        raise ArgumentError, 'response is not a syndication feed' unless response.feed_response?
+    module Parser
+      class << self
+        ##
+        # @param response [Html2rss::RequestService::Response]
+        # @return [Array<Hash{Symbol => Object}>] article hashes (may be empty)
+        # @raise [ArgumentError] when the response is not a syndication feed
+        def parse_response(response)
+          raise ArgumentError, 'response is not a syndication feed' unless response.feed_response?
+
+          parse(response.body, base_url: response.url)
+        end
 
-        parse(response.body, base_url: response.url)
-      end
+        ##
+        # @param body [String] raw RSS/Atom document
+        # @param base_url [String, Html2rss::Url, nil] fallback base for relative item links
+        # @return [Array<Hash{Symbol => Object}>]
+        def parse(body, base_url: nil)
+          feed = RSS::Parser.parse(body.to_s, false)
+
+          items_from(feed).filter_map { |item| article_hash(item, base_url:) }
+        rescue RSS::Error => error
+          Log.warn({
+            event_name: 'syndication.parse_failed',
+            error_class: error.class.name,
+            message: "Syndication::Parser: failed to parse feed (#{error.class}: #{error.message})"
+          }.to_json)
+          []
+        end
 
-      ##
-      # @param body [String] raw RSS/Atom document
-      # @param base_url [String, Html2rss::Url, nil] fallback base for relative item links
-      # @return [Array<Hash{Symbol => Object}>]
-      def parse(body, base_url: nil)
-        feed = RSS::Parser.parse(body.to_s, false)
-        return [] unless feed
-
-        items_from(feed, base_url:).filter_map { |item| article_hash(item, base_url:) }
-      rescue RSS::Error => error
-        Log.warn({
-          event_name: 'syndication.parse_failed',
-          error_class: error.class.name,
-          message: "Syndication::Parser: failed to parse feed (#{error.class}: #{error.message})"
-        }.to_json)
-        []
-      end
+        private
 
-      def items_from(feed, base_url:) # rubocop:disable Lint/UnusedMethodArgument -- base reserved for callers
-        case feed
-        when RSS::Rss then Array(feed.items)
-        when RSS::Atom::Feed then Array(feed.entries)
-        else []
+        def items_from(feed)
+          case feed
+          when RSS::Rss then feed.items
+          when RSS::Atom::Feed then feed.entries
+          else []
+          end
         end
-      end
-      module_function :items_from
-      private_class_method :items_from
 
-      def article_hash(item, base_url:)
-        case item
-        when RSS::Rss::Channel::Item then rss_item_hash(item, base_url:)
-        when RSS::Atom::Entry, RSS::Atom::Feed::Entry then atom_entry_hash(item, base_url:)
+        def article_hash(item, base_url:)
+          case item
+          when RSS::Rss::Channel::Item then rss_item_hash(item, base_url:)
+          when RSS::Atom::Feed::Entry then atom_entry_hash(item, base_url:)
+          end
         end
-      end
-      module_function :article_hash
-      private_class_method :article_hash
-
-      def rss_item_hash(item, base_url:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- RSS item mapping
-        url = absolute_url(item.link, base_url:)
-        return unless url
-
-        {
-          id: present(item.guid&.content) || url.to_s,
-          title: present(item.title),
-          description: present(item.description),
-          url:,
-          published_at: item.date || item.pubDate,
-          author: present(item.author),
-          categories: Array(item.categories).filter_map { |cat| present(cat.content) }
-        }.compact
-      end
-      module_function :rss_item_hash
-      private_class_method :rss_item_hash
-
-      def atom_entry_hash(entry, base_url:) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity -- Atom entry mapping
-        url = atom_link(entry, base_url:)
-        return unless url
-
-        {
-          id: present(entry.id&.content) || url.to_s,
-          title: present(atom_text(entry.title)),
-          description: present(atom_text(entry.content) || atom_text(entry.summary)),
-          url:,
-          published_at: entry.published&.content || entry.updated&.content,
-          author: present(atom_author(entry)),
-          categories: Array(entry.categories).filter_map { |cat| present(cat.term) }
-        }.compact
-      end
-      module_function :atom_entry_hash
-      private_class_method :atom_entry_hash
 
-      def atom_link(entry, base_url:)
-        links = Array(entry.links)
-        link = links.find { |node| node.rel.nil? || node.rel == 'alternate' } || links.first
-        absolute_url(link&.href, base_url:)
-      end
-      module_function :atom_link
-      private_class_method :atom_link
+        def rss_item_hash(item, base_url:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- RSS item mapping
+          url = absolute_url(item.link, base_url:)
+          return unless url
+
+          {
+            id: present(item.guid&.content) || url.to_s,
+            title: present(item.title),
+            description: present(item.description),
+            url:,
+            published_at: item.pubDate,
+            author: present(item.author),
+            categories: item.categories.filter_map { |cat| present(cat.content) }
+          }.compact
+        end
 
-      def atom_text(node)
-        return unless node.is_a?(RSS::Atom::TextConstruct) || node.is_a?(RSS::Atom::Feed::Entry::Content)
+        def atom_entry_hash(entry, base_url:) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity -- Atom entry mapping
+          url = atom_link(entry, base_url:)
+          return unless url
+
+          {
+            id: present(entry.id&.content) || url.to_s,
+            title: present(entry.title&.content),
+            description: present(entry.content&.content || entry.summary&.content),
+            url:,
+            published_at: entry.published&.content || entry.updated&.content,
+            author: atom_author(entry),
+            categories: entry.categories.filter_map { |cat| present(cat.term) }
+          }.compact
+        end
 
-        node.content
-      end
-      module_function :atom_text
-      private_class_method :atom_text
+        def atom_link(entry, base_url:)
+          links = entry.links
+          link = links.find { |node| node.rel.nil? || node.rel == 'alternate' } || links.first
+          absolute_url(link&.href, base_url:)
+        end
 
-      def atom_author(entry)
-        author = Array(entry.author).first
-        return unless author
+        def atom_author(entry)
+          author = entry.author
+          return unless author
 
-        present(author.name&.content) || present(author.email&.content)
-      end
-      module_function :atom_author
-      private_class_method :atom_author
-
-      def absolute_url(raw, base_url:)
-        value = present(raw)
-        return unless value
+          present(author.name&.content) || present(author.email&.content)
+        end
 
-        if base_url
-          Html2rss::Url.from_relative(value, base_url)
-        else
-          Html2rss::Url.from_absolute(value)
+        def absolute_url(raw, base_url:)
+          value = present(raw)
+          return unless value
+
+          if base_url
+            Html2rss::Url.from_relative(value, base_url)
+          else
+            Html2rss::Url.from_absolute(value)
+          end
+        rescue Html2rss::Url::InvalidUrlError
+          nil
         end
-      rescue ArgumentError
-        nil
-      end
-      module_function :absolute_url
-      private_class_method :absolute_url
 
-      def present(value)
-        text = value.to_s.strip
-        text unless text.empty?
+        def present(value)
+          text = value.to_s.strip
+          text unless text.empty?
+        end
       end
-      module_function :present
-      private_class_method :present
     end
   end
 end