html2rss 0.27.2 → 0.29.1

data/lib/html2rss/syndication/discovery.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Syndication
+    ##
+    # Finds a same-origin RSS/Atom URL for a page via head alternates and path guesses.
+    #
+    # Ports configs +probe_rss+ path/alternate logic onto {RequestSession#follow_up}.
+    module Discovery # rubocop:disable Metrics/ModuleLength -- discovery + path probes stay co-located
+      # Common feed path suffixes probed after head +rel=alternate+ hints.
+      DEFAULT_PATHS = CandidateCatalog::FEED_PATHS
+
+      LINK_TAG_RE = /<link\b[^>]*>/i
+      HREF_RE = /\bhref\s*=\s*["']([^"']+)["']/i
+      TYPE_RE = /\btype\s*=\s*["']([^"']+)["']/i
+      REL_RE = /\brel\s*=\s*["']([^"']+)["']/i
+      private_constant :LINK_TAG_RE, :HREF_RE, :TYPE_RE, :REL_RE
+
+      module_function
+
+      ##
+      # Probes candidates lazily and returns the first feedish URL.
+      #
+      # @param page_url [String, Html2rss::Url] page or origin URL
+      # @param request_session [Html2rss::RequestSession] shared session for follow-ups
+      # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML when available
+      # @param html [String, nil] raw HTML when a document is unavailable
+      # @param extra_paths [Array<String>] additional path guesses
+      # @param max_probes [Integer, nil] max candidate GETs (nil = uncapped)
+      # @return [Html2rss::Url, nil]
+      # rubocop:disable-next Metrics/ParameterLists -- discovery kwargs stay co-located
+      def best_feed_url(page_url:, request_session:, parsed_body: nil, html: nil, extra_paths: [],
+                        max_probes: nil)
+        best_feed_response(
+          page_url:, request_session:, parsed_body:, html:, extra_paths:, max_probes:
+        )&.url
+      end
+
+      ##
+      # Like {#best_feed_url} but returns the validated syndication response for parsing.
+      #
+      # @param page_url [String, Html2rss::Url]
+      # @param request_session [Html2rss::RequestSession]
+      # @param parsed_body [Nokogiri::HTML::Document, nil]
+      # @param html [String, nil]
+      # @param extra_paths [Array<String>]
+      # @param max_probes [Integer, nil] max candidate GETs (nil = uncapped)
+      # @return [Html2rss::RequestService::Response, nil]
+      # rubocop:disable-next Metrics/ParameterLists, Metrics/MethodLength -- discovery kwargs stay co-located
+      def best_feed_response(page_url:, request_session:, parsed_body: nil, html: nil, extra_paths: [],
+                             max_probes: nil)
+        page = Html2rss::Url.from_absolute(page_url)
+        candidates = candidate_urls(page_url: page, parsed_body:, html:, extra_paths:)
+        Log.debug(
+          "Syndication::Discovery: host=#{page.host} candidate_count=#{candidates.size}"
+        )
+
+        first_feedish_response(
+          candidates, request_session:, origin_url: page, max_probes:
+        ).tap do |response|
+          next unless response
+
+          Log.info("Syndication::Discovery: host=#{page.host} selected_feed_url=#{response.url}")
+        end
+      end
+
+      ##
+      # Ordered candidate feed URLs (head alternates first, then path guesses).
+      #
+      # @param page_url [String, Html2rss::Url]
+      # @param parsed_body [Nokogiri::HTML::Document, nil]
+      # @param html [String, nil]
+      # @param extra_paths [Array<String>]
+      # @return [Array<Html2rss::Url>]
+      def candidate_urls(page_url:, parsed_body: nil, html: nil, extra_paths: [])
+        page = Html2rss::Url.from_absolute(page_url)
+        (alternate_feed_urls(page, parsed_body:, html:) +
+          path_guess_urls(page, extra_paths:)).uniq
+      end
+
+      ##
+      # @param response [Html2rss::RequestService::Response, nil]
+      # @return [Boolean]
+      def feedish?(response)
+        return false unless response
+        return false unless successful_status?(response)
+
+        response.feed_response?
+      end
+
+      def successful_status?(response)
+        status = response.status
+        status.nil? || status.between?(200, 299)
+      end
+      module_function :successful_status?
+      private_class_method :successful_status?
+
+      ##
+      # Directory-relative feed path guesses for a page URL.
+      #
+      # @param page_url [String, Html2rss::Url]
+      # @return [Array<String>] relative paths
+      def page_dir_paths(page_url)
+        path = Html2rss::Url.from_absolute(page_url).path
+        return [] if path.nil? || path.empty? || path == '/'
+
+        dir = path.sub(%r{/[^/]*$}, '/')
+        ["#{dir}feed", "#{dir}rss.xml", "#{dir}atom.xml"]
+      end
+
+      def first_feedish_response(urls, request_session:, origin_url:, max_probes: nil)
+        scoped = max_probes.nil? ? urls.lazy : urls.lazy.take(max_probes)
+        scoped.filter_map do |url|
+          response = probe(url, request_session:, origin_url:)
+          response if feedish?(response)
+        end.first
+      end
+      module_function :first_feedish_response
+      private_class_method :first_feedish_response
+
+      def first_feedish_url(urls, request_session:, origin_url:, max_probes: nil)
+        first_feedish_response(urls, request_session:, origin_url:, max_probes:)&.url
+      end
+      module_function :first_feedish_url
+      private_class_method :first_feedish_url
+
+      def probe(url, request_session:, origin_url:)
+        request_session.follow_up(url:, relation: :auto_source, origin_url:)
+      rescue Html2rss::Error => error
+        Log.debug(
+          "Syndication::Discovery: probe skipped host=#{origin_url.host} " \
+          "(#{error.class})"
+        )
+        nil
+      end
+      module_function :probe
+      private_class_method :probe
+
+      def alternate_feed_urls(page, parsed_body:, html:)
+        from_document = feed_links_from_document(parsed_body, page)
+        return from_document if from_document.any?
+
+        alternate_feed_hrefs(html.to_s, page)
+      end
+      module_function :alternate_feed_urls
+      private_class_method :alternate_feed_urls
+
+      def feed_links_from_document(parsed_body, page)
+        return [] unless parsed_body.is_a?(Nokogiri::HTML::Document)
+
+        Html::FeedLink.from_document(parsed_body).filter_map do |link|
+          absolute_url(page, link.href)
+        end
+      end
+      module_function :feed_links_from_document
+      private_class_method :feed_links_from_document
+
+      def path_guess_urls(page, extra_paths:)
+        paths = (page_dir_paths(page) + DEFAULT_PATHS + Array(extra_paths)).uniq
+        paths.filter_map { |path| absolute_url(page, path) }
+      end
+      module_function :path_guess_urls
+      private_class_method :path_guess_urls
+
+      def alternate_feed_hrefs(html, page)
+        return [] if html.nil? || html.empty?
+
+        html.scan(LINK_TAG_RE).filter_map { |tag| href_from_alternate_link(tag, page) }.uniq
+      end
+      module_function :alternate_feed_hrefs
+      private_class_method :alternate_feed_hrefs
+
+      def href_from_alternate_link(tag, page)
+        return unless alternate_rel?(tag)
+
+        href = tag[HREF_RE, 1]
+        return if href.nil? || href.empty?
+        return unless syndication_type?(tag[TYPE_RE, 1]) || feed_like_href?(href)
+
+        absolute_url(page, href.strip)
+      end
+      module_function :href_from_alternate_link
+      private_class_method :href_from_alternate_link
+
+      def alternate_rel?(tag)
+        tag[REL_RE, 1].to_s.downcase.split(/\s+/).include?('alternate')
+      end
+      module_function :alternate_rel?
+      private_class_method :alternate_rel?
+
+      def syndication_type?(type)
+        return false if type.nil? || type.empty?
+
+        t = type.downcase
+        t.include?('rss+xml') || t.include?('atom+xml') || t.include?('rss') || t.include?('atom')
+      end
+      module_function :syndication_type?
+      private_class_method :syndication_type?
+
+      def feed_like_href?(href)
+        href.match?(/rss|atom|feed|\.xml/i)
+      end
+      module_function :feed_like_href?
+      private_class_method :feed_like_href?
+
+      def absolute_url(base, path)
+        Html2rss::Url.from_relative(path, base)
+      rescue ArgumentError
+        nil
+      end
+      module_function :absolute_url
+      private_class_method :absolute_url
+    end
+  end
+end

data/lib/html2rss/syndication/parser.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require 'rss'
+
+module Html2rss
+  module Syndication
+    ##
+    # Parses RSS 2.0 and Atom documents into AutoSource article hashes.
+    module Parser
+      module_function
+
+      ##
+      # @param response [Html2rss::RequestService::Response]
+      # @return [Array<Hash{Symbol => Object}>] article hashes (may be empty)
+      # @raise [ArgumentError] when the response is not a syndication feed
+      def parse_response(response)
+        raise ArgumentError, 'response is not a syndication feed' unless response.feed_response?
+
+        parse(response.body, base_url: response.url)
+      end
+
+      ##
+      # @param body [String] raw RSS/Atom document
+      # @param base_url [String, Html2rss::Url, nil] fallback base for relative item links
+      # @return [Array<Hash{Symbol => Object}>]
+      def parse(body, base_url: nil)
+        feed = RSS::Parser.parse(body.to_s, false)
+        return [] unless feed
+
+        items_from(feed, base_url:).filter_map { |item| article_hash(item, base_url:) }
+      rescue RSS::Error => error
+        Log.warn("Syndication::Parser: failed to parse feed (#{error.class}: #{error.message})")
+        []
+      end
+
+      def items_from(feed, base_url:) # rubocop:disable Lint/UnusedMethodArgument -- base reserved for callers
+        case feed
+        when RSS::Rss then Array(feed.items)
+        when RSS::Atom::Feed then Array(feed.entries)
+        else []
+        end
+      end
+      module_function :items_from
+      private_class_method :items_from
+
+      def article_hash(item, base_url:)
+        case item
+        when RSS::Rss::Channel::Item then rss_item_hash(item, base_url:)
+        when RSS::Atom::Entry, RSS::Atom::Feed::Entry then atom_entry_hash(item, base_url:)
+        end
+      end
+      module_function :article_hash
+      private_class_method :article_hash
+
+      def rss_item_hash(item, base_url:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- RSS item mapping
+        url = absolute_url(item.link, base_url:)
+        return unless url
+
+        {
+          id: present(item.guid&.content) || url.to_s,
+          title: present(item.title),
+          description: present(item.description),
+          url:,
+          published_at: item.date || item.pubDate,
+          author: present(item.author),
+          categories: Array(item.categories).filter_map { |cat| present(cat.content) }
+        }.compact
+      end
+      module_function :rss_item_hash
+      private_class_method :rss_item_hash
+
+      def atom_entry_hash(entry, base_url:) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity -- Atom entry mapping
+        url = atom_link(entry, base_url:)
+        return unless url
+
+        {
+          id: present(entry.id&.content) || url.to_s,
+          title: present(atom_text(entry.title)),
+          description: present(atom_text(entry.content) || atom_text(entry.summary)),
+          url:,
+          published_at: entry.published&.content || entry.updated&.content,
+          author: present(atom_author(entry)),
+          categories: Array(entry.categories).filter_map { |cat| present(cat.term) }
+        }.compact
+      end
+      module_function :atom_entry_hash
+      private_class_method :atom_entry_hash
+
+      def atom_link(entry, base_url:)
+        links = Array(entry.links)
+        link = links.find { |node| node.rel.nil? || node.rel == 'alternate' } || links.first
+        absolute_url(link&.href, base_url:)
+      end
+      module_function :atom_link
+      private_class_method :atom_link
+
+      def atom_text(node)
+        return if node.nil?
+
+        node.respond_to?(:content) ? node.content : node.to_s
+      end
+      module_function :atom_text
+      private_class_method :atom_text
+
+      def atom_author(entry)
+        author = Array(entry.author).first
+        return unless author
+
+        present(author.name&.content) || present(author.email&.content)
+      end
+      module_function :atom_author
+      private_class_method :atom_author
+
+      def absolute_url(raw, base_url:)
+        value = present(raw)
+        return unless value
+
+        if base_url
+          Html2rss::Url.from_relative(value, base_url)
+        else
+          Html2rss::Url.from_absolute(value)
+        end
+      rescue ArgumentError
+        nil
+      end
+      module_function :absolute_url
+      private_class_method :absolute_url
+
+      def present(value)
+        text = value.to_s.strip
+        text unless text.empty?
+      end
+      module_function :present
+      private_class_method :present
+    end
+  end
+end

data/lib/html2rss/syndication.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Native RSS/Atom discovery and parse (not HTML AutoSource ownership).
+  #
+  # {include:file:lib/html2rss/syndication/README.md}
+  module Syndication
+  end
+end

data/lib/html2rss/test/enhance_audit.rb

@@ -0,0 +1,192 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Test
+    ##
+    # Probes list-card +enhance+ value on cached listing HTML.
+    # Sole owner of enhance_gains metrics and enhance-specific warn-only warnings.
+    module EnhanceAudit # rubocop:disable Metrics/ModuleLength -- probe + compare + heuristics stay co-located
+      module_function
+
+      TRACKED_KEYS = %i[title url description published_at categories author image id enclosures].freeze
+      CURATOR_KEYS = %i[title url description published_at].freeze
+      private_constant :TRACKED_KEYS, :CURATOR_KEYS
+
+      ##
+      # Aggregate enhance probe metrics for one configuration test.
+      EnhanceGains = Data.define(:items_probed, :keys_added, :descriptions_added, :no_op)
+
+      ##
+      # Probe output merged into {QualityReport}.
+      AuditSlice = Data.define(:enhance_gains, :warnings)
+
+      IMAGE_FILENAME = /\.(jpe?g|png|gif|webp|avif|svg)\z/i
+      private_constant :IMAGE_FILENAME
+
+      ##
+      # @param response [Html2rss::RequestService::Response]
+      # @param selectors [Hash{Symbol => Object}]
+      # @param time_zone [String]
+      # @return [AuditSlice, nil] nil when selectors are absent
+      def probe(response:, selectors:, time_zone:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- per-item probe loop
+        return nil unless selectors&.dig(:items, :selector)
+
+        selector_engine = Selectors.new(response, selectors:, time_zone:)
+        gains = {
+          items_probed: 0,
+          keys_added: zero_keys_added.dup,
+          descriptions_added: 0,
+          no_op: true
+        }
+        warnings = []
+
+        response.parsed_body.css(selector_engine.items_selector).each do |item|
+          baseline = selector_engine.extract_article(item, response)
+          enhanced = selector_engine.enhance_article_hash(baseline.dup, item, response.url)
+          item_keys = keys_added(baseline, enhanced)
+          merge_keys_added!(gains, item_keys)
+          append_item_warnings!(warnings, item:, enhanced:, item_keys:)
+        end
+
+        finalize_gains!(gains)
+        warnings << :enhance_no_op if gains.fetch(:no_op)
+        AuditSlice.new(
+          enhance_gains: EnhanceGains.new(
+            items_probed: gains.fetch(:items_probed),
+            keys_added: gains.fetch(:keys_added).freeze,
+            descriptions_added: gains.fetch(:descriptions_added),
+            no_op: gains.fetch(:no_op)
+          ),
+          warnings: warnings.freeze
+        )
+      end
+
+      ##
+      # @param response [Html2rss::RequestService::Response]
+      # @param selectors [Hash{Symbol => Object}]
+      # @param time_zone [String]
+      # @return [Hash{Symbol => Object}, nil]
+      def compare(response:, selectors:, time_zone:) # rubocop:disable Metrics/MethodLength -- enhance off/on stats + delta
+        return nil unless selectors&.dig(:items, :selector)
+
+        off = extraction_stats(response, selectors, time_zone, enhance: false)
+        on = extraction_stats(response, selectors, time_zone, enhance: true)
+        delta_gains = probe(response:, selectors:, time_zone:)&.enhance_gains || empty_gains
+
+        {
+          enhance_off: off,
+          enhance_on: on,
+          delta: {
+            descriptions_gained: on[:descriptions_filled] - off[:descriptions_filled],
+            keys_added: delta_gains.keys_added,
+            no_op: delta_gains.no_op
+          }
+        }
+      end
+
+      ##
+      # @return [EnhanceGains] zeroed gains for compare fallback
+      def empty_gains
+        EnhanceGains.new(
+          items_probed: 0,
+          keys_added: zero_keys_added,
+          descriptions_added: 0,
+          no_op: true
+        )
+      end
+
+      def zero_keys_added
+        TRACKED_KEYS.to_h { |key| [key, 0] }.freeze
+      end
+      private_class_method :zero_keys_added
+
+      def extraction_stats(response, selectors, time_zone, enhance:)
+        merged = selectors_with_enhance(selectors, enhance:)
+        articles = Selectors.new(response, selectors: merged, time_zone:).articles
+        {
+          item_count: articles.size,
+          descriptions_filled: articles.count { |article| article.description.to_s.strip != '' }
+        }
+      end
+      private_class_method :extraction_stats
+
+      def selectors_with_enhance(selectors, enhance:)
+        merged = HashUtil.deep_dup(selectors)
+        merged[:items] = merged.fetch(:items, {}).merge(enhance:)
+        merged
+      end
+      private_class_method :selectors_with_enhance
+
+      def keys_added(baseline, enhanced)
+        enhanced.each_with_object({}) do |(key, value), counts|
+          next if value.nil?
+
+          baseline_value = baseline[key]
+          next if baseline.key?(key) && baseline_value
+
+          counts[key] = (counts[key] || 0) + 1
+        end
+      end
+      private_class_method :keys_added
+
+      def merge_keys_added!(gains, item_keys)
+        gains[:items_probed] += 1
+        item_keys.each do |key, count|
+          next unless TRACKED_KEYS.include?(key)
+
+          gains[:keys_added][key] += count
+        end
+        gains[:descriptions_added] += item_keys.fetch(:description, 0)
+      end
+      private_class_method :merge_keys_added!
+
+      def finalize_gains!(gains)
+        curator_total = CURATOR_KEYS.sum { |key| gains[:keys_added][key] }
+        gains[:no_op] = curator_total.zero?
+      end
+      private_class_method :finalize_gains!
+
+      def append_item_warnings!(warnings, item:, enhanced:, item_keys:)
+        curator_added = item_keys.slice(*CURATOR_KEYS)
+        return if curator_added.empty?
+        return unless curator_added.key?(:description)
+
+        description = enhanced[:description].to_s.strip
+        return if description.empty?
+
+        if category_only_description?(description, enhanced)
+          warnings << :enhance_category_only_description
+        elsif image_only_description?(description, item)
+          warnings << :enhance_image_only_description
+        end
+      end
+      private_class_method :append_item_warnings!
+
+      def category_only_description?(description, enhanced)
+        normalized = description.downcase
+        return true if Html2rss::Html::ArticleRules::Description::TYPE_CHIPS.include?(normalized)
+
+        categories = Array(enhanced[:categories]).map { |category| category.to_s.strip }
+        categories.any? { |category| category.casecmp?(description) }
+      end
+      private_class_method :category_only_description?
+
+      def image_only_description?(description, item)
+        images = item.css('img')
+        return description == images.first['alt'].to_s.strip if images.size == 1 && images.first['alt']
+
+        prose_shaped?(description) == false
+      end
+      private_class_method :image_only_description?
+
+      def prose_shaped?(text)
+        return false if text.match?(IMAGE_FILENAME)
+        return false if text.match?(%r{\Ahttps?://}i)
+        return false if text.match?(%r{\A/[\w./-]+\z})
+
+        text.match?(/[a-z]/i)
+      end
+      private_class_method :prose_shaped?
+    end
+  end
+end