html2rss 0.32.4 → 0.34.0

data/lib/html2rss/syndication/parser.rb

@@ -5,137 +5,138 @@ require 'rss'
 module Html2rss
   module Syndication
     ##
-    # Parses RSS 2.0 and Atom documents into AutoSource article hashes.
-    module Parser # rubocop:disable Metrics/ModuleLength -- feed formats + structured parse warn
-      module_function
-
-      ##
-      # @param response [Html2rss::RequestService::Response]
-      # @return [Array<Hash{Symbol => Object}>] article hashes (may be empty)
-      # @raise [ArgumentError] when the response is not a syndication feed
-      def parse_response(response)
-        raise ArgumentError, 'response is not a syndication feed' unless response.feed_response?
-
-        parse(response.body, base_url: response.url)
-      end
+    # Parses RSS 2.0, RSS 1.0 (RDF), and Atom documents into a {Feed}: AutoSource article hashes, with
+    # the media files and image each item declares ({ItemMedia}), and what the feed states about its
+    # channel ({ChannelFacts}), read from the same parse.
+    #
+    # An item's URL is its link (an Atom entry's +rel="alternate"+ link or a link without +rel+). A
+    # link-less item (a podcast episode on a hosting platform) takes its guid, Atom +id+ or RDF +about+
+    # when that is an absolute http(s) URL, else its first audio or video file; an item with none of
+    # these has no URL and is skipped. Links and files resolve against the feed URL through
+    # {Url.resolve_web}: a +javascript:+, +data:+ or +ftp:+ one counts as absent, so no item URL is one.
+    module Parser
+      class << self
+        ##
+        # @param response [Html2rss::RequestService::Response]
+        # @return [Feed]
+        # @raise [RequestService::UnsupportedResponseContentType] when the response is not a syndication feed
+        def parse_response(response)
+          unless response.feed_response?
+            raise RequestService::UnsupportedResponseContentType, 'response is not a syndication feed'
+          end
+
+          parse(response.body, base_url: response.url)
+        end
 
-      ##
-      # @param body [String] raw RSS/Atom document
-      # @param base_url [String, Html2rss::Url, nil] fallback base for relative item links
-      # @return [Array<Hash{Symbol => Object}>]
-      def parse(body, base_url: nil)
-        feed = RSS::Parser.parse(body.to_s, false)
-        return [] unless feed
-
-        items_from(feed, base_url:).filter_map { |item| article_hash(item, base_url:) }
-      rescue RSS::Error => error
-        Log.warn({
-          event_name: 'syndication.parse_failed',
-          error_class: error.class.name,
-          message: "Syndication::Parser: failed to parse feed (#{error.class}: #{error.message})"
-        }.to_json)
-        []
-      end
+        ##
+        # @param body [String] raw RSS/RDF/Atom document
+        # @param base_url [String, Html2rss::Url, nil] fallback base for relative item links
+        # @return [Feed] no articles and no channel facts when the document fails to parse
+        def parse(body, base_url: nil)
+          feed = RSS::Parser.parse(body.to_s, false)
+          Feed.new(articles: articles_from(feed, body, base_url:), channel: ChannelFacts.from(feed))
+        rescue RSS::Error => error
+          Log.warn({
+            event_name: 'syndication.parse_failed',
+            error_class: error.class.name,
+            message: "Syndication::Parser: failed to parse feed (#{error.class}: #{error.message})"
+          }.to_json)
+          Feed::NONE
+        end
 
-      def items_from(feed, base_url:) # rubocop:disable Lint/UnusedMethodArgument -- base reserved for callers
-        case feed
-        when RSS::Rss then Array(feed.items)
-        when RSS::Atom::Feed then Array(feed.entries)
-        else []
+        private
+
+        def articles_from(feed, body, base_url:)
+          items = items_from(feed)
+          media = ItemMedia.for_items(body, count: items.size, url: base_url)
+
+          items.zip(media).filter_map { |item, item_media| article_hash(item, item_media, base_url:) }
         end
-      end
-      module_function :items_from
-      private_class_method :items_from
 
-      def article_hash(item, base_url:)
-        case item
-        when RSS::Rss::Channel::Item then rss_item_hash(item, base_url:)
-        when RSS::Atom::Entry, RSS::Atom::Feed::Entry then atom_entry_hash(item, base_url:)
+        def items_from(feed)
+          case feed
+          when RSS::Rss, RSS::RDF then feed.items
+          when RSS::Atom::Feed then feed.entries
+          else []
+          end
         end
-      end
-      module_function :article_hash
-      private_class_method :article_hash
-
-      def rss_item_hash(item, base_url:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- RSS item mapping
-        url = absolute_url(item.link, base_url:)
-        return unless url
-
-        {
-          id: present(item.guid&.content) || url.to_s,
-          title: present(item.title),
-          description: present(item.description),
-          url:,
-          published_at: item.date || item.pubDate,
-          author: present(item.author),
-          categories: Array(item.categories).filter_map { |cat| present(cat.content) }
-        }.compact
-      end
-      module_function :rss_item_hash
-      private_class_method :rss_item_hash
-
-      def atom_entry_hash(entry, base_url:) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity -- Atom entry mapping
-        url = atom_link(entry, base_url:)
-        return unless url
-
-        {
-          id: present(entry.id&.content) || url.to_s,
-          title: present(atom_text(entry.title)),
-          description: present(atom_text(entry.content) || atom_text(entry.summary)),
-          url:,
-          published_at: entry.published&.content || entry.updated&.content,
-          author: present(atom_author(entry)),
-          categories: Array(entry.categories).filter_map { |cat| present(cat.term) }
-        }.compact
-      end
-      module_function :atom_entry_hash
-      private_class_method :atom_entry_hash
 
-      def atom_link(entry, base_url:)
-        links = Array(entry.links)
-        link = links.find { |node| node.rel.nil? || node.rel == 'alternate' } || links.first
-        absolute_url(link&.href, base_url:)
-      end
-      module_function :atom_link
-      private_class_method :atom_link
+        def article_hash(item, media, base_url:)
+          case item
+          when RSS::Rss::Channel::Item then rss_item_hash(item, media, base_url:)
+          when RSS::RDF::Item then rdf_item_hash(item, media, base_url:)
+          when RSS::Atom::Feed::Entry then atom_entry_hash(item, media, base_url:)
+          end
+        end
 
-      def atom_text(node)
-        return unless node.is_a?(RSS::Atom::TextConstruct) || node.is_a?(RSS::Atom::Feed::Entry::Content)
+        def rss_item_hash(item, media, base_url:)
+          article(
+            Url.resolve_web(item.link, base_url), media,
+            id: item.guid&.content, title: item.title, description: item.description,
+            published_at: item.pubDate, author: item.author, categories: item.categories.map(&:content)
+          )
+        end
 
-        node.content
-      end
-      module_function :atom_text
-      private_class_method :atom_text
+        def rdf_item_hash(item, media, base_url:)
+          article(
+            Url.resolve_web(item.link, base_url), media,
+            id: item.about, title: item.title, description: item.description,
+            published_at: item.dc_date, author: item.dc_creator, categories: item.dc_subjects.map(&:content)
+          )
+        end
+
+        def atom_entry_hash(entry, media, base_url:)
+          article(
+            atom_link(entry, base_url:), media,
+            id: entry.id&.content, title: entry.title&.content, description: atom_text(entry.content, entry.summary),
+            published_at: atom_text(entry.published, entry.updated),
+            author: atom_author(entry), categories: entry.categories.map(&:term)
+          )
+        end
 
-      def atom_author(entry)
-        author = Array(entry.author).first
-        return unless author
+        ##
+        # Single owner of the article hash shape: blank text becomes absent, +id+ falls back to the url,
+        # +media+ adds the item's +enclosures+ and +image+. The url is the item's http(s) link, else its id
+        # when that is an absolute http(s) URL, whatever +isPermaLink+ says (+false+ only denies a guaranteed
+        # permalink, and such a guid is mostly the item's page on the publisher or its host; a relative, +urn:+
+        # or +tag:+ id names no page), else its first audio or video file.
+        def article(link, media, id:, title:, description:, published_at:, author:, categories:) # rubocop:disable Metrics/ParameterLists -- the article shape
+          return unless (url = link || Url.resolve_web(id) || media_url(media))
+
+          {
+            id: present(id) || url.to_s,
+            title: present(title),
+            description: present(description),
+            url:, published_at:,
+            author: present(author),
+            categories: categories.filter_map { |category| present(category) },
+            **media
+          }.compact
+        end
 
-        present(author.name&.content) || present(author.email&.content)
-      end
-      module_function :atom_author
-      private_class_method :atom_author
+        def media_url(media)
+          media.fetch(:enclosures, []).find { |facts| Article::Enclosure.new(**facts).media? }&.fetch(:url)
+        end
 
-      def absolute_url(raw, base_url:)
-        value = present(raw)
-        return unless value
+        def atom_text(*nodes) = nodes.find { |node| node }&.content
 
-        if base_url
-          Html2rss::Url.from_relative(value, base_url)
-        else
-          Html2rss::Url.from_absolute(value)
+        def atom_link(entry, base_url:)
+          link = entry.links.find { |node| node.rel.nil? || node.rel == 'alternate' }
+          Url.resolve_web(link&.href, base_url)
         end
-      rescue ArgumentError
-        nil
-      end
-      module_function :absolute_url
-      private_class_method :absolute_url
 
-      def present(value)
-        text = value.to_s.strip
-        text unless text.empty?
+        def atom_author(entry)
+          author = entry.author
+          return unless author
+
+          present(author.name&.content) || present(author.email&.content)
+        end
+
+        def present(value)
+          text = value.to_s.strip
+          text unless text.empty?
+        end
       end
-      module_function :present
-      private_class_method :present
     end
   end
 end

data/lib/html2rss/test/enhance_audit.rb

@@ -6,8 +6,6 @@ module Html2rss
     # Probes list-card +enhance+ value on cached listing HTML.
     # Sole owner of enhance_gains metrics and enhance-specific warn-only warnings.
     module EnhanceAudit # rubocop:disable Metrics/ModuleLength -- probe + compare + heuristics stay co-located
-      module_function
-
       TRACKED_KEYS = %i[title url description published_at categories author image id enclosures].freeze
       CURATOR_KEYS = %i[title url description published_at].freeze
       private_constant :TRACKED_KEYS, :CURATOR_KEYS
@@ -28,7 +26,7 @@ module Html2rss
       # @param selectors [Hash{Symbol => Object}]
       # @param time_zone [String]
       # @return [AuditSlice, nil] nil when selectors are absent
-      def probe(response:, selectors:, time_zone:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- per-item probe loop
+      def self.probe(response:, selectors:, time_zone:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- per-item probe loop
         return nil unless selectors&.dig(:items, :selector)
 
         selector_engine = Selectors.new(response, selectors:, time_zone:)
@@ -40,7 +38,7 @@ module Html2rss
         }
         warnings = []
 
-        selector_engine.each_enhance_pair do |baseline, enhanced, item|
+        selector_engine.each_enhance_pair(linked_only: true) do |baseline, enhanced, item|
           item_keys = keys_added(baseline, enhanced)
           merge_keys_added!(gains, item_keys)
           append_item_warnings!(warnings, item:, enhanced:, item_keys:)
@@ -64,7 +62,7 @@ module Html2rss
       # @param selectors [Hash{Symbol => Object}]
       # @param time_zone [String]
       # @return [Hash{Symbol => Object}, nil]
-      def compare(response:, selectors:, time_zone:) # rubocop:disable Metrics/MethodLength -- enhance off/on stats + delta
+      def self.compare(response:, selectors:, time_zone:) # rubocop:disable Metrics/MethodLength -- enhance off/on stats + delta
         return nil unless selectors&.dig(:items, :selector)
 
         off = extraction_stats(response, selectors, time_zone, enhance: false)
@@ -84,7 +82,7 @@ module Html2rss
 
       ##
       # @return [EnhanceGains] zeroed gains for compare fallback
-      def empty_gains
+      def self.empty_gains
         EnhanceGains.new(
           items_probed: 0,
           keys_added: zero_keys_added,
@@ -93,12 +91,12 @@ module Html2rss
         )
       end
 
-      def zero_keys_added
+      def self.zero_keys_added
         TRACKED_KEYS.to_h { |key| [key, 0] }.freeze
       end
       private_class_method :zero_keys_added
 
-      def extraction_stats(response, selectors, time_zone, enhance:)
+      def self.extraction_stats(response, selectors, time_zone, enhance:)
         merged = selectors_with_enhance(selectors, enhance:)
         articles = Selectors.new(response, selectors: merged, time_zone:).articles
         {
@@ -108,14 +106,14 @@ module Html2rss
       end
       private_class_method :extraction_stats
 
-      def selectors_with_enhance(selectors, enhance:)
+      def self.selectors_with_enhance(selectors, enhance:)
         merged = HashUtil.deep_dup(selectors)
         merged[:items] = merged.fetch(:items, {}).merge(enhance:)
         merged
       end
       private_class_method :selectors_with_enhance
 
-      def keys_added(baseline, enhanced)
+      def self.keys_added(baseline, enhanced)
         enhanced.each_with_object({}) do |(key, value), counts|
           next if value.nil?
 
@@ -127,7 +125,7 @@ module Html2rss
       end
       private_class_method :keys_added
 
-      def merge_keys_added!(gains, item_keys)
+      def self.merge_keys_added!(gains, item_keys)
         gains[:items_probed] += 1
         item_keys.each do |key, count|
           next unless TRACKED_KEYS.include?(key)
@@ -138,13 +136,13 @@ module Html2rss
       end
       private_class_method :merge_keys_added!
 
-      def finalize_gains!(gains)
+      def self.finalize_gains!(gains)
         curator_total = CURATOR_KEYS.sum { |key| gains[:keys_added][key] }
         gains[:no_op] = curator_total.zero?
       end
       private_class_method :finalize_gains!
 
-      def append_item_warnings!(warnings, item:, enhanced:, item_keys:)
+      def self.append_item_warnings!(warnings, item:, enhanced:, item_keys:)
         curator_added = item_keys.slice(*CURATOR_KEYS)
         return if curator_added.empty?
         return unless curator_added.key?(:description)
@@ -160,16 +158,16 @@ module Html2rss
       end
       private_class_method :append_item_warnings!
 
-      def category_only_description?(description, enhanced)
+      def self.category_only_description?(description, enhanced)
         normalized = description.downcase
-        return true if Html2rss::Html::ArticleRules::Description::TYPE_CHIPS.include?(normalized)
+        return true if Html2rss::Article::Title::TYPE_MARKERS.include?(normalized)
 
         categories = Array(enhanced[:categories]).map { |category| category.to_s.strip }
         categories.any? { |category| category.casecmp?(description) }
       end
       private_class_method :category_only_description?
 
-      def image_only_description?(description, item)
+      def self.image_only_description?(description, item)
         images = item.css('img')
         return description == images.first['alt'].to_s.strip if images.size == 1 && images.first['alt']
 
@@ -177,7 +175,7 @@ module Html2rss
       end
       private_class_method :image_only_description?
 
-      def prose_shaped?(text)
+      def self.prose_shaped?(text)
         return false if text.match?(IMAGE_FILENAME)
         return false if text.match?(%r{\Ahttps?://}i)
         return false if text.match?(%r{\A/[\w./-]+\z})

data/lib/html2rss/test/item_sample.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Test
+    ##
+    # Builds the bounded +items+ preview of a test run from extracted articles,
+    # independent of the quality gate and of the rendered RSS.
+    module ItemSample
+      # Maximum characters kept in an item's plain-text description.
+      DESCRIPTION_LIMIT = 200
+
+      ##
+      # @param articles [Array<Html2rss::Article>] extracted articles
+      # @param limit [Integer] maximum number of items, >= 0
+      # @return [Array<Hash{Symbol => Object}>]
+      def self.call(articles, limit:)
+        articles.first(limit).map { |article| item_for(article) }
+      end
+
+      ##
+      # @param article [Html2rss::Article]
+      # @return [Hash{Symbol => Object}]
+      def self.item_for(article)
+        {
+          title: article.title.to_s.strip,
+          url: article.url&.to_s,
+          published_at: article.published_at&.iso8601,
+          description: plain_description(article.description),
+          image: article.image&.to_s,
+          categories: article.categories
+        }
+      end
+
+      ##
+      # @param html [String, nil]
+      # @return [String] whitespace-collapsed text truncated to {DESCRIPTION_LIMIT} characters
+      def self.plain_description(html)
+        text = Nokogiri::HTML.fragment(html).text.gsub(/\s+/, ' ').strip
+        text[0, DESCRIPTION_LIMIT]
+      end
+    end
+  end
+end

data/lib/html2rss/test/policy.rb

@@ -6,8 +6,6 @@ module Html2rss
     # Pass/fail classification for a configuration test after live extraction.
     # Owns min_items vs quality thresholds; {Test.call} orchestrates only.
     module Policy
-      module_function
-
       Outcome = Data.define(:passed, :failure_kind, :error_message)
 
       ##
@@ -28,7 +26,7 @@ module Html2rss
       # @param strict_quality [Boolean]
       # @param quality_report [QualityReport]
       # @return [Outcome]
-      def evaluate(item_count:, min_items:, strict_quality:, quality_report:)
+      def self.evaluate(item_count:, min_items:, strict_quality:, quality_report:)
         min_items_passed = item_count >= min_items
         quality_failed = strict_quality && min_items_passed && quality_failure?(quality_report)
         passed = min_items_passed && !quality_failed
@@ -39,7 +37,7 @@ module Html2rss
         Outcome.new(passed:, failure_kind:, error_message:)
       end
 
-      def log_strict_quality_failure(failure_kind, item_count)
+      def self.log_strict_quality_failure(failure_kind, item_count)
         Log.info("Test strict quality: failure_kind=#{failure_kind.to_sym} item_count=#{item_count}")
       end
       private_class_method :log_strict_quality_failure
@@ -47,7 +45,7 @@ module Html2rss
       ##
       # @param quality_report [QualityReport]
       # @return [Boolean]
-      def quality_failure?(quality_report)
+      def self.quality_failure?(quality_report)
         metrics = quality_report.metrics
         item_count = metrics[:item_count]
         return false if item_count.zero?
@@ -62,7 +60,7 @@ module Html2rss
       # @param min_items [Integer]
       # @param quality_report [QualityReport]
       # @return [Array(FailureKind, nil), Array(String, nil)]
-      def outcome_failure(min_items_passed:, quality_failed:, item_count:, min_items:, quality_report:)
+      def self.outcome_failure(min_items_passed:, quality_failed:, item_count:, min_items:, quality_report:)
         return [nil, nil] if min_items_passed && !quality_failed
         unless min_items_passed
           return [FailureKind[:min_items],
@@ -72,13 +70,13 @@ module Html2rss
         [FailureKind[:quality], quality_failure_message(quality_report)]
       end
 
-      def quality_failure_message(quality_report)
+      def self.quality_failure_message(quality_report)
         reasons = quality_failure_reasons(quality_report.metrics)
         "Feed quality check failed (#{reasons.join(', ')})"
       end
       private_class_method :quality_failure_message
 
-      def quality_failure_reasons(metrics)
+      def self.quality_failure_reasons(metrics)
         reasons = []
         reasons << 'duplicate_urls' if duplicate_urls_failure?(metrics)
         reasons << 'generic_titles' if junk_title_ratio_failure?(metrics)
@@ -87,17 +85,17 @@ module Html2rss
       end
       private_class_method :quality_failure_reasons
 
-      def duplicate_urls_failure?(metrics)
+      def self.duplicate_urls_failure?(metrics)
         metrics[:item_count] >= 2 && metrics[:unique_url_count] < 2
       end
       private_class_method :duplicate_urls_failure?
 
-      def junk_title_ratio_failure?(metrics)
+      def self.junk_title_ratio_failure?(metrics)
         metrics[:junk_title_count] > (metrics[:item_count] / 2.0)
       end
       private_class_method :junk_title_ratio_failure?
 
-      def short_title_failure?(metrics)
+      def self.short_title_failure?(metrics)
         metrics[:short_title_count].positive?
       end
       private_class_method :short_title_failure?