html2rss 0.33.0 → 0.34.0

data/lib/html2rss/syndication/item_media.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Syndication
+    ##
+    # Reads the media files and image a feed item declares, for {Parser}: the +rss+ library parses the
+    # item text but drops Media RSS, so this module reads the same document's item elements.
+    #
+    # Enclosures, in document order and once per URL: RSS 2.0 +enclosure+ (+url+, +type+, +length+),
+    # Atom +link rel="enclosure"+ (+href+, +type+, +length+) and Media RSS +media:content+, also inside
+    # +media:group+ (+url+, +type+, +fileSize+, +duration+). A Media RSS +medium+ (+audio+, +video+, +image+)
+    # is the file's kind, typing an untyped file ({Article::Enclosure::KIND_TYPES}); it never makes a watch or
+    # player page ({Article::Enclosure#player_page?}, judged without it) a file, except that +image+ keeps one.
+    # An item +itunes:duration+ is the duration of every file that declares none ({Article::Enclosure} keeps it
+    # only on audio and video). The item image is its +itunes:image+ (+href+, or the URL as its text), else its
+    # first +media:thumbnail+ (also inside +media:group+); without either, {Article} makes its first image file
+    # the item image. Every declared URL resolves against the feed URL through {Url.resolve_web}, so a
+    # +javascript:+, +data:+ or +ftp:+ file or image is dropped, never shipped nor made an item URL ({Parser}).
+    # Sizes, durations and kinds come from {Article::Enclosure}. Items match by namespace
+    # ({ITEMS}); a document without any media marker ({MEDIA_MARKERS}) is not parsed a second time.
+    module ItemMedia
+      # Namespace prefixes the item queries bind.
+      NAMESPACES = {
+        'rss1' => 'http://purl.org/rss/1.0/',
+        'rdf' => 'http://www.w3.org/1999/02/22-rdf-syntax-ns#',
+        'atom' => 'http://www.w3.org/2005/Atom',
+        'media' => 'http://search.yahoo.com/mrss/',
+        'itunes' => 'http://www.itunes.com/dtds/podcast-1.0.dtd'
+      }.freeze
+      # Item elements in document order, by format: RSS 2.0 +item+ (no namespace) under +rss/channel+,
+      # RSS 1.0 +item+ under +rdf:RDF+, Atom +entry+ under +feed+. Extension elements named +item+ never count.
+      ITEMS = '/rss/channel/item | /rdf:RDF/rss1:item | /atom:feed/atom:entry'
+      # Text only a document declaring item media contains: an RSS 2.0 enclosure, an Atom enclosure link,
+      # or the Media RSS or iTunes namespace. Any other document is not parsed a second time.
+      MEDIA_MARKERS = Regexp.union('<enclosure', /rel=["']enclosure["']/, NAMESPACES['media'], NAMESPACES['itunes'])
+      # Enclosure elements of one item, in document order.
+      ENCLOSURES = "enclosure | atom:link[@rel='enclosure'] | media:content | media:group/media:content"
+      # Thumbnail URLs of one item, in document order (blank ones skipped, like a blank iTunes image).
+      THUMBNAILS = 'media:thumbnail/@url[normalize-space()] | media:group/media:thumbnail/@url[normalize-space()]'
+      # The item's iTunes image URL: its +href+, else the URL written as the element text.
+      ITUNES_IMAGE = 'itunes:image/@href[normalize-space()] | ' \
+                     'itunes:image[not(normalize-space(@href))][normalize-space()]'
+
+      class << self
+        ##
+        # @param body [String] the feed document
+        # @param count [Integer] items the +rss+ library read from +body+
+        # @param url [String, Html2rss::Url, nil] the feed URL (nil when unknown): the base of relative file and
+        #   image URLs, and named in the log when items cannot be matched
+        # @return [Array<Hash{Symbol => Object}>] per item, in document order, its +enclosures+ and +image+
+        #   when present; all empty when the document declares no media or its item elements do not
+        #   number +count+ (logged)
+        def for_items(body, count:, url:)
+          return Array.new(count) { {} } unless MEDIA_MARKERS.match?(body)
+
+          nodes = Nokogiri::XML(body).xpath(ITEMS, NAMESPACES)
+          return nodes.map { |node| item_media(node, url) } if nodes.size == count
+
+          warn_mismatch(nodes.size, count, url)
+          Array.new(count) { {} }
+        end
+
+        private
+
+        def item_media(node, feed_url)
+          duration = Article::Enclosure.parse_duration(node.at_xpath('itunes:duration', NAMESPACES)&.text)
+          files = node.xpath(ENCLOSURES, NAMESPACES).filter_map { file(_1, duration, feed_url) }
+          declared = node.at_xpath(ITUNES_IMAGE, NAMESPACES) || node.at_xpath(THUMBNAILS, NAMESPACES)
+          { enclosures: files.uniq { _1[:url].to_s }, image: declared && Url.resolve_web(declared.content, feed_url) }
+            .reject { |_, value| value.nil? || value == [] }
+        end
+
+        # The file's facts: its declared type, its +medium+ as kind, its byte length and its own duration, else
+        # the item's.
+        #
+        # @return [Hash{Symbol => Object}, nil] nil without an http(s) URL or for a watch or player page not marked
+        #   an image
+        def file(element, item_duration, feed_url)
+          return unless (url = Url.resolve_web(attribute(element, 'url', 'href'), feed_url))
+
+          type = attribute(element, 'type')
+          bytes_length = Article::Enclosure.parse_size(attribute(element, 'length', 'fileSize'))
+          kind = Article::Enclosure::KIND_TYPES.each_key.find { _1.name == element['medium'] }
+          return if kind != :image && Article::Enclosure.new(url:, type:, bytes_length:).player_page?
+
+          duration = Article::Enclosure.parse_duration(attribute(element, 'duration')) || item_duration
+          { url:, type:, kind:, bytes_length:, duration: }.compact
+        end
+
+        def attribute(element, *names)
+          names.filter_map { element[_1]&.strip }.find { !_1.empty? }
+        end
+
+        def warn_mismatch(elements, count, url)
+          Log.warn({
+            event_name: 'syndication.item_media_mismatch', feed_url: url,
+            message: "Syndication::ItemMedia: #{elements} item elements for #{count} parsed items; no item media read"
+          }.to_json)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/syndication/parser.rb

@@ -5,87 +5,124 @@ require 'rss'
 module Html2rss
   module Syndication
     ##
-    # Parses RSS 2.0 and Atom documents into AutoSource article hashes.
+    # Parses RSS 2.0, RSS 1.0 (RDF), and Atom documents into a {Feed}: AutoSource article hashes, with
+    # the media files and image each item declares ({ItemMedia}), and what the feed states about its
+    # channel ({ChannelFacts}), read from the same parse.
+    #
+    # An item's URL is its link (an Atom entry's +rel="alternate"+ link or a link without +rel+). A
+    # link-less item (a podcast episode on a hosting platform) takes its guid, Atom +id+ or RDF +about+
+    # when that is an absolute http(s) URL, else its first audio or video file; an item with none of
+    # these has no URL and is skipped. Links and files resolve against the feed URL through
+    # {Url.resolve_web}: a +javascript:+, +data:+ or +ftp:+ one counts as absent, so no item URL is one.
     module Parser
       class << self
         ##
         # @param response [Html2rss::RequestService::Response]
-        # @return [Array<Hash{Symbol => Object}>] article hashes (may be empty)
-        # @raise [ArgumentError] when the response is not a syndication feed
+        # @return [Feed]
+        # @raise [RequestService::UnsupportedResponseContentType] when the response is not a syndication feed
         def parse_response(response)
-          raise ArgumentError, 'response is not a syndication feed' unless response.feed_response?
+          unless response.feed_response?
+            raise RequestService::UnsupportedResponseContentType, 'response is not a syndication feed'
+          end
 
           parse(response.body, base_url: response.url)
         end
 
         ##
-        # @param body [String] raw RSS/Atom document
+        # @param body [String] raw RSS/RDF/Atom document
         # @param base_url [String, Html2rss::Url, nil] fallback base for relative item links
-        # @return [Array<Hash{Symbol => Object}>]
+        # @return [Feed] no articles and no channel facts when the document fails to parse
         def parse(body, base_url: nil)
           feed = RSS::Parser.parse(body.to_s, false)
-
-          items_from(feed).filter_map { |item| article_hash(item, base_url:) }
+          Feed.new(articles: articles_from(feed, body, base_url:), channel: ChannelFacts.from(feed))
         rescue RSS::Error => error
           Log.warn({
             event_name: 'syndication.parse_failed',
             error_class: error.class.name,
             message: "Syndication::Parser: failed to parse feed (#{error.class}: #{error.message})"
           }.to_json)
-          []
+          Feed::NONE
         end
 
         private
 
+        def articles_from(feed, body, base_url:)
+          items = items_from(feed)
+          media = ItemMedia.for_items(body, count: items.size, url: base_url)
+
+          items.zip(media).filter_map { |item, item_media| article_hash(item, item_media, base_url:) }
+        end
+
         def items_from(feed)
           case feed
-          when RSS::Rss then feed.items
+          when RSS::Rss, RSS::RDF then feed.items
           when RSS::Atom::Feed then feed.entries
           else []
           end
         end
 
-        def article_hash(item, base_url:)
+        def article_hash(item, media, base_url:)
           case item
-          when RSS::Rss::Channel::Item then rss_item_hash(item, base_url:)
-          when RSS::Atom::Feed::Entry then atom_entry_hash(item, base_url:)
+          when RSS::Rss::Channel::Item then rss_item_hash(item, media, base_url:)
+          when RSS::RDF::Item then rdf_item_hash(item, media, base_url:)
+          when RSS::Atom::Feed::Entry then atom_entry_hash(item, media, base_url:)
           end
         end
 
-        def rss_item_hash(item, base_url:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- RSS item mapping
-          url = absolute_url(item.link, base_url:)
-          return unless url
+        def rss_item_hash(item, media, base_url:)
+          article(
+            Url.resolve_web(item.link, base_url), media,
+            id: item.guid&.content, title: item.title, description: item.description,
+            published_at: item.pubDate, author: item.author, categories: item.categories.map(&:content)
+          )
+        end
 
-          {
-            id: present(item.guid&.content) || url.to_s,
-            title: present(item.title),
-            description: present(item.description),
-            url:,
-            published_at: item.pubDate,
-            author: present(item.author),
-            categories: item.categories.filter_map { |cat| present(cat.content) }
-          }.compact
+        def rdf_item_hash(item, media, base_url:)
+          article(
+            Url.resolve_web(item.link, base_url), media,
+            id: item.about, title: item.title, description: item.description,
+            published_at: item.dc_date, author: item.dc_creator, categories: item.dc_subjects.map(&:content)
+          )
+        end
+
+        def atom_entry_hash(entry, media, base_url:)
+          article(
+            atom_link(entry, base_url:), media,
+            id: entry.id&.content, title: entry.title&.content, description: atom_text(entry.content, entry.summary),
+            published_at: atom_text(entry.published, entry.updated),
+            author: atom_author(entry), categories: entry.categories.map(&:term)
+          )
         end
 
-        def atom_entry_hash(entry, base_url:) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity -- Atom entry mapping
-          url = atom_link(entry, base_url:)
-          return unless url
+        ##
+        # Single owner of the article hash shape: blank text becomes absent, +id+ falls back to the url,
+        # +media+ adds the item's +enclosures+ and +image+. The url is the item's http(s) link, else its id
+        # when that is an absolute http(s) URL, whatever +isPermaLink+ says (+false+ only denies a guaranteed
+        # permalink, and such a guid is mostly the item's page on the publisher or its host; a relative, +urn:+
+        # or +tag:+ id names no page), else its first audio or video file.
+        def article(link, media, id:, title:, description:, published_at:, author:, categories:) # rubocop:disable Metrics/ParameterLists -- the article shape
+          return unless (url = link || Url.resolve_web(id) || media_url(media))
 
           {
-            id: present(entry.id&.content) || url.to_s,
-            title: present(entry.title&.content),
-            description: present(entry.content&.content || entry.summary&.content),
-            url:,
-            published_at: entry.published&.content || entry.updated&.content,
-            author: atom_author(entry),
-            categories: entry.categories.filter_map { |cat| present(cat.term) }
+            id: present(id) || url.to_s,
+            title: present(title),
+            description: present(description),
+            url:, published_at:,
+            author: present(author),
+            categories: categories.filter_map { |category| present(category) },
+            **media
           }.compact
         end
 
+        def media_url(media)
+          media.fetch(:enclosures, []).find { |facts| Article::Enclosure.new(**facts).media? }&.fetch(:url)
+        end
+
+        def atom_text(*nodes) = nodes.find { |node| node }&.content
+
         def atom_link(entry, base_url:)
-          links = entry.links
-          link = links.find { |node| node.rel.nil? || node.rel == 'alternate' } || links.first
-          absolute_url(link&.href, base_url:)
+          link = entry.links.find { |node| node.rel.nil? || node.rel == 'alternate' }
+          Url.resolve_web(link&.href, base_url)
         end
 
         def atom_author(entry)
@@ -95,19 +132,6 @@ module Html2rss
           present(author.name&.content) || present(author.email&.content)
         end
 
-        def absolute_url(raw, base_url:)
-          value = present(raw)
-          return unless value
-
-          if base_url
-            Html2rss::Url.from_relative(value, base_url)
-          else
-            Html2rss::Url.from_absolute(value)
-          end
-        rescue Html2rss::Url::InvalidUrlError
-          nil
-        end
-
         def present(value)
           text = value.to_s.strip
           text unless text.empty?

data/lib/html2rss/test/enhance_audit.rb

@@ -38,7 +38,7 @@ module Html2rss
         }
         warnings = []
 
-        selector_engine.each_enhance_pair do |baseline, enhanced, item|
+        selector_engine.each_enhance_pair(linked_only: true) do |baseline, enhanced, item|
           item_keys = keys_added(baseline, enhanced)
           merge_keys_added!(gains, item_keys)
           append_item_warnings!(warnings, item:, enhanced:, item_keys:)
@@ -160,7 +160,7 @@ module Html2rss
 
       def self.category_only_description?(description, enhanced)
         normalized = description.downcase
-        return true if Html2rss::Html::ArticleRules::Description::TYPE_CHIPS.include?(normalized)
+        return true if Html2rss::Article::Title::TYPE_MARKERS.include?(normalized)
 
         categories = Array(enhanced[:categories]).map { |category| category.to_s.strip }
         categories.any? { |category| category.casecmp?(description) }

data/lib/html2rss/test/item_sample.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Test
+    ##
+    # Builds the bounded +items+ preview of a test run from extracted articles,
+    # independent of the quality gate and of the rendered RSS.
+    module ItemSample
+      # Maximum characters kept in an item's plain-text description.
+      DESCRIPTION_LIMIT = 200
+
+      ##
+      # @param articles [Array<Html2rss::Article>] extracted articles
+      # @param limit [Integer] maximum number of items, >= 0
+      # @return [Array<Hash{Symbol => Object}>]
+      def self.call(articles, limit:)
+        articles.first(limit).map { |article| item_for(article) }
+      end
+
+      ##
+      # @param article [Html2rss::Article]
+      # @return [Hash{Symbol => Object}]
+      def self.item_for(article)
+        {
+          title: article.title.to_s.strip,
+          url: article.url&.to_s,
+          published_at: article.published_at&.iso8601,
+          description: plain_description(article.description),
+          image: article.image&.to_s,
+          categories: article.categories
+        }
+      end
+
+      ##
+      # @param html [String, nil]
+      # @return [String] whitespace-collapsed text truncated to {DESCRIPTION_LIMIT} characters
+      def self.plain_description(html)
+        text = Nokogiri::HTML.fragment(html).text.gsub(/\s+/, ' ').strip
+        text[0, DESCRIPTION_LIMIT]
+      end
+    end
+  end
+end

data/lib/html2rss/test.rb

@@ -17,7 +17,7 @@ module Html2rss
       ##
       # @param name [Symbol, String]
       # @return [FailureKind]
-      def self.[](name) = ALL.fetch(name.to_sym) { raise ArgumentError, "unknown failure kind: #{name.inspect}" }
+      def self.[](name) = ALL.fetch(name.to_sym) { raise InvalidOption, "unknown failure kind: #{name.inspect}" }
 
       ##
       # @return [Boolean]
@@ -78,6 +78,7 @@ module Html2rss
       :success,
       :item_count,
       :sample_items,
+      :items,
       :channel_title,
       :channel_url,
       :strategy_used,
@@ -100,6 +101,8 @@ module Html2rss
       # @param success [Boolean]
       # @param item_count [Integer]
       # @param sample_items [Array<Hash>]
+      # @param items [Array<Hash>] up to +sample_limit+ items (title, url, published_at, description, image,
+      #   categories) built from extracted articles regardless of the quality gate
       # @param channel_title [String, nil]
       # @param channel_url [String, nil]
       # @param strategy_used [Symbol, nil]
@@ -113,7 +116,7 @@ module Html2rss
       # @param response_body [String, nil] fetched page HTML copied from the pipeline response
       def initialize(success:, item_count:, sample_items:, channel_title:, channel_url:, # rubocop:disable Metrics/ParameterLists
                      strategy_used:, duration_seconds:, validation_issues:, error_message:,
-                     failure_kind:, rss:, quality_report: nil, enhance_compare: nil, response_body: nil)
+                     failure_kind:, rss:, items: [], quality_report: nil, enhance_compare: nil, response_body: nil)
         super
       end
 
@@ -136,6 +139,7 @@ module Html2rss
           success:,
           item_count:,
           sample_items:,
+          items:,
           channel_title:,
           channel_url:,
           strategy_used:,
@@ -160,15 +164,18 @@ module Html2rss
     # @param strategy [Symbol, nil] optional strategy override
     # @param strict_quality [Boolean] when true, fail on ship-quality audit thresholds
     # @param compare_enhance [Boolean] diagnostic enhance off vs on comparison on cached HTML
+    # @param sample_limit [Integer] maximum number of {Result#items} (default 3)
     # @return [Html2rss::Test::Result]
     def self.call(config_input, feed_name = nil, min_items: 1, params: {}, strategy: nil, # rubocop:disable Metrics/ParameterLists
-                  strict_quality: false, compare_enhance: false)
+                  strict_quality: false, compare_enhance: false, sample_limit: 3)
       raw_config, validation = Config.resolve_and_validate(config_input, feed_name:, params:)
       return validation_failure_result(validation, raw_config) unless validation.success?
 
       config = build_test_config(raw_config, strategy, params)
       started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      execute_timed_pipeline(raw_config, config, started, min_items:, strict_quality:, compare_enhance:)
+      execute_timed_pipeline(raw_config, config, started, min_items:, strict_quality:, compare_enhance:, sample_limit:)
+    rescue Html2rss::ContractError
+      raise
     rescue Html2rss::Error => error
       duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
       execution_failure_result(error, raw_config, duration)
@@ -193,19 +200,20 @@ module Html2rss
     private_class_method :compile_quality_report
 
     def self.execute_timed_pipeline(raw_config, config, started, min_items:, strict_quality:, # rubocop:disable Metrics/ParameterLists
-                                    compare_enhance:)
+                                    compare_enhance:, sample_limit:)
       feed_result, pipeline_outcome = extract_feed(raw_config)
       duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
       quality_report = compile_quality_report(raw_config, config, feed_result, pipeline_outcome)
       enhance_compare = build_enhance_compare(config, pipeline_outcome) if compare_enhance
 
       build_test_result(raw_config, feed_result, duration, quality_report, enhance_compare,
-                        min_items:, strict_quality:, response_body: pipeline_outcome.response.body)
+                        min_items:, strict_quality:, response_body: pipeline_outcome.response.body,
+                        items: ItemSample.call(pipeline_outcome.articles, limit: sample_limit))
     end
     private_class_method :execute_timed_pipeline
 
     def self.build_test_result(raw_config, feed_result, duration, quality_report, enhance_compare, # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/ParameterLists
-                               min_items:, strict_quality:, response_body:)
+                               min_items:, strict_quality:, response_body:, items:)
       rss_doc = feed_result.to_rss
       outcome = Policy.evaluate(item_count: rss_doc.items.size, min_items:, strict_quality:, quality_report:)
       strategy_used = feed_result.status.selected_strategy || raw_config[:strategy] ||
@@ -215,6 +223,7 @@ module Html2rss
         success: outcome.passed,
         item_count: rss_doc.items.size,
         sample_items: extract_samples(rss_doc.items),
+        items:,
         channel_title: feed_result.channel_title,
         channel_url: raw_config.dig(:channel, :url).to_s,
         strategy_used:,
@@ -246,14 +255,16 @@ module Html2rss
     # @param config [Html2rss::Config]
     # @param feed_result [Html2rss::FeedResult]
     # @param pipeline_outcome [Html2rss::FeedPipeline::PipelineOutcome]
-    # @param probe_native_feed [Boolean] when false, skip syndication discovery (apply ship gate)
+    # @param probe_native_feed [Boolean] when false, skip syndication discovery (apply ship gate);
+    #   when true, discovery reads the page in +pipeline_outcome.response+ (no extra page fetch)
     # @return [QualityReport]
     def self.quality_report_for(items, channel_url:, config:, feed_result:, pipeline_outcome:, # rubocop:disable Metrics/ParameterLists
                                 probe_native_feed: true)
       audit = AutoSource::Cleanup.audit_feed_items(items)
-      native_feed = probe_native_feed ? probe_native_feed_url(channel_url, config)&.to_s : nil
+      native_feed = probe_native_feed_url(channel_url, config, pipeline_outcome.response)&.to_s if probe_native_feed
       report = QualityReport.from_audit(audit, native_feed:)
       report = append_url_mismatch_warning(report, channel_url, feed_result)
+      report = append_items_without_url_warning(report, pipeline_outcome)
       merge_enhance_audit(report, config, pipeline_outcome)
     end
 
@@ -313,6 +324,21 @@ module Html2rss
     end
     private_class_method :enhance_gains_to_h
 
+    # Selector items dropped for having no URL ({Selectors#admission_drops}) leave nothing for the audit to
+    # see, so the warning names the +url+ selector as the place to look.
+    def self.append_items_without_url_warning(report, pipeline_outcome)
+      dropped = pipeline_outcome.admission_drops.fetch(Selectors::MISSING_URL_REASON, 0)
+      return report unless dropped.positive?
+
+      QualityReport.new(
+        warnings: (report.warnings + [:items_without_url]).freeze,
+        metrics: report.metrics.merge(items_without_url: dropped),
+        native_feed: report.native_feed,
+        defer_reason: report.defer_reason
+      )
+    end
+    private_class_method :append_items_without_url_warning
+
     def self.append_url_mismatch_warning(report, channel_url, feed_result)
       return report unless url_mismatch?(channel_url, feed_result)
 
@@ -340,15 +366,17 @@ module Html2rss
     end
     private_class_method :url_mismatch?
 
-    def self.probe_native_feed_url(channel_url, config)
+    # The page the pipeline fetched (where any redirect landed) is the page probed, as in Recon and Capture: no
+    # extra page fetch.
+    def self.probe_native_feed_url(channel_url, config, response)
       return nil if channel_url.to_s.empty?
 
       resources = FeedPipeline::RuntimePolicy.resources_for(config)
       strategy = FeedPipeline::StrategyPlan.concrete_for_diagnostic(config.strategy)
-      session = RequestSession.build(
-        config:, strategy:, budget: resources.budget, policy: resources.policy
-      )
-      Syndication::Discovery.best_feed_url(page_url: channel_url, request_session: session)
+      session = RequestSession.build(config:, strategy:, budget: resources.budget, policy: resources.policy)
+      Syndication::Discovery.native_feed_url(response:, request_session: session)
+    rescue Html2rss::ContractError
+      raise
     rescue Html2rss::Error
       nil
     end