html2rss 0.33.0 → 0.34.0

data/lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb

@@ -8,8 +8,15 @@ module Html2rss
         module ArticleNormalizer # rubocop:disable Metrics/ModuleLength -- field keys + URL/scalar unwrap stay co-located
           # Preferred keys when extracting title-like values from state payloads.
           TITLE_KEYS = %i[title headline name text].freeze
+          # Title keys that mark an article; a bare +name+ also labels people and other entities.
+          ARTICLE_TITLE_KEYS = (TITLE_KEYS - %i[name]).freeze
+          # Explicit kicker field names (app-state +overline+/+kickerText+, German CMS +topline+/+dachzeile+) that
+          # sit beside a headline. The one closed list: no standard Schema/JSON-LD field exists, so none is guessed.
+          KICKER_KEYS = %i[overline kicker kickerText topline dachzeile eyebrow].freeze
           # Preferred keys when extracting URL-like values from state payloads.
-          URL_KEYS = %i[url link href permalink slug path canonicalUrl shortUrl alias self].freeze
+          URL_KEYS = %i[url link href permalink slug path canonicalUrl shortUrl alias self links].freeze
+          # URL keys an entry itself can own; +links+ is an envelope sibling of the node, never the entry's own URL.
+          DIRECT_URL_KEYS = (URL_KEYS - %i[links]).freeze
           # Preferred keys when extracting description-like values from state payloads.
           DESCRIPTION_KEYS = %i[description summary excerpt dek subheading abstract].freeze
           # Preferred keys when extracting image-like values from state payloads.
@@ -27,6 +34,8 @@ module Html2rss
           SCALAR_KEYS = %i[rendered value text].freeze
           # Snake `_at`/`_on` or camel `At`/`On` suffix after a closed published-at miss.
           DATE_KEY_SUFFIX = /_at\z|_on\z|[a-z]At\z|[a-z]On\z/
+          # Levels below a wrapper entry searched for its article node.
+          MAX_NODE_DEPTH = 2
 
           class << self
             # rubocop:disable Metrics/MethodLength
@@ -34,28 +43,60 @@ module Html2rss
             # @param base_url [String, Html2rss::Url] base URL for relative link resolution
             # @return [Hash{Symbol => Object, nil}] normalized article hash for downstream extraction
             def normalise(entry, base_url:)
-              return unless entry.is_a?(Hash)
-
-              title = string(ValueFinder.fetch(entry, TITLE_KEYS))
-              description = string(ValueFinder.fetch(entry, DESCRIPTION_KEYS))
-              article_url = resolve_link(entry, keys: URL_KEYS, base_url:,
-                                                log_key: 'JsonState: invalid URL encountered')
+              sources = article_sources(entry) # nil (skipped entry) yields no URL below
+              title = string(ValueFinder.fetch(sources, TITLE_KEYS))
+              kicker = title && string(ValueFinder.fetch(sources, KICKER_KEYS))
+              description = string(ValueFinder.fetch(sources, DESCRIPTION_KEYS))
+              article_url = resolve_link(sources, keys: URL_KEYS, base_url:,
+                                                  log_key: 'JsonState: invalid URL encountered')
               return unless article_url
               return if title.nil? && description.nil?
 
               {
                 title:,
+                kicker:,
                 description:,
                 url: article_url,
-                image: resolve_link(entry, keys: IMAGE_KEYS, base_url:,
-                                           log_key: 'JsonState: invalid image URL encountered'),
-                published_at: published_at(entry),
-                categories: categories(entry),
-                id: identifier(entry, article_url)
+                image: resolve_link(sources, keys: IMAGE_KEYS, base_url:,
+                                             log_key: 'JsonState: invalid image URL encountered'),
+                published_at: published_at(sources),
+                categories: categories(sources),
+                id: identifier(sources, article_url)
               }.compact
             end
             # rubocop:enable Metrics/MethodLength
 
+            # Locates the hashes holding an entry's article fields, article node first.
+            #
+            # An entry with its own title or description is its own article node. A wrapper entry
+            # delegates to the only nested Hash (up to {MAX_NODE_DEPTH} levels down) carrying a title
+            # and URL, else to the only nested Hash carrying a title; the entry stays the fallback
+            # source (e.g. an envelope's sibling +links+). A wrapper entry with its own URL is the
+            # link: it delegates only to the only nested Hash carrying an article title (never a bare
+            # +name+), and its URL wins over the node's. An entry whose only URL is +links+ is not that
+            # link: the node leads and +links+ stays a fallback. Competing nested nodes skip the entry, so an
+            # author object holding a name and URL never becomes the article.
+            #
+            # @param entry [Object] raw article entry candidate
+            # @return [Array<Hash>, nil] field sources in lookup order; nil when not a Hash or ambiguous
+            def article_sources(entry)
+              return unless entry.is_a?(Hash)
+              return [entry] if text?(entry)
+
+              nodes = candidate_nodes(entry)
+              return [entry] if nodes.empty?
+              return unless nodes.one?
+
+              node = nodes.first
+              own_url?(entry) && url?(node) ? [entry, node] : [node, entry]
+            end
+
+            # @param sources [Hash, Array<Hash>, nil] entry or field sources from {article_sources}
+            # @return [Boolean] whether the sources hold a title and a URL value
+            def linked?(sources)
+              title?(sources) && url?(sources)
+            end
+
             # @param value [Object] candidate scalar, or Hash wrapping +rendered+/+value+/+text+
             # @return [String, nil] normalized non-empty string (HTML preserved for descriptions)
             def string(value)
@@ -80,15 +121,13 @@ module Html2rss
               value unless value.is_a?(Hash) || value.is_a?(Array)
             end
 
-            # @param entry [Hash] raw article entry candidate
+            # @param sources [Hash, Array<Hash>] entry or field sources from {article_sources}
             # @param keys [Array<Symbol>] preferred link keys
             # @param base_url [String, Html2rss::Url] base URL for relative link resolution
             # @param log_key [String] structured log message key
             # @return [Html2rss::Url, nil] resolved absolute URL
-            def resolve_link(entry, keys:, base_url:, log_key:)
-              value = unwrap_url(ValueFinder.fetch(entry, keys), keys) ||
-                      unwrap_url(ValueFinder.fetch(entry[:links], keys), keys)
-              string = string(value)
+            def resolve_link(sources, keys:, base_url:, log_key:)
+              string = string(unwrap_url(ValueFinder.fetch(sources, keys), keys))
               return unless string
 
               Url.from_relative(string, base_url)
@@ -97,38 +136,36 @@ module Html2rss
               nil
             end
 
-            # @param value [Object] candidate URL value, possibly a nested Hash
+            # @param value [Object] candidate URL value, possibly a nested Hash or a links Array
             # @param keys [Array<Symbol>] preferred link keys
-            # @return [Object, nil] scalar URL candidate after at most {MAX_URL_UNWRAP} Hash unwraps
+            # @return [Object, nil] scalar URL candidate after at most {MAX_URL_UNWRAP} unwraps
             def unwrap_url(value, keys)
               depth = 0
-              while value.is_a?(Hash) && depth < MAX_URL_UNWRAP
+              while (value.is_a?(Hash) || value.is_a?(Array)) && depth < MAX_URL_UNWRAP
                 value = ValueFinder.fetch(value, keys)
                 depth += 1
               end
-              value unless value.is_a?(Hash)
+              value unless value.is_a?(Hash) || value.is_a?(Array)
             end
 
-            # @param entry [Hash] raw article entry candidate
+            # @param sources [Hash, Array<Hash>] entry or field sources from {article_sources}
             # @return [String, nil] publication timestamp string or unix timestamp digits
-            def published_at(entry)
-              value = ValueFinder.fetch(entry, PUBLISHED_AT_KEYS)
-              string(value) || string(suffix_date_value(entry))
+            def published_at(sources)
+              value = ValueFinder.fetch(sources, PUBLISHED_AT_KEYS)
+              string(value) || string(suffix_date_value(sources))
             end
 
-            # @param hash [Hash, nil] entry or nested attributes/data hash
+            # @param sources [Hash, Array<Hash>, nil] entry or field sources, searched in order
             # @return [Object, nil] first string or numeric value whose key matches {DATE_KEY_SUFFIX}
-            def suffix_date_value(hash)
-              return unless hash.is_a?(Hash)
-
-              hash.each do |key, value|
-                next unless date_suffix_key?(key)
-                next unless value.is_a?(String) || value.is_a?(Numeric)
-
-                return value
+            def suffix_date_value(sources)
+              case sources
+              when Hash
+                sources.each do |key, value|
+                  return value if date_suffix_key?(key) && (value.is_a?(String) || value.is_a?(Numeric))
+                end
+                nil
+              when Array then sources.filter_map { suffix_date_value(_1) }.first
               end
-
-              suffix_date_value(hash[:attributes]) || suffix_date_value(hash[:data])
             end
 
             # @param key [Symbol, String] candidate published-at key
@@ -138,10 +175,10 @@ module Html2rss
             end
 
             # rubocop:disable Metrics/MethodLength
-            # @param entry [Hash] raw article entry candidate
+            # @param sources [Hash, Array<Hash>] entry or field sources from {article_sources}
             # @return [Array<String>, nil] normalized unique categories
-            def categories(entry)
-              raw = ValueFinder.fetch(entry, CATEGORY_KEYS)
+            def categories(sources)
+              raw = ValueFinder.fetch(sources, CATEGORY_KEYS)
               names = case raw
                       when Array then raw
                       when Hash then raw.values
@@ -163,14 +200,54 @@ module Html2rss
             end
             # rubocop:enable Metrics/MethodLength
 
-            # @param entry [Hash] raw article entry candidate
+            # @param sources [Hash, Array<Hash>] entry or field sources from {article_sources}
             # @param article_url [Html2rss::Url] resolved article URL
             # @return [String] stable article identifier fallbacking to resolved URL
-            def identifier(entry, article_url)
-              value = ValueFinder.fetch(entry, ID_KEYS)
+            def identifier(sources, article_url)
+              value = ValueFinder.fetch(sources, ID_KEYS)
               value = ValueFinder.fetch(value, ID_KEYS) if value.is_a?(Hash)
               string(value) || article_url.to_s
             end
+
+            private
+
+            # Shape checks (key holds a value), not usability checks: normalise decides usability.
+            def text?(hash)
+              title?(hash) || !ValueFinder.fetch(hash, DESCRIPTION_KEYS).nil?
+            end
+
+            def title?(object)
+              !ValueFinder.fetch(object, TITLE_KEYS).nil?
+            end
+
+            def url?(object)
+              !ValueFinder.fetch(object, URL_KEYS).nil?
+            end
+
+            def own_url?(entry)
+              !ValueFinder.fetch(entry, DIRECT_URL_KEYS).nil?
+            end
+
+            # Nested nodes that may hold the article fields of a wrapper entry.
+            def candidate_nodes(entry)
+              titled = titled_nodes(entry, MAX_NODE_DEPTH)
+              return titled.reject { ValueFinder.fetch(_1, ARTICLE_TITLE_KEYS).nil? } if own_url?(entry)
+
+              linked = titled.select { url?(_1) }
+              linked.empty? ? titled : linked
+            end
+
+            # Titled Hash values up to +depth+ levels down; a titled node's children are its own fields.
+            def titled_nodes(hash, depth)
+              return [] if depth.zero?
+
+              hash.each_value.flat_map do |value|
+                next [] unless value.is_a?(Hash)
+                next value if title?(value)
+
+                titled_nodes(value, depth - 1)
+              end
+            end
           end
         end
       end

data/lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb

@@ -30,26 +30,15 @@ module Html2rss
             end
 
             # @param array [Array<Object>] candidate list of entries
-            # @return [Boolean] whether array includes hash entries with title and URL fields
+            # @return [Boolean] whether array holds at least one title+url hash
             def array_of_articles?(array)
-              array.any? do |element|
-                next unless element.is_a?(Hash)
-
-                title_from(element) && url_from(element)
-              end
+              array.any? { linked_entry?(_1) }
             end
 
-            # @param object [Hash] article candidate object
-            # @return [Object, nil] detected title-like value
-            def title_from(object)
-              ValueFinder.fetch(object, ArticleNormalizer::TITLE_KEYS)
-            end
+            private
 
-            # @param object [Hash] article candidate object
-            # @return [Object, nil] detected URL-like value (including links.self)
-            def url_from(object)
-              ValueFinder.fetch(object, ArticleNormalizer::URL_KEYS) ||
-                ValueFinder.fetch(object[:links], ArticleNormalizer::URL_KEYS)
+            def linked_entry?(element)
+              ArticleNormalizer.linked?(ArticleNormalizer.article_sources(element))
             end
           end
         end

data/lib/html2rss/auto_source/scraper/json_state/value_finder.rb

@@ -19,14 +19,13 @@ module Html2rss
 
             # @param hash [Hash] hash candidate traversed during key lookup
             # @param keys [Array<Symbol>] keys to probe in order
-            # @return [Object, nil] first matching value from hash or nested metadata
+            # @return [Object, nil] value of the first present key
             def fetch_from_hash(hash, keys)
               keys.each do |key|
                 return hash[key] if hash.key?(key)
               end
 
-              fetch_nested(hash[:attributes], keys) ||
-                fetch_nested(hash[:data], keys)
+              nil
             end
 
             # @param array [Array] array whose entries may contain target keys
@@ -40,13 +39,6 @@ module Html2rss
 
               nil
             end
-
-            # @param value [Hash, Array, nil] nested value to recurse into
-            # @param keys [Array<Symbol>] keys to probe in order
-            # @return [Object, nil] matching nested value
-            def fetch_nested(value, keys)
-              fetch(value, keys) if value
-            end
           end
         end
       end

data/lib/html2rss/auto_source/scraper/json_state.rb

@@ -21,6 +21,7 @@ module Html2rss
 
         class << self
           # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML document
+          # @return [Boolean] whether any embedded JSON document holds candidate articles
           def articles?(parsed_body)
             return false unless parsed_body
 
@@ -62,9 +63,7 @@ module Html2rss
 
         # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML document
         # @param url [String, Html2rss::Url] page URL used to resolve relative links
-        # @param _opts [Hash] scraper-specific options
-        # @option _opts [Object] :_reserved reserved for future scraper-specific options
-        def initialize(parsed_body, url:, **_opts)
+        def initialize(parsed_body, url:)
           @parsed_body = parsed_body
           @url = url
         end

data/lib/html2rss/auto_source/scraper/meta_oembed.rb

@@ -60,10 +60,8 @@ module Html2rss
         # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
         # @param url [String, Html2rss::Url] base page URL
         # @param request_session [Html2rss::RequestSession, nil] shared request session for follow-up fetches
-        # @param _opts [Hash] unused scraper-specific options
-        # @option _opts [Object] :_reserved reserved for future scraper-specific options
         # @return [void]
-        def initialize(parsed_body, url:, request_session: nil, **_opts)
+        def initialize(parsed_body, url:, request_session: nil)
           @parsed_body = parsed_body
           @url = Html2rss::Url.from_absolute(url)
           @request_session = request_session
@@ -131,6 +129,8 @@ module Html2rss
 
           response = request_session.follow_up(url: oembed_url, relation: :auto_source, origin_url: url)
           parse_oembed_response(response)
+        rescue Html2rss::ContractError
+          raise
         rescue Html2rss::Error, RequestService::UnsupportedResponseContentType, JSON::ParserError => error
           Log.warn({
             event_name: 'meta_oembed.fetch_failed',
@@ -181,26 +181,18 @@ module Html2rss
         end
 
         # @param meta_data [Hash{Symbol => Object}] extracted OpenGraph meta tags
-        # @return [Array<Hash{Symbol => Object}>, nil] og:video / og:audio enclosures (page-type embeds skipped)
+        # @return [Array<Hash{Symbol => Object}>, nil] og:video / og:audio enclosures (page-type embeds skipped; no
+        #   kind, since an og:video is often a player page rather than a file)
         def media_enclosures(meta_data)
           found = %i[video audio].filter_map do |kind|
             type = meta_data[:"#{kind}_type"]
             next if type && !type.start_with?("#{kind}/")
 
-            media_enclosure(meta_data[kind], type)
+            Html2rss::Html::ArticleRules::Enclosure.from_media(meta_data[kind], type, url)
           end
           found unless found.empty?
         end
 
-        # @param src [String, nil] raw media URL
-        # @param type [String, nil] declared MIME type
-        # @return [Hash{Symbol => Object}, nil] enclosure attributes
-        def media_enclosure(src, type)
-          ::Html2rss::Html::ArticleRules::Enclosure.from_media(src, type, url)
-        rescue Html2rss::Url::InvalidUrlError
-          nil
-        end
-
         # @param raw_url [String, nil] raw relative or absolute URL string
         # @return [Html2rss::Url, nil] resolved absolute URL or nil
         def resolve_url(raw_url)

data/lib/html2rss/auto_source/scraper/microdata.rb

@@ -24,6 +24,7 @@ module Html2rss
 
         class << self
           # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML document
+          # @return [Boolean] whether any supported Microdata root exists
           def articles?(parsed_body)
             supported_roots(parsed_body).any?
           end
@@ -37,6 +38,7 @@ module Html2rss
           end
 
           # @param node [Nokogiri::XML::Element] itemscope candidate node
+          # @return [Boolean] whether the node is a supported top-level root
           def supported_root?(node)
             supported_type_name(node) && top_level_item?(node)
           end
@@ -59,6 +61,7 @@ module Html2rss
           end
 
           # @param node [Nokogiri::XML::Element] itemscope candidate node
+          # @return [Boolean] whether the node is not nested in another item property
           def top_level_item?(node)
             return false if node.attribute('itemprop')
 
@@ -78,10 +81,8 @@ module Html2rss
         # @param parsed_body [Nokogiri::HTML5::Document, Nokogiri::HTML4::Document, Nokogiri::XML::Node, nil]
         #   the parsed response body to inspect for top-level Microdata items.
         # @param url [Html2rss::Url] the absolute page URL used to resolve relative links.
-        # @param _opts [Hash] unused scraper-specific options.
-        # @option _opts [Object] :_reserved reserved for future scraper-specific options
         # @return [void]
-        def initialize(parsed_body, url:, **_opts)
+        def initialize(parsed_body, url:)
           @parsed_body = parsed_body
           @url = url
         end
@@ -447,7 +448,8 @@ module Html2rss
               image: ValueNormalizer.image_value(properties.delete(:image), properties.delete(:thumbnailUrl),
                                                  properties.delete(:thumbnail)),
               video: properties.delete(:video),
-              audio: properties.delete(:audio)
+              audio: properties.delete(:audio),
+              associatedMedia: properties.delete(:associatedMedia)
             }
           end
 

data/lib/html2rss/auto_source/scraper/microformats2.rb

@@ -35,10 +35,8 @@ module Html2rss
 
         # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
         # @param url [String, Html2rss::Url] base page URL
-        # @param _opts [Hash] unused scraper-specific options
-        # @option _opts [Object] :_reserved reserved for future options
         # @return [void]
-        def initialize(parsed_body, url:, **_opts)
+        def initialize(parsed_body, url:)
           @parsed_body = parsed_body
           @url = Html2rss::Url.from_absolute(url)
         end

data/lib/html2rss/auto_source/scraper/native_feed.rb

@@ -5,9 +5,15 @@ module Html2rss
     module Scraper
       ##
       # Promotes a page's native RSS/Atom feed via {Syndication::Discovery} + {Syndication::Parser}.
+      # After {#each}, {#channel} holds what that feed states about its channel and {#feed_url} where
+      # it was read, the provenance {Cleanup} admits the feed's own domain by.
       class NativeFeed
         include Enumerable
 
+        # Href shapes that Discovery's path probing can still resolve when the link type is not a feed MIME type.
+        FEEDISH_HREF = /rss|atom|feed|\.xml/i
+        private_constant :FEEDISH_HREF
+
         # @return [Symbol] scraper config key
         def self.options_key = :native_feed
 
@@ -27,25 +33,16 @@ module Html2rss
         def self.articles?(parsed_body)
           return false unless parsed_body.is_a?(Nokogiri::HTML::Document)
 
-          return true if ::Html2rss::Html::FeedLink.from_document(parsed_body).any?
+          return true if Html2rss::Html::FeedLink.from_document(parsed_body).any?
 
-          parsed_body.css('head link[rel~="alternate"][href]').any? do |node|
-            href = node['href'].to_s
-            ::Html2rss::Html::Probe.mime_match?(
-              node['type'],
-              ::Html2rss::Html::Probe::APPLICATION_RSS_XML,
-              ::Html2rss::Html::Probe::APPLICATION_ATOM_XML
-            ) || href.match?(/rss|atom|feed|\.xml/i)
-          end
+          parsed_body.css('head link[rel~="alternate"][href]').any? { |node| node['href'].match?(FEEDISH_HREF) }
         end
 
         ##
         # @param parsed_body [Nokogiri::HTML::Document]
         # @param url [String, Html2rss::Url]
         # @param request_session [Html2rss::RequestSession, nil]
-        # @param _opts [Hash]
-        # @option _opts [Object] :_reserved reserved for future scraper-specific options
-        def initialize(parsed_body, url:, request_session: nil, **_opts)
+        def initialize(parsed_body, url:, request_session: nil)
           @parsed_body = parsed_body
           @url = Html2rss::Url.from_absolute(url)
           @request_session = request_session
@@ -54,6 +51,16 @@ module Html2rss
         # @return [Boolean]
         def extractable? = self.class.articles?(parsed_body)
 
+        ##
+        # @return [Syndication::ChannelFacts, nil] what the followed feed states about its channel;
+        #   nil before {#each} or when no feed was read
+        def channel = @feed&.channel
+
+        ##
+        # @return [Html2rss::Url, nil] URL of the feed read (after redirects); nil before {#each} or
+        #   when no feed was read
+        attr_reader :feed_url
+
         ##
         # @yieldparam article [Hash{Symbol => Object}]
         # @return [Enumerator, void]
@@ -61,7 +68,7 @@ module Html2rss
         def each(&)
           return enum_for(:each) unless block_given?
 
-          articles = fetch_articles
+          articles = (@feed = fetch_feed).articles
           if articles.empty?
             Log.info({
               event_name: 'auto_source.native_feed.empty',
@@ -84,8 +91,8 @@ module Html2rss
         attr_reader :parsed_body, :url, :request_session
 
         # rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength -- discovery + parse + structured warn
-        def fetch_articles
-          return [] unless request_session
+        def fetch_feed
+          return Syndication::Feed::NONE unless request_session
 
           response = Syndication::Discovery.best_feed_response(
             page_url: url,
@@ -93,9 +100,12 @@ module Html2rss
             parsed_body:,
             max_probes: self.class.request_slots
           )
-          return [] unless response
+          return Syndication::Feed::NONE unless response
 
+          @feed_url = response.url
           Syndication::Parser.parse_response(response)
+        rescue Html2rss::ContractError
+          raise
         rescue Html2rss::Error => error
           Log.warn({
             event_name: 'auto_source.native_feed.failed',
@@ -103,7 +113,7 @@ module Html2rss
             error_class: error.class.name,
             message: "#{self.class}: host=#{url.host} failed (#{error.class}: #{error.message})"
           }.to_json)
-          []
+          Syndication::Feed::NONE
         end
       end
     end

data/lib/html2rss/auto_source/scraper/schema/item_list.rb

@@ -49,11 +49,13 @@ module Html2rss
 
           # @param object [Hash] candidate schema object
           # @param from_list_item [Boolean] whether object came from a ListItem unwrap
-          # @return [Boolean] whether to emit this object as an article (also bare Event, JobPosting and video items)
+          # @return [Boolean] whether to emit this object: a ListItem unwrap admits everything except pure
+          #   entities ({Thing::ENTITY_TYPES}); a bare element must be an article, media, Event or JobPosting type
           def emit?(object, from_list_item:)
-            return true if from_list_item
+            types = Schema.normalize_types(object[:@type])
+            return types.empty? || !types.subset?(Thing::ENTITY_TYPES) if from_list_item
 
-            Schema.normalize_types(object[:@type]).intersect?(Thing::SUPPORTED_TYPES | Thing::ITEM_LIST_TYPES)
+            types.intersect?(Thing::ITEM_TYPES | Thing::ITEM_LIST_TYPES)
           end
         end
       end