html2rss 0.23.0 → 0.24.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f53482ae1084593193664ab8301bd451f53f68b4afabc8f0a336d9ae6dcd357c
-  data.tar.gz: 65ef2c8261a216040484134f71de9b8579378d6f7aa20100d0c79cf684efcd65
+  metadata.gz: 8e71e2d9112602603e9cbdf23907d2ad6baabc633c22bd9305094c2ab405f9ad
+  data.tar.gz: 1239aecc1f56a8ab4f5f83990671d4655127ad481907552e89fcce30af7d25ef
 SHA512:
-  metadata.gz: 18c6b5dfb6f23753b1c4885339b16505800b91235e65f397d7195d2301b7c32974c12be0087c35c2e63a5bd08185518ea49c6bd252cc68f22521814c6f6e74f8
-  data.tar.gz: 458fff5c3579ea95190e458a9cf5814418467fdfc37cedc1244faf4d7c7eb6582640b9d8465329f68eac3624c33c1691514da25c1ce1a7cb6ddb2c7999c38638
+  metadata.gz: fe0e1fe5a2b81886b0bda2f9087e12c42a5b309d105bcd7c1ea2268690740ee36390ad54bafb94757d758204968617569f981f489afe05572f41be28c1e4025a
+  data.tar.gz: cfad45944b508e5fafaae484cdb625e33ee43e336a2ab74b336f398c7eaab306432cf06d524ddb81b7ec5a002e77c065d7af491a6b245d0a2f5b2df6098aeda2

data/README.md

@@ -13,7 +13,7 @@ Most people looking for a first working feed should start with `html2rss-web`, r
 Detailed usage guides, reference docs, and the feed directory live on the project website:
 
 - [Ruby gem documentation](https://html2rss.github.io/ruby-gem)
-- [Request strategies](https://html2rss.github.io/ruby-gem/reference/strategy) (`auto`, `faraday`, `botasaurus`, `browserless`)
+- [Request strategies](https://html2rss.github.io/ruby-gem/reference/strategy) (`auto` = `faraday` → `botasaurus`; pin `browserless` explicitly)
 - [Selectors & pagination](https://html2rss.github.io/ruby-gem/reference/selectors#paginated-feeds)
 - [Web application](https://html2rss.github.io/web-application)
 - [Feed directory](https://html2rss.github.io/feed-directory)
@@ -26,7 +26,7 @@ Cloud development: [Open in GitHub Codespaces](https://github.com/codespaces/new
 ## Architecture
 
 1. **Config** — loads and validates configuration (YAML/hash); schema via `html2rss schema` / `schema/html2rss-config.schema.json`
-2. **RequestService** — fetches pages (`faraday`, `botasaurus`, or `browserless`)
+2. **RequestService** — fetches pages (`faraday`, `botasaurus`, or explicit `browserless`)
 3. **Selectors** — extracts content via CSS selectors with extractors/post-processors
 4. **AutoSource** — auto-detects content (Schema.org, JSON state, semantic HTML, structural patterns)
 5. **FeedBuilder** — assembles Article objects and renders feeds (RSS 2.0 / JSON Feed 1.1)

data/lib/html2rss/article.rb

@@ -1,14 +1,14 @@
 # frozen_string_literal: true
 
 require 'zlib'
-require 'sanitize'
-require 'nokogiri'
 
 module Html2rss
   ##
   # Article is a simple data object representing an article extracted from a page.
   # It is enumerable and responds to all keys specified in PROVIDED_KEYS.
-  # rubocop:disable Metrics/ClassLength
+  #
+  # Description and enclosure wire presentation live in {FeedBuilder::ItemPresentation}.
+  # rubocop:disable Metrics/ClassLength -- value object retains Marshal + defensive freeze helpers
   class Article
     include Enumerable
     include Comparable
@@ -26,9 +26,9 @@ module Html2rss
     # @param options [Hash{Symbol => String}]
     # @option options [String] :id stable article identifier
     # @option options [String] :title article title
-    # @option options [String] :description article description/content
+    # @option options [String] :description raw extracted description/content (not feed-rendered HTML)
     # @option options [String, Html2rss::Url] :url canonical article URL
-    # @option options [String, Html2rss::Url] :image image URL for fallback enclosure rendering
+    # @option options [String, Html2rss::Url] :image image URL for description / JSON Feed +image+
     # @option options [String] :author author name
     # @option options [String] :guid explicit GUID override
     # @option options [String, Time, DateTime] :published_at publication timestamp
@@ -36,9 +36,9 @@ module Html2rss
     # @option options [Array<String>] :categories category labels
     # @option options [Class] :scraper scraper class that produced the article
     def initialize(**options)
-      @to_h = options.each_with_object({}) { |(k, v), h| h[k] = v.freeze if v }.freeze
+      @to_h = options.each_with_object({}) { |(key, value), hash| hash[key] = freeze_option(value) }.freeze
 
-      @description = @url = @image = @guid = @enclosures = @enclosure = @categories = @published_at = NOT_SET
+      @url = @image = @guid = @enclosures = @categories = @published_at = NOT_SET
 
       return unless (unknown_keys = options.keys - PROVIDED_KEYS).any?
 
@@ -65,18 +65,10 @@ module Html2rss
     # @return [String, nil] article title
     def title = blank_string_to_nil(@to_h[:title])
 
-    # @return [String] rendered article description
-    def description
-      return @description unless @description == NOT_SET
-
-      @description = Html::Rendering::DescriptionBuilder.new(
-        base: @to_h[:description],
-        title:,
-        url:,
-        enclosures:,
-        image:
-      ).call
-    end
+    # Raw extracted description — feed HTML enrichment is {FeedBuilder::ItemPresentation.description_for}.
+    #
+    # @return [String, nil]
+    def description = blank_string_to_nil(@to_h[:description])
 
     # @return [Url, nil]
     def url
@@ -117,21 +109,7 @@ module Html2rss
 
       @enclosures = Array(@to_h[:enclosures])
                     .map { |enclosure| Enclosure.new(**enclosure) }
-    end
-
-    # @return [Html2rss::Article::Enclosure, nil]
-    def enclosure
-      return @enclosure unless @enclosure == NOT_SET
-
-      @enclosure = case (object = @to_h[:enclosures]&.first)
-                   when Hash
-                     Enclosure.new(**object)
-                   when nil
-                     Enclosure.new(url: image) if image
-                   else
-                     Log.warn "Article: unknown enclosure type: #{object.class}"
-                     nil
-                   end
+                    .freeze
     end
 
     # @return [Array<String>] normalized, unique category names
@@ -142,7 +120,7 @@ module Html2rss
         categories.map! { |category| category.to_s.strip }
         categories.reject!(&:empty?)
         categories.uniq!
-      end
+      end.freeze
     end
 
     # Parses and returns the published_at time.
@@ -171,6 +149,35 @@ module Html2rss
 
     private
 
+    ##
+    # Marshal only the constructor payload so lazy +NOT_SET+ sentinels do not break
+    # {FeedResult} cache round-trips.
+    #
+    # @return [Hash{Symbol => Object}]
+    def marshal_dump = @to_h
+
+    ##
+    # @param payload [Hash{Symbol => Object}] constructor options from {#marshal_dump}
+    # @return [void]
+    def marshal_load(payload)
+      initialize(**payload)
+    end
+
+    def freeze_option(value)
+      case value
+      when String then value.dup.freeze
+      when Array then freeze_array_option(value)
+      when Hash then value.transform_values { freeze_option(_1) }.freeze
+      else value
+      end
+    end
+
+    def freeze_array_option(value)
+      value.map do |entry|
+        entry.is_a?(Hash) ? entry.transform_values { freeze_option(_1) }.freeze : freeze_option(entry)
+      end.freeze
+    end
+
     def dedup_from_url
       return unless (value = url)
 

data/lib/html2rss/auto_source/scraper/schema/category_extractor.rb

@@ -7,138 +7,53 @@ module Html2rss
         ##
         # Extracts categories from Schema.org structured data.
         module CategoryExtractor
+          # Schema.org fields checked for category-like string values.
+          CATEGORY_FIELDS = %i[articleSection keywords categories tags].freeze
+
           ##
-          # Extracts categories from a schema object.
-          #
           # @param schema_object [Hash] The schema object
           # @return [Array<String>] Array of category strings
           def self.call(schema_object)
             Set.new.tap do |categories|
-              extract_field_categories!(categories, schema_object)
-              extract_about_categories!(categories, schema_object)
+              CATEGORY_FIELDS.each { |field| add_field(categories, schema_object[field]) }
+              add_about(categories, schema_object[:about])
             end.to_a
           end
 
-          ##
-          # Extracts categories from keywords, categories, and tags fields.
-          #
-          # @param schema_object [Hash] The schema object
-          # @return [Set<String>] Set of category strings
-          def self.extract_field_categories(schema_object)
-            Set.new.tap { |categories| extract_field_categories!(categories, schema_object) }
-          end
-
-          ##
-          # Extracts categories from keywords, categories, and tags fields.
-          #
-          # @param categories [Set<String>] Accumulator set
-          # @param schema_object [Hash] The schema object
-          # @return [void]
-          def self.extract_field_categories!(categories, schema_object)
-            %i[keywords categories tags].each do |field|
-              extract_field_value!(categories, schema_object[field])
-            end
-          end
+          class << self
+            private
 
-          ##
-          # Extracts categories from the about field.
-          #
-          # @param schema_object [Hash] The schema object
-          # @return [Set<String>] Set of category strings
-          def self.extract_about_categories(schema_object)
-            Set.new.tap { |categories| extract_about_categories!(categories, schema_object) }
-          end
-
-          ##
-          # Extracts categories from the about field.
-          #
-          # @param categories [Set<String>] Accumulator set
-          # @param schema_object [Hash] The schema object
-          # @return [void]
-          def self.extract_about_categories!(categories, schema_object)
-            about = schema_object[:about]
-            return unless about
-
-            if about.is_a?(Array)
-              extract_about_array!(categories, about)
-            elsif about.is_a?(String)
-              extract_string_categories!(categories, about)
+            def add_field(categories, value)
+              case value
+              when Array
+                value.each { |item| add_string(categories, item.to_s) }
+              when String
+                split_string(categories, value)
+              end
             end
-          end
-
-          ##
-          # Extracts categories from a single field value.
-          #
-          # @param schema_object [Hash] The schema object
-          # @param field [String] The field name
-          # @return [Set<String>] Set of category strings
-          def self.extract_field_value(schema_object, field)
-            Set.new.tap { |categories| extract_field_value!(categories, schema_object[field.to_sym]) }
-          end
 
-          ##
-          # Extracts categories from a single field value.
-          #
-          # @param categories [Set<String>] Accumulator set
-          # @param value [Object] The field value
-          # @return [void]
-          def self.extract_field_value!(categories, value)
-            return unless value
-
-            if value.is_a?(Array)
-              value.each do |item|
-                s = item.to_s
-                categories.add(s) unless s.empty?
+            def add_about(categories, about)
+              case about
+              when Array then add_about_items(categories, about)
+              when String then split_string(categories, about)
               end
-            elsif value.is_a?(String)
-              extract_string_categories!(categories, value)
             end
-          end
 
-          ##
-          # Extracts categories from an about array.
-          #
-          # @param about [Array] The about array
-          # @return [Set<String>] Set of category strings
-          def self.extract_about_array(about)
-            Set.new.tap { |categories| extract_about_array!(categories, about) }
-          end
-
-          ##
-          # Extracts categories from an about array.
-          #
-          # @param categories [Set<String>] Accumulator set
-          # @param about [Array] The about array
-          # @return [void]
-          def self.extract_about_array!(categories, about)
-            about.each do |item|
-              if item.is_a?(Hash) && item[:name]
-                categories.add(item[:name].to_s)
-              elsif item.is_a?(String)
-                categories.add(item)
+            def add_about_items(categories, about)
+              about.each do |item|
+                case item
+                when Hash then add_string(categories, item[:name].to_s)
+                when String then add_string(categories, item)
+                end
               end
             end
-          end
 
-          ##
-          # Extracts categories from a string by splitting on separators.
-          #
-          # @param string [String] source string that may contain category delimiters
-          # @return [Set<String>] Set of category strings
-          def self.extract_string_categories(string)
-            Set.new.tap { |categories| extract_string_categories!(categories, string) }
-          end
+            def split_string(categories, string)
+              string.split(/[,;|]/).each { |part| add_string(categories, part.strip) }
+            end
 
-          ##
-          # Extracts categories from a string by splitting on separators.
-          #
-          # @param categories [Set<String>] Accumulator set
-          # @param string [String] source string that may contain category delimiters
-          # @return [void]
-          def self.extract_string_categories!(categories, string)
-            string.split(/[,;|]/).each do |part|
-              s = part.strip
-              categories.add(s) unless s.empty?
+            def add_string(categories, string)
+              categories.add(string) unless string.empty?
             end
           end
         end

data/lib/html2rss/auto_source/scraper/schema/item_list.rb

@@ -5,28 +5,67 @@ module Html2rss
     module Scraper
       class Schema
         ##
-        # Handles schema.org ItemList objects, which contain
-        # 1. itemListElements, and/or
-        # 2. interesting attributes, i.e. description, url, image, itself.
+        # Walks schema.org ItemList objects and yields article-shaped elements only.
+        # The ItemList container itself is never emitted.
         #
         # @see https://schema.org/ItemList
         class ItemList < Thing
           # Schema.org type names handled by the ItemList extractor.
           SUPPORTED_TYPES = Set['ItemList']
 
-          # @return [Array<Hash>] the scraped article hashes with DEFAULT_ATTRIBUTES
+          # @return [Array<Hash>] article hashes from list elements (no parent container)
           def call
-            hashes = [super]
-
-            return hashes unless (elements = @schema_object[:itemListElement])
+            elements = @schema_object[:itemListElement]
+            return [] unless elements
 
             elements = [elements] unless elements.is_a?(Array)
+            elements.filter_map { |element| materialize_element(element) }
+          end
+
+          private
+
+          # @param element [Object] raw itemListElement entry
+          # @return [Hash, nil] scraped article hash when emit-worthy
+          def materialize_element(element)
+            object, from_list_item = unwrap_list_item(element)
+            return unless object.is_a?(Hash)
+            return unless emit?(object, from_list_item:)
+
+            article = Thing.new(object, url: base_url || '').call
+            titleize_list_item_stub!(article) if from_list_item
+            article
+          end
+
+          # @param element [Object] raw list entry
+          # @return [Array(Hash, Boolean)] schema object and whether it came from ListItem
+          def unwrap_list_item(element)
+            return [element, false] unless element.is_a?(Hash)
+
+            types = Schema.normalize_types(element[:@type])
+            return [element, false] unless types.include?('ListItem')
+
+            item = element[:item]
+            [item.is_a?(Hash) ? item : element, true]
+          end
+
+          # @param object [Hash] candidate schema object
+          # @param from_list_item [Boolean] whether object came from a ListItem unwrap
+          # @return [Boolean] whether to emit this object as an article
+          def emit?(object, from_list_item:)
+            return true if from_list_item
+
+            Schema.normalize_types(object[:@type]).intersect?(Thing::SUPPORTED_TYPES)
+          end
 
-            elements.each do |schema_object|
-              hashes << ListItem.new(schema_object, url: base_url || '').call
-            end
+          # URL-only ListItem stubs historically used a titleized path as title.
+          #
+          # @param article [Hash] scraped article hash
+          # @return [void]
+          def titleize_list_item_stub!(article)
+            return unless article[:title].to_s.empty?
+            return unless (article_url = article[:url])
 
-            hashes
+            article[:title] = article_url.titleized
           end
         end
       end

data/lib/html2rss/auto_source/scraper/schema/thing.rb

@@ -40,8 +40,13 @@ module Html2rss
             @id = id.to_s.empty? ? nil : id
           end
 
-          # @return [String, nil] article title
-          def title = schema_object[:title]
+          # @return [String, nil] article title (schema.org: headline → name → alternativeHeadline; legacy title)
+          def title
+            schema_object.values_at(:headline, :name, :alternativeHeadline, :title)
+                         .lazy
+                         .map { _1.to_s.strip }
+                         .find { !_1.empty? }
+          end
 
           # @return [String, nil] longest available description field
           def description
@@ -74,8 +79,11 @@ module Html2rss
             @image = img_url ? Url.from_relative(img_url, base_url || img_url) : nil
           end
 
-          # @return [String, nil] published-at timestamp string
-          def published_at = schema_object[:datePublished]
+          # @return [String, nil] published-at timestamp (datePublished, else dateModified)
+          def published_at
+            value = schema_object[:datePublished] || schema_object[:dateModified]
+            value.to_s.empty? ? nil : value
+          end
 
           # @return [Array<String>, nil] extracted category labels
           def categories
@@ -107,7 +115,8 @@ module Html2rss
           # @return [String, nil] image URL string
           def image_url_from(obj)
             return obj if obj.is_a?(String)
-            return unless obj.is_a?(Hash) && obj[:@type] == 'ImageObject'
+            return unless obj.is_a?(Hash)
+            return unless Schema.normalize_types(obj[:@type]).include?('ImageObject')
 
             obj[:url] || obj[:contentUrl]
           end

data/lib/html2rss/auto_source/scraper/schema.rb

@@ -18,13 +18,25 @@ module Html2rss
         # Selector for JSON-LD script tags containing Schema.org objects.
         TAG_SELECTOR = 'script[type="application/ld+json"]'
 
-        # Pre-compiled regex union for supported schema types.
-        # Performs a single pass over script tag text instead of multiple regex matches.
+        # Pre-compiled regex for supported schema types (short name or schema.org URL; string or array @type).
+        # Allows preceding entries in a JSON @type array (e.g. ["WebPage","NewsArticle"]).
         SUPPORTED_TYPES_RE = begin
           types = Thing::SUPPORTED_TYPES | ItemList::SUPPORTED_TYPES
-          /"@type"\s*:\s*"(?:#{Regexp.union(types.to_a).source})"/
+          type_re = Regexp.union(types.to_a)
+          %r{"@type"\s*:\s*(?:\[\s*(?:"[^"]*"\s*,\s*)*)?"(?:https?://schema\.org/)?(?:#{type_re.source})"}
         end.freeze
 
+        # Matches a leading schema.org URL prefix on @type values (http or https).
+        SCHEMA_ORG_PREFIX_RE = %r{\Ahttps?://schema\.org/}i
+
+        # Prefer these keys when recursively walking unsupported container objects.
+        COLLECTION_KEYS = %i[itemListElement blogPost mainEntity hasPart].freeze
+
+        # Container types that must never be emitted as feed items (walk children only).
+        DENIED_CONTAINER_TYPES = Set[
+          'ItemList', 'Blog', 'BreadcrumbList', 'WebPage', 'CollectionPage'
+        ].freeze
+
         # @return [Symbol] scraper config key
         def self.options_key = :schema
 
@@ -46,19 +58,17 @@ module Html2rss
           # of all supported schema objects
           # by recursively traversing the given `object`.
           #
+          # Prefers collection keys ({COLLECTION_KEYS}) when walking containers.
+          #
           # @param object [Hash, Array, Nokogiri::XML::Element]
           # @return [Array<Hash>] the schema_objects, or an empty array
           # :reek:DuplicateMethodCall
           def from(object)
             case object
-            when Nokogiri::XML::Element
-              from(parse_script_tag(object))
-            when Hash
-              supported_schema_object?(object) ? [object] : object.values.flat_map { |item| from(item) }
-            when Array
-              object.flat_map { |item| from(item) }
-            else
-              []
+            when Nokogiri::XML::Element then from(parse_script_tag(object))
+            when Hash then from_hash(object)
+            when Array then object.flat_map { |item| from(item) }
+            else []
             end
           end
 
@@ -72,20 +82,53 @@ module Html2rss
           # @param schema_object [Hash{Symbol => Object}] schema object with an @type key
           # @return [Scraper::Schema::Thing, Scraper::Schema::ItemList, nil] a class responding to `#call`
           def scraper_for_schema_object(schema_object)
-            type = schema_object[:@type]
+            types = normalize_types(schema_object[:@type])
+
+            # Prefer article/list extractors over denied containers for multi-type @type arrays
+            # (e.g. ["NewsArticle","WebPage"]).
+            return ItemList if types.intersect?(ItemList::SUPPORTED_TYPES)
+            return Thing if types.intersect?(Thing::SUPPORTED_TYPES)
+            return nil if types.intersect?(DENIED_CONTAINER_TYPES)
+
+            Log.debug("#{name}: unsupported schema object @type=#{schema_object[:@type].inspect}")
+            nil
+          end
 
-            if Thing::SUPPORTED_TYPES.member?(type)
-              Thing
-            elsif ItemList::SUPPORTED_TYPES.member?(type)
-              ItemList
+          # Normalizes Schema.org `@type` wire forms to short type names.
+          #
+          # Handles String, Symbol, Array, and strips `https://schema.org/` / `http://schema.org/` prefixes.
+          #
+          # @param object [String, Symbol, Array, nil] raw `@type` value
+          # @return [Set<String>] short type names (e.g. "NewsArticle")
+          # @api private
+          def normalize_types(object)
+            case object
+            when Array
+              object.each_with_object(Set.new) { |item, set| set.merge(normalize_types(item)) }
+            when String, Symbol
+              short = object.to_s.sub(SCHEMA_ORG_PREFIX_RE, '')
+              short.empty? ? Set.new : Set[short]
             else
-              Log.debug("#{name}: unsupported schema object @type=#{type.inspect}")
-              nil
+              Set.new
             end
           end
 
           private
 
+          # @param hash [Hash] candidate schema object
+          # @return [Array<Hash>] the object itself or nested supported objects
+          def from_hash(hash)
+            supported_schema_object?(hash) ? [hash] : walk_hash_values(hash)
+          end
+
+          # @param hash [Hash] schema object that is not itself extractable
+          # @return [Array<Hash>] nested supported objects
+          def walk_hash_values(hash)
+            preferred = COLLECTION_KEYS.filter_map { |key| hash[key] }
+            rest = hash.except(*COLLECTION_KEYS).values
+            (preferred + rest).flat_map { |item| from(item) }
+          end
+
           def parse_script_tag(script_tag)
             JSON.parse(script_tag.text, symbolize_names: true)
           rescue JSON::ParserError => error