html2rss 0.32.3 → 0.33.0

data/lib/html2rss/auto_source/scraper/schema/category_extractor.rb

@@ -8,7 +8,7 @@ module Html2rss
         # Extracts categories from Schema.org structured data.
         module CategoryExtractor
           # Schema.org fields checked for category-like string values.
-          CATEGORY_FIELDS = %i[articleSection keywords categories tags].freeze
+          CATEGORY_FIELDS = %i[articleSection keywords categories tags occupationalCategory industry].freeze
 
           ##
           # @param schema_object [Hash] The schema object
@@ -26,15 +26,20 @@ module Html2rss
             def add_field(categories, value)
               case value
               when Array
-                value.each { |item| add_string(categories, item.to_s) }
+                value.each { |item| add_item(categories, item) }
               when String
                 split_string(categories, value)
               end
             end
 
+            def add_item(categories, item)
+              item.is_a?(String) ? split_string(categories, item) : add_string(categories, item.to_s)
+            end
+
             def add_about(categories, about)
               case about
               when Array then add_about_items(categories, about)
+              when Hash then add_about_items(categories, [about])
               when String then split_string(categories, about)
               end
             end

data/lib/html2rss/auto_source/scraper/schema/item_list.rb

@@ -49,11 +49,11 @@ module Html2rss
 
           # @param object [Hash] candidate schema object
           # @param from_list_item [Boolean] whether object came from a ListItem unwrap
-          # @return [Boolean] whether to emit this object as an article
+          # @return [Boolean] whether to emit this object as an article (also bare Event, JobPosting and video items)
           def emit?(object, from_list_item:)
             return true if from_list_item
 
-            Schema.normalize_types(object[:@type]).intersect?(Thing::SUPPORTED_TYPES)
+            Schema.normalize_types(object[:@type]).intersect?(Thing::SUPPORTED_TYPES | Thing::ITEM_LIST_TYPES)
           end
         end
       end

data/lib/html2rss/auto_source/scraper/schema/thing.rb

@@ -10,7 +10,7 @@ module Html2rss
         # A Thing is kind of the 'base class' for Schema.org schema_objects.
         #
         # @see https://schema.org/Thing
-        class Thing
+        class Thing # rubocop:disable Metrics/ClassLength -- one accessor per schema.org field
           # Supported Schema.org `@type` values mapped to article extraction.
           SUPPORTED_TYPES = %w[
             AdvertiserContentArticle AnalysisNewsArticle APIReference Article
@@ -19,6 +19,12 @@ module Html2rss
             ReviewNewsArticle SatiricalArticle ScholarlyArticle SocialMediaPosting TechArticle
           ].to_set.freeze
 
+          # Schema.org types emitted only as elements of an ItemList (never standalone).
+          ITEM_LIST_TYPES = Set['Event', 'JobPosting', 'VideoObject', 'PodcastEpisode'].freeze
+
+          # Schema.org types whose own `contentUrl` is the media enclosure.
+          MEDIA_TYPES = Set['VideoObject'].freeze
+
           # @param schema_object [Hash{Symbol => Object}] parsed schema.org object
           # @param url [String, Html2rss::Url, nil] base URL used for relative normalization
           def initialize(schema_object, url:)
@@ -28,7 +34,7 @@ module Html2rss
 
           # @return [Hash{Symbol => Object}] scraped article attributes
           def call
-            { id:, title:, description:, author:, url:, image:, published_at:, categories: }
+            { id:, title:, description:, author:, url:, image:, published_at:, categories:, enclosures: }
           end
 
           # @return [String, nil] stable schema object identifier
@@ -43,31 +49,29 @@ module Html2rss
           def title
             schema_object.values_at(:headline, :name, :alternativeHeadline, :title)
                          .lazy
-                         .map { _1.to_s.strip }
+                         .map { first_scalar(_1).to_s.strip }
                          .find { !_1.empty? }
           end
 
           # @return [String, nil] longest available description field
           def description
-            schema_object.values_at(:description, :articleBody, :abstract, :schema_object_body).max_by { _1.to_s.size }
+            schema_object.values_at(:description, :articleBody, :abstract, :schema_object_body)
+                         .map { first_scalar(_1) }
+                         .max_by { _1.to_s.size }
           end
 
           # @return [String, nil] extracted author or publisher name
           def author
-            extract_author_name(schema_object[:author]) || extract_author_name(schema_object[:publisher])
+            extract_author_name(schema_object[:author]) || extract_author_name(schema_object[:publisher]) ||
+              extract_author_name(schema_object[:hiringOrganization]) ||
+              extract_author_name(schema_object[:organizer])
           end
 
           # @return [Html2rss::Url, nil] the URL of the schema object
           def url
             return @url if defined?(@url)
 
-            url = schema_object[:url]
-            if url.to_s.empty?
-              Log.debug("Schema#Thing.url: no url in schema_object: #{schema_object.inspect}")
-              return @url = nil
-            end
-
-            @url = Url.from_relative(url, base_url || url)
+            @url = build_url(first_scalar(schema_object[:url])) || fallback_url
           end
 
           # @return [Html2rss::Url, nil] normalized article image URL
@@ -78,10 +82,11 @@ module Html2rss
             @image = img_url ? Url.from_relative(img_url, base_url || img_url) : nil
           end
 
-          # @return [String, nil] published-at timestamp (datePublished, else dateModified)
+          # @return [String, nil] first non-empty of datePublished, datePosted, dateCreated, dateModified, uploadDate
           def published_at
-            value = schema_object[:datePublished] || schema_object[:dateModified]
-            value.to_s.empty? ? nil : value
+            schema_object.values_at(:datePublished, :datePosted, :dateCreated, :dateModified, :uploadDate)
+                         .map { first_scalar(_1) }
+                         .find { !_1.to_s.empty? }
           end
 
           # @return [Array<String>, nil] extracted category labels
@@ -89,15 +94,84 @@ module Html2rss
             @categories ||= CategoryExtractor.call(schema_object)
           end
 
+          # @return [Array<Hash{Symbol => Object}>, nil] enclosures from VideoObject/AudioObject `contentUrl`
+          def enclosures
+            nodes = Array(schema_object.values_at(:video, :audio)).flatten
+            nodes << schema_object if own_media?
+            found = nodes.filter_map { media_enclosure(_1) }
+            found unless found.empty?
+          end
+
           attr_reader :schema_object, :base_url
 
           # @return [Array<String>] normalized image URL candidates
           def image_urls
-            @image_urls ||= schema_object.values_at(:image, :thumbnailUrl).filter_map { image_url_from(_1) }
+            @image_urls ||= schema_object.values_at(:image, :thumbnailUrl, :thumbnail).filter_map { image_url_from(_1) }
           end
 
           private
 
+          # @return [Boolean] whether the object itself is a media object carrying its own `contentUrl`
+          def own_media?
+            Schema.normalize_types(schema_object[:@type]).intersect?(MEDIA_TYPES)
+          end
+
+          # @param node [Object] VideoObject/AudioObject node
+          # @return [Hash{Symbol => Object}, nil] enclosure attributes, or nil without a usable absolute contentUrl
+          def media_enclosure(node)
+            return unless node.is_a?(Hash) && node[:contentUrl].is_a?(String)
+
+            build_media_enclosure(node[:contentUrl], node[:encodingFormat])
+          end
+
+          # @param src [String] contentUrl
+          # @param type [Object] encodingFormat (used when a String)
+          # @return [Hash{Symbol => Object}, nil] enclosure attributes with an absolute url
+          def build_media_enclosure(src, type)
+            found = ::Html2rss::Html::ArticleRules::Enclosure.from_media(src, (type if type.is_a?(String)),
+                                                                         base_url || src)
+            found if found&.dig(:url)&.absolute?
+          rescue Url::InvalidUrlError
+            nil
+          end
+
+          # @param value [Object] url candidate
+          # @return [Html2rss::Url, nil] absolute url, or nil when missing or unusable
+          def build_url(value)
+            return Url.from_relative(value, base_url || value) if value.is_a?(String) && !value.empty?
+
+            Log.debug("Schema#Thing.url: no usable url in schema_object: #{schema_object.inspect}")
+            nil
+          rescue Url::InvalidUrlError
+            Log.debug("Schema#Thing.url: unusable url #{value.inspect}")
+            nil
+          end
+
+          # @return [Html2rss::Url, nil] first absolute http(s) mainEntityOfPage or @id, without fragment
+          def fallback_url
+            main = schema_object[:mainEntityOfPage]
+            candidates = main.is_a?(Hash) ? [main[:@id], main[:url]] : [main]
+            (candidates + [schema_object[:@id]]).lazy.filter_map { web_url(_1) }.first
+          end
+
+          # @param value [Object] url candidate
+          # @return [Html2rss::Url, nil] fragment-free url when value is an absolute http(s) string
+          def web_url(value)
+            return unless value.is_a?(String) && value.match?(%r{\Ahttps?://}i)
+
+            Url.from_absolute(value.strip).without_fragment
+          rescue Url::InvalidUrlError
+            nil
+          end
+
+          # @param value [Object] schema value that may be wrapped in a list (multilingual or multi-valued fields)
+          # @return [Object, nil] the value, or the first non-empty String of a list
+          def first_scalar(value)
+            return value unless value.is_a?(Array)
+
+            value.find { _1.is_a?(String) && !_1.strip.empty? }
+          end
+
           # @param obj [String, Hash, Array, nil] schema author or publisher field
           # @return [String, nil] extracted author name
           def extract_author_name(obj)
@@ -110,12 +184,15 @@ module Html2rss
             end
           end
 
-          # @param obj [String, Hash, nil] image node
-          # @return [String, nil] image URL string
+          # @param obj [String, Hash, Array, nil] image node or list of image nodes
+          # @return [String, nil] image URL string (first usable entry for lists)
           def image_url_from(obj)
             return obj if obj.is_a?(String)
+            return obj.filter_map { image_url_from(_1) }.first if obj.is_a?(Array)
             return unless obj.is_a?(Hash)
-            return unless Schema.normalize_types(obj[:@type]).include?('ImageObject')
+
+            types = Schema.normalize_types(obj[:@type])
+            return unless types.empty? || types.include?('ImageObject')
 
             obj[:url] || obj[:contentUrl]
           end
@@ -129,7 +206,7 @@ module Html2rss
 
             norm_url = normalized_id_url(text, reference_url:)
             reference_url && norm_url.host == reference_url.host ? normalized_id_value(norm_url) : text
-          rescue ArgumentError
+          rescue Html2rss::Url::InvalidUrlError
             text
           end
 
@@ -153,7 +230,7 @@ module Html2rss
           # @return [Html2rss::Url, nil] normalized absolute URL for schema resolution
           def normalized_base_url(url)
             Url.from_absolute(url) unless url.to_s.strip.empty?
-          rescue ArgumentError
+          rescue Html2rss::Url::InvalidUrlError
             nil
           end
         end

data/lib/html2rss/auto_source/scraper/schema.rb

@@ -25,7 +25,8 @@ module Html2rss
 
         # Canonical Schema.org type names keyed by folded wire forms (case-insensitive lookup).
         CANONICAL_BY_DOWNCASE = begin
-          canonical_types = Thing::SUPPORTED_TYPES | ItemList::SUPPORTED_TYPES | DENIED_CONTAINER_TYPES | Set['Product']
+          canonical_types = Thing::SUPPORTED_TYPES | ItemList::SUPPORTED_TYPES | Thing::ITEM_LIST_TYPES |
+                            DENIED_CONTAINER_TYPES | Set['Product']
           canonical_types.to_h { |type| [::Html2rss::Html::Probe.fold(type), type] }.freeze
         end.freeze
 
@@ -43,6 +44,9 @@ module Html2rss
         # Shared empty type set for nil/unsupported wire forms (avoids per-call Set.new).
         EMPTY_TYPES = Set.new.freeze
 
+        # Keys whose `{"@id": ...}` stubs are resolved against sibling nodes of the same JSON-LD document.
+        REFERENCE_KEYS = %i[author publisher image].freeze
+
         # @return [Symbol] scraper config key
         def self.options_key = :schema
 
@@ -84,6 +88,18 @@ module Html2rss
             end
           end
 
+          ##
+          # Like {from} for a script tag, but resolves `@id` stubs for {REFERENCE_KEYS} against the same document.
+          #
+          # @param script [Nokogiri::XML::Element] schema JSON-LD script tag
+          # @return [Array<Hash>] the schema_objects with resolved references
+          # @api private
+          def from_script(script)
+            document = parse_script_tag(script)
+            index = node_index(document)
+            from(document).map { |object| resolve_references(object, index) }
+          end
+
           # @param object [Hash{Symbol => Object}] schema candidate object
           # @return [Boolean] whether an extractor exists for the candidate object
           def supported_schema_object?(object)
@@ -139,6 +155,37 @@ module Html2rss
 
           private
 
+          # @param object [Hash, Array, Object] parsed JSON-LD value
+          # @param index [Hash{String => Hash}] accumulator of nodes (more than just `@id`) keyed by `@id`
+          # @return [Hash{String => Hash}] nodes keyed by `@id`
+          def node_index(object, index = {})
+            case object
+            when Hash
+              index[object[:@id]] ||= object if object[:@id].is_a?(String) && object.size > 1
+              object.each_value { node_index(_1, index) }
+            when Array then object.each { node_index(_1, index) }
+            end
+            index
+          end
+
+          # @param object [Hash{Symbol => Object}] schema object
+          # @param index [Hash{String => Hash}] nodes keyed by `@id`
+          # @return [Hash{Symbol => Object}] object with reference stubs replaced where a sibling node exists
+          def resolve_references(object, index)
+            object.to_h { |key, value| [key, REFERENCE_KEYS.include?(key) ? resolve_stub(value, index) : value] }
+          end
+
+          # @param value [Object] author, publisher or image value
+          # @param index [Hash{String => Hash}] nodes keyed by `@id`
+          # @return [Object] the referenced node for an `{"@id": ...}` stub, else the value unchanged
+          def resolve_stub(value, index)
+            case value
+            when Array then value.map { resolve_stub(_1, index) }
+            when Hash then value.keys == [:@id] ? index.fetch(value[:@id], value) : value
+            else value
+            end
+          end
+
           # @param hash [Hash] candidate schema object
           # @return [Array<Hash>] the object itself or nested supported objects
           def from_hash(hash)
@@ -200,7 +247,7 @@ module Html2rss
 
         def schema_objects
           ::Html2rss::Html::Probe.scripts(@parsed_body, ::Html2rss::Html::Probe::APPLICATION_LD_JSON)
-                                 .flat_map { |tag| Schema.from(tag) }
+                                 .flat_map { |tag| Schema.from_script(tag) }
         end
 
         attr_reader :parsed_body, :url

data/lib/html2rss/auto_source/scraper/semantic_html.rb

@@ -26,8 +26,6 @@ module Html2rss
           return false unless parsed_body
 
           new(parsed_body, url: 'https://example.com').extractable?
-        rescue ArgumentError
-          false
         end
 
         # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML (when +document:+ omitted)
@@ -57,8 +55,6 @@ module Html2rss
         # @return [Boolean]
         def extractable?
           ranked_segments.any?
-        rescue ArgumentError
-          false
         end
 
         private
@@ -69,21 +65,21 @@ module Html2rss
             entries = deduplicator.call(ranked_segments)
             entries.filter_map { |ranked| deduplicator.article_for(ranked) }
           end
-        rescue ArgumentError
-          []
         end
 
         def ranked_segments
-          @ranked_segments ||= begin
-            segments = Segmenter.call(
-              document,
-              base_url: @url,
-              strategy: :semantic,
-              permit_unanchored: @permit_unanchored,
-              link_resolver:
-            )
-            Scoring::Engine.new(link_resolver:).rank_top(segments, limit: TOP_K)
-          end
+          @ranked_segments ||= document ? rank_document : []
+        end
+
+        def rank_document
+          segments = Segmenter.call(
+            document,
+            base_url: @url,
+            strategy: :semantic,
+            permit_unanchored: @permit_unanchored,
+            link_resolver:
+          )
+          Scoring::Engine.new(link_resolver:).rank_top(segments, limit: TOP_K)
         end
 
         def link_resolver

data/lib/html2rss/auto_source/scraper/sitemap/parser.rb

@@ -149,10 +149,7 @@ module Html2rss
           def stale_date?(published_at)
             return false unless published_at
 
-            time = Time.parse(published_at)
-            (Time.now.utc - time) > (max_age_days * 86_400)
-          rescue ArgumentError
-            false
+            (Time.now.utc - Time.parse(published_at)) > (max_age_days * 86_400)
           end
         end
       end

data/lib/html2rss/auto_source/scraper/sitemap.rb

@@ -7,7 +7,7 @@ module Html2rss
     module Scraper
       ##
       # Scrapes articles from XML sitemaps explicitly linked in HTML or discovered via standard sitemap paths.
-      class Sitemap # rubocop:disable Metrics/ClassLength -- scraper + structured warn paths
+      class Sitemap
         include Enumerable
 
         # Selector for sitemap link tags in HTML head
@@ -65,13 +65,7 @@ module Html2rss
         def each
           return enum_for(:each) unless block_given?
 
-          entries = fetch_and_parse_entries
-          return if entries.empty?
-
-          entries.each do |entry|
-            article_hash = article_from_entry(entry)
-            yield article_hash if article_hash
-          end
+          fetch_and_parse_entries.each { |entry| yield article_from_entry(entry) }
         end
 
         private
@@ -137,7 +131,7 @@ module Html2rss
 
         # rubocop:disable-next Metrics/MethodLength -- structured fetch-failed warn
         def fetch_xml(target_url)
-          return unless request_session && target_url
+          return unless request_session
 
           response = request_session.follow_up(url: target_url, relation: :auto_source, origin_url: url)
           response&.body
@@ -160,8 +154,6 @@ module Html2rss
         end
 
         def article_from_entry(entry)
-          return unless entry.url
-
           {
             url: entry.url,
             title: entry.title,

data/lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb

@@ -154,7 +154,7 @@ module Html2rss
             end
 
             def archive_like?
-              category_archive? || tag_archive? || author_archive? || date_archive? || body_classes.include?('archive')
+              body_classes.include?('archive')
             end
 
             def singular_like?
@@ -178,7 +178,7 @@ module Html2rss
 
               canonical_url = Html2rss::Url.from_relative(href, url)
               same_origin_url?(canonical_url, url) ? canonical_url : url
-            rescue ArgumentError
+            rescue Html2rss::Url::InvalidUrlError
               url
             end
 

data/lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb

@@ -54,7 +54,7 @@ module Html2rss
             return log_missing_api_root if href.empty?
 
             Html2rss::Url.from_relative(href, page_url)
-          rescue ArgumentError => error
+          rescue Html2rss::Url::InvalidUrlError => error
             logger.warn("#{WordpressApi}: invalid WordPress API endpoint #{href.inspect} (#{error.message})")
             nil
           end
@@ -110,13 +110,11 @@ module Html2rss
           def normalized_api_path(path)
             segments = path.to_s.split('/').reject(&:empty?)
             normalized_path = "/#{segments.join('/')}"
-            normalized_path = '/' if normalized_path == '/'
             normalized_path.end_with?('/') ? normalized_path : "#{normalized_path}/"
           end
 
           def normalized_rest_route(route)
             value = route.to_s
-            value = '/' if value.empty?
             value = "/#{value}" unless value.start_with?('/')
             trim_trailing_slashes(value)
           end

data/lib/html2rss/auto_source/scraper/wordpress_api.rb

@@ -132,8 +132,8 @@ module Html2rss
           absolute_link(post[:link])
         end
 
-        def article_id(_post, article_url)
-          root_path_query_id(article_url) || present_string(article_url.path) || article_url.to_s
+        def article_id(article_url)
+          root_path_query_id(article_url) || present_string(article_url.path)
         end
 
         def article_title(post)
@@ -154,7 +154,7 @@ module Html2rss
 
         def article_attributes(post, article_url)
           {
-            id: article_id(post, article_url),
+            id: article_id(article_url),
             title: article_title(post),
             description: article_description(post),
             url: article_url,
@@ -168,7 +168,7 @@ module Html2rss
           return unless value
 
           Html2rss::Url.from_relative(value, url)
-        rescue ArgumentError
+        rescue Html2rss::Url::InvalidUrlError
           nil
         end
 

data/lib/html2rss/auto_source/scraper/xhr_articles.rb

@@ -35,7 +35,7 @@ module Html2rss
 
         # @return [Boolean] true when any captured body contains article-like arrays
         def extractable?
-          parsed_bodies.any? { |doc| JsonState::CandidateDetector.candidate_array?(doc) }
+          parsed_bodies.any?
         end
 
         # @yield [Hash{Symbol => Object}] normalized article hash

data/lib/html2rss/auto_source/scraper.rb

@@ -117,15 +117,6 @@ module Html2rss
         HEURISTIC_SCRAPERS.intersect?(tier)
       end
 
-      ##
-      # @param parsed_body [Nokogiri::HTML::Document]
-      # @return [SST::Document, nil]
-      def self.normalize_sst(parsed_body)
-        SST::Normalizer.call(parsed_body)
-      rescue SST::Normalizer::EmptyTree
-        nil
-      end
-
       ##
       # Builds a scraper when enabled; returns nil when disabled.
       #
@@ -220,8 +211,6 @@ module Html2rss
       private_class_method :app_shell_surface?
 
       def self.high_entropy_surface?(parsed_body)
-        return false if parsed_body.nil?
-
         parsed_body.css('body a[href]').size >= HIGH_ENTROPY_MIN_ANCHORS
       end
       private_class_method :high_entropy_surface?

data/lib/html2rss/auto_source/segmenter/cluster.rb

@@ -7,12 +7,10 @@ module Html2rss
       # Class / structure clustering for anchorless pages
       # (port of Discovery::DomClustering + OverlapResolver; group scoring owned by Scoring::ClusterScorer).
       module Cluster
-        module_function
-
         ##
         # @param segmenter [Segmenter]
         # @return [Array<Segment>]
-        def call(segmenter)
+        def self.call(segmenter)
           primary = PrimaryLink.new(segmenter)
           nodes = best_group_nodes(segmenter)
           nodes.each_with_index.filter_map do |node, position|
@@ -23,7 +21,7 @@ module Html2rss
           end
         end
 
-        def best_group_nodes(segmenter)
+        def self.best_group_nodes(segmenter)
           groups = collect_groups(segmenter)
           return [] if groups.empty?
 
@@ -33,10 +31,9 @@ module Html2rss
           final_groups = resolver.filter_1_to_1_overlap(non_containers)
           scorer.select_best_group(final_groups)
         end
-        module_function :best_group_nodes
         private_class_method :best_group_nodes
 
-        def collect_groups(segmenter) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
+        def self.collect_groups(segmenter) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
           class_groups = Hash.new { |hash, key| hash[key] = [] }
           structure_groups = Hash.new { |hash, key| hash[key] = [] }
           min = segmenter.minimum_selector_frequency
@@ -57,24 +54,21 @@ module Html2rss
 
           structure_groups.select { |_, nodes| nodes.size >= min }
         end
-        module_function :collect_groups
         private_class_method :collect_groups
 
-        def normalize_class(class_names)
+        def self.normalize_class(class_names)
           return '' if class_names.empty?
 
           class_names.map { ::Html2rss::Html::Probe.fold(_1) }.sort.join(' ')
         end
-        module_function :normalize_class
         private_class_method :normalize_class
 
-        def structure_signature(node)
+        def self.structure_signature(node)
           kids = node.children
           return '' if kids.empty?
 
           kids.map { |child| child.name.to_s }.join('>')
         end
-        module_function :structure_signature
         private_class_method :structure_signature
 
         ##