html2rss 0.18.0 → 0.19.1

data/lib/html2rss/auto_source/scraper/microdata.rb

@@ -7,31 +7,43 @@ module Html2rss
       class Microdata
         include Enumerable
 
+        # Selector matching nodes that define a microdata item scope.
         ITEM_SELECTOR = '[itemscope][itemtype]'
+        # Schema.org types supported for article extraction via Microdata.
         SUPPORTED_TYPES = (Schema::Thing::SUPPORTED_TYPES | Set['Product']).freeze
+        # Attribute names checked first for microdata property values.
         VALUE_ATTRIBUTES = %w[content datetime href src data value].freeze
 
+        # @return [Symbol] scraper config key
         def self.options_key = :microdata
 
         class << self
+          # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML document
           def articles?(parsed_body)
             supported_roots(parsed_body).any?
           end
 
+          # @param parsed_body [Nokogiri::HTML::Document, nil] parsed HTML document
+          # @return [Array<Nokogiri::XML::Element>] top-level supported Microdata roots
           def supported_roots(parsed_body)
             return [] unless parsed_body
 
             parsed_body.css(ITEM_SELECTOR).select { supported_root?(_1) }
           end
 
+          # @param node [Nokogiri::XML::Element] itemscope candidate node
           def supported_root?(node)
             supported_type_name(node) && top_level_item?(node)
           end
 
+          # @param node [Nokogiri::XML::Element] itemscope candidate node
+          # @return [String, nil] supported schema type name when present
           def supported_type_name(node)
             normalized_types(node['itemtype']).find { SUPPORTED_TYPES.include?(_1) }
           end
 
+          # @param itemtype [String, nil] raw itemtype attribute value
+          # @return [Array<String>] normalized schema type names
           def normalized_types(itemtype)
             itemtype.to_s.split.filter_map do |value|
               type = value.split('/').last.to_s.split('#').last.to_s
@@ -39,6 +51,7 @@ module Html2rss
             end
           end
 
+          # @param node [Nokogiri::XML::Element] itemscope candidate node
           def top_level_item?(node)
             return false if node.attribute('itemprop')
 
@@ -53,6 +66,7 @@ module Html2rss
         #   the parsed response body to inspect for top-level Microdata items.
         # @param url [Html2rss::Url] the absolute page URL used to resolve relative links.
         # @param _opts [Hash] unused scraper-specific options.
+        # @option _opts [Object] :_reserved reserved for future scraper-specific options
         # @return [void]
         def initialize(parsed_body, url:, **_opts)
           @parsed_body = parsed_body
@@ -62,7 +76,7 @@ module Html2rss
         ##
         # Iterates over normalized article hashes extracted from supported Microdata roots.
         #
-        # @yieldparam article [Hash<Symbol, Object>] the normalized article attributes.
+        # @yieldparam article [Hash{Symbol => Object}] the normalized article attributes.
         # @return [Enumerator, void] an enumerator when no block is given.
         def each
           return enum_for(:each) unless block_given?
@@ -77,6 +91,8 @@ module Html2rss
 
         attr_reader :parsed_body, :url
 
+        # @param root [Nokogiri::XML::Element] supported Microdata root node
+        # @return [Hash{Symbol => Object}, nil] normalized article hash
         def article_from(root)
           schema_object = SchemaObjectBuilder.call(root)
           return unless schema_object
@@ -87,6 +103,8 @@ module Html2rss
           article
         end
 
+        # @param article [Hash{Symbol => Object}] normalized article hash
+        # @return [Boolean] whether article contains required fields
         def valid_article?(article)
           return false unless article[:url]
 
@@ -97,12 +115,17 @@ module Html2rss
         module ItemParser
           module_function
 
+          # @param root [Nokogiri::XML::Element] microdata root node
+          # @return [Hash{Symbol => Object}] extracted direct properties
           def call(root)
             {}.tap do |properties|
               direct_properties(root).each { append_properties!(properties, _1) }
             end
           end
 
+          # @param properties [Hash{Symbol => Object}] accumulator hash for parsed properties
+          # @param node [Nokogiri::XML::Element] itemprop node
+          # @return [void]
           def append_properties!(properties, node)
             value = property_value(node)
             return if blank_value?(value)
@@ -112,16 +135,23 @@ module Html2rss
             end
           end
 
+          # @param root [Nokogiri::XML::Element] microdata root node
+          # @return [Array<Nokogiri::XML::Element>] direct property nodes for the root
           def direct_properties(root)
             root.css('[itemprop]').select { direct_property?(root, _1) }
           end
 
+          # @param root [Nokogiri::XML::Element] microdata root node
+          # @param node [Nokogiri::XML::Element] candidate itemprop node
+          # @return [Boolean] whether the node belongs directly to the current root item
           def direct_property?(root, node)
             return false if node == root
 
             node.ancestors.take_while { _1 != root }.none? { |ancestor| ancestor.attribute('itemscope') }
           end
 
+          # @param node [Nokogiri::XML::Element] itemprop node
+          # @return [Array<String>] normalized property names
           def property_names(node)
             node['itemprop'].to_s.split.filter_map do |name|
               stripped = name.strip
@@ -129,6 +159,8 @@ module Html2rss
             end
           end
 
+          # @param node [Nokogiri::XML::Element] itemprop node
+          # @return [Object, nil] parsed property value
           def property_value(node)
             value = if node.attribute('itemscope')
                       nested_item(node)
@@ -139,6 +171,8 @@ module Html2rss
             value unless blank_value?(value)
           end
 
+          # @param node [Nokogiri::XML::Element] nested itemscope node
+          # @return [Hash{Symbol => Object}] nested parsed microdata item
           def nested_item(node)
             item = call(node)
             itemtype = node['itemtype']
@@ -148,6 +182,8 @@ module Html2rss
             item
           end
 
+          # @param node [Nokogiri::XML::Element] itemprop node
+          # @return [String, nil] first present attribute value
           def attribute_value(node)
             VALUE_ATTRIBUTES.each do |attribute|
               value = node[attribute]
@@ -157,11 +193,17 @@ module Html2rss
             nil
           end
 
+          # @param node [Nokogiri::XML::Element] itemprop node
+          # @return [String, nil] normalized text content
           def text_value(node)
             value = node.text.to_s.strip
             value unless value.empty?
           end
 
+          # @param properties [Hash{Symbol => Object}] accumulator hash for parsed properties
+          # @param key [Symbol] target property key
+          # @param value [Object] parsed property value to assign for the key
+          # @return [void]
           def append(properties, key, value)
             return if blank_value?(value)
 
@@ -173,6 +215,8 @@ module Html2rss
             properties[key] = Array(properties[key]) << value
           end
 
+          # @param value [Object] candidate value
+          # @return [Boolean] whether value is blank for microdata extraction purposes
           def blank_value?(value)
             case value
             when nil then true
@@ -182,6 +226,8 @@ module Html2rss
             end
           end
 
+          # @param value [Object] candidate value
+          # @return [Boolean] whether value is present for microdata extraction purposes
           def present?(value)
             !blank_value?(value)
           end
@@ -192,6 +238,8 @@ module Html2rss
         module ValueNormalizer
           module_function
 
+          # @param values [Array<Object>] value candidates
+          # @return [String, nil] first URL-like value converted to string
           def url_value(*values)
             values.each do |value|
               candidate = extract_nested_value(value, :url, :@id)
@@ -201,6 +249,8 @@ module Html2rss
             nil
           end
 
+          # @param values [Array<Object>] value candidates
+          # @return [String, Hash, nil] first normalized image candidate
           def image_value(*values)
             values.each do |value|
               candidate = normalize_image(value)
@@ -210,6 +260,8 @@ module Html2rss
             nil
           end
 
+          # @param value [Object] image candidate value
+          # @return [String, Hash, nil] normalized image-like value
           def normalize_image(value)
             candidate = unwrap(value)
             return unless present?(candidate)
@@ -219,6 +271,8 @@ module Html2rss
             candidate.to_s
           end
 
+          # @param value [Object] about candidate value
+          # @return [Array<String, Hash>, nil] normalized about values
           def normalize_about(value)
             candidate = unwrap(value)
             items = candidate.is_a?(Array) ? candidate : [candidate]
@@ -226,6 +280,8 @@ module Html2rss
             values unless values.empty?
           end
 
+          # @param item [Object] single about item
+          # @return [String, Hash, nil] normalized about item
           def normalize_about_item(item)
             case item
             when Hash
@@ -235,6 +291,8 @@ module Html2rss
             end
           end
 
+          # @param value [Object] scalar or array candidate
+          # @return [String, Array<String>, nil] normalized scalar or string array
           def string_or_array(value)
             candidate = unwrap(value)
             return unless present?(candidate)
@@ -245,15 +303,21 @@ module Html2rss
             result unless result.empty?
           end
 
+          # @param values [Array<Object>] value candidates
+          # @return [Array<String>, nil] normalized unique string values
           def array_value(*values)
             result = values.flat_map { string_values(Array(unwrap(_1))) }.uniq
             result unless result.empty?
           end
 
+          # @param values [Array<Object>] candidate scalar values collected from microdata arrays
+          # @return [Array<String>] normalized string values
           def string_values(values)
             values.filter_map { stringify(_1) }
           end
 
+          # @param values [Array<Object>] value candidates
+          # @return [String, nil] first present string-like value
           def first_string(*values)
             values.each do |value|
               candidate = stringify(unwrap(value))
@@ -263,6 +327,9 @@ module Html2rss
             nil
           end
 
+          # @param value [Object] nested container or scalar
+          # @param keys [Array<Symbol>] nested keys to probe in order
+          # @return [Object, nil] first matching nested value
           def extract_nested_value(value, *keys)
             candidate = unwrap(value)
             return candidate unless candidate.is_a?(Hash)
@@ -275,10 +342,14 @@ module Html2rss
             nil
           end
 
+          # @param value [Object] scalar or array candidate
+          # @return [Object] first array element or the original value
           def unwrap(value)
             value.is_a?(Array) ? value.first : value
           end
 
+          # @param value [Object] scalar candidate normalized to string output
+          # @return [String, nil] normalized string representation
           def stringify(value)
             return unless present?(value)
             return value if value.is_a?(String)
@@ -287,6 +358,8 @@ module Html2rss
             value.to_s
           end
 
+          # @param value [Object] candidate value
+          # @return [Boolean] whether value is present
           def present?(value)
             case value
             when nil then false
@@ -304,6 +377,8 @@ module Html2rss
 
           extend ValueNormalizer
 
+          # @param root [Nokogiri::XML::Element] supported microdata root node
+          # @return [Hash{Symbol => Object}, nil] compact schema-like object
           def call(root)
             type = Microdata.supported_type_name(root)
             return unless type
@@ -311,12 +386,20 @@ module Html2rss
             compact_object(type, root, ItemParser.call(root))
           end
 
+          # @param type [String] schema type inferred from itemtype
+          # @param root [Nokogiri::XML::Element] supported microdata root node
+          # @param properties [Hash{Symbol => Object}] parsed microdata properties
+          # @return [Hash{Symbol => Object}] normalized schema-like object
           def compact_object(type, root, properties)
             object = base_attributes(type, root, properties)
             merge_categories!(object, properties)
             object.compact
           end
 
+          # @param type [String] schema type inferred from itemtype
+          # @param root [Nokogiri::XML::Element] supported microdata root node
+          # @param properties [Hash{Symbol => Object}] parsed microdata properties
+          # @return [Hash{Symbol => Object}] base schema attributes before category merging
           def base_attributes(type, root, properties)
             identifier = first_string(root['itemid'], properties.delete(:identifier))
 
@@ -328,10 +411,14 @@ module Html2rss
               .merge(media_attributes(properties))
           end
 
+          # @param properties [Hash{Symbol => Object}] parsed microdata properties
+          # @return [String, nil] normalized title
           def title(properties)
             first_string(properties.delete(:headline), properties.delete(:title), properties.delete(:name))
           end
 
+          # @param properties [Hash{Symbol => Object}] parsed microdata properties
+          # @return [Hash{Symbol => Object}] normalized text attributes
           def text_attributes(properties)
             {
               title: title(properties),
@@ -342,18 +429,26 @@ module Html2rss
             }
           end
 
+          # @param properties [Hash{Symbol => Object}] parsed microdata properties
+          # @param identifier [String, nil] identifier candidate for fallback URL handling
+          # @return [Hash{Symbol => Object}] normalized link attributes
           def link_attributes(properties, identifier)
             {
               url: url(properties, identifier)
             }
           end
 
+          # @param properties [Hash{Symbol => Object}] parsed microdata properties
+          # @return [Hash{Symbol => Object}] normalized media attributes
           def media_attributes(properties)
             {
               image: image_value(properties.delete(:image), properties.delete(:thumbnailUrl))
             }
           end
 
+          # @param properties [Hash{Symbol => Object}] parsed microdata properties
+          # @param fallback_id [String, nil] identifier candidate for fallback URL handling
+          # @return [String, nil] normalized URL candidate
           def url(properties, fallback_id)
             url_value(
               properties.delete(:url),
@@ -362,6 +457,8 @@ module Html2rss
             )
           end
 
+          # @param fallback_id [String, nil] identifier candidate for fallback URL handling
+          # @return [String, nil] fallback URL candidate when identifier looks URL-like
           def url_fallback(fallback_id)
             value = first_string(fallback_id)
             return unless value
@@ -371,6 +468,8 @@ module Html2rss
             nil
           end
 
+          # @param properties [Hash{Symbol => Object}] parsed microdata properties
+          # @return [String, nil] normalized published-at value
           def published_at(properties)
             first_string(
               properties.delete(:datePublished),
@@ -380,6 +479,9 @@ module Html2rss
             )
           end
 
+          # @param object [Hash{Symbol => Object}] schema-like output object
+          # @param properties [Hash{Symbol => Object}] parsed microdata properties
+          # @return [void]
           def merge_categories!(object, properties)
             categories = array_value(properties.delete(:categories), properties.delete(:articleSection))
             assign_if_present(object, :categories, categories)
@@ -388,6 +490,10 @@ module Html2rss
             assign_if_present(object, :about, normalize_about(properties.delete(:about)))
           end
 
+          # @param object [Hash{Symbol => Object}] schema-like output object
+          # @param key [Symbol] target attribute key
+          # @param value [Object] value to assign when present
+          # @return [void]
           def assign_if_present(object, key, value)
             object[key] = value if value
           end

data/lib/html2rss/auto_source/scraper/schema/category_extractor.rb

@@ -90,7 +90,7 @@ module Html2rss
           ##
           # Extracts categories from a string by splitting on separators.
           #
-          # @param string [String] The string to process
+          # @param string [String] source string that may contain category delimiters
           # @return [Set<String>] Set of category strings
           def self.extract_string_categories(string)
             Set.new(string.split(/[,;|]/).map(&:strip).reject(&:empty?))

data/lib/html2rss/auto_source/scraper/schema/item_list.rb

@@ -11,6 +11,7 @@ module Html2rss
         #
         # @see https://schema.org/ItemList
         class ItemList < Thing
+          # Schema.org type names handled by the ItemList extractor.
           SUPPORTED_TYPES = Set['ItemList']
 
           # @return [Array<Hash>] the scraped article hashes with DEFAULT_ATTRIBUTES

data/lib/html2rss/auto_source/scraper/schema/list_item.rb

@@ -5,11 +5,13 @@ module Html2rss
     module Scraper
       class Schema
         ##
-        #
         # @see https://schema.org/ListItem
         class ListItem < Thing
+          # @return [String, nil] stable list-item identifier
           def id =          (id = (schema_object.dig(:item, :@id) || super).to_s).empty? ? nil : id
+          # @return [String, nil] list-item title
           def title =       schema_object.dig(:item, :name) || super || url&.titleized
+          # @return [String, nil] list-item description
           def description = schema_object.dig(:item, :description) || super
 
           # @return [Html2rss::Url, nil]

data/lib/html2rss/auto_source/scraper/schema/thing.rb

@@ -11,6 +11,7 @@ module Html2rss
         #
         # @see https://schema.org/Thing
         class Thing
+          # Supported Schema.org `@type` values mapped to article extraction.
           SUPPORTED_TYPES = %w[
             AdvertiserContentArticle
             AnalysisNewsArticle
@@ -32,8 +33,11 @@ module Html2rss
             TechArticle
           ].to_set.freeze
 
+          # Attributes exposed by `#call` in generated article hashes.
           DEFAULT_ATTRIBUTES = %i[id title description url image published_at categories].freeze
 
+          # @param schema_object [Hash{Symbol => Object}] parsed schema.org object
+          # @param url [String, Html2rss::Url, nil] base URL used for relative normalization
           def initialize(schema_object, url:)
             @schema_object = schema_object
             @base_url = normalized_base_url(url)
@@ -46,6 +50,7 @@ module Html2rss
             end
           end
 
+          # @return [String, nil] stable schema object identifier
           def id
             return @id if defined?(@id)
 
@@ -56,8 +61,10 @@ module Html2rss
             @id = id
           end
 
+          # @return [String, nil] article title
           def title = schema_object[:title]
 
+          # @return [String, nil] longest available description field
           def description
             schema_object.values_at(:description, :schema_object_body, :abstract)
                          .max_by { |string| string.to_s.size }
@@ -74,14 +81,17 @@ module Html2rss
             Url.from_relative(url, base_url || url)
           end
 
+          # @return [Html2rss::Url, nil] normalized article image URL
           def image
             if (image_url = image_urls.first)
               Url.from_relative(image_url, base_url || image_url)
             end
           end
 
+          # @return [String, nil] published-at timestamp string
           def published_at = schema_object[:datePublished]
 
+          # @return [Array<String>, nil] extracted category labels
           def categories
             return @categories if defined?(@categories)
 
@@ -90,6 +100,7 @@ module Html2rss
 
           attr_reader :schema_object, :base_url
 
+          # @return [Array<String>] normalized image URL candidates
           def image_urls
             schema_object.values_at(:image, :thumbnailUrl).filter_map do |object|
               next unless object
@@ -102,6 +113,9 @@ module Html2rss
             end
           end
 
+          # @param value [String, Symbol, nil] candidate schema identifier
+          # @param reference_url [Html2rss::Url, nil] URL used for same-origin normalization
+          # @return [String, nil] normalized identifier value
           def normalized_id(value, reference_url:)
             text = value.to_s
             return if text.empty?
@@ -114,6 +128,9 @@ module Html2rss
             text
           end
 
+          # @param text [String] raw identifier text
+          # @param reference_url [Html2rss::Url, nil] URL used to resolve relative IDs
+          # @return [Html2rss::Url] normalized identifier URL
           def normalized_id_url(text, reference_url:)
             if text.start_with?('/')
               Url.from_relative(text, reference_url || text)
@@ -122,6 +139,8 @@ module Html2rss
             end
           end
 
+          # @param url [Html2rss::Url] normalized identifier URL
+          # @return [String, nil] path/query portion used as stable ID
           def normalized_id_value(url)
             path = url.path.to_s
             return "#{path}?#{url.query}" if (path.empty? || path == '/') && !url.query.to_s.empty?
@@ -130,6 +149,8 @@ module Html2rss
             url.query
           end
 
+          # @param url [String, Html2rss::Url, nil] candidate page URL
+          # @return [Html2rss::Url, nil] normalized absolute URL for schema resolution
           def normalized_base_url(url)
             return if url.to_s.strip.empty?
 

data/lib/html2rss/auto_source/scraper/schema.rb

@@ -8,24 +8,28 @@ module Html2rss
     module Scraper
       ##
       # Scrapes articles from Schema.org objects, by looking for the objects in:
-
       # <script type="application/ld+json"> "schema" tags.
       #
-      # See:
-      # 1. https://schema.org/docs/full.html
-      # 2. https://developers.google.com/search/docs/appearance/structured-data/article#microdata
+      # @see https://schema.org/docs/full.html
+      # @see https://developers.google.com/search/docs/appearance/structured-data/article#microdata
       class Schema
         include Enumerable
 
+        # Selector for JSON-LD script tags containing Schema.org objects.
         TAG_SELECTOR = 'script[type="application/ld+json"]'
 
+        # @return [Symbol] scraper config key
         def self.options_key = :schema
 
         class << self
+          # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
+          # @return [Boolean] whether the page includes supported schema types
           def articles?(parsed_body)
             parsed_body.css(TAG_SELECTOR).any? { |script| supported_schema_type?(script) }
           end
 
+          # @param script [Nokogiri::XML::Element] schema JSON-LD script tag
+          # @return [Boolean] whether the tag references a supported schema type
           def supported_schema_type?(script)
             supported_types = Thing::SUPPORTED_TYPES | ItemList::SUPPORTED_TYPES
             supported_types.any? { |type| script.text.match?(/"@type"\s*:\s*"#{Regexp.escape(type)}"/) }
@@ -52,11 +56,14 @@ module Html2rss
             end
           end
 
+          # @param object [Hash{Symbol => Object}] schema candidate object
+          # @return [Boolean] whether an extractor exists for the candidate object
           def supported_schema_object?(object)
             scraper_for_schema_object(object) ? true : false
           end
 
           ##
+          # @param schema_object [Hash{Symbol => Object}] schema object with an @type key
           # @return [Scraper::Schema::Thing, Scraper::Schema::ItemList, nil] a class responding to `#call`
           def scraper_for_schema_object(schema_object)
             type = schema_object[:@type]
@@ -81,6 +88,10 @@ module Html2rss
           end
         end
 
+        # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
+        # @param url [String, Html2rss::Url] base page URL
+        # @param opts [Hash] scraper-specific options
+        # @option opts [Object] :_reserved reserved for future scraper-specific options
         def initialize(parsed_body, url:, **opts)
           @parsed_body = parsed_body
           @url = url

data/lib/html2rss/auto_source/scraper/semantic_html/anchor_selector.rb

@@ -25,7 +25,9 @@ module Html2rss
             :score
           )
 
+          # Comma-separated heading selector used for heading/anchor matching.
           HEADING_SELECTOR = HtmlExtractor::HEADING_TAGS.join(',').freeze
+          # Path segments that usually represent utility navigation rather than article content.
           UTILITY_PATH_SEGMENTS = %w[
             about account author category comment comments contact feedback help
             login newsletter profile register search settings share signup subscribe
@@ -40,11 +42,14 @@ module Html2rss
             logout
             user users
           ].to_set.freeze
+          # Path segments that signal content-like destinations.
           CONTENT_PATH_SEGMENTS = %w[
             article articles news post posts story stories update updates
           ].to_set.freeze
+          # Ancestor tags that usually indicate navigation/utility regions.
           UTILITY_LANDMARK_TAGS = %w[nav aside footer menu].freeze
 
+          # @param base_url [String, Html2rss::Url] page URL used to normalize href destinations
           def initialize(base_url)
             @base_url = base_url
           end

data/lib/html2rss/auto_source/scraper/semantic_html.rb

@@ -20,8 +20,10 @@ module Html2rss
       class SemanticHtml
         include Enumerable
 
+        # Container plus selected anchor chosen for extraction.
         Entry = Data.define(:container, :selected_anchor)
 
+        # Candidate semantic container selectors used to locate extractable blocks.
         CONTAINER_SELECTORS = [
           'article:not(:has(article))',
           'section:not(:has(section))',
@@ -45,6 +47,8 @@ module Html2rss
         # @param parsed_body [Nokogiri::HTML::Document] parsed HTML document
         # @param url [String, Html2rss::Url] base url
         # @param extractor [Class] extractor class used for article extraction
+        # @param _opts [Hash] scraper-specific options
+        # @option _opts [Object] :_reserved reserved for future scraper-specific options
         def initialize(parsed_body, url:, extractor: HtmlExtractor, **_opts)
           @parsed_body = parsed_body
           @url = url