html2rss 0.30.0 → 0.31.0

data/lib/html2rss/selectors.rb

@@ -12,18 +12,19 @@ module Html2rss
   #
   # Additionally, it uniquely offers the capability to convert JSON into XML,
   # extending its versatility for diverse data processing workflows.
-  class Selectors # rubocop:disable Metrics/ClassLength
+  #
+  # {include:file:lib/html2rss/selectors/README.md}
+  class Selectors # rubocop:disable Metrics/ClassLength -- public API + private field dispatch
+    # Raised when a selector key is missing or not allowed for extraction.
     class InvalidSelectorName < Html2rss::Error; end
 
     include Enumerable
 
-    # A context instance passed to item extractors and post-processors.
-    # When built via {ItemScope#context_for}, +item_scope+ carries the per-item
-    # extraction base_url for nested selects (e.g. Template).
-    Context = Struct.new('Context', :options, :item, :config, :scraper, :item_scope, keyword_init: true)
-
+    # Wire spelling of the default items selector.
+    # Runtime CSS is {Html::Navigator::MAIN_ANCHOR_SELECTOR}.
+    DEFAULT_ITEMS_SELECTOR = 'a[href]'
     # Default selectors options merged into user configuration.
-    DEFAULT_CONFIG = { items: { enhance: true } }.freeze
+    DEFAULT_CONFIG = { items: { selector: DEFAULT_ITEMS_SELECTOR, enhance: true } }.freeze
 
     # Selector key that points to the root list of article nodes.
     ITEMS_SELECTOR_KEY = :items
@@ -35,7 +36,12 @@ module Html2rss
     # +:enclosure+ stays singular in YAML; Article stores +:enclosures+.
     SELECTOR_TO_ARTICLE_KEY = { enclosure: :enclosures }.freeze
     # Selector keys that may be copied onto an Article (PROVIDED_KEYS + mapped aliases).
-    SELECTABLE_SELECTOR_KEYS = (Html2rss::Article::PROVIDED_KEYS + SELECTOR_TO_ARTICLE_KEY.keys).to_set.freeze
+    SELECTABLE_SELECTOR_KEYS = Set[*(Html2rss::Article::PROVIDED_KEYS + SELECTOR_TO_ARTICLE_KEY.keys)].freeze
+    # ArticleExtractor keys that enhance can fill. Skip the extractor when all are already present.
+    ENHANCEABLE_KEYS = %i[title url description published_at enclosures categories].freeze
+    # Per-field extraction plan compiled once from the static selector config.
+    CompiledField = Data.define(:name, :article_key, :config, :extractor_class, :post_process_steps, :special)
+    private_constant :CompiledField
 
     ##
     # Initializes a new Selectors instance.
@@ -48,7 +54,10 @@ module Html2rss
       @url = response.url
       @selectors = selectors
       @time_zone = time_zone
-      @rss_item_attributes = @selectors.keys.select { |key| SELECTABLE_SELECTOR_KEYS.include?(key) }
+      @rss_item_attributes = @selectors.keys.select { SELECTABLE_SELECTOR_KEYS.include?(_1) }
+      @parsed_body = nil
+      @compiled_fields = compile_selector_fields
+      @compiled_item_fields = @rss_item_attributes.filter_map { @compiled_fields[_1] }
     end
 
     ##
@@ -71,56 +80,130 @@ module Html2rss
       enhance = enhance?
 
       parsed_body.css(items_selector).each do |item|
-        article_hash = extract_article(item, response)
+        article_hash = extract_article(item)
 
-        enhance_article_hash(article_hash, item, response.url) if enhance
+        enhance_article_hash(article_hash, item) if enhance
 
         yield Html2rss::Article.new(**article_hash, scraper: self.class)
       end
     end
 
     ##
-    # Returns the CSS selector for the items.
-    # @return [String] the CSS selector for the items
-    def items_selector = @selectors.dig(ITEMS_SELECTOR_KEY, :selector)
+    # Yields +baseline+, +enhanced+, and the item node per list entry for enhance diagnostics.
+    # Consumed by {Test::EnhanceAudit} so the extraction loop has one owner.
+    #
+    # @yieldparam baseline [Hash]
+    # @yieldparam enhanced [Hash]
+    # @yieldparam item [Nokogiri::XML::Element]
+    # @return [Enumerator] if no block is given
+    def each_enhance_pair
+      return enum_for(:each_enhance_pair) unless block_given?
 
-    ## @return [Boolean] whether to enhance the article hash with auto_source's semantic HTML extraction.
-    def enhance? = !!@selectors.dig(ITEMS_SELECTOR_KEY, :enhance)
+      parsed_body.css(items_selector).each do |item|
+        baseline = extract_article(item)
+        enhanced = enhance_article_hash(baseline.dup, item)
+        yield baseline, enhanced, item
+      end
+    end
 
     ##
-    # Extracts an article hash for a given item element.
+    # Selects the value for a given attribute from an HTML element.
     #
-    # @param item [Nokogiri::XML::Element] The element to extract from.
-    # @param page_response [RequestService::Response] response used for selector extraction context
-    # @return [Hash] Hash of attributes for the article.
-    def extract_article(item, page_response = response)
-      scope = item_scope_for(item, page_response.url)
-      hash = @rss_item_attributes.each_with_object({}) do |selector_key, h|
-        value = scope.select(selector_key)
+    # @param name [Symbol, String] Name of the attribute.
+    # @param item [Nokogiri::XML::Element] The HTML element to process.
+    # @param base_url [String, Html2rss::Url] base URL for relative extraction values
+    # @return [Object, Array<Object>] The selected value(s).
+    # @raise [InvalidSelectorName] If the attribute name is invalid or not defined.
+    def select(name, item, base_url: @url)
+      select_in_scope(name, item_env_for(item, base_url))
+    end
+
+    ##
+    # Selects the value for a given attribute within an existing {ItemEnv}.
+    # Used by {ItemEnv#select} so nested selects reuse one env per extraction pass.
+    #
+    # @param name [Symbol, String] Name of the attribute.
+    # @param scope [ItemEnv] Per-item extraction scope.
+    # @return [Object, Array<Object>] The selected value(s).
+    # @raise [InvalidSelectorName] If the attribute name is invalid or not defined.
+    def select_in_scope(name, scope)
+      name = name.to_sym
+      raise InvalidSelectorName, "Attribute selector '#{name}' is reserved for items." if name == ITEMS_SELECTOR_KEY
+
+      field = @compiled_fields[name]
+      return select_compiled(field, scope) if field
+      return select_field(name, scope:, config: nil) if name == :url
+
+      raise InvalidSelectorName, "Selector for '#{name}' is not defined."
+    end
+
+    ##
+    # Resolves a selector name to +[key, config]+. Used by category field dispatch.
+    #
+    # @param name [Symbol, String]
+    # @param allow_nil [Boolean]
+    # @return [Array(Symbol, Hash, nil)]
+    def selector_config_for(name, allow_nil: false)
+      selector_key = name.to_sym
+
+      return [selector_key, @selectors[selector_key]] if @selectors.key?(selector_key)
+      return [selector_key, nil] if allow_nil
+
+      raise InvalidSelectorName, "Selector for '#{selector_key}' is not defined."
+    end
+
+    private
+
+    attr_reader :response
+
+    def items_selector
+      selector = @selectors.dig(ITEMS_SELECTOR_KEY, :selector)
+      return Html::Navigator::MAIN_ANCHOR_SELECTOR if default_items_selector?(selector)
+
+      selector
+    end
+
+    def default_items_selector?(selector)
+      text = selector.to_s.strip
+      text.empty? || text == DEFAULT_ITEMS_SELECTOR
+    end
+
+    def enhance? = !!@selectors.dig(ITEMS_SELECTOR_KEY, :enhance)
+
+    def parsed_body
+      @parsed_body ||= build_parsed_body(response)
+    end
+
+    def build_parsed_body(page_response)
+      if page_response.json_response?
+        fragment = JsonXml.call(page_response.parsed_body)
+        Nokogiri::HTML5.fragment(fragment)
+      else
+        page_response.parsed_body
+      end
+    end
+
+    def extract_article(item)
+      scope = item_env_for(item, @url)
+      hash = @compiled_item_fields.each_with_object({}) do |field, h|
+        value = select_compiled(field, scope)
         next if value.nil?
 
-        article_key = SELECTOR_TO_ARTICLE_KEY.fetch(selector_key, selector_key)
+        article_key = field.article_key
         h[article_key] = article_key == :enclosures ? wrap_enclosure_value(value) : value
       end
 
-      hash[:url] ||= default_item_url(item, page_response.url) if anchor_element?(item)
+      hash[:url] ||= default_item_url(item) if anchor_element?(item)
       hash
     end
 
-    ##
-    # Enhances the article hash using semantic HTML extraction.
-    # Only adds keys that are missing from the original hash.
-    #
-    # @param article_hash [Hash] The original article hash.
-    # @param article_tag [Nokogiri::XML::Element] HTML element to extract additional info from.
-    # @param base_url [String, Html2rss::Url] base URL for normalization during enhancement
-    # @return [Hash] The enhanced article hash.
-    # rubocop:disable-next Metrics/MethodLength
-    def enhance_article_hash(article_hash, article_tag, base_url = @url)
+    def enhance_article_hash(article_hash, article_tag) # rubocop:disable Metrics/MethodLength -- merge extracted keys
+      return article_hash if enhanceable_fields_present?(article_hash)
+
       selected_anchor = Html2rss::Html::Navigator.main_anchor_for(article_tag)
       extracted = Html2rss::Html::ArticleExtractor.call(
         article_tag,
-        base_url:,
+        base_url: @url,
         selected_anchor:,
         fallback_anchorless: true,
         time_zone: @time_zone
@@ -134,63 +217,77 @@ module Html2rss
       end
     end
 
-    ##
-    # Selects the value for a given attribute from an HTML element.
-    #
-    # @param name [Symbol, String] Name of the attribute.
-    # @param item [Nokogiri::XML::Element] The HTML element to process.
-    # @param base_url [String, Html2rss::Url] base URL for relative extraction values
-    # @return [Object, Array<Object>] The selected value(s).
-    # @raise [InvalidSelectorName] If the attribute name is invalid or not defined.
-    def select(name, item, base_url: @url)
-      select_in_scope(name, item_scope_for(item, base_url))
+    def enhanceable_fields_present?(article_hash)
+      ENHANCEABLE_KEYS.all? { |key| article_hash[key] }
     end
 
-    ##
-    # Selects the value for a given attribute within an existing {ItemScope}.
-    # Used by {ItemScope#select} so nested selects reuse one scope per extraction pass.
-    #
-    # @param name [Symbol, String] Name of the attribute.
-    # @param scope [ItemScope] Per-item extraction scope.
-    # @return [Object, Array<Object>] The selected value(s).
-    # @raise [InvalidSelectorName] If the attribute name is invalid or not defined.
-    def select_in_scope(name, scope)
-      name = name.to_sym
-
-      raise InvalidSelectorName, "Attribute selector '#{name}' is reserved for items." if name == ITEMS_SELECTOR_KEY
+    def wrap_enclosure_value(value)
+      value.is_a?(Array) ? value : [value]
+    end
 
-      selector_key, config = selector_config_for(name, allow_nil: name == :url)
+    def select_field(selector_key, scope:, config:)
       return default_item_url(scope.item, scope.base_url) if fallback_url_selector?(selector_key, config, scope.item)
       raise InvalidSelectorName, "Selector for '#{selector_key}' is not defined." if config.nil?
 
-      dispatch_select(selector_key, scope:, config:)
+      if SPECIAL_ATTRIBUTES.member?(selector_key)
+        select_special(selector_key, scope:, config:)
+      else
+        select_regular(selector_key, scope:, config:)
+      end
     end
 
-    private
+    def select_compiled(field, scope)
+      if fallback_url_selector?(field.name, field.config, scope.item)
+        return default_item_url(scope.item, scope.base_url)
+      end
+      raise InvalidSelectorName, "Selector for '#{field.name}' is not defined." if field.config.nil?
 
-    attr_reader :response
+      if field.special
+        select_special(field.name, scope:, config: field.config)
+      else
+        select_regular_compiled(field, scope)
+      end
+    end
 
-    def item_scope_for(item, base_url)
-      ItemScope.new(
-        item:,
-        base_url:,
-        scraper: self,
-        channel: channel_context(base_url)
+    def compile_selector_fields
+      @selectors.each_key.with_object({}) do |name, acc|
+        next if name == ITEMS_SELECTOR_KEY
+
+        acc[name] = compile_field(name)
+      end
+    end
+
+    def compile_field(name)
+      config = @selectors[name]
+      CompiledField.new(
+        name:,
+        article_key: SELECTOR_TO_ARTICLE_KEY.fetch(name, name),
+        config:,
+        extractor_class: extractor_class_for(config),
+        post_process_steps: compile_post_process_steps(config),
+        special: SPECIAL_ATTRIBUTES.member?(name)
       )
     end
 
-    def parsed_body
-      parsed_body_for(response)
+    def extractor_class_for(config)
+      return unless config.is_a?(Hash)
+
+      Extractors::NAME_TO_CLASS[config[:extractor]&.to_sym || Extractors::DEFAULT_EXTRACTOR]
     end
 
-    def parsed_body_for(page_response)
-      @parsed_bodies ||= {}
-      @parsed_bodies[page_response.url] ||= if page_response.json_response?
-                                              fragment = ObjectToXmlConverter.new(page_response.parsed_body).call
-                                              Nokogiri::HTML5.fragment(fragment)
-                                            else
-                                              page_response.parsed_body
-                                            end
+    def compile_post_process_steps(config)
+      return unless config.is_a?(Hash) && config[:post_process]
+
+      steps = config[:post_process]
+      (steps.is_a?(Array) ? steps : [steps]).map { |step| compile_post_process_step(step) }
+    end
+
+    def compile_post_process_step(step)
+      klass = PostProcessors::NAME_TO_CLASS[step[:name].to_sym]
+      raise PostProcessors::UnknownPostProcessorName, "Unknown name '#{step[:name]}'" unless klass
+
+      klass.validate_options!(StepEnv.new(step_config: step, time_zone: @time_zone))
+      [klass, step]
     end
 
     def select_special(name, scope:, config:)
@@ -205,27 +302,55 @@ module Html2rss
     end
 
     def select_regular(_name, scope:, config:)
-      @merged_configs ||= {}
-      merged_config = @merged_configs[[config.object_id, scope.base_url]] ||=
-        config.merge(channel: scope.channel).freeze
-      value = Extractors.get(merged_config, scope.item)
-
-      if value && (post_process_steps = config[:post_process])
-        steps = post_process_steps.is_a?(Array) ? post_process_steps : [post_process_steps]
-        value = post_process(scope, value, steps)
+      value = Extractors.call(config, scope.item, base_url: scope.base_url)
+
+      apply_post_process_steps(scope:, value:, post_process_steps: config[:post_process])
+    end
+
+    def select_regular_compiled(field, scope)
+      value = field.extractor_class.new(
+        scope.item,
+        selector: field.config[:selector],
+        attribute: field.config[:attribute],
+        static: field.config[:static],
+        base_url: scope.base_url
+      ).call
+
+      apply_compiled_post_process(scope, value, field.post_process_steps)
+    end
+
+    def apply_post_process_steps(scope:, value:, post_process_steps:)
+      return value unless value && post_process_steps
+
+      steps = post_process_steps.is_a?(Array) ? post_process_steps : [post_process_steps]
+      post_process(scope, value, steps)
+    end
+
+    def post_process(scope, value, post_process_steps)
+      post_process_steps.each do |step_config|
+        value = PostProcessors.call(step_config[:name], value, scope.context_for(step_config:))
       end
 
       value
     end
 
-    def post_process(scope, value, post_process_steps)
-      post_process_steps.each do |options|
-        value = PostProcessors.get(options[:name], value, scope.context_for(options:))
+    def apply_compiled_post_process(scope, value, steps)
+      return value unless value && steps
+
+      steps.each do |klass, step_config|
+        value = klass.new(value, scope.context_for(step_config:)).call
       end
 
       value
     end
 
+    # Flattens configured category selector names into discrete category strings.
+    # When a selector matches multiple nodes, each node is extracted independently
+    # (with optional post-processors) so multi-tag UIs become multiple categories.
+    #
+    # @param category_selectors [Array, String, Symbol]
+    # @param scope [ItemEnv]
+    # @return [Array<String>]
     def select_categories(category_selectors:, scope:)
       Array(category_selectors).flat_map do |selector_name|
         extract_category_values(selector_name, scope:)
@@ -233,17 +358,17 @@ module Html2rss
     end
 
     def extract_category_values(selector_name, scope:)
-      selector_key, config = selector_config_for(selector_name, allow_nil: true)
+      _selector_key, config = selector_config_for(selector_name, allow_nil: true)
       return [] unless config
 
       nodes = extract_nodes(item: scope.item, config:)
-      return Array(select_regular(selector_key, scope:, config:)) unless node_set_with_multiple_elements?(nodes)
+      return Array(scope.select(selector_name)) unless node_set_with_multiple_elements?(nodes)
 
       Array(nodes).flat_map { |node| extract_categories_from_node(node, scope:, config:) }
     end
 
     def extract_categories_from_node(node, scope:, config:)
-      values = Extractors.get(category_node_options(config, scope:), node)
+      values = Extractors.call(config, node, base_url: scope.base_url, selector: nil)
       values = apply_post_process_steps(scope:, value: values, post_process_steps: config[:post_process])
 
       Array(values).filter_map { |category| extract_category_text(category) }
@@ -265,49 +390,12 @@ module Html2rss
       nodes.is_a?(Nokogiri::XML::NodeSet) && nodes.length > 1
     end
 
-    def category_node_options(selector_config, scope:)
-      @category_node_configs ||= {}
-      @category_node_configs[[selector_config.object_id, scope.base_url]] ||= selector_config.merge(
-        channel: scope.channel,
-        selector: nil
-      ).freeze
-    end
-
-    def apply_post_process_steps(scope:, value:, post_process_steps:)
-      return value unless value && post_process_steps
-
-      steps = post_process_steps.is_a?(Array) ? post_process_steps : [post_process_steps]
-      post_process(scope, value, steps)
-    end
-
-    def selector_config_for(name, allow_nil: false)
-      selector_key = name.to_sym
-
-      return [selector_key, @selectors[selector_key]] if @selectors.key?(selector_key)
-      return [selector_key, nil] if allow_nil
-
-      raise InvalidSelectorName, "Selector for '#{selector_key}' is not defined."
-    end
-
     def extract_nodes(item:, config:)
-      return unless config.respond_to?(:[]) && config[:selector]
+      return unless config.is_a?(Hash) && config[:selector]
 
       Extractors.element(item, config[:selector])
     end
 
-    def channel_context(base_url)
-      @channel_contexts ||= {}
-      @channel_contexts[base_url] ||= { url: base_url, time_zone: @time_zone }.freeze
-    end
-
-    # Keep a single enclosure Hash as one list entry; +Array(hash)+ would split pairs.
-    #
-    # @param value [Hash, Array] enclosure hash or list of hashes
-    # @return [Array]
-    def wrap_enclosure_value(value)
-      value.is_a?(Array) ? value : [value]
-    end
-
     # @return [Hash, nil] enclosure details, or nil when the selector yields nothing.
     def enclosure(scope:, config:)
       selected = select_regular(:enclosure, scope:, config:)
@@ -318,27 +406,28 @@ module Html2rss
       { url:, type: config[:content_type] }
     end
 
-    def anchor_element?(item)
-      item.respond_to?(:name) && item.name.to_s.casecmp('a').zero?
-    end
-
     def fallback_url_selector?(selector_key, config, item)
       selector_key == :url && config.nil? && anchor_element?(item)
     end
 
-    def dispatch_select(selector_key, scope:, config:)
-      if SPECIAL_ATTRIBUTES.member?(selector_key)
-        select_special(selector_key, scope:, config:)
-      else
-        select_regular(selector_key, scope:, config:)
-      end
+    def anchor_element?(item)
+      item.name.to_s.casecmp('a').zero?
     end
 
-    def default_item_url(item, base_url)
+    def default_item_url(item, base_url = @url)
       href = item['href'].to_s.strip
       return if href.empty?
 
       Url.from_relative(href, base_url)
     end
+
+    def item_env_for(item, base_url)
+      ItemEnv.new(
+        item:,
+        base_url:,
+        scraper: self,
+        time_zone: @time_zone
+      )
+    end
   end
 end

data/lib/html2rss/sst/attrs.rb

@@ -11,11 +11,13 @@ module Html2rss
       EMPTY_RAW = {}.freeze
     end
 
-    ##
-    # Typed HTML attributes for an SST node. Absent fields are +nil+.
     Attrs = Data.define(
       :href, :src, :id, :class_names, :datetime, :itemprop, :style, :srcset, :type, :raw
-    ) do
+    )
+
+    ##
+    # Typed HTML attributes for an SST node. Absent fields are +nil+.
+    class Attrs
       class << self
         ##
         # @param href [String, nil]

data/lib/html2rss/sst/document.rb

@@ -2,9 +2,11 @@
 
 module Html2rss
   module SST
+    Document = Data.define(:root, :index, :degraded, :node_count)
+
     ##
     # Immutable SST document: root node plus relationship index.
-    Document = Data.define(:root, :index, :degraded, :node_count) do
+    class Document
       ##
       # @param root [Node]
       # @param index [Index]

data/lib/html2rss/sst/index.rb

@@ -63,7 +63,9 @@ module Html2rss
       def memo_word_count(node)
         return @word_count[node] if @word_count.key?(node)
 
-        @word_count[node] = memo_visible_text(node).to_s.scan(/\p{Alnum}+/).size
+        count = 0
+        memo_visible_text(node).to_s.scan(/\p{Alnum}+/) { count += 1 }
+        @word_count[node] = count
       end
 
       ##

data/lib/html2rss/sst/node.rb

@@ -2,10 +2,12 @@
 
 module Html2rss
   module SST
+    Node = Data.define(:name, :attrs, :own_text, :children, :tag_path)
+
     ##
     # Immutable simplified semantic tree node. Predicates and traversal live on
     # the type so Scoring/Segmenter never touch Nokogiri.
-    Node = Data.define(:name, :attrs, :own_text, :children, :tag_path) do
+    class Node
       ##
       # @param name [Symbol, String]
       # @param attrs [Attrs, nil]
@@ -86,11 +88,26 @@ module Html2rss
       #
       # @yieldparam node [Node]
       # @return [Enumerator, Array<Node>]
-      def each_node(&block)
-        return enum_for(:each_node) unless block
+      def each_node(&)
+        return enum_for(:each_node) unless block_given?
 
         yield self
-        children.each { |child| child.each_node(&block) }
+        children.each { |child| child.each_node(&) }
+      end
+
+      ##
+      # Non-allocating traversal that counts descendants matching the block.
+      # When no block is given, counts all descendants.
+      #
+      # @yieldparam node [Node]
+      # @return [Integer]
+      def count_descendants(&)
+        total = 0
+        children.each do |child|
+          total += 1 if !block_given? || yield(child)
+          total += child.count_descendants(&)
+        end
+        total
       end
 
       ##
@@ -101,14 +118,14 @@ module Html2rss
 
       ##
       # @return [Array<Node>]
-      def find_all(&predicate)
-        each_node.select(&predicate)
+      def find_all(&)
+        each_node.select(&)
       end
 
       ##
       # @return [Node, nil]
-      def find(&predicate)
-        each_node.find(&predicate)
+      def find(&)
+        each_node.find(&)
       end
 
       ##
@@ -132,14 +149,16 @@ module Html2rss
           return index.memo_word_count(self)
         end
 
-        visible_text.to_s.scan(/\p{Alnum}+/).size
+        count = 0
+        visible_text.to_s.scan(/\p{Alnum}+/) { count += 1 }
+        count
       end
 
       ##
       # @return [Float] words per descendant link
       def text_density
         words = word_count
-        links = descendants.count(&:link?)
+        links = count_descendants(&:link?)
         links.zero? ? words.to_f : words.to_f / links
       end
     end