html2rss 0.22.2 → 0.24.0

data/lib/html2rss/selectors.rb

@@ -18,7 +18,9 @@ module Html2rss
     include Enumerable
 
     # A context instance passed to item extractors and post-processors.
-    Context = Struct.new('Context', :options, :item, :config, :scraper, keyword_init: true)
+    # When built via {ItemScope#context_for}, +item_scope+ carries the per-item
+    # extraction base_url for nested selects (e.g. Template).
+    Context = Struct.new('Context', :options, :item, :config, :scraper, :item_scope, keyword_init: true)
 
     # Default selectors options merged into user configuration.
     DEFAULT_CONFIG = { items: { enhance: true } }.freeze
@@ -29,9 +31,11 @@ module Html2rss
     ITEM_TAGS = %i[title url description author comments published_at guid enclosure categories].freeze
     # Item attributes that require dedicated extraction logic.
     SPECIAL_ATTRIBUTES = Set[:guid, :enclosure, :categories].freeze
-
-    # Mapping of new attribute names to their legacy names for backward compatibility.
-    RENAMED_ATTRIBUTES = { published_at: %i[updated pubDate] }.freeze
+    # Config selector keys that map onto a different {Article} attribute.
+    # +:enclosure+ stays singular in YAML; Article stores +:enclosures+.
+    SELECTOR_TO_ARTICLE_KEY = { enclosure: :enclosures }.freeze
+    # Selector keys that may be copied onto an Article (PROVIDED_KEYS + mapped aliases).
+    SELECTABLE_SELECTOR_KEYS = (Html2rss::Article::PROVIDED_KEYS + SELECTOR_TO_ARTICLE_KEY.keys).to_set.freeze
 
     ##
     # Initializes a new Selectors instance.
@@ -46,14 +50,14 @@ module Html2rss
       @time_zone = time_zone
 
       prepare_selectors!
-      @rss_item_attributes = @selectors.keys & Html2rss::RssBuilder::Article::PROVIDED_KEYS
+      @rss_item_attributes = @selectors.keys.select { |key| SELECTABLE_SELECTOR_KEYS.include?(key) }
     end
 
     ##
     # Returns articles extracted from the response.
     # Reverses order if config specifies reverse ordering.
     #
-    # @return [Array<Html2rss::RssBuilder::Article>]
+    # @return [Array<Html2rss::Article>]
     def articles
       @articles ||= @selectors.dig(ITEMS_SELECTOR_KEY, :order) == 'reverse' ? to_a.tap(&:reverse!) : to_a
     end
@@ -61,7 +65,7 @@ module Html2rss
     ##
     # Iterates over each scraped article.
     #
-    # @yield [article] Gives each article as an Html2rss::RssBuilder::Article.
+    # @yield [article] Gives each article as an Html2rss::Article.
     # @return [Enumerator] An enumerator if no block is given.
     def each(&)
       return enum_for(:each) unless block_given?
@@ -73,7 +77,7 @@ module Html2rss
 
         enhance_article_hash(article_hash, item, response.url) if enhance
 
-        yield Html2rss::RssBuilder::Article.new(**article_hash, scraper: self.class)
+        yield Html2rss::Article.new(**article_hash, scraper: self.class)
       end
     end
 
@@ -92,7 +96,14 @@ module Html2rss
     # @param page_response [RequestService::Response] response used for selector extraction context
     # @return [Hash] Hash of attributes for the article.
     def extract_article(item, page_response = response)
-      @rss_item_attributes.to_h { |key| [key, select(key, item, base_url: page_response.url)] }.compact
+      scope = item_scope_for(item, page_response.url)
+      @rss_item_attributes.each_with_object({}) do |selector_key, hash|
+        value = scope.select(selector_key)
+        next if value.nil?
+
+        article_key = SELECTOR_TO_ARTICLE_KEY.fetch(selector_key, selector_key)
+        hash[article_key] = article_key == :enclosures ? wrap_enclosure_value(value) : value
+      end
     end
 
     ##
@@ -105,13 +116,13 @@ module Html2rss
     # @return [Hash] The enhanced article hash.
     # rubocop:disable Metrics/MethodLength
     def enhance_article_hash(article_hash, article_tag, base_url = @url)
-      selected_anchor = HtmlExtractor.main_anchor_for(article_tag)
-      extracted = HtmlExtractor.new(
+      selected_anchor = Html2rss::Html::Navigator.main_anchor_for(article_tag)
+      extracted = Html2rss::Html::ArticleExtractor.call(
         article_tag,
         base_url:,
         selected_anchor:,
         fallback_anchorless: true
-      ).call
+      )
       return article_hash unless extracted
 
       extracted.each_with_object(article_hash) do |(key, value), hash|
@@ -131,6 +142,18 @@ module Html2rss
     # @return [Object, Array<Object>] The selected value(s).
     # @raise [InvalidSelectorName] If the attribute name is invalid or not defined.
     def select(name, item, base_url: @url)
+      select_in_scope(name, item_scope_for(item, base_url))
+    end
+
+    ##
+    # Selects the value for a given attribute within an existing {ItemScope}.
+    # Used by {ItemScope#select} so nested selects reuse one scope per extraction pass.
+    #
+    # @param name [Symbol, String] Name of the attribute.
+    # @param scope [ItemScope] Per-item extraction scope.
+    # @return [Object, Array<Object>] The selected value(s).
+    # @raise [InvalidSelectorName] If the attribute name is invalid or not defined.
+    def select_in_scope(name, scope)
       name = name.to_sym
 
       raise InvalidSelectorName, "Attribute selector '#{name}' is reserved for items." if name == ITEMS_SELECTOR_KEY
@@ -138,9 +161,9 @@ module Html2rss
       selector_key, config = selector_config_for(name)
 
       if SPECIAL_ATTRIBUTES.member?(selector_key)
-        select_special(selector_key, item:, config:, base_url:)
+        select_special(selector_key, scope:, config:)
       else
-        select_regular(selector_key, item:, config:, base_url:)
+        select_regular(selector_key, scope:, config:)
       end
     end
 
@@ -148,10 +171,18 @@ module Html2rss
 
     attr_reader :response
 
+    def item_scope_for(item, base_url)
+      ItemScope.new(
+        item:,
+        base_url:,
+        scraper: self,
+        channel: channel_context(base_url)
+      )
+    end
+
     def prepare_selectors!
       validate_url_and_link_exclusivity!
       fix_url_and_link!
-      handle_renamed_attributes!
     end
 
     def validate_url_and_link_exclusivity!
@@ -167,17 +198,6 @@ module Html2rss
       @selectors[:url] = @selectors[:link]
     end
 
-    def handle_renamed_attributes!
-      RENAMED_ATTRIBUTES.each_pair do |new_name, old_names|
-        old_names.each do |old_name|
-          next unless @selectors.key?(old_name)
-
-          Html2rss::Log.warn("Selector '#{old_name}' is deprecated. Please rename to '#{new_name}'.")
-          @selectors[new_name] ||= @selectors.delete(old_name)
-        end
-      end
-    end
-
     def parsed_body
       parsed_body_for(response)
     end
@@ -192,64 +212,58 @@ module Html2rss
                                             end
     end
 
-    def select_special(name, item:, config:, base_url:)
+    def select_special(name, scope:, config:)
       case name
       when :enclosure
-        enclosure(item:, config:, base_url:)
+        enclosure(scope:, config:)
       when :guid
-        Array(config).map { |selector_name| select(selector_name, item, base_url:) }
+        Array(config).map { |selector_name| scope.select(selector_name) }
       when :categories
-        select_categories(category_selectors: config, item:, base_url:)
+        select_categories(category_selectors: config, scope:)
       end
     end
 
-    def select_regular(_name, item:, config:, base_url:)
+    def select_regular(_name, scope:, config:)
       @merged_configs ||= {}
-      merged_config = @merged_configs[[config.object_id, base_url]] ||=
-        config.merge(channel: channel_context(base_url)).freeze
-      value = Extractors.get(merged_config, item)
+      merged_config = @merged_configs[[config.object_id, scope.base_url]] ||=
+        config.merge(channel: scope.channel).freeze
+      value = Extractors.get(merged_config, scope.item)
 
       if value && (post_process_steps = config[:post_process])
         steps = post_process_steps.is_a?(Array) ? post_process_steps : [post_process_steps]
-        value = post_process(item, value, steps, base_url:)
+        value = post_process(scope, value, steps)
       end
 
       value
     end
 
-    def post_process(item, value, post_process_steps, base_url:)
-      pp_context = channel_post_process_context(base_url)
+    def post_process(scope, value, post_process_steps)
       post_process_steps.each do |options|
-        context = Context.new(config: pp_context,
-                              item:, scraper: self, options:)
-
-        value = PostProcessors.get(options[:name], value, context)
+        value = PostProcessors.get(options[:name], value, scope.context_for(options:))
       end
 
       value
     end
 
-    def select_categories(category_selectors:, item:, base_url:)
+    def select_categories(category_selectors:, scope:)
       Array(category_selectors).flat_map do |selector_name|
-        extract_category_values(selector_name, item:, base_url:)
+        extract_category_values(selector_name, scope:)
       end
     end
 
-    def extract_category_values(selector_name, item:, base_url:)
+    def extract_category_values(selector_name, scope:)
       selector_key, config = selector_config_for(selector_name, allow_nil: true)
       return [] unless config
 
-      nodes = extract_nodes(item:, config:)
-      unless node_set_with_multiple_elements?(nodes)
-        return Array(select_regular(selector_key, item:, config:, base_url:))
-      end
+      nodes = extract_nodes(item: scope.item, config:)
+      return Array(select_regular(selector_key, scope:, config:)) unless node_set_with_multiple_elements?(nodes)
 
-      Array(nodes).flat_map { |node| extract_categories_from_node(node, item:, config:, base_url:) }
+      Array(nodes).flat_map { |node| extract_categories_from_node(node, scope:, config:) }
     end
 
-    def extract_categories_from_node(node, item:, config:, base_url:)
-      values = Extractors.get(category_node_options(config, base_url:), node)
-      values = apply_post_process_steps(item:, value: values, post_process_steps: config[:post_process], base_url:)
+    def extract_categories_from_node(node, scope:, config:)
+      values = Extractors.get(category_node_options(config, scope:), node)
+      values = apply_post_process_steps(scope:, value: values, post_process_steps: config[:post_process])
 
       Array(values).filter_map { |category| extract_category_text(category) }
     end
@@ -257,7 +271,7 @@ module Html2rss
     def extract_category_text(category)
       text = case category
              when Nokogiri::XML::Node, Nokogiri::XML::NodeSet
-               HtmlExtractor.extract_visible_text(category)
+               Html2rss::Html::Navigator.extract_visible_text(category)
              else
                category&.to_s
              end
@@ -270,19 +284,19 @@ module Html2rss
       nodes.is_a?(Nokogiri::XML::NodeSet) && nodes.length > 1
     end
 
-    def category_node_options(selector_config, base_url:)
+    def category_node_options(selector_config, scope:)
       @category_node_configs ||= {}
-      @category_node_configs[[selector_config.object_id, base_url]] ||= selector_config.merge(
-        channel: channel_context(base_url),
+      @category_node_configs[[selector_config.object_id, scope.base_url]] ||= selector_config.merge(
+        channel: scope.channel,
         selector: nil
       ).freeze
     end
 
-    def apply_post_process_steps(item:, value:, post_process_steps:, base_url:)
+    def apply_post_process_steps(scope:, value:, post_process_steps:)
       return value unless value && post_process_steps
 
       steps = post_process_steps.is_a?(Array) ? post_process_steps : [post_process_steps]
-      post_process(item, value, steps, base_url:)
+      post_process(scope, value, steps)
     end
 
     def selector_config_for(name, allow_nil: false)
@@ -305,14 +319,20 @@ module Html2rss
       @channel_contexts[base_url] ||= { url: base_url, time_zone: @time_zone }.freeze
     end
 
-    def channel_post_process_context(base_url)
-      @channel_pp_contexts ||= {}
-      @channel_pp_contexts[base_url] ||= { channel: channel_context(base_url) }.freeze
+    # Keep a single enclosure Hash as one list entry; +Array(hash)+ would split pairs.
+    #
+    # @param value [Hash, Array] enclosure hash or list of hashes
+    # @return [Array]
+    def wrap_enclosure_value(value)
+      value.is_a?(Array) ? value : [value]
     end
 
-    # @return [Hash] enclosure details.
-    def enclosure(item:, config:, base_url:)
-      url = Url.from_relative(select_regular(:enclosure, item:, config:, base_url:), base_url)
+    # @return [Hash, nil] enclosure details, or nil when the selector yields nothing.
+    def enclosure(scope:, config:)
+      selected = select_regular(:enclosure, scope:, config:)
+      return if selected.nil? || selected.to_s.strip.empty?
+
+      url = Url.from_relative(selected, scope.base_url)
 
       { url:, type: config[:content_type] }
     end

data/lib/html2rss/status.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Shared RSS +generator+ / JSON Feed +user_comment+ formatter string.
+  #
+  # Exposed publicly via {FeedResult#status}. Safe to log without reading articles.
+  # Stable telemetry payload for cross-repo consumers (e.g. html2rss-web observability).
+  # Tallies and counters are validated and frozen at construction (including Marshal load).
+  Status = Data.define(
+    :version, :scraper_tallies, :dedup_dropped, :selected_strategy, :attempt_count, :strategy_attempts
+  ) do
+    class << self
+      ##
+      # Builds status from extracted articles and scrape telemetry.
+      #
+      # @param articles [Array<Html2rss::Article>] articles kept after deduplication
+      # @param dedup_dropped [Integer] number of articles removed by deduplication
+      # @param selected_strategy [Symbol, nil] concrete strategy that succeeded under +:auto+ (else +nil+)
+      # @param attempt_count [Integer] auto-fallback attempt count (0 when not under +:auto+)
+      # @param strategy_attempts [Array<Hash>] auto-fallback attempt hashes (empty outside +:auto+)
+      # @return [Html2rss::Status]
+      def build(articles:, dedup_dropped: 0, selected_strategy: nil, attempt_count: 0, strategy_attempts: [])
+        tallies = articles.filter_map(&:scraper).tally.transform_keys { |klass| scraper_name(klass) }
+        new(
+          version: Html2rss::VERSION,
+          scraper_tallies: tallies,
+          dedup_dropped:,
+          selected_strategy:,
+          attempt_count:,
+          strategy_attempts:
+        )
+      end
+
+      ##
+      # @param klass [Class, #to_s] scraper class
+      # @return [String] short scraper label for tallies / generator text
+      def scraper_name(klass)
+        klass.to_s.gsub(/(?:Html2rss|Scraper)::/, '')
+      end
+    end
+
+    ##
+    # @param version [String]
+    # @param scraper_tallies [Hash{String => Integer}]
+    # @param dedup_dropped [Integer]
+    # @param selected_strategy [Symbol, nil]
+    # @param attempt_count [Integer]
+    # @param strategy_attempts [Array<Hash>]
+    # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength -- Status Data.define members
+    def initialize(
+      version:, scraper_tallies:, dedup_dropped:, selected_strategy: nil, attempt_count: 0, strategy_attempts: []
+    )
+      dedup = Integer(dedup_dropped)
+      attempts = Integer(attempt_count)
+      validate_counters!(dedup:, attempts:, selected_strategy:)
+
+      super(
+        version: version.to_s.dup.freeze,
+        scraper_tallies: freeze_tallies(scraper_tallies),
+        dedup_dropped: dedup,
+        selected_strategy:,
+        attempt_count: attempts,
+        strategy_attempts: freeze_attempts(strategy_attempts)
+      )
+    end
+    # rubocop:enable Metrics/ParameterLists, Metrics/MethodLength
+
+    ##
+    # Observability hash for web (+scraper_status+). Omits empty/absent optional keys:
+    # +:scraper_tallies+ when empty, +:selected_strategy+ when +nil+, +:attempt_count+ when zero,
+    # +:strategy_attempts+ when empty.
+    # Data members remain available via readers even when omitted here.
+    #
+    # @return [Hash{Symbol => Object}] always +:version+ (String), +:dedup_dropped+ (Integer);
+    #   optionally +:scraper_tallies+, +:selected_strategy+, +:attempt_count+, +:strategy_attempts+
+    def to_h
+      {
+        version:,
+        dedup_dropped:,
+        **(scraper_tallies.any? ? { scraper_tallies: } : {}),
+        **(selected_strategy.nil? ? {} : { selected_strategy: }),
+        **(attempt_count.positive? ? { attempt_count: } : {}),
+        **(strategy_attempts.any? ? { strategy_attempts: } : {})
+      }
+    end
+
+    ##
+    # Formats the RSS +generator+ string and JSON Feed +user_comment+.
+    # Scraper-focused only — auto strategy summary stays on {#to_h}, not this string.
+    # Omits the +(scrapers: …)+ clause when tallies are empty.
+    #
+    # @return [String]
+    def to_generator_comment
+      return "html2rss V. #{version}" if scraper_tallies.empty?
+
+      counts = scraper_tallies.map { |name, count| "#{name} (#{count})" }
+      "html2rss V. #{version} (scrapers: #{counts.join(', ')})"
+    end
+
+    private
+
+    def marshal_dump
+      [version, scraper_tallies, dedup_dropped, selected_strategy, attempt_count, strategy_attempts]
+    end
+
+    def marshal_load((version, scraper_tallies, dedup_dropped, selected_strategy, attempt_count, strategy_attempts))
+      initialize(version:, scraper_tallies:, dedup_dropped:, selected_strategy:, attempt_count:, strategy_attempts:)
+    end
+
+    def freeze_tallies(tallies)
+      tallies.to_h.transform_keys(&:to_s).transform_values { |count| Integer(count) }.freeze
+    end
+
+    def freeze_attempts(attempts)
+      Array(attempts).map { |attempt| attempt.to_h.freeze }.freeze
+    end
+
+    def validate_counters!(dedup:, attempts:, selected_strategy:)
+      raise ArgumentError, 'dedup_dropped must be >= 0' if dedup.negative?
+      raise ArgumentError, 'attempt_count must be >= 0' if attempts.negative?
+
+      unless selected_strategy.nil? || selected_strategy.is_a?(Symbol)
+        raise ArgumentError, 'selected_strategy must be a Symbol or nil'
+      end
+
+      return unless selected_strategy && attempts < 1
+
+      raise ArgumentError, 'attempt_count must be >= 1 when selected_strategy is set'
+    end
+  end
+end

data/lib/html2rss/version.rb

@@ -1,9 +1,7 @@
 # frozen_string_literal: true
 
-##
-# The Html2rss namespace.
 module Html2rss
   # Current application version.
-  VERSION = '0.22.2'
+  VERSION = '0.24.0'
   public_constant :VERSION
 end

data/lib/html2rss.rb

@@ -8,7 +8,7 @@ loader.setup
 
 require 'logger'
 require 'forwardable'
-require 'html2rss/configuration'
+require 'html2rss/defaults'
 
 ##
 # The Html2rss namespace.
@@ -34,22 +34,34 @@ module Html2rss
     Config.load_yaml(file, feed_name)
   end
 
+  ##
+  # Returns an opaque, Marshal-cacheable result of one scrape.
+  #
+  # Prefer this when the same scrape must render as RSS and JSON Feed (e.g. web cache).
+  #
+  # @param raw_config [Hash{Symbol => Object}] feed configuration
+  # @return [Html2rss::FeedResult]
+  def self.feed_result(raw_config)
+    FeedPipeline.new(raw_config).to_result
+  end
+
   ##
   # Returns an RSS object generated from the provided configuration.
   #
   # @param raw_config [Hash{Symbol => Object}] feed configuration
   # @return [RSS::Rss] generated RSS feed
   def self.feed(raw_config)
-    FeedPipeline.new(raw_config).to_rss
+    feed_result(raw_config).to_rss
   end
 
   ##
   # Returns a JSONFeed 1.1 hash generated from the provided configuration.
   #
   # @param raw_config [Hash{Symbol => Object}] feed configuration
+  # @param feed_url [String, nil] optional self URL for the feed (JSON Feed +feed_url+)
   # @return [Hash] JSONFeed-compliant hash
-  def self.json_feed(raw_config)
-    FeedPipeline.new(raw_config).to_json_feed
+  def self.json_feed(raw_config, feed_url: nil)
+    feed_result(raw_config).to_json_feed(feed_url:)
   end
 
   # rubocop:disable Metrics/ParameterLists
@@ -61,14 +73,14 @@ module Html2rss
   # @param strategy [Symbol] request strategy to use
   # @param items_selector [String, nil] optional selector hint for item extraction
   # @param max_redirects [Integer, nil] optional redirect limit override
-  # @param max_requests [Integer, nil] optional request budget override
+  # @param max_requests [Integer] optional request budget override (default: 4 for sitemap sub-fetches)
   # @param local_file_path [String, nil] optional local HTML file path
   # @return [RSS::Rss] generated RSS feed
   def self.auto_source(url,
                        strategy: :auto,
                        items_selector: nil,
                        max_redirects: nil,
-                       max_requests: nil,
+                       max_requests: 4,
                        local_file_path: nil)
     feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:, local_file_path:))
   end
@@ -80,14 +92,14 @@ module Html2rss
   # @param strategy [Symbol] request strategy to use
   # @param items_selector [String, nil] optional selector hint for item extraction
   # @param max_redirects [Integer, nil] optional redirect limit override
-  # @param max_requests [Integer, nil] optional request budget override
+  # @param max_requests [Integer] optional request budget override (default: 4 for sitemap sub-fetches)
   # @param local_file_path [String, nil] optional local HTML file path
   # @return [Hash] JSONFeed-compliant hash
   def self.auto_json_feed(url,
                           strategy: :auto,
                           items_selector: nil,
                           max_redirects: nil,
-                          max_requests: nil,
+                          max_requests: 4,
                           local_file_path: nil)
     json_feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
                                        local_file_path:))
@@ -98,26 +110,26 @@ module Html2rss
   # rubocop:disable ThreadSafety/ClassInstanceVariable
   class << self
     ##
-    # @return [Html2rss::Configuration] the global configuration instance
-    def configuration
-      @configuration ||= Configuration.new.freeze
+    # @return [Html2rss::Defaults] the global defaults instance
+    def defaults
+      @defaults ||= Defaults.new.freeze
     end
 
     ##
     # Configures global library defaults.
     #
-    # @yieldparam config [Html2rss::Configuration]
-    # @return [Html2rss::Configuration] the frozen configuration
+    # @yieldparam config [Html2rss::Defaults]
+    # @return [Html2rss::Defaults] the frozen defaults
     def configure
-      config = configuration.dup
+      config = defaults.dup
       yield config
-      @configuration = config.freeze
+      @defaults = config.freeze
     end
 
     ##
     # @return [Object] the logger
     def logger
-      configuration.logger
+      defaults.logger
     end
 
     ##
@@ -129,12 +141,12 @@ module Html2rss
     private
 
     ##
-    # Resets the global configuration to defaults (mainly for testing).
+    # Resets the global defaults (mainly for testing).
     #
     # @return [void]
-    def reset_configuration!
-      @configuration = nil
-      logger.level = configuration.log_level if logger.respond_to?(:level=)
+    def reset_defaults!
+      @defaults = nil
+      logger.level = defaults.log_level if logger.respond_to?(:level=)
     end
   end
   # rubocop:enable ThreadSafety/ClassInstanceVariable
@@ -156,7 +168,7 @@ module Html2rss
     end
 
     def shortcut_request_controls(strategy:, max_redirects:, max_requests:)
-      RequestControls.new(
+      Config::RequestControls.new(
         strategy:,
         max_redirects:,
         max_requests:,
@@ -173,7 +185,7 @@ module Html2rss
     end
   end
 
-  logger.level = configuration.log_level if logger.respond_to?(:level=)
+  logger.level = defaults.log_level if logger.respond_to?(:level=)
 end
 
 loader.eager_load