html2rss 0.22.2 → 0.23.0

data/lib/html2rss/selectors/post_processors/template.rb

@@ -42,6 +42,10 @@ module Html2rss
 
           string = context[:options]&.dig(:string).to_s
           raise InvalidType, 'The `string` template is absent.' if string.empty?
+
+          return if context.item_scope
+
+          raise MissingOption, 'The post-processor context is missing `item_scope`.', [], cause: nil
         end
 
         ##
@@ -51,8 +55,6 @@ module Html2rss
           super
 
           @options = context[:options] || {}
-          @scraper = context[:scraper]
-          @item = context[:item]
           @string = @options[:string].to_s
           @getter = ->(key) { item_value(key) }
         end
@@ -69,7 +71,9 @@ module Html2rss
         # @return [String]
         def item_value(key)
           key = key.to_sym
-          key == :self ? value : @scraper.select(key, @item)
+          return value if key == :self
+
+          @context.item_scope.select(key)
         end
       end
     end

data/lib/html2rss/selectors.rb

@@ -18,7 +18,9 @@ module Html2rss
     include Enumerable
 
     # A context instance passed to item extractors and post-processors.
-    Context = Struct.new('Context', :options, :item, :config, :scraper, keyword_init: true)
+    # When built via {ItemScope#context_for}, +item_scope+ carries the per-item
+    # extraction base_url for nested selects (e.g. Template).
+    Context = Struct.new('Context', :options, :item, :config, :scraper, :item_scope, keyword_init: true)
 
     # Default selectors options merged into user configuration.
     DEFAULT_CONFIG = { items: { enhance: true } }.freeze
@@ -30,9 +32,6 @@ module Html2rss
     # Item attributes that require dedicated extraction logic.
     SPECIAL_ATTRIBUTES = Set[:guid, :enclosure, :categories].freeze
 
-    # Mapping of new attribute names to their legacy names for backward compatibility.
-    RENAMED_ATTRIBUTES = { published_at: %i[updated pubDate] }.freeze
-
     ##
     # Initializes a new Selectors instance.
     #
@@ -46,14 +45,14 @@ module Html2rss
       @time_zone = time_zone
 
       prepare_selectors!
-      @rss_item_attributes = @selectors.keys & Html2rss::RssBuilder::Article::PROVIDED_KEYS
+      @rss_item_attributes = @selectors.keys & Html2rss::Article::PROVIDED_KEYS
     end
 
     ##
     # Returns articles extracted from the response.
     # Reverses order if config specifies reverse ordering.
     #
-    # @return [Array<Html2rss::RssBuilder::Article>]
+    # @return [Array<Html2rss::Article>]
     def articles
       @articles ||= @selectors.dig(ITEMS_SELECTOR_KEY, :order) == 'reverse' ? to_a.tap(&:reverse!) : to_a
     end
@@ -61,7 +60,7 @@ module Html2rss
     ##
     # Iterates over each scraped article.
     #
-    # @yield [article] Gives each article as an Html2rss::RssBuilder::Article.
+    # @yield [article] Gives each article as an Html2rss::Article.
     # @return [Enumerator] An enumerator if no block is given.
     def each(&)
       return enum_for(:each) unless block_given?
@@ -73,7 +72,7 @@ module Html2rss
 
         enhance_article_hash(article_hash, item, response.url) if enhance
 
-        yield Html2rss::RssBuilder::Article.new(**article_hash, scraper: self.class)
+        yield Html2rss::Article.new(**article_hash, scraper: self.class)
       end
     end
 
@@ -92,7 +91,8 @@ module Html2rss
     # @param page_response [RequestService::Response] response used for selector extraction context
     # @return [Hash] Hash of attributes for the article.
     def extract_article(item, page_response = response)
-      @rss_item_attributes.to_h { |key| [key, select(key, item, base_url: page_response.url)] }.compact
+      scope = item_scope_for(item, page_response.url)
+      @rss_item_attributes.to_h { |key| [key, scope.select(key)] }.compact
     end
 
     ##
@@ -105,13 +105,13 @@ module Html2rss
     # @return [Hash] The enhanced article hash.
     # rubocop:disable Metrics/MethodLength
     def enhance_article_hash(article_hash, article_tag, base_url = @url)
-      selected_anchor = HtmlExtractor.main_anchor_for(article_tag)
-      extracted = HtmlExtractor.new(
+      selected_anchor = Html2rss::Html::Navigator.main_anchor_for(article_tag)
+      extracted = Html2rss::Html::ArticleExtractor.call(
         article_tag,
         base_url:,
         selected_anchor:,
         fallback_anchorless: true
-      ).call
+      )
       return article_hash unless extracted
 
       extracted.each_with_object(article_hash) do |(key, value), hash|
@@ -131,6 +131,18 @@ module Html2rss
     # @return [Object, Array<Object>] The selected value(s).
     # @raise [InvalidSelectorName] If the attribute name is invalid or not defined.
     def select(name, item, base_url: @url)
+      select_in_scope(name, item_scope_for(item, base_url))
+    end
+
+    ##
+    # Selects the value for a given attribute within an existing {ItemScope}.
+    # Used by {ItemScope#select} so nested selects reuse one scope per extraction pass.
+    #
+    # @param name [Symbol, String] Name of the attribute.
+    # @param scope [ItemScope] Per-item extraction scope.
+    # @return [Object, Array<Object>] The selected value(s).
+    # @raise [InvalidSelectorName] If the attribute name is invalid or not defined.
+    def select_in_scope(name, scope)
       name = name.to_sym
 
       raise InvalidSelectorName, "Attribute selector '#{name}' is reserved for items." if name == ITEMS_SELECTOR_KEY
@@ -138,9 +150,9 @@ module Html2rss
       selector_key, config = selector_config_for(name)
 
       if SPECIAL_ATTRIBUTES.member?(selector_key)
-        select_special(selector_key, item:, config:, base_url:)
+        select_special(selector_key, scope:, config:)
       else
-        select_regular(selector_key, item:, config:, base_url:)
+        select_regular(selector_key, scope:, config:)
       end
     end
 
@@ -148,10 +160,18 @@ module Html2rss
 
     attr_reader :response
 
+    def item_scope_for(item, base_url)
+      ItemScope.new(
+        item:,
+        base_url:,
+        scraper: self,
+        channel: channel_context(base_url)
+      )
+    end
+
     def prepare_selectors!
       validate_url_and_link_exclusivity!
       fix_url_and_link!
-      handle_renamed_attributes!
     end
 
     def validate_url_and_link_exclusivity!
@@ -167,17 +187,6 @@ module Html2rss
       @selectors[:url] = @selectors[:link]
     end
 
-    def handle_renamed_attributes!
-      RENAMED_ATTRIBUTES.each_pair do |new_name, old_names|
-        old_names.each do |old_name|
-          next unless @selectors.key?(old_name)
-
-          Html2rss::Log.warn("Selector '#{old_name}' is deprecated. Please rename to '#{new_name}'.")
-          @selectors[new_name] ||= @selectors.delete(old_name)
-        end
-      end
-    end
-
     def parsed_body
       parsed_body_for(response)
     end
@@ -192,64 +201,58 @@ module Html2rss
                                             end
     end
 
-    def select_special(name, item:, config:, base_url:)
+    def select_special(name, scope:, config:)
       case name
       when :enclosure
-        enclosure(item:, config:, base_url:)
+        enclosure(scope:, config:)
       when :guid
-        Array(config).map { |selector_name| select(selector_name, item, base_url:) }
+        Array(config).map { |selector_name| scope.select(selector_name) }
       when :categories
-        select_categories(category_selectors: config, item:, base_url:)
+        select_categories(category_selectors: config, scope:)
       end
     end
 
-    def select_regular(_name, item:, config:, base_url:)
+    def select_regular(_name, scope:, config:)
       @merged_configs ||= {}
-      merged_config = @merged_configs[[config.object_id, base_url]] ||=
-        config.merge(channel: channel_context(base_url)).freeze
-      value = Extractors.get(merged_config, item)
+      merged_config = @merged_configs[[config.object_id, scope.base_url]] ||=
+        config.merge(channel: scope.channel).freeze
+      value = Extractors.get(merged_config, scope.item)
 
       if value && (post_process_steps = config[:post_process])
         steps = post_process_steps.is_a?(Array) ? post_process_steps : [post_process_steps]
-        value = post_process(item, value, steps, base_url:)
+        value = post_process(scope, value, steps)
       end
 
       value
     end
 
-    def post_process(item, value, post_process_steps, base_url:)
-      pp_context = channel_post_process_context(base_url)
+    def post_process(scope, value, post_process_steps)
       post_process_steps.each do |options|
-        context = Context.new(config: pp_context,
-                              item:, scraper: self, options:)
-
-        value = PostProcessors.get(options[:name], value, context)
+        value = PostProcessors.get(options[:name], value, scope.context_for(options:))
       end
 
       value
     end
 
-    def select_categories(category_selectors:, item:, base_url:)
+    def select_categories(category_selectors:, scope:)
       Array(category_selectors).flat_map do |selector_name|
-        extract_category_values(selector_name, item:, base_url:)
+        extract_category_values(selector_name, scope:)
       end
     end
 
-    def extract_category_values(selector_name, item:, base_url:)
+    def extract_category_values(selector_name, scope:)
       selector_key, config = selector_config_for(selector_name, allow_nil: true)
       return [] unless config
 
-      nodes = extract_nodes(item:, config:)
-      unless node_set_with_multiple_elements?(nodes)
-        return Array(select_regular(selector_key, item:, config:, base_url:))
-      end
+      nodes = extract_nodes(item: scope.item, config:)
+      return Array(select_regular(selector_key, scope:, config:)) unless node_set_with_multiple_elements?(nodes)
 
-      Array(nodes).flat_map { |node| extract_categories_from_node(node, item:, config:, base_url:) }
+      Array(nodes).flat_map { |node| extract_categories_from_node(node, scope:, config:) }
     end
 
-    def extract_categories_from_node(node, item:, config:, base_url:)
-      values = Extractors.get(category_node_options(config, base_url:), node)
-      values = apply_post_process_steps(item:, value: values, post_process_steps: config[:post_process], base_url:)
+    def extract_categories_from_node(node, scope:, config:)
+      values = Extractors.get(category_node_options(config, scope:), node)
+      values = apply_post_process_steps(scope:, value: values, post_process_steps: config[:post_process])
 
       Array(values).filter_map { |category| extract_category_text(category) }
     end
@@ -257,7 +260,7 @@ module Html2rss
     def extract_category_text(category)
       text = case category
              when Nokogiri::XML::Node, Nokogiri::XML::NodeSet
-               HtmlExtractor.extract_visible_text(category)
+               Html2rss::Html::Navigator.extract_visible_text(category)
              else
                category&.to_s
              end
@@ -270,19 +273,19 @@ module Html2rss
       nodes.is_a?(Nokogiri::XML::NodeSet) && nodes.length > 1
     end
 
-    def category_node_options(selector_config, base_url:)
+    def category_node_options(selector_config, scope:)
       @category_node_configs ||= {}
-      @category_node_configs[[selector_config.object_id, base_url]] ||= selector_config.merge(
-        channel: channel_context(base_url),
+      @category_node_configs[[selector_config.object_id, scope.base_url]] ||= selector_config.merge(
+        channel: scope.channel,
         selector: nil
       ).freeze
     end
 
-    def apply_post_process_steps(item:, value:, post_process_steps:, base_url:)
+    def apply_post_process_steps(scope:, value:, post_process_steps:)
       return value unless value && post_process_steps
 
       steps = post_process_steps.is_a?(Array) ? post_process_steps : [post_process_steps]
-      post_process(item, value, steps, base_url:)
+      post_process(scope, value, steps)
     end
 
     def selector_config_for(name, allow_nil: false)
@@ -305,14 +308,9 @@ module Html2rss
       @channel_contexts[base_url] ||= { url: base_url, time_zone: @time_zone }.freeze
     end
 
-    def channel_post_process_context(base_url)
-      @channel_pp_contexts ||= {}
-      @channel_pp_contexts[base_url] ||= { channel: channel_context(base_url) }.freeze
-    end
-
     # @return [Hash] enclosure details.
-    def enclosure(item:, config:, base_url:)
-      url = Url.from_relative(select_regular(:enclosure, item:, config:, base_url:), base_url)
+    def enclosure(scope:, config:)
+      url = Url.from_relative(select_regular(:enclosure, scope:, config:), scope.base_url)
 
       { url:, type: config[:content_type] }
     end

data/lib/html2rss/version.rb

@@ -1,9 +1,7 @@
 # frozen_string_literal: true
 
-##
-# The Html2rss namespace.
 module Html2rss
   # Current application version.
-  VERSION = '0.22.2'
+  VERSION = '0.23.0'
   public_constant :VERSION
 end

data/lib/html2rss.rb

@@ -8,7 +8,7 @@ loader.setup
 
 require 'logger'
 require 'forwardable'
-require 'html2rss/configuration'
+require 'html2rss/defaults'
 
 ##
 # The Html2rss namespace.
@@ -61,14 +61,14 @@ module Html2rss
   # @param strategy [Symbol] request strategy to use
   # @param items_selector [String, nil] optional selector hint for item extraction
   # @param max_redirects [Integer, nil] optional redirect limit override
-  # @param max_requests [Integer, nil] optional request budget override
+  # @param max_requests [Integer] optional request budget override (default: 4 for sitemap sub-fetches)
   # @param local_file_path [String, nil] optional local HTML file path
   # @return [RSS::Rss] generated RSS feed
   def self.auto_source(url,
                        strategy: :auto,
                        items_selector: nil,
                        max_redirects: nil,
-                       max_requests: nil,
+                       max_requests: 4,
                        local_file_path: nil)
     feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:, local_file_path:))
   end
@@ -80,14 +80,14 @@ module Html2rss
   # @param strategy [Symbol] request strategy to use
   # @param items_selector [String, nil] optional selector hint for item extraction
   # @param max_redirects [Integer, nil] optional redirect limit override
-  # @param max_requests [Integer, nil] optional request budget override
+  # @param max_requests [Integer] optional request budget override (default: 4 for sitemap sub-fetches)
   # @param local_file_path [String, nil] optional local HTML file path
   # @return [Hash] JSONFeed-compliant hash
   def self.auto_json_feed(url,
                           strategy: :auto,
                           items_selector: nil,
                           max_redirects: nil,
-                          max_requests: nil,
+                          max_requests: 4,
                           local_file_path: nil)
     json_feed(build_auto_source_config(url:, strategy:, items_selector:, max_redirects:, max_requests:,
                                        local_file_path:))
@@ -98,26 +98,26 @@ module Html2rss
   # rubocop:disable ThreadSafety/ClassInstanceVariable
   class << self
     ##
-    # @return [Html2rss::Configuration] the global configuration instance
-    def configuration
-      @configuration ||= Configuration.new.freeze
+    # @return [Html2rss::Defaults] the global defaults instance
+    def defaults
+      @defaults ||= Defaults.new.freeze
     end
 
     ##
     # Configures global library defaults.
     #
-    # @yieldparam config [Html2rss::Configuration]
-    # @return [Html2rss::Configuration] the frozen configuration
+    # @yieldparam config [Html2rss::Defaults]
+    # @return [Html2rss::Defaults] the frozen defaults
     def configure
-      config = configuration.dup
+      config = defaults.dup
       yield config
-      @configuration = config.freeze
+      @defaults = config.freeze
     end
 
     ##
     # @return [Object] the logger
     def logger
-      configuration.logger
+      defaults.logger
     end
 
     ##
@@ -129,12 +129,12 @@ module Html2rss
     private
 
     ##
-    # Resets the global configuration to defaults (mainly for testing).
+    # Resets the global defaults (mainly for testing).
     #
     # @return [void]
-    def reset_configuration!
-      @configuration = nil
-      logger.level = configuration.log_level if logger.respond_to?(:level=)
+    def reset_defaults!
+      @defaults = nil
+      logger.level = defaults.log_level if logger.respond_to?(:level=)
     end
   end
   # rubocop:enable ThreadSafety/ClassInstanceVariable
@@ -156,7 +156,7 @@ module Html2rss
     end
 
     def shortcut_request_controls(strategy:, max_redirects:, max_requests:)
-      RequestControls.new(
+      Config::RequestControls.new(
         strategy:,
         max_redirects:,
         max_requests:,
@@ -173,7 +173,7 @@ module Html2rss
     end
   end
 
-  logger.level = configuration.log_level if logger.respond_to?(:level=)
+  logger.level = defaults.log_level if logger.respond_to?(:level=)
 end
 
 loader.eager_load

data/schema/html2rss-config.schema.json

@@ -109,6 +109,31 @@
               },
               "required": []
             },
+            "sitemap": {
+              "type": "object",
+              "properties": {
+                "enabled": {
+                  "type": "boolean",
+                  "not": {
+                    "type": "null"
+                  }
+                },
+                "min_priority": {
+                  "type": "number",
+                  "not": {
+                    "type": "null"
+                  }
+                },
+                "max_age_days": {
+                  "type": "integer",
+                  "not": {
+                    "type": "null"
+                  },
+                  "exclusiveMinimum": 0
+                }
+              },
+              "required": []
+            },
             "schema": {
               "type": "object",
               "properties": {
@@ -133,6 +158,18 @@
               },
               "required": []
             },
+            "microformats2": {
+              "type": "object",
+              "properties": {
+                "enabled": {
+                  "type": "boolean",
+                  "not": {
+                    "type": "null"
+                  }
+                }
+              },
+              "required": []
+            },
             "json_state": {
               "type": "object",
               "properties": {
@@ -145,6 +182,18 @@
               },
               "required": []
             },
+            "meta_oembed": {
+              "type": "object",
+              "properties": {
+                "enabled": {
+                  "type": "boolean",
+                  "not": {
+                    "type": "null"
+                  }
+                }
+              },
+              "required": []
+            },
             "semantic_html": {
               "type": "object",
               "properties": {
@@ -224,15 +273,26 @@
           "wordpress_api": {
             "enabled": true
           },
+          "sitemap": {
+            "enabled": true,
+            "min_priority": 0.3,
+            "max_age_days": 30
+          },
           "schema": {
             "enabled": true
           },
           "microdata": {
             "enabled": true
           },
+          "microformats2": {
+            "enabled": true
+          },
           "json_state": {
             "enabled": true
           },
+          "meta_oembed": {
+            "enabled": true
+          },
           "semantic_html": {
             "enabled": true,
             "fallback_anchorless": true
@@ -277,18 +337,58 @@
               }
             },
             "pagination": {
-              "type": "object",
-              "properties": {
-                "max_pages": {
+              "description": "Pagination configuration or maximum page count integer.",
+              "oneOf": [
+                {
                   "type": "integer",
-                  "not": {
-                    "type": "null"
-                  },
                   "exclusiveMinimum": 0
+                },
+                {
+                  "type": "object",
+                  "properties": {
+                    "strategy": {
+                      "type": "string",
+                      "enum": [
+                        "rel_next",
+                        "custom_selector",
+                        "url_template",
+                        "offset",
+                        "json_cursor"
+                      ]
+                    },
+                    "max_pages": {
+                      "type": "integer",
+                      "exclusiveMinimum": 0
+                    },
+                    "selector": {
+                      "type": "string"
+                    },
+                    "param": {
+                      "type": "string"
+                    },
+                    "start_page": {
+                      "type": "integer"
+                    },
+                    "step": {
+                      "type": "integer",
+                      "exclusiveMinimum": 0
+                    },
+                    "start_offset": {
+                      "type": "integer"
+                    },
+                    "increment": {
+                      "type": "integer",
+                      "exclusiveMinimum": 0
+                    },
+                    "cursor_path": {
+                      "type": "string"
+                    },
+                    "next_url_path": {
+                      "type": "string"
+                    }
+                  },
+                  "additionalProperties": true
                 }
-              },
-              "required": [
-                "max_pages"
               ]
             }
           },