html2rss 0.22.2 → 0.24.0

data/lib/html2rss/auto_source/scraper.rb

@@ -22,9 +22,12 @@ module Html2rss
       # Ordered scraper classes considered during auto-source extraction.
       SCRAPERS = [
         WordpressApi,
+        Sitemap,
         Schema,
         Microdata,
+        Microformats2,
         JsonState,
+        MetaOembed,
         SemanticHtml,
         Html
       ].freeze
@@ -67,12 +70,14 @@ module Html2rss
 
       ##
       # Returns an array of scraper classes that claim to find articles in the parsed body.
-      # @param parsed_body [Nokogiri::HTML::Document] The parsed HTML body.
+      # @param parsed_body [Nokogiri::HTML::Document] The parsed HTML document.
       # @param opts [Hash] The options hash.
       # @option opts [Hash] :wordpress_api scraper toggle and configuration
       # @option opts [Hash] :schema scraper toggle and configuration
       # @option opts [Hash] :microdata scraper toggle and configuration
+      # @option opts [Hash] :microformats2 scraper toggle and configuration
       # @option opts [Hash] :json_state scraper toggle and configuration
+      # @option opts [Hash] :meta_oembed scraper toggle and configuration
       # @option opts [Hash] :semantic_html scraper toggle and configuration
       # @option opts [Hash] :html scraper toggle and configuration
       # @return [Array<Class>] An array of scraper classes that can handle the parsed body.
@@ -86,14 +91,16 @@ module Html2rss
       end
 
       # Returns scraper instances ready for extraction.
-      # @param parsed_body [Nokogiri::HTML::Document] The parsed HTML body.
+      # @param parsed_body [Nokogiri::HTML::Document] The parsed HTML document.
       # @param url [String, Html2rss::Url] The page url.
       # @param request_session [Html2rss::RequestSession, nil] Shared follow-up session.
       # @param opts [Hash] The options hash.
       # @option opts [Hash] :wordpress_api scraper toggle and configuration
       # @option opts [Hash] :schema scraper toggle and configuration
       # @option opts [Hash] :microdata scraper toggle and configuration
+      # @option opts [Hash] :microformats2 scraper toggle and configuration
       # @option opts [Hash] :json_state scraper toggle and configuration
+      # @option opts [Hash] :meta_oembed scraper toggle and configuration
       # @option opts [Hash] :semantic_html scraper toggle and configuration
       # @option opts [Hash] :html scraper toggle and configuration
       # @return [Array<Object>] An array of scraper instances that can handle the parsed body.
@@ -139,7 +146,7 @@ module Html2rss
       private_class_method :classify_no_scraper_surface
 
       def self.blocked_surface?(parsed_body)
-        Html2rss::BlockedSurface.interstitial?(parsed_body.to_html)
+        Html2rss::RequestService::BlockedSurface.interstitial?(parsed_body.to_html)
       end
       private_class_method :blocked_surface?
 

data/lib/html2rss/auto_source.rb

@@ -22,15 +22,26 @@ module Html2rss
         wordpress_api: {
           enabled: true
         },
+        sitemap: {
+          enabled: true,
+          min_priority: Discovery::Sitemap::DEFAULT_MIN_PRIORITY,
+          max_age_days: Discovery::Sitemap::DEFAULT_MAX_AGE_DAYS
+        },
         schema: {
           enabled: true
         },
         microdata: {
           enabled: true
         },
+        microformats2: {
+          enabled: true
+        },
         json_state: {
           enabled: true
         },
+        meta_oembed: {
+          enabled: true
+        },
         semantic_html: {
           enabled: true,
           fallback_anchorless: true
@@ -45,39 +56,23 @@ module Html2rss
       cleanup: Cleanup::DEFAULT_CONFIG
     }.freeze
 
-    SCRAPER_CONFIG = proc do
-      optional(:wordpress_api).hash do
-        optional(:enabled).filled(:bool)
-      end
-      optional(:schema).hash do
-        optional(:enabled).filled(:bool)
-      end
-      optional(:microdata).hash do
-        optional(:enabled).filled(:bool)
-      end
-      optional(:json_state).hash do
-        optional(:enabled).filled(:bool)
-      end
-      optional(:semantic_html).hash do
-        optional(:enabled).filled(:bool)
-        optional(:fallback_anchorless).filled(:bool)
-      end
-      optional(:html).hash do
-        optional(:enabled).filled(:bool)
-        optional(:minimum_selector_frequency).filled(:integer, gt?: 0)
-        optional(:use_top_selectors).filled(:integer, gt?: 0)
-        optional(:fallback_anchorless).filled(:bool)
-      end
-    end.freeze
-    private_constant :SCRAPER_CONFIG
-
-    # Runtime schema used to validate auto-source config values.
-    Config = Dry::Schema.Params do
-      optional(:scraper).hash(&SCRAPER_CONFIG)
+    class << self
+      ##
+      # Returns the sum of required request slots for all enabled scrapers in the config.
+      #
+      # @param config [Hash, nil] auto_source configuration hash
+      # @return [Integer] total request slots required by scrapers
+      def request_slots_for(config)
+        return 0 unless config
 
-      optional(:cleanup).hash do
-        optional(:keep_different_domain).filled(:bool)
-        optional(:min_words_title).filled(:integer, gt?: 0)
+        Scraper::SCRAPERS.sum do |scraper|
+          if config.dig(:scraper, scraper.options_key, :enabled)
+            opts = config.dig(:scraper, scraper.options_key)
+            scraper.respond_to?(:request_slots) ? scraper.request_slots(opts) : 0
+          else
+            0
+          end
+        end
       end
     end
 
@@ -98,7 +93,7 @@ module Html2rss
     ##
     # Extracts article candidates by selecting every scraper that can explain the
     # page shape, running those scrapers, and normalizing the resulting hashes
-    # into `RssBuilder::Article` objects.
+    # into `Article` objects.
     #
     # The contributor-facing flow is:
     # 1. choose scraper instances that match the page
@@ -108,7 +103,7 @@ module Html2rss
     # Scrapers with expensive precomputation, such as `SemanticHtml`, keep that
     # state on the instance so detection and extraction can reuse the same work.
     #
-    # @return [Array<Html2rss::RssBuilder::Article>] extracted articles
+    # @return [Array<Html2rss::Article>] extracted articles
     def articles
       @articles ||= extract_articles
     rescue Html2rss::AutoSource::Scraper::NoScraperFound => error
@@ -133,7 +128,7 @@ module Html2rss
 
     def run_scraper(instance)
       instance.each.map do |article_hash|
-        RssBuilder::Article.new(**article_hash, scraper: instance.class)
+        Article.new(**article_hash, scraper: instance.class)
       end
     end
 

data/lib/html2rss/channel.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+require 'time'
+
+# rubocop:disable Metrics/ModuleLength -- Channel value object owns extract + freeze + Marshal
+module Html2rss
+  ##
+  # Materialized channel metadata for feed builders and {FeedResult}.
+  #
+  # Extract once via {Channel.from_response}; builders consume this value object and
+  # do not own channel extraction. Safe to Marshal (no Response / Nokogiri retained).
+  # Strings and image URL are normalized and frozen at construction (including Marshal load).
+  #
+  # @!attribute [r] title
+  #   @return [String]
+  # @!attribute [r] url
+  #   @return [Html2rss::Url]
+  # @!attribute [r] description
+  #   @return [String]
+  # @!attribute [r] language
+  #   @return [String, nil]
+  # @!attribute [r] ttl
+  #   @return [Integer]
+  # @!attribute [r] last_build_date
+  #   @return [Time]
+  # @!attribute [r] image
+  #   @return [Html2rss::Url, nil]
+  # @!attribute [r] author
+  #   @return [String, nil]
+  Channel = Data.define(:title, :url, :description, :language, :ttl, :last_build_date, :image, :author) do
+    class << self
+      ##
+      # Materializes channel attributes from an HTTP response and optional overrides.
+      #
+      # @param response [Html2rss::RequestService::Response]
+      # @param overrides [Hash{Symbol => Object}] optional overrides for channel attributes
+      # @return [Html2rss::Channel]
+      def from_response(response, overrides: {})
+        new(
+          title: title_for(response, overrides),
+          url: url_for(response),
+          description: description_for(response, overrides),
+          language: language_for(response, overrides),
+          ttl: ttl_for(response, overrides),
+          last_build_date: last_build_date_for(response),
+          image: image_for(response, overrides),
+          author: author_for(response, overrides)
+        )
+      end
+
+      private
+
+      # Fallback RSS ttl (in minutes) when no cache directives are present.
+      def default_ttl_in_minutes = 360
+
+      # Description template used when no explicit or discovered description exists.
+      def default_description_template = 'Latest items from %<url>s'
+
+      def url_for(response) = Html2rss::Url.from_absolute(response.url)
+
+      def title_for(response, overrides)
+        return overrides[:title] if overrides[:title]
+
+        title = parsed_title(response)
+        return title if title
+
+        url_for(response).channel_titleized
+      end
+
+      def parsed_title(response)
+        return unless html_response?(response)
+
+        title = parsed_body(response).at_css('head > title')&.text.to_s
+        return if title.empty?
+
+        title.gsub(/\s+/, ' ').strip
+      end
+
+      def description_for(response, overrides)
+        return overrides[:description] unless overrides[:description].to_s.empty?
+
+        description = meta_description(response)
+        return format(default_description_template, url: url_for(response)) if description.to_s.empty?
+
+        description
+      end
+
+      def meta_description(response)
+        return unless html_response?(response)
+
+        parsed_body(response).at_css('meta[name="description"]')&.[]('content')
+      end
+
+      def ttl_for(response, overrides)
+        calculated = if overrides[:ttl]
+                       overrides[:ttl].to_i
+                     elsif (max_age = headers(response)['cache-control']&.match(/max-age=(\d+)/)&.[](1))
+                       max_age.to_i.fdiv(60).ceil
+                     else
+                       default_ttl_in_minutes
+                     end
+
+        min_ttl = Html2rss.defaults.min_ttl
+        min_ttl ? [calculated, min_ttl].max : calculated
+      end
+
+      def language_for(response, overrides)
+        return overrides[:language] if overrides[:language]
+
+        if (language_code = headers(response)['content-language']&.match(/^([a-z]{2})/))
+          return language_code[0]
+        end
+
+        return unless html_response?(response)
+
+        parsed_body(response)['lang'] || parsed_body(response).at_css('[lang]')&.[]('lang')
+      end
+
+      def author_for(response, overrides)
+        return overrides[:author] if overrides[:author]
+        return unless html_response?(response)
+
+        parsed_body(response).at_css('meta[name="author"]')&.[]('content')
+      end
+
+      def last_build_date_for(response)
+        header = headers(response)['last-modified']
+        return Time.now unless header
+
+        Time.httpdate(header)
+      rescue ArgumentError
+        Time.now
+      end
+
+      def image_for(response, overrides)
+        return overrides[:image] if overrides[:image]
+        return unless html_response?(response)
+
+        if (image_url = parsed_body(response).at_css('meta[property="og:image"]')&.[]('content'))
+          Url.sanitize(image_url)
+        end
+      end
+
+      def parsed_body(response) = response.parsed_body
+      def headers(response) = response.headers
+      def html_response?(response) = response.html_response?
+    end
+
+    ##
+    # @param title [String]
+    # @param url [Html2rss::Url, String]
+    # @param description [String]
+    # @param language [String, nil]
+    # @param ttl [Integer]
+    # @param last_build_date [Time, String]
+    # @param image [Html2rss::Url, String, nil]
+    # @param author [String, nil]
+    # rubocop:disable Metrics/ParameterLists -- Data.define members are the channel contract
+    def initialize(title:, url:, description:, language:, ttl:, last_build_date:, image:, author:)
+      super(
+        title: freeze_string(title),
+        url: url.is_a?(Url) ? url : Url.from_absolute(url),
+        description: freeze_string(description),
+        language: language && freeze_string(language),
+        ttl: Integer(ttl),
+        last_build_date: normalize_last_build_date(last_build_date),
+        image: normalize_image(image),
+        author: author && freeze_string(author)
+      )
+    end
+    # rubocop:enable Metrics/ParameterLists
+
+    private
+
+    def marshal_dump = [title, url, description, language, ttl, last_build_date, image, author]
+
+    def marshal_load((title, url, description, language, ttl, last_build_date, image, author))
+      initialize(title:, url:, description:, language:, ttl:, last_build_date:, image:, author:)
+    end
+
+    def freeze_string(value) = value.to_s.dup.freeze
+
+    def normalize_image(value)
+      return value if value.is_a?(Url)
+      return if value.nil?
+
+      Url.sanitize(value.to_s)
+    end
+
+    def normalize_last_build_date(value)
+      return value if value.is_a?(Time)
+      raise ArgumentError, 'last_build_date must be a Time or HTTP-date String' unless value.is_a?(String)
+
+      Time.httpdate(value)
+    end
+  end
+  # rubocop:enable Metrics/ModuleLength
+end

data/lib/html2rss/cli.rb

@@ -4,21 +4,20 @@ require 'fileutils'
 require 'json'
 require 'thor'
 
-##
-# The Html2rss namespace / command line interface.
 module Html2rss
   ##
   # The Html2rss command line interface.
   class CLI < Thor # rubocop:disable Metrics/ClassLength
     check_unknown_options!
-    # Ordered fallback chain attempted by auto strategy.
+    # Ordered fallback chain attempted by the feed-level :auto plan.
     #
     # @return [Array<Symbol>]
     AUTO_FALLBACK_CHAIN = Html2rss::FeedPipeline::AutoFallback::CHAIN.freeze
-    # Supported CLI strategy option values.
+    # Supported CLI strategy plan option values (:auto plus concrete strategies).
     #
     # @return [Array<String>]
-    STRATEGY_OPTION_ENUM = (['auto'] + Html2rss::RequestService.strategy_names).uniq.freeze
+    STRATEGY_OPTION_ENUM = Html2rss::FeedPipeline::StrategyPlan.accepted_names.map(&:to_s).freeze
+
     # User-facing strategy help text that reflects the current fallback chain.
     #
     # @return [String]
@@ -188,7 +187,7 @@ module Html2rss
     end
 
     def request_controls
-      Html2rss::RequestControls.new(
+      Html2rss::Config::RequestControls.new(
         strategy: options[:strategy]&.to_sym,
         max_redirects: options[:max_redirects],
         max_requests: options[:max_requests],

data/lib/html2rss/config/auto_source_contract.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'dry-validation'
+
+module Html2rss
+  class Config
+    # Runtime source of truth for validating auto-source config values.
+    AutoSourceContract = Dry::Schema.Params do # rubocop:disable Metrics/BlockLength
+      optional(:scraper).hash do # rubocop:disable Metrics/BlockLength
+        optional(:wordpress_api).hash do
+          optional(:enabled).filled(:bool)
+        end
+        optional(:sitemap).hash do
+          optional(:enabled).filled(:bool)
+          optional(:min_priority).filled(:float)
+          optional(:max_age_days).filled(:integer, gt?: 0)
+        end
+        optional(:schema).hash do
+          optional(:enabled).filled(:bool)
+        end
+        optional(:microdata).hash do
+          optional(:enabled).filled(:bool)
+        end
+        optional(:microformats2).hash do
+          optional(:enabled).filled(:bool)
+        end
+        optional(:json_state).hash do
+          optional(:enabled).filled(:bool)
+        end
+        optional(:meta_oembed).hash do
+          optional(:enabled).filled(:bool)
+        end
+        optional(:semantic_html).hash do
+          optional(:enabled).filled(:bool)
+          optional(:fallback_anchorless).filled(:bool)
+        end
+        optional(:html).hash do
+          optional(:enabled).filled(:bool)
+          optional(:minimum_selector_frequency).filled(:integer, gt?: 0)
+          optional(:use_top_selectors).filled(:integer, gt?: 0)
+          optional(:fallback_anchorless).filled(:bool)
+        end
+      end
+
+      optional(:cleanup).hash do
+        optional(:keep_different_domain).filled(:bool)
+        optional(:min_words_title).filled(:integer, gt?: 0)
+      end
+    end
+  end
+end

data/lib/html2rss/config/request_controls.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Config
+    ##
+    # Wire provenance for request control values (explicit vs default).
+    class RequestControls
+      # Request-control keys accepted at the top level of feed config.
+      TOP_LEVEL_KEYS = %i[strategy].freeze
+      # Request-control keys accepted under the nested `request` config.
+      REQUEST_KEYS = %i[max_redirects max_requests total_timeout_seconds].freeze
+
+      ##
+      # @param config [Hash{Symbol => Object}] raw config input
+      # @return [RequestControls] request controls extracted from the config hash
+      def self.from_config(config)
+        HashUtil.assert_symbol_keys!(config, context: 'config', deep: false)
+        HashUtil.assert_symbol_keys!(config[:request], context: 'config[:request]') if config[:request].is_a?(Hash)
+
+        new(
+          strategy: config[:strategy],
+          max_redirects: request_value_for(config, :max_redirects),
+          max_requests: request_value_for(config, :max_requests),
+          total_timeout_seconds: request_value_for(config, :total_timeout_seconds),
+          explicit_keys: explicit_keys_for(config)
+        )
+      end
+
+      def self.explicit_keys_for(config)
+        TOP_LEVEL_KEYS.filter { config.key?(_1) } +
+          REQUEST_KEYS.filter { request_key?(config, _1) }
+      end
+
+      def self.request_value_for(config, key)
+        request_config = config[:request]
+        return nil unless request_config.is_a?(Hash)
+
+        request_config[key]
+      end
+
+      def self.request_key?(config, key)
+        request_config = config[:request]
+        request_config.is_a?(Hash) && request_config.key?(key)
+      end
+
+      private_class_method :explicit_keys_for, :request_value_for, :request_key?
+
+      ##
+      # @param strategy [Symbol, nil] effective request strategy
+      # @param max_redirects [Integer, nil] effective redirect limit
+      # @param max_requests [Integer, nil] effective request budget
+      # @param total_timeout_seconds [Integer, nil] effective request timeout
+      # @param explicit_keys [Array<Symbol>] controls explicitly supplied by the caller
+      def initialize(strategy: nil, max_redirects: nil, max_requests: nil,
+                     total_timeout_seconds: nil, explicit_keys: [])
+        @strategy = strategy
+        @max_redirects = max_redirects
+        @max_requests = max_requests
+        @total_timeout_seconds = total_timeout_seconds
+        @explicit_keys = explicit_keys.map(&:to_sym).uniq.freeze
+        freeze
+      end
+
+      ##
+      # @return [Symbol, nil] effective request strategy
+      attr_reader :strategy
+
+      ##
+      # @return [Integer, nil] effective redirect limit
+      attr_reader :max_redirects
+
+      ##
+      # @return [Integer, nil] effective request budget
+      attr_reader :max_requests
+
+      ##
+      # @return [Integer, nil] effective request timeout
+      attr_reader :total_timeout_seconds
+
+      ##
+      # @param name [Symbol, String] request control name
+      # @return [Boolean] whether the control was explicitly supplied
+      def explicit?(name)
+        explicit_keys.include?(name.to_sym)
+      end
+
+      ##
+      # @param strategy [Symbol, nil] validated request strategy
+      # @param max_redirects [Integer, nil] validated redirect limit
+      # @param max_requests [Integer, nil] validated request budget
+      # @param total_timeout_seconds [Integer, nil] validated request timeout
+      # @return [RequestControls] controls updated with validated effective values
+      def with_effective_values(strategy:, max_redirects:, max_requests:, total_timeout_seconds:)
+        self.class.new(
+          strategy:,
+          max_redirects:,
+          max_requests:,
+          total_timeout_seconds:,
+          explicit_keys:
+        )
+      end
+
+      ##
+      # Applies only explicitly set controls to the provided config hash.
+      #
+      # @param config [Hash{Symbol => Object}] mutable config hash
+      # @return [Hash{Symbol => Object}] the same hash with explicit controls written
+      def apply_to(config)
+        config[:strategy] = strategy if explicit?(:strategy)
+        apply_request_value(config, :max_redirects, max_redirects)
+        apply_request_value(config, :max_requests, max_requests)
+        apply_request_value(config, :total_timeout_seconds, total_timeout_seconds)
+        config
+      end
+
+      private
+
+      attr_reader :explicit_keys
+
+      def apply_request_value(config, key, value)
+        return unless explicit?(key)
+
+        ensure_request_config!(config)
+        config[:request][key] = value
+      end
+
+      def ensure_request_config!(config)
+        request_config = config[:request]
+        return config[:request] = {} if request_config.nil?
+        return if request_config.is_a?(Hash)
+
+        raise ArgumentError, 'request config must be a hash'
+      end
+    end
+  end
+end

data/lib/html2rss/config/request_headers.rb

@@ -41,7 +41,7 @@ module Html2rss
         # @return [Hash{String => String}] the default header set merged with global defaults
         def browser_defaults
           defaults = DEFAULT_HEADERS.dup
-          global_headers = Html2rss.configuration.headers
+          global_headers = Html2rss.defaults.headers
           global_headers = global_headers.call if global_headers.respond_to?(:call)
 
           if global_headers.is_a?(Hash)