html2rss 0.22.2 → 0.23.0

data/lib/html2rss/auto_source/scraper.rb

@@ -22,9 +22,12 @@ module Html2rss
       # Ordered scraper classes considered during auto-source extraction.
       SCRAPERS = [
         WordpressApi,
+        Sitemap,
         Schema,
         Microdata,
+        Microformats2,
         JsonState,
+        MetaOembed,
         SemanticHtml,
         Html
       ].freeze
@@ -67,12 +70,14 @@ module Html2rss
 
       ##
       # Returns an array of scraper classes that claim to find articles in the parsed body.
-      # @param parsed_body [Nokogiri::HTML::Document] The parsed HTML body.
+      # @param parsed_body [Nokogiri::HTML::Document] The parsed HTML document.
       # @param opts [Hash] The options hash.
       # @option opts [Hash] :wordpress_api scraper toggle and configuration
       # @option opts [Hash] :schema scraper toggle and configuration
       # @option opts [Hash] :microdata scraper toggle and configuration
+      # @option opts [Hash] :microformats2 scraper toggle and configuration
       # @option opts [Hash] :json_state scraper toggle and configuration
+      # @option opts [Hash] :meta_oembed scraper toggle and configuration
       # @option opts [Hash] :semantic_html scraper toggle and configuration
       # @option opts [Hash] :html scraper toggle and configuration
       # @return [Array<Class>] An array of scraper classes that can handle the parsed body.
@@ -86,14 +91,16 @@ module Html2rss
       end
 
       # Returns scraper instances ready for extraction.
-      # @param parsed_body [Nokogiri::HTML::Document] The parsed HTML body.
+      # @param parsed_body [Nokogiri::HTML::Document] The parsed HTML document.
       # @param url [String, Html2rss::Url] The page url.
       # @param request_session [Html2rss::RequestSession, nil] Shared follow-up session.
       # @param opts [Hash] The options hash.
       # @option opts [Hash] :wordpress_api scraper toggle and configuration
       # @option opts [Hash] :schema scraper toggle and configuration
       # @option opts [Hash] :microdata scraper toggle and configuration
+      # @option opts [Hash] :microformats2 scraper toggle and configuration
       # @option opts [Hash] :json_state scraper toggle and configuration
+      # @option opts [Hash] :meta_oembed scraper toggle and configuration
       # @option opts [Hash] :semantic_html scraper toggle and configuration
       # @option opts [Hash] :html scraper toggle and configuration
       # @return [Array<Object>] An array of scraper instances that can handle the parsed body.
@@ -139,7 +146,7 @@ module Html2rss
       private_class_method :classify_no_scraper_surface
 
       def self.blocked_surface?(parsed_body)
-        Html2rss::BlockedSurface.interstitial?(parsed_body.to_html)
+        Html2rss::RequestService::BlockedSurface.interstitial?(parsed_body.to_html)
       end
       private_class_method :blocked_surface?
 

data/lib/html2rss/auto_source.rb

@@ -22,15 +22,26 @@ module Html2rss
         wordpress_api: {
           enabled: true
         },
+        sitemap: {
+          enabled: true,
+          min_priority: Discovery::Sitemap::DEFAULT_MIN_PRIORITY,
+          max_age_days: Discovery::Sitemap::DEFAULT_MAX_AGE_DAYS
+        },
         schema: {
           enabled: true
         },
         microdata: {
           enabled: true
         },
+        microformats2: {
+          enabled: true
+        },
         json_state: {
           enabled: true
         },
+        meta_oembed: {
+          enabled: true
+        },
         semantic_html: {
           enabled: true,
           fallback_anchorless: true
@@ -45,39 +56,23 @@ module Html2rss
       cleanup: Cleanup::DEFAULT_CONFIG
     }.freeze
 
-    SCRAPER_CONFIG = proc do
-      optional(:wordpress_api).hash do
-        optional(:enabled).filled(:bool)
-      end
-      optional(:schema).hash do
-        optional(:enabled).filled(:bool)
-      end
-      optional(:microdata).hash do
-        optional(:enabled).filled(:bool)
-      end
-      optional(:json_state).hash do
-        optional(:enabled).filled(:bool)
-      end
-      optional(:semantic_html).hash do
-        optional(:enabled).filled(:bool)
-        optional(:fallback_anchorless).filled(:bool)
-      end
-      optional(:html).hash do
-        optional(:enabled).filled(:bool)
-        optional(:minimum_selector_frequency).filled(:integer, gt?: 0)
-        optional(:use_top_selectors).filled(:integer, gt?: 0)
-        optional(:fallback_anchorless).filled(:bool)
-      end
-    end.freeze
-    private_constant :SCRAPER_CONFIG
-
-    # Runtime schema used to validate auto-source config values.
-    Config = Dry::Schema.Params do
-      optional(:scraper).hash(&SCRAPER_CONFIG)
+    class << self
+      ##
+      # Returns the sum of required request slots for all enabled scrapers in the config.
+      #
+      # @param config [Hash, nil] auto_source configuration hash
+      # @return [Integer] total request slots required by scrapers
+      def request_slots_for(config)
+        return 0 unless config
 
-      optional(:cleanup).hash do
-        optional(:keep_different_domain).filled(:bool)
-        optional(:min_words_title).filled(:integer, gt?: 0)
+        Scraper::SCRAPERS.sum do |scraper|
+          if config.dig(:scraper, scraper.options_key, :enabled)
+            opts = config.dig(:scraper, scraper.options_key)
+            scraper.respond_to?(:request_slots) ? scraper.request_slots(opts) : 0
+          else
+            0
+          end
+        end
       end
     end
 
@@ -98,7 +93,7 @@ module Html2rss
     ##
     # Extracts article candidates by selecting every scraper that can explain the
     # page shape, running those scrapers, and normalizing the resulting hashes
-    # into `RssBuilder::Article` objects.
+    # into `Article` objects.
     #
     # The contributor-facing flow is:
     # 1. choose scraper instances that match the page
@@ -108,7 +103,7 @@ module Html2rss
     # Scrapers with expensive precomputation, such as `SemanticHtml`, keep that
     # state on the instance so detection and extraction can reuse the same work.
     #
-    # @return [Array<Html2rss::RssBuilder::Article>] extracted articles
+    # @return [Array<Html2rss::Article>] extracted articles
     def articles
       @articles ||= extract_articles
     rescue Html2rss::AutoSource::Scraper::NoScraperFound => error
@@ -133,7 +128,7 @@ module Html2rss
 
     def run_scraper(instance)
       instance.each.map do |article_hash|
-        RssBuilder::Article.new(**article_hash, scraper: instance.class)
+        Article.new(**article_hash, scraper: instance.class)
       end
     end
 

data/lib/html2rss/channel.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Channel metadata for feed builders, extracted from the HTML document head
+  # and HTTP response (with optional config overrides).
+  #
+  # Format adapters ({FeedBuilder::Rss}, {FeedBuilder::JsonFeed}) consume this type; they do
+  # not own channel extraction.
+  class Channel
+    # Fallback RSS ttl (in minutes) when no cache directives are present.
+    DEFAULT_TTL_IN_MINUTES = 360
+    # Description template used when no explicit or discovered description exists.
+    DEFAULT_DESCRIPTION_TEMPLATE = 'Latest items from %<url>s'
+
+    ##
+    # @param response [Html2rss::RequestService::Response]
+    # @param overrides [Hash{Symbol => String}] optional overrides for channel attributes
+    def initialize(response, overrides: {})
+      @response = response
+      @overrides = overrides
+    end
+
+    # @return [String] channel title derived from overrides, document title, or URL
+    def title
+      @title ||= fetch_title
+    end
+
+    # @return [Html2rss::Url] canonical channel URL
+    def url = @url ||= Html2rss::Url.from_absolute(@response.url)
+
+    # @return [String] channel description text
+    def description
+      return overrides[:description] unless overrides[:description].to_s.empty?
+
+      description = parsed_body.at_css('meta[name="description"]')&.[]('content') if html_response?
+
+      return format(DEFAULT_DESCRIPTION_TEMPLATE, url:) if description.to_s.empty?
+
+      description
+    end
+
+    # @return [Integer] cache time-to-live in minutes
+    def ttl
+      calculated = if overrides[:ttl]
+                     overrides[:ttl].to_i
+                   elsif (max_age = headers['cache-control']&.match(/max-age=(\d+)/)&.[](1))
+                     max_age.to_i.fdiv(60).ceil
+                   else
+                     DEFAULT_TTL_IN_MINUTES
+                   end
+
+      min_ttl = Html2rss.defaults.min_ttl
+      min_ttl ? [calculated, min_ttl].max : calculated
+    end
+
+    # @return [String, nil] ISO-like language code when available
+    def language
+      return overrides[:language] if overrides[:language]
+
+      if (language_code = headers['content-language']&.match(/^([a-z]{2})/))
+        return language_code[0]
+      end
+
+      return unless html_response?
+
+      parsed_body['lang'] || parsed_body.at_css('[lang]')&.[]('lang')
+    end
+
+    # @return [String, nil] channel author metadata
+    def author
+      return overrides[:author] if overrides[:author]
+
+      return unless html_response?
+
+      parsed_body.at_css('meta[name="author"]')&.[]('content')
+    end
+
+    # @return [String, Time] source last-modified timestamp or current time fallback
+    def last_build_date = headers['last-modified'] || Time.now
+
+    # @return [Html2rss::Url, nil] channel image URL
+    def image
+      return overrides[:image] if overrides[:image]
+
+      return unless html_response?
+
+      if (image_url = parsed_body.at_css('meta[property="og:image"]')&.[]('content'))
+        Url.sanitize(image_url)
+      end
+    end
+
+    private
+
+    attr_reader :overrides
+
+    def parsed_body = @parsed_body ||= @response.parsed_body
+    def headers = @headers ||= @response.headers
+    def html_response? = @html_response ||= @response.html_response?
+
+    def fetch_title
+      override_title = overrides[:title]
+      return override_title if override_title
+      return parsed_title if parsed_title
+
+      url.channel_titleized
+    end
+
+    def parsed_title
+      return unless html_response?
+
+      title = parsed_body.at_css('head > title')&.text.to_s
+      return if title.empty?
+
+      title.gsub(/\s+/, ' ').strip
+    end
+  end
+end

data/lib/html2rss/cli.rb

@@ -4,21 +4,20 @@ require 'fileutils'
 require 'json'
 require 'thor'
 
-##
-# The Html2rss namespace / command line interface.
 module Html2rss
   ##
   # The Html2rss command line interface.
   class CLI < Thor # rubocop:disable Metrics/ClassLength
     check_unknown_options!
-    # Ordered fallback chain attempted by auto strategy.
+    # Ordered fallback chain attempted by the feed-level :auto plan.
     #
     # @return [Array<Symbol>]
     AUTO_FALLBACK_CHAIN = Html2rss::FeedPipeline::AutoFallback::CHAIN.freeze
-    # Supported CLI strategy option values.
+    # Supported CLI strategy plan option values (:auto plus concrete strategies).
     #
     # @return [Array<String>]
-    STRATEGY_OPTION_ENUM = (['auto'] + Html2rss::RequestService.strategy_names).uniq.freeze
+    STRATEGY_OPTION_ENUM = Html2rss::FeedPipeline::StrategyPlan.accepted_names.map(&:to_s).freeze
+
     # User-facing strategy help text that reflects the current fallback chain.
     #
     # @return [String]
@@ -188,7 +187,7 @@ module Html2rss
     end
 
     def request_controls
-      Html2rss::RequestControls.new(
+      Html2rss::Config::RequestControls.new(
         strategy: options[:strategy]&.to_sym,
         max_redirects: options[:max_redirects],
         max_requests: options[:max_requests],

data/lib/html2rss/config/auto_source_contract.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require 'dry-validation'
+
+module Html2rss
+  class Config
+    # Runtime source of truth for validating auto-source config values.
+    AutoSourceContract = Dry::Schema.Params do # rubocop:disable Metrics/BlockLength
+      optional(:scraper).hash do # rubocop:disable Metrics/BlockLength
+        optional(:wordpress_api).hash do
+          optional(:enabled).filled(:bool)
+        end
+        optional(:sitemap).hash do
+          optional(:enabled).filled(:bool)
+          optional(:min_priority).filled(:float)
+          optional(:max_age_days).filled(:integer, gt?: 0)
+        end
+        optional(:schema).hash do
+          optional(:enabled).filled(:bool)
+        end
+        optional(:microdata).hash do
+          optional(:enabled).filled(:bool)
+        end
+        optional(:microformats2).hash do
+          optional(:enabled).filled(:bool)
+        end
+        optional(:json_state).hash do
+          optional(:enabled).filled(:bool)
+        end
+        optional(:meta_oembed).hash do
+          optional(:enabled).filled(:bool)
+        end
+        optional(:semantic_html).hash do
+          optional(:enabled).filled(:bool)
+          optional(:fallback_anchorless).filled(:bool)
+        end
+        optional(:html).hash do
+          optional(:enabled).filled(:bool)
+          optional(:minimum_selector_frequency).filled(:integer, gt?: 0)
+          optional(:use_top_selectors).filled(:integer, gt?: 0)
+          optional(:fallback_anchorless).filled(:bool)
+        end
+      end
+
+      optional(:cleanup).hash do
+        optional(:keep_different_domain).filled(:bool)
+        optional(:min_words_title).filled(:integer, gt?: 0)
+      end
+    end
+  end
+end

data/lib/html2rss/config/request_controls.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Config
+    ##
+    # Wire provenance for request control values (explicit vs default).
+    class RequestControls
+      # Request-control keys accepted at the top level of feed config.
+      TOP_LEVEL_KEYS = %i[strategy].freeze
+      # Request-control keys accepted under the nested `request` config.
+      REQUEST_KEYS = %i[max_redirects max_requests total_timeout_seconds].freeze
+
+      ##
+      # @param config [Hash{Symbol => Object}] raw config input
+      # @return [RequestControls] request controls extracted from the config hash
+      def self.from_config(config)
+        HashUtil.assert_symbol_keys!(config, context: 'config', deep: false)
+        HashUtil.assert_symbol_keys!(config[:request], context: 'config[:request]') if config[:request].is_a?(Hash)
+
+        new(
+          strategy: config[:strategy],
+          max_redirects: request_value_for(config, :max_redirects),
+          max_requests: request_value_for(config, :max_requests),
+          total_timeout_seconds: request_value_for(config, :total_timeout_seconds),
+          explicit_keys: explicit_keys_for(config)
+        )
+      end
+
+      def self.explicit_keys_for(config)
+        TOP_LEVEL_KEYS.filter { config.key?(_1) } +
+          REQUEST_KEYS.filter { request_key?(config, _1) }
+      end
+
+      def self.request_value_for(config, key)
+        request_config = config[:request]
+        return nil unless request_config.is_a?(Hash)
+
+        request_config[key]
+      end
+
+      def self.request_key?(config, key)
+        request_config = config[:request]
+        request_config.is_a?(Hash) && request_config.key?(key)
+      end
+
+      private_class_method :explicit_keys_for, :request_value_for, :request_key?
+
+      ##
+      # @param strategy [Symbol, nil] effective request strategy
+      # @param max_redirects [Integer, nil] effective redirect limit
+      # @param max_requests [Integer, nil] effective request budget
+      # @param total_timeout_seconds [Integer, nil] effective request timeout
+      # @param explicit_keys [Array<Symbol>] controls explicitly supplied by the caller
+      def initialize(strategy: nil, max_redirects: nil, max_requests: nil,
+                     total_timeout_seconds: nil, explicit_keys: [])
+        @strategy = strategy
+        @max_redirects = max_redirects
+        @max_requests = max_requests
+        @total_timeout_seconds = total_timeout_seconds
+        @explicit_keys = explicit_keys.map(&:to_sym).uniq.freeze
+        freeze
+      end
+
+      ##
+      # @return [Symbol, nil] effective request strategy
+      attr_reader :strategy
+
+      ##
+      # @return [Integer, nil] effective redirect limit
+      attr_reader :max_redirects
+
+      ##
+      # @return [Integer, nil] effective request budget
+      attr_reader :max_requests
+
+      ##
+      # @return [Integer, nil] effective request timeout
+      attr_reader :total_timeout_seconds
+
+      ##
+      # @param name [Symbol, String] request control name
+      # @return [Boolean] whether the control was explicitly supplied
+      def explicit?(name)
+        explicit_keys.include?(name.to_sym)
+      end
+
+      ##
+      # @param strategy [Symbol, nil] validated request strategy
+      # @param max_redirects [Integer, nil] validated redirect limit
+      # @param max_requests [Integer, nil] validated request budget
+      # @param total_timeout_seconds [Integer, nil] validated request timeout
+      # @return [RequestControls] controls updated with validated effective values
+      def with_effective_values(strategy:, max_redirects:, max_requests:, total_timeout_seconds:)
+        self.class.new(
+          strategy:,
+          max_redirects:,
+          max_requests:,
+          total_timeout_seconds:,
+          explicit_keys:
+        )
+      end
+
+      ##
+      # Applies only explicitly set controls to the provided config hash.
+      #
+      # @param config [Hash{Symbol => Object}] mutable config hash
+      # @return [Hash{Symbol => Object}] the same hash with explicit controls written
+      def apply_to(config)
+        config[:strategy] = strategy if explicit?(:strategy)
+        apply_request_value(config, :max_redirects, max_redirects)
+        apply_request_value(config, :max_requests, max_requests)
+        apply_request_value(config, :total_timeout_seconds, total_timeout_seconds)
+        config
+      end
+
+      private
+
+      attr_reader :explicit_keys
+
+      def apply_request_value(config, key, value)
+        return unless explicit?(key)
+
+        ensure_request_config!(config)
+        config[:request][key] = value
+      end
+
+      def ensure_request_config!(config)
+        request_config = config[:request]
+        return config[:request] = {} if request_config.nil?
+        return if request_config.is_a?(Hash)
+
+        raise ArgumentError, 'request config must be a hash'
+      end
+    end
+  end
+end

data/lib/html2rss/config/request_headers.rb

@@ -41,7 +41,7 @@ module Html2rss
         # @return [Hash{String => String}] the default header set merged with global defaults
         def browser_defaults
           defaults = DEFAULT_HEADERS.dup
-          global_headers = Html2rss.configuration.headers
+          global_headers = Html2rss.defaults.headers
           global_headers = global_headers.call if global_headers.respond_to?(:call)
 
           if global_headers.is_a?(Hash)