html2rss 0.16.0 → 0.18.0

data/lib/html2rss/config/request_headers.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Config
+    ##
+    # Normalizes HTTP headers for outgoing requests.
+    # Ensures a browser-like baseline while respecting caller overrides.
+    class RequestHeaders
+      DEFAULT_ACCEPT = %w[
+        text/html
+        application/xhtml+xml
+        application/xml;q=0.9
+        image/avif
+        image/webp
+        image/apng
+        */*;q=0.8
+      ].join(',')
+
+      DEFAULT_USER_AGENT = [
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64)',
+        'AppleWebKit/537.36 (KHTML, like Gecko)',
+        'Chrome/123.0.0.0',
+        'Safari/537.36'
+      ].join(' ')
+
+      DEFAULT_HEADERS = {
+        'Accept' => DEFAULT_ACCEPT,
+        'Cache-Control' => 'max-age=0',
+        'Connection' => 'keep-alive',
+        'Sec-Fetch-Dest' => 'document',
+        'Sec-Fetch-Mode' => 'navigate',
+        'Sec-Fetch-Site' => 'none',
+        'Sec-Fetch-User' => '?1',
+        'Upgrade-Insecure-Requests' => '1',
+        'User-Agent' => DEFAULT_USER_AGENT
+      }.freeze
+
+      class << self
+        ##
+        # @return [Hash<String, String>] the unmodified default header set
+        def browser_defaults
+          DEFAULT_HEADERS.dup
+        end
+
+        ##
+        # Normalizes the provided headers while applying Html2rss defaults.
+        #
+        # @param headers [Hash, nil] caller provided headers
+        # @param channel_language [String, nil] language defined on the channel
+        # @param url [String] request URL used to infer the Host header
+        # @return [Hash<String, String>] normalized HTTP headers
+        def normalize(headers, channel_language:, url:)
+          new(headers || {}, channel_language:, url:).to_h
+        end
+      end
+
+      def initialize(headers, channel_language:, url:)
+        @headers = headers
+        @channel_language = channel_language
+        @url = url
+      end
+
+      ##
+      # @return [Hash<String, String>] normalized HTTP headers
+      def to_h
+        defaults = DEFAULT_HEADERS.dup
+        normalized = normalize_custom_headers(headers)
+
+        accept_override = normalized.delete('Accept')
+        defaults.merge!(normalized)
+
+        defaults['Accept'] = normalize_accept(accept_override)
+        defaults['Accept-Language'] = build_accept_language
+        defaults['Host'] ||= request_host
+
+        defaults.compact
+      end
+
+      private
+
+      attr_reader :headers, :channel_language, :url
+
+      def normalize_custom_headers(custom)
+        custom.transform_keys { canonicalize(_1) }
+      end
+
+      def canonicalize(key)
+        key.to_s.split('-').map!(&:capitalize).join('-')
+      end
+
+      def normalize_accept(override)
+        return DEFAULT_ACCEPT if override.nil? || override.empty?
+
+        values = accept_values(DEFAULT_ACCEPT)
+
+        accept_values(override).reverse_each do |value|
+          next if values.include?(value)
+
+          values.unshift(value)
+        end
+
+        values.join(',')
+      end
+
+      def accept_values(header)
+        header.split(',').map!(&:strip).reject(&:empty?)
+      end
+
+      def build_accept_language
+        language = channel_language.to_s.strip
+        return 'en-US,en;q=0.9' if language.empty?
+
+        normalized = language.tr('_', '-')
+        primary, region = normalized.split('-', 2)
+        primary = primary.downcase
+        region = region&.upcase
+
+        return primary if region.nil?
+
+        "#{primary}-#{region},#{primary};q=0.9"
+      end
+
+      def request_host
+        return nil if url.nil? || url.empty?
+
+        Html2rss::Url.from_absolute(url).host
+      end
+    end
+  end
+end

data/lib/html2rss/config/schema.rb

@@ -0,0 +1,208 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Config
+    ##
+    # Builds the exported configuration JSON Schema from the runtime validators.
+    module Schema
+      module_function
+
+      SCHEMA_FILENAME = 'html2rss-config.schema.json'
+
+      ##
+      # Returns the exported configuration JSON Schema.
+      #
+      # @return [Hash<String, Object>] JSON Schema represented as a Ruby hash
+      def json_schema
+        load_json_schema_extension!
+        Builder.call
+      end
+
+      ##
+      # Resolves the packaged schema path used by downstream tools.
+      #
+      # @return [String] absolute path to the packaged JSON schema file
+      def path
+        search_path = File.expand_path(__dir__)
+
+        loop do
+          candidate = File.join(search_path, 'schema', SCHEMA_FILENAME)
+          return candidate if File.exist?(candidate)
+
+          parent_path = File.dirname(search_path)
+          break if parent_path == search_path
+
+          search_path = parent_path
+        end
+
+        File.expand_path("../../../schema/#{SCHEMA_FILENAME}", __dir__)
+      end
+
+      def load_json_schema_extension!
+        require 'dry/schema/extensions/json_schema'
+        Dry::Schema.load_extensions(:json_schema)
+      end
+
+      ##
+      # Orchestrates schema assembly from runtime validator contracts plus
+      # client-facing overlays.
+      class Builder
+        class << self
+          def call
+            new.call
+          end
+        end
+
+        def call
+          schema = validator_schema
+          apply_top_level(schema)
+          assign_properties(schema.fetch(:properties))
+          DeepStringifier.call(schema)
+        end
+
+        private
+
+        def validator_schema
+          Html2rss::Config::Validator.new.schema.json_schema(loose: true)
+        end
+
+        def apply_top_level(schema)
+          schema['$schema'] = 'https://json-schema.org/draft/2020-12/schema'
+          schema[:anyOf] = [
+            { 'required' => ['selectors'] },
+            { 'required' => ['auto_source'] }
+          ]
+        end
+
+        def assign_properties(properties)
+          properties.merge!(
+            headers: Components.headers,
+            stylesheets: Components.stylesheets,
+            auto_source: Components.auto_source,
+            selectors: Components.selectors
+          )
+          properties.delete(:dynamic_params_error)
+        end
+      end
+
+      ##
+      # Exposes schema fragments that populate the top-level configuration schema.
+      module Components
+        module_function
+
+        def headers
+          {
+            type: 'object',
+            description: 'HTTP headers applied to every request.',
+            additionalProperties: { type: 'string' }
+          }
+        end
+
+        def stylesheets
+          {
+            type: 'array',
+            description: 'Collection of stylesheets to attach to the RSS feed.',
+            items: Html2rss::Config::Validator::StylesheetConfig.json_schema(loose: true)
+          }
+        end
+
+        def auto_source
+          schema = Html2rss::AutoSource::Config.json_schema(loose: true)
+          schema[:default] = DeepStringifier.call(Html2rss::AutoSource::DEFAULT_CONFIG)
+          schema
+        end
+
+        def selectors
+          Selectors.schema
+        end
+      end
+
+      ##
+      # Provides schema fragments that document selector configuration.
+      module Selectors
+        module_function
+
+        RESERVED_SELECTOR_PATTERN = '^(?!items$|enclosure$|guid$|categories$).+$'
+
+        def schema
+          {
+            type: 'object',
+            description: 'Selectors used to extract article attributes.',
+            properties: selector_properties,
+            patternProperties: pattern_properties,
+            additionalProperties: true
+          }
+        end
+
+        # rubocop:disable Layout/LineLength
+        def selector_properties
+          {
+            items: items_schema,
+            enclosure: enclosure_schema,
+            guid: reference_array('List of selector keys used to build the GUID. Each entry must reference a sibling selector key; runtime validation enforces those references.'),
+            categories: reference_array('List of selector keys whose values will be used as categories. Each entry must reference a sibling selector key; runtime validation enforces those references.')
+          }
+        end
+        # rubocop:enable Layout/LineLength
+
+        def pattern_properties
+          { RESERVED_SELECTOR_PATTERN => dynamic_selector_schema }
+        end
+
+        def dynamic_selector_schema
+          Html2rss::Selectors::Config::Selector.new.schema.json_schema(loose: true).merge(
+            description: 'Dynamic selector definition keyed by attribute name.'
+          )
+        end
+
+        def items_schema
+          Html2rss::Selectors::Config::Items.new.schema.json_schema(loose: true).merge(
+            description: 'Defines the items selector and optional enhancement settings.'
+          )
+        end
+
+        def enclosure_schema
+          Html2rss::Selectors::Config::Enclosure.new.schema.json_schema(loose: true).merge(
+            description: 'Describes enclosure extraction settings.'
+          )
+        end
+
+        # JSON Schema can enforce non-empty reference arrays, while runtime
+        # validation remains authoritative for checking that each entry points
+        # to an existing sibling selector key.
+        def reference_array(description)
+          {
+            type: 'array',
+            description:,
+            minItems: 1,
+            items: {
+              type: 'string',
+              description: 'Selector key defined elsewhere in this object.'
+            }
+          }
+        end
+      end
+
+      ##
+      # Converts nested hash keys to strings so the resulting schema serializes cleanly.
+      module DeepStringifier
+        module_function
+
+        def call(object)
+          case object
+          when Hash
+            stringify_hash(object)
+          when Array
+            object.map { |value| call(value) }
+          else
+            object
+          end
+        end
+
+        def stringify_hash(object)
+          object.to_h { |key, value| [key.to_s, call(value)] }
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/config/validator.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require 'dry-validation'
+
+module Html2rss
+  class Config
+    # Validates the configuration hash using Dry::Validation.
+    # The configuration options adhere to the documented schema in README.md.
+    class Validator < Dry::Validation::Contract
+      URI_REGEXP = Url::URI_REGEXP
+      STYLESHEET_TYPES = RssBuilder::Stylesheet::TYPES
+      LANGUAGE_FORMAT_REGEX = /\A[a-z]{2}(-[A-Z]{2})?\z/
+
+      ChannelConfig = Dry::Schema.Params do
+        required(:url).filled(:string, format?: URI_REGEXP)
+        optional(:title).maybe(:string)
+        optional(:description).maybe(:string)
+        optional(:language).maybe(:string, format?: LANGUAGE_FORMAT_REGEX)
+        optional(:ttl).maybe(:integer, gt?: 0)
+        optional(:time_zone).maybe(:string)
+      end
+
+      StylesheetConfig = Dry::Schema.Params do
+        required(:href).filled(:string)
+        required(:type).filled(:string, included_in?: STYLESHEET_TYPES)
+        optional(:media).maybe(:string)
+      end
+
+      BrowserlessPreloadClickSelectorConfig = Dry::Schema.Params do
+        required(:selector).filled(:string)
+        optional(:max_clicks).filled(:integer, gt?: 0)
+        optional(:wait_after_ms).filled(:integer, gteq?: 0)
+      end
+
+      BrowserlessPreloadScrollConfig = Dry::Schema.Params do
+        optional(:iterations).filled(:integer, gt?: 0)
+        optional(:wait_after_ms).filled(:integer, gteq?: 0)
+      end
+
+      BrowserlessPreloadConfig = Dry::Schema.Params do
+        optional(:wait_after_ms).filled(:integer, gteq?: 0)
+        optional(:click_selectors).array(BrowserlessPreloadClickSelectorConfig)
+        optional(:scroll_down).hash(BrowserlessPreloadScrollConfig)
+      end
+
+      BrowserlessRequestConfig = Dry::Schema.Params do
+        optional(:preload).hash(BrowserlessPreloadConfig)
+      end
+
+      RequestConfig = Dry::Schema.Params do
+        optional(:max_redirects).filled(:integer, gteq?: 0)
+        optional(:max_requests).filled(:integer, gt?: 0)
+        optional(:browserless).hash(BrowserlessRequestConfig)
+      end
+
+      params do
+        required(:strategy).filled(:symbol)
+        required(:channel).hash(ChannelConfig)
+        optional(:headers).hash
+        optional(:stylesheets).array(StylesheetConfig)
+        optional(:auto_source).hash(AutoSource::Config)
+        optional(:selectors).hash
+        optional(:dynamic_params_error).maybe(:string)
+        optional(:request).hash(RequestConfig)
+      end
+
+      rule(:headers) do
+        value&.each do |key, header_value|
+          unless header_value.is_a?(String)
+            key([:headers, key]).failure("must be a String, but got #{header_value.class}")
+          end
+        end
+      end
+
+      rule(:dynamic_params_error) do
+        base.failure(value) if value
+      end
+
+      # Ensure at least one of :selectors or :auto_source is present.
+      rule(:selectors, :auto_source) do
+        unless values.key?(:selectors) || values.key?(:auto_source)
+          base.failure("Configuration must include at least 'selectors' or 'auto_source'")
+        end
+      end
+
+      rule(:selectors) do
+        next unless value
+
+        errors = Html2rss::Selectors::Config.call(value).errors
+        errors.each { |error| key(:selectors).failure(error.text) } unless errors.empty?
+      end
+
+      # URL validation delegated to Url class
+      rule(:channel) do
+        next unless values[:channel]&.key?(:url)
+
+        url_string = values[:channel][:url]
+        next if url_string.nil? || url_string.empty?
+
+        begin
+          Html2rss::Url.for_channel(url_string)
+        rescue ArgumentError => error
+          key(%i[channel url]).failure(error.message)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/config.rb

@@ -1,82 +1,133 @@
 # frozen_string_literal: true
 
-require 'forwardable'
+require 'json'
+require 'yaml'
 
 module Html2rss
   ##
-  # The Config class abstracts from the config data structure and
-  # provides default values.
+  # The provided configuration is used to generate the RSS feed.
+  # This class provides methods to load and process configuration from a YAML file,
+  # supporting both single and multiple feed configurations.
+  #
+  # Configuration is validated during initialization.
   class Config
-    extend Forwardable
+    class InvalidConfig < Html2rss::Error; end
+    extend ClassMethods
 
     ##
-    # The Error class to be thrown when a feed config requires params, but none
-    # were passed to Config.
-    class ParamsMissing < Html2rss::Error; end
-
-    ##
-    # Thrown when the feed config does not contain a value at `:channel`.
-    class ChannelMissing < Html2rss::Error; end
-
-    def_delegator :@channel, :author
-    def_delegator :@channel, :ttl
-    def_delegator :@channel, :title
-    def_delegator :@channel, :language
-    def_delegator :@channel, :description
-    def_delegator :@channel, :url
-    def_delegator :@channel, :url, :link
-    def_delegator :@channel, :time_zone
-    def_delegator :@channel, :json?
-    def_delegator :@channel, :strategy
-
-    def_delegator :@selectors, :item_selector_names
-    def_delegator :@selectors, :selector?
-    def_delegator :@selectors, :category_selector_names
-    def_delegator :@selectors, :guid_selector_names
-    def_delegator :@selectors, :items_order
-    def_delegator :@selectors, :selector_string
-
-    ##
-    # Initializes the Config object with feed configuration, global settings, and parameters.
+    # Initializes the configuration object.
     #
-    # @param feed_config [Hash<Symbol, Object>] The configuration hash containing `:channel` and `:selectors`.
-    # @param global [Hash<Symbol, Object>] Global settings hash.
-    # @param params [Hash<Symbol, String>] Parameters hash.
-    def initialize(feed_config, global = {}, params = {})
-      channel_config = feed_config[:channel]
-      raise ChannelMissing, 'Channel configuration is missing in feed_config' unless channel_config
-
-      @channel = Channel.new(channel_config, params:)
-      @selectors = Selectors.new(feed_config[:selectors])
-      @global = global
+    # Processes deprecated attributes, applies default values, and validates the configuration.
+    #
+    # @param config [Hash<Symbol, Object>] the configuration hash.
+    # @raise [InvalidConfig] if the configuration fails validation.
+    def initialize(config)
+      @request_controls = RequestControls.from_config(config)
+      prepared_config = Preparer.new.call(config)
+      validated_config = validated_config_for(prepared_config)
+
+      @config = validated_config.freeze
+      @request_controls = request_controls.with_effective_values(
+        strategy: validated_config[:strategy],
+        max_redirects: validated_config.dig(:request, :max_redirects),
+        max_requests: validated_config.dig(:request, :max_requests)
+      )
     end
 
+    def strategy = request_controls.strategy
+    def max_redirects = request_controls.max_redirects
+    def max_requests = request_controls.max_requests
+    def stylesheets = config[:stylesheets]
+
     ##
-    # Retrieves selector attributes merged with channel attributes.
-    #
-    # @param name [Symbol] Selector name.
-    # @return [Hash<Symbol, Object>] Merged attributes hash.
-    def selector_attributes_with_channel(name)
-      @selectors.selector(name).to_h.merge(channel: @channel)
+    # @return [Boolean] whether max_requests was explicitly configured by the caller
+    def explicit_max_requests?
+      request_controls.explicit?(:max_requests)
     end
 
     ##
-    # Retrieves headers merged from global settings and channel headers.
-    #
-    # @return [Hash] Merged headers hash.
-    def headers
-      @global.fetch(:headers, {}).merge(@channel.headers)
+    # @return [Html2rss::RequestControls] request controls with provenance
+    attr_reader :request_controls
+
+    def headers = config[:headers]
+    def channel = config[:channel]
+    def url = config.dig(:channel, :url)
+    def time_zone = config.dig(:channel, :time_zone)
+
+    def request = config[:request]
+
+    def selectors = config[:selectors]
+    def auto_source = config[:auto_source]
+
+    private
+
+    attr_reader :config
+
+    # Normalizes raw config input before validation.
+    class Preparer
+      ##
+      # @param config [Hash<Symbol, Object>] raw config input
+      # @return [Hash<Symbol, Object>] config with defaults and deprecations applied
+      def call(config)
+        config = config.dup if config.frozen?
+
+        config = handle_deprecated_channel_attributes(config)
+        config = apply_default_config(config)
+        config = apply_default_selectors_config(config) if config[:selectors]
+        config = apply_default_auto_source_config(config) if config[:auto_source]
+
+        config
+      end
+
+      private
+
+      def handle_deprecated_channel_attributes(config)
+        { strategy: RequestService.default_strategy_name, headers: {} }.each_pair do |key, default_value|
+          if !config[key] && (value = config.dig(:channel, key))
+            Log.warn("The `channel.#{key}` key is deprecated. Please move the definition of `#{key}` to the top level.")
+            config[key] = value
+          end
+
+          config[key] ||= default_value
+        end
+
+        config
+      end
+
+      def apply_default_config(config)
+        deep_merge(Config.default_config, config)
+      end
+
+      def apply_default_selectors_config(config)
+        deep_merge({ selectors: Selectors::DEFAULT_CONFIG }, config)
+      end
+
+      def apply_default_auto_source_config(config)
+        deep_merge({ auto_source: Html2rss::AutoSource::DEFAULT_CONFIG }, config)
+      end
+
+      def deep_merge(base_config, override_config)
+        base_config.merge(override_config) do |_key, oldval, newval|
+          oldval.is_a?(Hash) && newval.is_a?(Hash) ? deep_merge(oldval, newval) : newval
+        end
+      end
     end
 
-    ##
-    # Retrieves stylesheets from global settings.
-    #
-    # @return [Array<Stylesheet>] Array of Stylesheet structs.
-    def stylesheets
-      @global.fetch(:stylesheets, []).map { |attributes| Html2rss::RssBuilder::Stylesheet.new(**attributes) }
+    def validated_config_for(config)
+      validator = Validator.new.call(config)
+
+      raise InvalidConfig, "Invalid configuration: #{validator.errors.to_h}" unless validator.success?
+
+      normalized_headers(validator.to_h)
     end
 
-    # Provides read-only access to the channel object.
-    attr_reader :channel
+    def normalized_headers(validated_config)
+      validated_config[:headers] = RequestHeaders.normalize(
+        validated_config[:headers],
+        channel_language: validated_config.dig(:channel, :language),
+        url: validated_config.dig(:channel, :url)
+      )
+      validated_config
+    end
   end
 end

data/lib/html2rss/error.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Html2rss
+  # The Html2rss::Error base class.
+  class Error < StandardError; end
+end

data/lib/html2rss/html_extractor/date_extractor.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class HtmlExtractor
+    # Extracts the earliest date from an article_tag.
+    class DateExtractor
+      # @return [DateTime, nil]
+      def self.call(article_tag)
+        times = article_tag.css('[datetime]').filter_map do |tag|
+          DateTime.parse(tag['datetime'])
+        rescue ArgumentError, TypeError
+          nil
+        end
+
+        times.min
+      end
+    end
+  end
+end