html2rss 0.17.0 → 0.19.0

data/lib/html2rss/cli.rb

@@ -1,46 +1,217 @@
 # frozen_string_literal: true
 
-require_relative '../html2rss'
+require 'fileutils'
+require 'json'
 require 'thor'
 
 ##
 # The Html2rss namespace / command line interface.
 module Html2rss
-  Log = Logger.new($stderr)
-
   ##
   # The Html2rss command line interface.
-  class CLI < Thor
+  class CLI < Thor # rubocop:disable Metrics/ClassLength
+    check_unknown_options!
+    # Ordered fallback chain attempted by auto strategy.
+    #
+    # @return [Array<Symbol>]
+    AUTO_FALLBACK_CHAIN = Html2rss::FeedPipeline::AutoFallback::CHAIN.freeze
+    # Supported CLI strategy option values.
+    #
+    # @return [Array<String>]
+    STRATEGY_OPTION_ENUM = (['auto'] + Html2rss::RequestService.strategy_names).uniq.freeze
+    # User-facing strategy help text that reflects the current fallback chain.
+    #
+    # @return [String]
+    STRATEGY_OPTION_DESC = [
+      'Optional request strategy (defaults to auto; auto tries',
+      "#{AUTO_FALLBACK_CHAIN.join(' -> ')})"
+    ].join(' ').freeze
+
+    # @return [Boolean] whether Thor should terminate process on command failures
     def self.exit_on_failure?
       true
     end
 
-    desc 'feed YAML_FILE [FEED_NAME] [param=value ...]', 'Print RSS built from the YAML_FILE file to stdout'
+    desc 'feed YAML_FILE [feed_name]', 'Print RSS built from the YAML_FILE file to stdout'
+    method_option :params,
+                  type: :hash,
+                  optional: true,
+                  required: false,
+                  default: {}
+    method_option :strategy,
+                  type: :string,
+                  desc: STRATEGY_OPTION_DESC,
+                  enum: STRATEGY_OPTION_ENUM
+    method_option :max_redirects,
+                  type: :numeric,
+                  desc: 'Maximum redirects to follow per request'
+    method_option :max_requests,
+                  type: :numeric,
+                  desc: 'Maximum requests to allow for this feed build'
+    # @param yaml_file [String] path to YAML config
+    # @param feed_name [String, nil] optional named feed in multi-feed config
+    # @return [void]
+    def feed(yaml_file, feed_name = nil)
+      config = Html2rss.config_from_yaml_file(yaml_file, feed_name)
+      config[:params] = options[:params] || {}
+      apply_runtime_request_overrides!(config)
+
+      puts(execute_feed { Html2rss.feed(config) })
+    end
+
+    desc 'auto [URL]', 'Automatically sources an RSS feed from the URL'
+    method_option :strategy,
+                  type: :string,
+                  desc: STRATEGY_OPTION_DESC,
+                  enum: STRATEGY_OPTION_ENUM
+    method_option :format,
+                  type: :string,
+                  desc: 'Output format for the auto-sourced feed',
+                  enum: %w[rss jsonfeed],
+                  default: 'rss'
+    method_option :items_selector, type: :string, desc: 'CSS selector for items (will be enhanced) (optional)'
+    method_option :max_redirects,
+                  type: :numeric,
+                  desc: 'Maximum redirects to follow per request'
+    method_option :max_requests,
+                  type: :numeric,
+                  desc: 'Maximum requests to allow for this feed build'
+    # @param url [String] source page URL for auto discovery
+    # @return [void]
+    def auto(url) # rubocop:disable Metrics/MethodLength
+      format = options.fetch(:format, 'rss')
+      source_method = format == 'jsonfeed' ? Html2rss.method(:auto_json_feed) : Html2rss.method(:auto_source)
+
+      result = execute_feed do
+        source_method.call(
+          url,
+          strategy: current_strategy,
+          items_selector: options[:items_selector],
+          max_redirects: options[:max_redirects],
+          max_requests: options[:max_requests]
+        )
+      end
+
+      puts(format == 'jsonfeed' ? JSON.pretty_generate(result) : result)
+    end
+
+    desc 'schema', 'Print the exported config JSON Schema'
+    method_option :pretty,
+                  type: :boolean,
+                  desc: 'Pretty-print the schema JSON',
+                  default: true
+    method_option :write,
+                  type: :string,
+                  desc: 'Write the schema JSON to the given file path'
     ##
-    # Prints the feed to STDOUT.
+    # Prints or writes the exported configuration JSON Schema.
     #
-    # @param yaml_file [String] Path to the YAML configuration file.
-    # @param options [Array<String>] Additional options including feed name and parameters.
-    # @return [nil]
-    def feed(yaml_file, *options)
-      raise "File '#{yaml_file}' does not exist" unless File.exist?(yaml_file)
+    # @return [void]
+    def schema
+      schema_json = Html2rss::Config.json_schema_json(pretty: options.fetch(:pretty, true))
 
-      feed_name = options.shift unless options.first&.include?('=')
-      params = options.to_h { |opt| opt.split('=', 2) }
+      if options[:write]
+        FileUtils.mkdir_p(File.dirname(options[:write]))
+        File.write(options[:write], "#{schema_json}\n")
+        puts options[:write]
+        return
+      end
 
-      puts Html2rss.feed_from_yaml_config(yaml_file, feed_name, params:)
+      puts schema_json
     end
 
-    desc 'auto URL', 'Automatically sources an RSS feed from the URL'
-    method_option :strategy,
-                  type: :string,
-                  desc: 'The strategy to request the URL',
-                  enum: RequestService.strategy_names,
-                  default: RequestService.default_strategy_name
-    def auto(url)
-      strategy = options.fetch(:strategy) { RequestService.default_strategy_name }.to_sym
+    desc 'validate YAML_FILE [feed_name]', 'Validate a YAML config with the runtime validator'
+    method_option :params,
+                  type: :hash,
+                  optional: true,
+                  required: false,
+                  default: {}
+    ##
+    # Validates a YAML config and prints the result.
+    #
+    # @param yaml_file [String] the YAML file to validate
+    # @param feed_name [String, nil] optional feed name for multi-feed files
+    # @return [void]
+    def validate(yaml_file, feed_name = nil)
+      result = Html2rss::Config.validate_yaml(yaml_file, feed_name, params: options[:params] || {})
+
+      raise Thor::Error, "Invalid configuration: #{result.errors.to_h}" unless result.success?
+
+      puts 'Configuration is valid'
+    end
+
+    private
+
+    def apply_runtime_request_overrides!(config)
+      clear_blank_request_overrides!(config)
+      request_controls.apply_to(config)
+    end
+
+    def clear_blank_request_overrides!(config)
+      config.delete(:strategy) if config[:strategy].nil?
+
+      request_config = config[:request]
+      return unless request_config.is_a?(Hash)
+
+      %i[max_redirects max_requests].each do |key|
+        request_config.delete(key) if request_config[key].nil?
+      end
+      config.delete(:request) if request_config.empty?
+    end
+
+    def request_controls
+      Html2rss::RequestControls.new(
+        strategy: options[:strategy]&.to_sym,
+        max_redirects: options[:max_redirects],
+        max_requests: options[:max_requests],
+        explicit_keys: explicit_request_control_keys
+      )
+    end
+
+    def explicit_request_control_keys
+      keys = []
+      keys << :strategy if options[:strategy]
+      keys << :max_redirects unless options[:max_redirects].nil?
+      keys << :max_requests unless options[:max_requests].nil?
+      keys
+    end
+
+    def current_strategy
+      options[:strategy]&.to_sym || :auto
+    end
+
+    def current_max_redirects
+      options.fetch(:max_redirects, Html2rss::RequestService::Policy::DEFAULTS[:max_redirects])
+    end
+
+    def current_max_requests
+      options.fetch(:max_requests, Html2rss::RequestService::Policy::DEFAULTS[:max_requests])
+    end
+
+    def suggested_max_redirects
+      current_max_redirects + 1
+    end
+
+    def suggested_max_requests
+      current_max_requests + 1
+    end
 
-      puts Html2rss.auto_source(url, strategy:)
+    def execute_feed # rubocop:disable Metrics/MethodLength
+      yield
+    rescue Faraday::FollowRedirects::RedirectLimitReached => error
+      raise Thor::Error,
+            "#{error.message}. retry with --max-redirects #{suggested_max_redirects} or use the final URL directly."
+    rescue Html2rss::RequestService::RequestBudgetExceeded => error
+      raise Thor::Error,
+            "#{error.message}. retry with --max-requests #{suggested_max_requests} " \
+            'or increase request.max_requests in the config.'
+    rescue Html2rss::RequestService::BrowserlessConfigurationError,
+           Html2rss::RequestService::BrowserlessConnectionFailed,
+           Html2rss::RequestService::BotasaurusConfigurationError,
+           Html2rss::RequestService::BotasaurusConnectionFailed,
+           Html2rss::RequestService::BlockedSurfaceDetected,
+           Html2rss::NoFeedItemsExtracted => error
+      raise Thor::Error, error.message
     end
   end
 end

data/lib/html2rss/config/class_methods.rb

@@ -0,0 +1,178 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Config
+    ##
+    # Public class-level helpers for loading, validating, and exporting config.
+    module ClassMethods
+      # Sentinel to differentiate omitted params from explicit `nil`.
+      UNSET = Object.new.freeze
+
+      ##
+      # Returns the exported JSON Schema for html2rss configuration.
+      #
+      # @return [Hash{String => Object}] JSON Schema represented as a Ruby hash
+      def json_schema
+        Schema.json_schema
+      end
+
+      ##
+      # Returns the exported JSON Schema as JSON.
+      #
+      # @param pretty [Boolean] whether to pretty-print the JSON output
+      # @return [String] serialized JSON Schema
+      def json_schema_json(pretty: true)
+        pretty ? JSON.pretty_generate(json_schema) : JSON.generate(json_schema)
+      end
+
+      ##
+      # Validates a configuration hash with the runtime validator.
+      #
+      # @param config [Hash{Symbol => Object}] the configuration hash
+      # @param params [Hash{Symbol => Object}, Hash{String => Object}, nil] dynamic parameters for string formatting
+      # @return [Dry::Validation::Result] validation result after defaults and deprecations are applied
+      def validate(config, params: UNSET)
+        prepared_config = prepare_for_validation(resolve_effective_config(config, params:))
+
+        Validator.new.call(prepared_config)
+      rescue DynamicParams::ParamsMissing => error
+        prepared_config = prepare_for_validation(HashUtil.deep_symbolize_keys(config, context: 'config'))
+        prepared_config[:dynamic_params_error] = error.message
+
+        Validator.new.call(prepared_config)
+      end
+
+      ##
+      # Returns the packaged JSON Schema file path.
+      #
+      # @return [String] absolute path to the packaged JSON Schema file
+      def schema_path
+        Schema.path
+      end
+
+      ##
+      # Loads and validates a YAML configuration file.
+      #
+      # @param file [String] the YAML file to load
+      # @param feed_name [String, nil] optional feed name for multi-feed files
+      # @param multiple_feeds_key [Symbol] key under which multiple feeds are defined
+      # @param params [Hash{Symbol => Object}, Hash{String => Object}, nil] dynamic parameters for string formatting
+      # @return [Dry::Validation::Result] validation result after defaults and deprecations are applied
+      def validate_yaml(file, feed_name = nil, multiple_feeds_key: MultipleFeedsConfig::CONFIG_KEY_FEEDS, params: UNSET)
+        validate(load_yaml(file, feed_name, multiple_feeds_key:), params:)
+      end
+
+      ##
+      # Loads the feed configuration from a YAML file.
+      #
+      # Supports multiple feeds defined under the specified key (default :feeds).
+      #
+      # @param file [String] the YAML file to load.
+      # @param feed_name [String, nil] the feed name when using multiple feeds.
+      # @param multiple_feeds_key [Symbol] the key under which multiple feeds are defined.
+      # @return [Hash{Symbol => Object}] the configuration hash.
+      # @raise [ArgumentError] if the file doesn't exist or feed is not found.
+      # rubocop:disable Metrics/MethodLength
+      def load_yaml(file, feed_name = nil, multiple_feeds_key: MultipleFeedsConfig::CONFIG_KEY_FEEDS)
+        raise ArgumentError, "File '#{file}' does not exist" unless File.exist?(file)
+        raise ArgumentError, "`#{multiple_feeds_key}` is a reserved feed name" if feed_name == multiple_feeds_key
+
+        yaml = YAML.safe_load_file(file, symbolize_names: true)
+
+        return yaml unless yaml.key?(multiple_feeds_key)
+
+        unless feed_name
+          available_feeds = yaml.fetch(multiple_feeds_key).keys.join(', ')
+          raise ArgumentError,
+                "Feed name is required under `#{multiple_feeds_key}`. Available feeds: #{available_feeds}"
+        end
+
+        config = yaml.dig(multiple_feeds_key, feed_name.to_sym)
+        raise ArgumentError, "Feed '#{feed_name}' not found under `#{multiple_feeds_key}` key." unless config
+
+        MultipleFeedsConfig.to_single_feed(config, yaml, multiple_feeds_key:)
+      end
+      # rubocop:enable Metrics/MethodLength
+
+      ##
+      # Processes the provided configuration hash, applying dynamic parameters if given,
+      # and returns a new configuration object.
+      #
+      # @param config [Hash{Symbol => Object}] the configuration hash.
+      # @param params [Hash{Symbol => Object}, Hash{String => Object}, nil] dynamic parameters for string formatting.
+      # @return [Html2rss::Config] the configuration object.
+      def from_hash(config, params: UNSET)
+        new(resolve_effective_config(config, params:))
+      end
+
+      ##
+      # Builds a top-level auto-source feed config for the public shortcut APIs.
+      #
+      # @param url [String] source page URL
+      # @param items_selector [String, nil] optional selector hint for item extraction
+      # @param request_controls [Html2rss::RequestControls, nil] explicit request controls to write
+      # @return [Hash{Symbol => Object}] feed config hash ready for {from_hash}
+      def auto_source_config(url:, items_selector: nil, request_controls: nil)
+        config = {
+          channel: default_config[:channel].merge(url:),
+          auto_source: AutoSource::DEFAULT_CONFIG
+        }
+
+        request_controls ||= Html2rss::RequestControls.new
+        request_controls.apply_to(config)
+
+        config[:selectors] = { items: { selector: items_selector, enhance: true } } if items_selector
+        config
+      end
+
+      ##
+      # Provides a default configuration.
+      #
+      # @return [Hash{Symbol => Object}] a hash with default configuration values.
+      def default_config
+        {
+          strategy: default_strategy_name,
+          request: {
+            max_redirects: RequestService::Policy::DEFAULTS[:max_redirects],
+            max_requests: RequestService::Policy::DEFAULTS[:max_requests]
+          },
+          channel: { time_zone: 'UTC' },
+          headers: RequestHeaders.browser_defaults,
+          stylesheets: []
+        }
+      end
+
+      # @return [Symbol] the default strategy for feed orchestration
+      def default_strategy_name
+        :auto
+      end
+
+      private
+
+      def resolve_effective_config(config, params:)
+        effective_config = HashUtil.deep_symbolize_keys(config, context: 'config')
+        resolved_params = parameter_defaults(effective_config)
+        unless params.equal?(UNSET) || params.nil?
+          resolved_params.merge!(HashUtil.deep_symbolize_keys(params, context: 'params'))
+        end
+
+        effective_config[:headers] = DynamicParams.call(effective_config[:headers], resolved_params)
+        effective_config[:channel] = DynamicParams.call(effective_config[:channel], resolved_params)
+
+        effective_config
+      end
+
+      def parameter_defaults(config)
+        config.fetch(:parameters, {})
+              .filter_map do |name, definition|
+                [name, definition[:default]] if definition.is_a?(Hash) && definition.key?(:default)
+              end
+              .to_h
+      end
+
+      def prepare_for_validation(config)
+        Config::Preparer.new.call(HashUtil.deep_dup(config))
+      end
+    end
+  end
+end

data/lib/html2rss/config/dynamic_params.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Config
+    # Processes and applies dynamic parameter formatting in configuration values.
+    class DynamicParams
+      # Raised when string interpolation references an unavailable parameter.
+      class ParamsMissing < Html2rss::Error; end
+
+      class << self
+        # Recursively traverses the given value and formats any strings containing
+        # placeholders with values from the provided params.
+        #
+        # @param value [String, Hash, Enumerable, Object] value that may contain parameter placeholders
+        # @param params [Hash] The parameters for substitution.
+        # @param getter [Proc, nil] Optional proc to retrieve a key's value.
+        # @param replace_missing_with [Object, nil] Value to substitute if a key is missing.
+        # @return [Object] The processed value.
+        def call(value, params = {}, getter: nil, replace_missing_with: nil)
+          case value
+          when String
+            from_string(value, params, getter:, replace_missing_with:)
+          when Hash
+            from_hash(value, params, getter:, replace_missing_with:)
+          when Enumerable
+            from_enumerable(value, params, getter:, replace_missing_with:)
+          else
+            value
+          end
+        end
+
+        private
+
+        def format_params(params, getter:, replace_missing_with:)
+          Hash.new do |hash, key|
+            hash[key] = if getter
+                          getter.call(key)
+                        else
+                          params.fetch(key.to_sym) { params[key.to_s] }
+                        end
+            hash[key] = replace_missing_with if hash[key].nil? && !replace_missing_with.nil?
+            hash[key]
+          end
+        end
+
+        def from_string(string, params, getter:, replace_missing_with:)
+          # Return the original string if no format placeholders are found.
+          return string unless /%\{[^{}]*\}|%<[^<>]*>/.match?(string)
+
+          mapping = format_params(params, getter:, replace_missing_with:)
+          format(string, mapping)
+        rescue KeyError => error
+          raise ParamsMissing, "Missing parameter for formatting: #{error.message}" if replace_missing_with.nil?
+
+          string
+        end
+
+        def from_hash(hash, params, getter:, replace_missing_with:)
+          HashUtil.deep_symbolize_keys(hash, context: 'dynamic params hash').to_h do |key, value|
+            [key, call(value, params, getter:, replace_missing_with:)]
+          end
+        end
+
+        def from_enumerable(enumerable, params, getter:, replace_missing_with:)
+          enumerable.map { |value| call(value, params, getter:, replace_missing_with:) }
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/config/multiple_feeds_config.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Config
+    # Handles multiple feeds within a single configuration hash.
+    # Individual feed configurations should be placed under the :feeds key,
+    # where each feed name is the key for its feed configuration.
+    # All global configuration keys (outside :feeds) are merged into each feed's settings.
+    class MultipleFeedsConfig
+      # Reserved YAML key under which multiple named feed configs are defined.
+      CONFIG_KEY_FEEDS = :feeds
+
+      class << self
+        # Merges global configuration into each feed's configuration.
+        #
+        # @param config [Hash] The feed-specific configuration.
+        # @param yaml [Hash] The full YAML configuration.
+        # @param multiple_feeds_key [Symbol] The key under which multiple feeds are defined.
+        # @return [Hash] The merged configuration.
+        def to_single_feed(config, yaml, multiple_feeds_key: CONFIG_KEY_FEEDS)
+          global_keys = yaml.keys - [multiple_feeds_key]
+          global_keys.each do |key|
+            config[key] = merge_key(config, yaml, key)
+          end
+          config
+        end
+
+        private
+
+        # Merges a specific global key from the YAML configuration into the feed configuration.
+        #
+        # @param config [Hash] The feed-specific configuration.
+        # @param yaml [Hash] The full YAML configuration.
+        # @param key [Symbol] The global configuration key to merge.
+        # @return [Object] The merged value for the key.
+        def merge_key(config, yaml, key)
+          global_value = yaml.fetch(key, nil)
+          local_value = config[key]
+          case local_value
+          when Hash
+            global_value.is_a?(Hash) ? HashUtil.deep_merge(global_value, local_value) : local_value
+          when Array
+            global_value.is_a?(Array) ? global_value + local_value : local_value
+          else
+            local_value.nil? ? global_value : local_value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/config/request_headers.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Config
+    ##
+    # Normalizes HTTP headers for outgoing requests.
+    # Ensures a browser-like baseline while respecting caller overrides.
+    class RequestHeaders
+      # Browser-like default `Accept` header value.
+      DEFAULT_ACCEPT = %w[
+        text/html
+        application/xhtml+xml
+        application/xml;q=0.9
+        image/avif
+        image/webp
+        image/apng
+        */*;q=0.8
+      ].join(',')
+
+      # Browser-like default `User-Agent` header value.
+      DEFAULT_USER_AGENT = [
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64)',
+        'AppleWebKit/537.36 (KHTML, like Gecko)',
+        'Chrome/123.0.0.0',
+        'Safari/537.36'
+      ].join(' ')
+
+      # Baseline browser-like header set used for outbound requests.
+      DEFAULT_HEADERS = {
+        'Accept' => DEFAULT_ACCEPT,
+        'Cache-Control' => 'max-age=0',
+        'Connection' => 'keep-alive',
+        'Sec-Fetch-Dest' => 'document',
+        'Sec-Fetch-Mode' => 'navigate',
+        'Sec-Fetch-Site' => 'none',
+        'Sec-Fetch-User' => '?1',
+        'Upgrade-Insecure-Requests' => '1',
+        'User-Agent' => DEFAULT_USER_AGENT
+      }.freeze
+
+      class << self
+        ##
+        # @return [Hash{String => String}] the unmodified default header set
+        def browser_defaults
+          DEFAULT_HEADERS.dup
+        end
+
+        ##
+        # Normalizes the provided headers while applying Html2rss defaults.
+        #
+        # @param headers [Hash, nil] caller provided headers
+        # @param channel_language [String, nil] language defined on the channel
+        # @param url [String] request URL used to infer the Host header
+        # @return [Hash{String => String}] normalized HTTP headers
+        def normalize(headers, channel_language:, url:)
+          new(headers || {}, channel_language:, url:).to_h
+        end
+      end
+
+      # @param headers [Hash{String, Symbol => String}] caller-provided headers
+      # @param channel_language [String, nil] channel language hint for Accept-Language
+      # @param url [String, Html2rss::Url, nil] request URL used to infer Host
+      def initialize(headers, channel_language:, url:)
+        @headers = headers
+        @channel_language = channel_language
+        @url = url
+      end
+
+      ##
+      # @return [Hash{String => String}] normalized HTTP headers
+      def to_h
+        defaults = DEFAULT_HEADERS.dup
+        normalized = normalize_custom_headers(headers)
+
+        accept_override = normalized.delete('Accept')
+        defaults.merge!(normalized)
+
+        defaults['Accept'] = normalize_accept(accept_override)
+        defaults['Accept-Language'] = build_accept_language
+        defaults['Host'] ||= request_host
+
+        defaults.compact
+      end
+
+      private
+
+      attr_reader :headers, :channel_language, :url
+
+      def normalize_custom_headers(custom)
+        custom.transform_keys { canonicalize(_1) }
+      end
+
+      def canonicalize(key)
+        key.to_s.split('-').map!(&:capitalize).join('-')
+      end
+
+      def normalize_accept(override)
+        return DEFAULT_ACCEPT if override.nil? || override.empty?
+
+        values = accept_values(DEFAULT_ACCEPT)
+
+        accept_values(override).reverse_each do |value|
+          next if values.include?(value)
+
+          values.unshift(value)
+        end
+
+        values.join(',')
+      end
+
+      def accept_values(header)
+        header.split(',').map!(&:strip).reject(&:empty?)
+      end
+
+      def build_accept_language
+        language = channel_language.to_s.strip
+        return 'en-US,en;q=0.9' if language.empty?
+
+        normalized = language.tr('_', '-')
+        primary, region = normalized.split('-', 2)
+        primary = primary.downcase
+        region = region&.upcase
+
+        return primary if region.nil?
+
+        "#{primary}-#{region},#{primary};q=0.9"
+      end
+
+      def request_host
+        return nil if url.nil? || url.empty?
+
+        Html2rss::Url.from_absolute(url).host
+      end
+    end
+  end
+end