html2rss 0.17.0 → 0.18.0

data/lib/html2rss/blocked_surface.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Shared anti-bot/interstitial signatures used by request and auto-source flows.
+  #
+  # This module centralizes signature matching so request-time guards and
+  # auto-source surface classification stay consistent.
+  module BlockedSurface
+    INTERSTITIAL_SIGNATURES = [
+      {
+        key: :cloudflare_interstitial,
+        min_matches: 2,
+        patterns: [
+          %r{<title>\s*just a moment\.\.\.\s*</title>}i,
+          /checking your browser before accessing/i,
+          /please (?:enable|turn on) javascript and cookies/i,
+          %r{cdn-cgi/challenge-platform}i,
+          /cloudflare ray id/i
+        ],
+        message: 'Blocked surface detected: Cloudflare anti-bot interstitial page. ' \
+                 'Retry with --strategy browserless, try a more specific public listing URL, ' \
+                 'or run from an environment that can complete anti-bot checks.'
+      }
+    ].freeze
+
+    ##
+    # Returns the first matching interstitial signature for the provided body.
+    #
+    # @param body [String, nil] response body candidate
+    # @return [Hash, nil] signature hash when matched, otherwise nil
+    def self.interstitial_signature_for(body)
+      normalized_body = normalize_body(body)
+      INTERSTITIAL_SIGNATURES.find { |signature| interstitial_signature_match?(normalized_body, signature) }
+    end
+
+    ##
+    # @param body [String, nil] response body candidate
+    # @return [Boolean] true when body matches a known interstitial signature
+    def self.interstitial?(body)
+      !interstitial_signature_for(body).nil?
+    end
+
+    def self.interstitial_signature_match?(body, signature)
+      min_matches = signature.fetch(:min_matches, 1)
+      matches = 0
+
+      signature.fetch(:patterns).each do |pattern|
+        matches += 1 if pattern.match?(body)
+        return true if matches >= min_matches
+      end
+
+      false
+    end
+    private_class_method :interstitial_signature_match?
+
+    def self.normalize_body(body)
+      body.to_s.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: '')
+    rescue Encoding::CompatibilityError, Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError
+      body.to_s.force_encoding(Encoding::UTF_8).scrub
+    end
+    private_class_method :normalize_body
+  end
+end

data/lib/html2rss/category_extractor.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # CategoryExtractor is responsible for extracting categories from HTML elements
+  # by looking for CSS class names containing common category-related terms.
+  class CategoryExtractor
+    # Common category-related terms to look for in class names
+    CATEGORY_TERMS = %w[category tag topic section label theme subject].freeze
+
+    # CSS selectors to find elements with category-related class names
+    CATEGORY_SELECTORS = CATEGORY_TERMS.map { |term| "[class*=\"#{term}\"]" }.freeze
+
+    # Regex pattern for matching category-related attribute names
+    CATEGORY_ATTR_PATTERN = /#{CATEGORY_TERMS.join('|')}/i
+
+    ##
+    # Extracts categories from the given article tag by looking for elements
+    # with class names containing common category-related terms.
+    #
+    # @param article_tag [Nokogiri::XML::Element] The article element to extract categories from
+    # @return [Array<String>] Array of category strings, empty if none found
+    def self.call(article_tag)
+      return [] unless article_tag
+
+      # Single optimized traversal that extracts all category types
+      extract_all_categories(article_tag)
+        .map(&:strip)
+        .reject(&:empty?)
+    end
+
+    ##
+    # Optimized single DOM traversal that extracts all category types.
+    #
+    # @param article_tag [Nokogiri::XML::Element] The article element
+    # @return [Set<String>] Set of category strings
+    def self.extract_all_categories(article_tag)
+      Set.new.tap do |categories|
+        article_tag.css('*').each do |element|
+          # Extract text categories from elements with category-related class names
+          categories.merge(extract_text_categories(element)) if element['class']&.match?(CATEGORY_ATTR_PATTERN)
+
+          # Extract data categories from all elements
+          categories.merge(extract_element_data_categories(element))
+        end
+      end
+    end
+
+    ##
+    # Extracts categories from data attributes of a single element.
+    #
+    # @param element [Nokogiri::XML::Element] The element to process
+    # @return [Set<String>] Set of category strings
+    def self.extract_element_data_categories(element)
+      Set.new.tap do |categories|
+        element.attributes.each_value do |attr|
+          next unless attr.name.match?(CATEGORY_ATTR_PATTERN)
+
+          value = attr.value&.strip
+          categories.add(value) if value && !value.empty?
+        end
+      end
+    end
+
+    ##
+    # Extracts text-based categories from elements, splitting content into discrete values.
+    #
+    # @param element [Nokogiri::XML::Element] The element to process
+    # @return [Set<String>] Set of category strings
+    def self.extract_text_categories(element)
+      anchor_values = element.css('a').filter_map do |node|
+        HtmlExtractor.extract_visible_text(node)
+      end
+      return Set.new(anchor_values.reject(&:empty?)) if anchor_values.any?
+
+      text = HtmlExtractor.extract_visible_text(element)
+      return Set.new unless text
+
+      Set.new(text.split(/\n+/).map(&:strip).reject(&:empty?))
+    end
+  end
+end

data/lib/html2rss/cli.rb

@@ -1,46 +1,193 @@
 # frozen_string_literal: true
 
-require_relative '../html2rss'
+require 'fileutils'
+require 'json'
 require 'thor'
 
 ##
 # The Html2rss namespace / command line interface.
 module Html2rss
-  Log = Logger.new($stderr)
-
   ##
   # The Html2rss command line interface.
-  class CLI < Thor
+  class CLI < Thor # rubocop:disable Metrics/ClassLength
+    check_unknown_options!
+
     def self.exit_on_failure?
       true
     end
 
-    desc 'feed YAML_FILE [FEED_NAME] [param=value ...]', 'Print RSS built from the YAML_FILE file to stdout'
-    ##
-    # Prints the feed to STDOUT.
-    #
-    # @param yaml_file [String] Path to the YAML configuration file.
-    # @param options [Array<String>] Additional options including feed name and parameters.
-    # @return [nil]
-    def feed(yaml_file, *options)
-      raise "File '#{yaml_file}' does not exist" unless File.exist?(yaml_file)
-
-      feed_name = options.shift unless options.first&.include?('=')
-      params = options.to_h { |opt| opt.split('=', 2) }
+    desc 'feed YAML_FILE [feed_name]', 'Print RSS built from the YAML_FILE file to stdout'
+    method_option :params,
+                  type: :hash,
+                  optional: true,
+                  required: false,
+                  default: {}
+    method_option :strategy,
+                  type: :string,
+                  desc: 'The strategy to request the URL',
+                  enum: %w[faraday browserless]
+    method_option :max_redirects,
+                  type: :numeric,
+                  desc: 'Maximum redirects to follow per request'
+    method_option :max_requests,
+                  type: :numeric,
+                  desc: 'Maximum requests to allow for this feed build'
+    def feed(yaml_file, feed_name = nil)
+      config = Html2rss.config_from_yaml_file(yaml_file, feed_name)
+      config[:params] = options[:params] || {}
+      apply_runtime_request_overrides!(config)
 
-      puts Html2rss.feed_from_yaml_config(yaml_file, feed_name, params:)
+      puts(execute_feed { Html2rss.feed(config) })
     end
 
-    desc 'auto URL', 'Automatically sources an RSS feed from the URL'
+    desc 'auto [URL]', 'Automatically sources an RSS feed from the URL'
     method_option :strategy,
                   type: :string,
                   desc: 'The strategy to request the URL',
-                  enum: RequestService.strategy_names,
-                  default: RequestService.default_strategy_name
-    def auto(url)
-      strategy = options.fetch(:strategy) { RequestService.default_strategy_name }.to_sym
+                  enum: %w[faraday browserless]
+    method_option :format,
+                  type: :string,
+                  desc: 'Output format for the auto-sourced feed',
+                  enum: %w[rss jsonfeed],
+                  default: 'rss'
+    method_option :items_selector, type: :string, desc: 'CSS selector for items (will be enhanced) (optional)'
+    method_option :max_redirects,
+                  type: :numeric,
+                  desc: 'Maximum redirects to follow per request'
+    method_option :max_requests,
+                  type: :numeric,
+                  desc: 'Maximum requests to allow for this feed build'
+    def auto(url) # rubocop:disable Metrics/MethodLength
+      format = options.fetch(:format, 'rss')
+      source_method = format == 'jsonfeed' ? Html2rss.method(:auto_json_feed) : Html2rss.method(:auto_source)
+
+      result = execute_feed do
+        source_method.call(
+          url,
+          strategy: current_strategy,
+          items_selector: options[:items_selector],
+          max_redirects: options[:max_redirects],
+          max_requests: options[:max_requests]
+        )
+      end
+
+      puts(format == 'jsonfeed' ? JSON.pretty_generate(result) : result)
+    end
+
+    desc 'schema', 'Print the exported config JSON Schema'
+    method_option :pretty,
+                  type: :boolean,
+                  desc: 'Pretty-print the schema JSON',
+                  default: true
+    method_option :write,
+                  type: :string,
+                  desc: 'Write the schema JSON to the given file path'
+    ##
+    # Prints or writes the exported configuration JSON Schema.
+    #
+    # @return [void]
+    def schema
+      schema_json = Html2rss::Config.json_schema_json(pretty: options.fetch(:pretty, true))
+
+      if options[:write]
+        FileUtils.mkdir_p(File.dirname(options[:write]))
+        File.write(options[:write], "#{schema_json}\n")
+        puts options[:write]
+        return
+      end
+
+      puts schema_json
+    end
+
+    desc 'validate YAML_FILE [feed_name]', 'Validate a YAML config with the runtime validator'
+    method_option :params,
+                  type: :hash,
+                  optional: true,
+                  required: false,
+                  default: {}
+    ##
+    # Validates a YAML config and prints the result.
+    #
+    # @param yaml_file [String] the YAML file to validate
+    # @param feed_name [String, nil] optional feed name for multi-feed files
+    # @return [void]
+    def validate(yaml_file, feed_name = nil)
+      result = Html2rss::Config.validate_yaml(yaml_file, feed_name, params: options[:params] || {})
+
+      raise Thor::Error, "Invalid configuration: #{result.errors.to_h}" unless result.success?
+
+      puts 'Configuration is valid'
+    end
+
+    private
+
+    def apply_runtime_request_overrides!(config)
+      clear_blank_request_overrides!(config)
+      request_controls.apply_to(config)
+    end
+
+    def clear_blank_request_overrides!(config)
+      config.delete(:strategy) if config[:strategy].nil?
+
+      request_config = config[:request]
+      return unless request_config.is_a?(Hash)
+
+      %i[max_redirects max_requests].each do |key|
+        request_config.delete(key) if request_config[key].nil?
+      end
+      config.delete(:request) if request_config.empty?
+    end
+
+    def request_controls
+      Html2rss::RequestControls.new(
+        strategy: options[:strategy]&.to_sym,
+        max_redirects: options[:max_redirects],
+        max_requests: options[:max_requests],
+        explicit_keys: explicit_request_control_keys
+      )
+    end
+
+    def explicit_request_control_keys
+      keys = []
+      keys << :strategy if options[:strategy]
+      keys << :max_redirects unless options[:max_redirects].nil?
+      keys << :max_requests unless options[:max_requests].nil?
+      keys
+    end
+
+    def current_strategy
+      options[:strategy]&.to_sym || :faraday
+    end
+
+    def current_max_redirects
+      options.fetch(:max_redirects, Html2rss::RequestService::Policy::DEFAULTS[:max_redirects])
+    end
+
+    def current_max_requests
+      options.fetch(:max_requests, Html2rss::RequestService::Policy::DEFAULTS[:max_requests])
+    end
+
+    def suggested_max_redirects
+      current_max_redirects + 1
+    end
+
+    def suggested_max_requests
+      current_max_requests + 1
+    end
 
-      puts Html2rss.auto_source(url, strategy:)
+    def execute_feed # rubocop:disable Metrics/MethodLength
+      yield
+    rescue Faraday::FollowRedirects::RedirectLimitReached => error
+      raise Thor::Error,
+            "#{error.message}. retry with --max-redirects #{suggested_max_redirects} or use the final URL directly."
+    rescue Html2rss::RequestService::RequestBudgetExceeded => error
+      raise Thor::Error,
+            "#{error.message}. retry with --max-requests #{suggested_max_requests} " \
+            'or increase request.max_requests in the config.'
+    rescue Html2rss::RequestService::BrowserlessConfigurationError,
+           Html2rss::RequestService::BrowserlessConnectionFailed,
+           Html2rss::RequestService::BlockedSurfaceDetected => error
+      raise Thor::Error, error.message
     end
   end
 end

data/lib/html2rss/config/class_methods.rb

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Config
+    ##
+    # Public class-level helpers for loading, validating, and exporting config.
+    module ClassMethods
+      UNSET = Object.new.freeze
+
+      ##
+      # Returns the exported JSON Schema for html2rss configuration.
+      #
+      # @return [Hash<String, Object>] JSON Schema represented as a Ruby hash
+      def json_schema
+        Schema.json_schema
+      end
+
+      ##
+      # Returns the exported JSON Schema as JSON.
+      #
+      # @param pretty [Boolean] whether to pretty-print the JSON output
+      # @return [String] serialized JSON Schema
+      def json_schema_json(pretty: true)
+        pretty ? JSON.pretty_generate(json_schema) : JSON.generate(json_schema)
+      end
+
+      ##
+      # Validates a configuration hash with the runtime validator.
+      #
+      # @param config [Hash<Symbol, Object>] the configuration hash
+      # @param params [Hash<Symbol, Object>, Hash<String, Object>, nil] dynamic parameters for string formatting
+      # @return [Dry::Validation::Result] validation result after defaults and deprecations are applied
+      def validate(config, params: UNSET)
+        prepared_config = prepare_for_validation(resolve_effective_config(config, params:))
+
+        Validator.new.call(prepared_config)
+      rescue DynamicParams::ParamsMissing => error
+        prepared_config = prepare_for_validation(deep_dup(config))
+        prepared_config[:dynamic_params_error] = error.message
+
+        Validator.new.call(prepared_config)
+      end
+
+      ##
+      # Returns the packaged JSON Schema file path.
+      #
+      # @return [String] absolute path to the packaged JSON Schema file
+      def schema_path
+        Schema.path
+      end
+
+      ##
+      # Loads and validates a YAML configuration file.
+      #
+      # @param file [String] the YAML file to load
+      # @param feed_name [String, nil] optional feed name for multi-feed files
+      # @param multiple_feeds_key [Symbol] key under which multiple feeds are defined
+      # @param params [Hash<Symbol, Object>, Hash<String, Object>, nil] dynamic parameters for string formatting
+      # @return [Dry::Validation::Result] validation result after defaults and deprecations are applied
+      def validate_yaml(file, feed_name = nil, multiple_feeds_key: MultipleFeedsConfig::CONFIG_KEY_FEEDS, params: UNSET)
+        validate(load_yaml(file, feed_name, multiple_feeds_key:), params:)
+      end
+
+      ##
+      # Loads the feed configuration from a YAML file.
+      #
+      # Supports multiple feeds defined under the specified key (default :feeds).
+      #
+      # @param file [String] the YAML file to load.
+      # @param feed_name [String, nil] the feed name when using multiple feeds.
+      # @param multiple_feeds_key [Symbol] the key under which multiple feeds are defined.
+      # @return [Hash<Symbol, Object>] the configuration hash.
+      # @raise [ArgumentError] if the file doesn't exist or feed is not found.
+      # rubocop:disable Metrics/MethodLength
+      def load_yaml(file, feed_name = nil, multiple_feeds_key: MultipleFeedsConfig::CONFIG_KEY_FEEDS)
+        raise ArgumentError, "File '#{file}' does not exist" unless File.exist?(file)
+        raise ArgumentError, "`#{multiple_feeds_key}` is a reserved feed name" if feed_name == multiple_feeds_key
+
+        yaml = YAML.safe_load_file(file, symbolize_names: true)
+
+        return yaml unless yaml.key?(multiple_feeds_key)
+
+        unless feed_name
+          available_feeds = yaml.fetch(multiple_feeds_key).keys.join(', ')
+          raise ArgumentError,
+                "Feed name is required under `#{multiple_feeds_key}`. Available feeds: #{available_feeds}"
+        end
+
+        config = yaml.dig(multiple_feeds_key, feed_name.to_sym)
+        raise ArgumentError, "Feed '#{feed_name}' not found under `#{multiple_feeds_key}` key." unless config
+
+        MultipleFeedsConfig.to_single_feed(config, yaml, multiple_feeds_key:)
+      end
+      # rubocop:enable Metrics/MethodLength
+
+      ##
+      # Processes the provided configuration hash, applying dynamic parameters if given,
+      # and returns a new configuration object.
+      #
+      # @param config [Hash<Symbol, Object>] the configuration hash.
+      # @param params [Hash<Symbol, Object>, Hash<String, Object>, nil] dynamic parameters for string formatting.
+      # @return [Html2rss::Config] the configuration object.
+      def from_hash(config, params: UNSET)
+        new(resolve_effective_config(config, params:))
+      end
+
+      ##
+      # Builds a top-level auto-source feed config for the public shortcut APIs.
+      #
+      # @param url [String] source page URL
+      # @param items_selector [String, nil] optional selector hint for item extraction
+      # @param request_controls [Html2rss::RequestControls, nil] explicit request controls to write
+      # @return [Hash<Symbol, Object>] feed config hash ready for {from_hash}
+      def auto_source_config(url:, items_selector: nil, request_controls: nil)
+        config = {
+          channel: default_config[:channel].merge(url:),
+          auto_source: AutoSource::DEFAULT_CONFIG
+        }
+
+        request_controls ||= Html2rss::RequestControls.new
+        request_controls.apply_to(config)
+
+        config[:selectors] = { items: { selector: items_selector, enhance: true } } if items_selector
+        config
+      end
+
+      ##
+      # Provides a default configuration.
+      #
+      # @return [Hash<Symbol, Object>] a hash with default configuration values.
+      def default_config
+        {
+          strategy: RequestService.default_strategy_name,
+          request: {
+            max_redirects: RequestService::Policy::DEFAULTS[:max_redirects],
+            max_requests: RequestService::Policy::DEFAULTS[:max_requests]
+          },
+          channel: { time_zone: 'UTC' },
+          headers: RequestHeaders.browser_defaults,
+          stylesheets: []
+        }
+      end
+
+      private
+
+      def resolve_effective_config(config, params:)
+        effective_config = deep_dup(config)
+        resolved_params = parameter_defaults(effective_config)
+        resolved_params.merge!(params) unless params.equal?(UNSET) || params.nil?
+
+        DynamicParams.call(effective_config[:headers], resolved_params)
+        DynamicParams.call(effective_config[:channel], resolved_params)
+
+        effective_config
+      end
+
+      def parameter_defaults(config)
+        config.fetch(:parameters, {})
+              .filter_map do |name, definition|
+                [name, definition[:default]] if definition.is_a?(Hash) && definition.key?(:default)
+              end
+              .to_h
+      end
+
+      def prepare_for_validation(config)
+        Config::Preparer.new.call(deep_dup(config))
+      end
+
+      # rubocop:disable Metrics/MethodLength
+      def deep_dup(object)
+        case object
+        when Hash
+          object.transform_values do |value|
+            deep_dup(value)
+          end
+        when Array
+          object.map { |value| deep_dup(value) }
+        else
+          begin
+            object.dup
+          rescue TypeError
+            object
+          end
+        end
+      end
+      # rubocop:enable Metrics/MethodLength
+    end
+  end
+end

data/lib/html2rss/config/dynamic_params.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Config
+    # Processes and applies dynamic parameter formatting in configuration values.
+    class DynamicParams
+      class ParamsMissing < Html2rss::Error; end
+
+      class << self
+        # Recursively traverses the given value and formats any strings containing
+        # placeholders with values from the provided params.
+        #
+        # @param value [String, Hash, Enumerable, Object] The value to process.
+        # @param params [Hash] The parameters for substitution.
+        # @param getter [Proc, nil] Optional proc to retrieve a key's value.
+        # @param replace_missing_with [Object, nil] Value to substitute if a key is missing.
+        # @return [Object] The processed value.
+        def call(value, params = {}, getter: nil, replace_missing_with: nil)
+          case value
+          when String
+            from_string(value, params, getter:, replace_missing_with:)
+          when Hash
+            from_hash(value, params, getter:, replace_missing_with:)
+          when Enumerable
+            from_enumerable(value, params, getter:, replace_missing_with:)
+          else
+            value
+          end
+        end
+
+        private
+
+        def format_params(params, getter:, replace_missing_with:)
+          Hash.new do |hash, key|
+            hash[key] = if getter
+                          getter.call(key)
+                        else
+                          params.fetch(key.to_sym) { params[key.to_s] }
+                        end
+            hash[key] = replace_missing_with if hash[key].nil? && !replace_missing_with.nil?
+            hash[key]
+          end
+        end
+
+        def from_string(string, params, getter:, replace_missing_with:)
+          # Return the original string if no format placeholders are found.
+          return string unless /%\{[^{}]*\}|%<[^<>]*>/.match?(string)
+
+          mapping = format_params(params, getter:, replace_missing_with:)
+          format(string, mapping)
+        rescue KeyError => error
+          raise ParamsMissing, "Missing parameter for formatting: #{error.message}" if replace_missing_with.nil?
+
+          string
+        end
+
+        def from_hash(hash, params, getter:, replace_missing_with:)
+          hash.transform_keys!(&:to_sym)
+          hash.transform_values! { |value| call(value, params, getter:, replace_missing_with:) }
+        end
+
+        def from_enumerable(enumerable, params, getter:, replace_missing_with:)
+          enumerable.map! { |value| call(value, params, getter:, replace_missing_with:) }
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/config/multiple_feeds_config.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Config
+    # Handles multiple feeds within a single configuration hash.
+    # Individual feed configurations should be placed under the :feeds key,
+    # where each feed name is the key for its feed configuration.
+    # All global configuration keys (outside :feeds) are merged into each feed's settings.
+    class MultipleFeedsConfig
+      CONFIG_KEY_FEEDS = :feeds
+
+      class << self
+        # Merges global configuration into each feed's configuration.
+        #
+        # @param config [Hash] The feed-specific configuration.
+        # @param yaml [Hash] The full YAML configuration.
+        # @param multiple_feeds_key [Symbol] The key under which multiple feeds are defined.
+        # @return [Hash] The merged configuration.
+        def to_single_feed(config, yaml, multiple_feeds_key: CONFIG_KEY_FEEDS)
+          global_keys = yaml.keys - [multiple_feeds_key]
+          global_keys.each do |key|
+            config[key] = merge_key(config, yaml, key)
+          end
+          config
+        end
+
+        private
+
+        # Merges a specific global key from the YAML configuration into the feed configuration.
+        #
+        # @param config [Hash] The feed-specific configuration.
+        # @param yaml [Hash] The full YAML configuration.
+        # @param key [Symbol] The global configuration key to merge.
+        # @return [Object] The merged value for the key.
+        def merge_key(config, yaml, key)
+          global_value = yaml.fetch(key, nil)
+          local_value = config[key]
+          case local_value
+          when Hash
+            global_value.is_a?(Hash) ? global_value.merge(local_value) : local_value
+          when Array
+            global_value.is_a?(Array) ? global_value + local_value : local_value
+          else
+            global_value
+          end
+        end
+      end
+    end
+  end
+end