html2rss 0.30.0 → 0.31.0

data/lib/html2rss/selectors/{object_to_xml_converter.rb → json_xml.rb}

@@ -5,32 +5,28 @@ require 'cgi'
 module Html2rss
   class Selectors
     ##
-    # A naive implementation of "Object to XML": converts a Ruby object to XML format.
-    class ObjectToXmlConverter
+    # Internal data converter helper: converts a Ruby object (e.g. parsed JSON response)
+    # to an XML format string so Nokogiri HTML5 CSS selectors can run on a fragment.
+    module JsonXml
       # Wrapper tags used for top-level collection conversion.
       OBJECT_TO_XML_TAGS = {
         hash: ['<object>', '</object>'],
         array: ['<array>', '</array>']
       }.freeze
 
-      ##
-      # @param object [Object] any Ruby object (Hash, Array, String, Symbol, etc.)
-      def initialize(object)
-        @object = object
-      end
+      module_function
 
       ##
       # Converts the object to XML format.
       #
+      # @param object [Object] any Ruby object (Hash, Array, String, Symbol, etc.)
       # @return [String] representing the object in XML
-      def call
-        object_to_xml(@object).tap do |converted|
-          Html2rss::Log.debug("#{self.class}: converted object to XML (#{converted.bytesize} bytes)")
+      def call(object)
+        object_to_xml(object).tap do |converted|
+          Html2rss::Log.debug("#{self}: converted object to XML (#{converted.bytesize} bytes)")
         end
       end
 
-      private
-
       def object_to_xml(object)
         case object
         when Hash
@@ -41,6 +37,7 @@ module Html2rss
           CGI.escapeHTML(object.to_s)
         end
       end
+      private_class_method :object_to_xml
 
       def hash_to_xml(object)
         prefix, suffix = OBJECT_TO_XML_TAGS[:hash]
@@ -50,6 +47,7 @@ module Html2rss
 
         "#{prefix}#{inner_xml}#{suffix}"
       end
+      private_class_method :hash_to_xml
 
       def array_to_xml(object)
         prefix, suffix = OBJECT_TO_XML_TAGS[:array]
@@ -57,6 +55,7 @@ module Html2rss
 
         "#{prefix}#{inner_xml}#{suffix}"
       end
+      private_class_method :array_to_xml
     end
   end
 end

data/lib/html2rss/selectors/option_spec.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class Selectors
+    OptionSpec = Data.define(:name, :type, :required)
+
+    ##
+    # One config-facing option on an extractor or post-processor strategy.
+    #
+    # Strategies own an +OPTIONS+ Array of these. This type is also the
+    # introspection SoT (+for+ / +expectation_for+); {SchemaExport} maps Ruby
+    # types to JSON Schema only.
+    class OptionSpec
+      # Maps Ruby option types to JSON Schema +type+ strings.
+      RUBY_TO_JSON_TYPE = {
+        String => 'string',
+        Integer => 'integer',
+        Hash => 'object',
+        Array => 'array'
+      }.freeze
+
+      # Human-readable labels for validation error messages.
+      TYPE_LABELS = {
+        Integer => 'an integer',
+        String => 'a string',
+        Hash => 'a hash'
+      }.freeze
+
+      # @param name [Symbol, String]
+      # @param type [Class, Array<Class>]
+      # @param required [Boolean]
+      def initialize(name:, type:, required: true)
+        super(name: name.to_sym, type:, required:)
+      end
+
+      ##
+      # @param actual [Object]
+      # @return [Boolean]
+      def valid_type?(actual)
+        self.class.valid_type?(actual, type)
+      end
+
+      ##
+      # @param optional [Boolean]
+      # @return [String]
+      def error_message(optional: !required)
+        suffix = optional ? ' or omitted' : ''
+        "`#{name}` must be #{type_failure_label}#{suffix}"
+      end
+
+      ##
+      # @return [String, Array<String>]
+      def json_type
+        self.class.json_type_for(type)
+      end
+
+      private
+
+      def type_failure_label
+        case type
+        when Array then type.map { |t| TYPE_LABELS.fetch(t) { "a #{t}" } }.join(' or ')
+        else TYPE_LABELS.fetch(type) { "a #{type}" }
+        end
+      end
+
+      class << self
+        ##
+        # @param actual [Object]
+        # @param type_or_types [Class, Array<Class>]
+        # @return [Boolean]
+        def valid_type?(actual, type_or_types)
+          case type_or_types
+          when Array then type_or_types.any? { |t| actual.is_a?(t) }
+          else actual.is_a?(type_or_types)
+          end
+        end
+
+        ##
+        # @param ruby_type [Class, Array<Class>]
+        # @return [String, Array<String>]
+        def json_type_for(ruby_type)
+          case ruby_type
+          when Array then ruby_type.map { |t| json_type_for(t) }
+          else
+            RUBY_TO_JSON_TYPE.fetch(ruby_type) do
+              raise ArgumentError, "unsupported OPTIONS type mapping for #{ruby_type}"
+            end
+          end
+        end
+
+        ##
+        # @param klass [Class] extractor or post-processor class
+        # @return [Array<OptionSpec>]
+        def for(klass)
+          klass::OPTIONS
+        end
+
+        ##
+        # Ruby-typed expectation for a leaf under a selector / post-process parent.
+        #
+        # @param parent [Hash] selector or post-process step hash
+        # @param leaf [Symbol] option key under +parent+
+        # @return [Hash{Symbol => Object}, nil] +{ type:, required: }+ with Ruby +type+
+        def expectation_for(parent:, leaf:)
+          option = option_for(parent, leaf)
+          return unless option
+
+          { type: option.type, required: option.required }
+        end
+
+        ##
+        # Resolves an OptionSpec for a parent hash and option leaf name.
+        #
+        # @param parent [Hash] selector or post-process step hash
+        # @param leaf [Symbol, String]
+        # @return [OptionSpec, nil]
+        def option_for(parent, leaf)
+          leaf = leaf.to_sym
+          if (name = parent[:extractor] || parent['extractor'])
+            klass = Extractors::NAME_TO_CLASS[name.to_sym]
+            return find_option(klass, leaf) if klass
+          end
+
+          if (name = parent[:name] || parent['name'])
+            klass = PostProcessors::NAME_TO_CLASS[name.to_sym]
+            return find_option(klass, leaf) if klass
+          end
+
+          nil
+        end
+
+        private
+
+        def find_option(klass, leaf)
+          self.for(klass).find { |spec| spec.name == leaf }
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/selectors/post_processors/base.rb

@@ -4,61 +4,109 @@ module Html2rss
   class Selectors
     module PostProcessors
       ##
-      # All post processors must inherit from this base class and implement `self.validate_args!` and `#get`.
+      # All post processors must inherit from this base class and implement `#call`.
+      # Declare +VALUE_TYPE+ when the extracted value must be a fixed Ruby type
+      # (+nil+ when any type is accepted); override +validate_args!+ only for
+      # semantic checks beyond that. Every processor owns +OPTIONS+ (+[]+ when none).
       class Base
-        # Validates the presence of required options in the context
-        #
-        # @param keys [Array<Symbol>] the keys to check for presence
-        # @param context [Selectors::Context] the context containing options
-        # @return [void]
-        # @raise [MissingOption] if any key is missing
-        def self.expect_options(keys, context)
-          keys.each do |key|
-            unless (options = context[:options]).key?(key)
-              raise MissingOption, "The `#{key}` option is missing in: #{options.inspect}", [],
-                    cause: nil
-            end
-          end
-        end
+        # Config-facing options contract (empty on the abstract base).
+        OPTIONS = [].freeze
+
+        # Expected Ruby class for the extracted value; +nil+ skips the type check.
+        VALUE_TYPE = nil
 
         # Asserts that the value is of the expected type(s)
         #
         # @param value [Object] the value to check
         # @param types [Array<Class>, Class] the expected type(s)
         # @param name [String] the name of the option being checked
-        # @param context [Selectors::Context] call-site context used for richer validation errors
+        # @param context [Selectors::StepEnv] call-site context used for richer validation errors
         # @return [void]
         # @raise [InvalidType] if the value is not of the expected type(s)
         def self.assert_type(value, types, name, context:)
-          return if Array(types).any? { |type| value.is_a?(type) }
+          return if Selectors::OptionSpec.valid_type?(value, types)
 
-          options = if context.respond_to?(:options)
-                      context.options
-                    else
-                      { file: File.basename(caller(1, 1).first.split(':').first) }
-                    end
           message = "The type of `#{name}` must be #{Array(types).join(' or ')}, " \
-                    "but is: #{value.class} in: #{options.inspect}"
+                    "but is: #{value.class} in: #{context.step_config.inspect}"
           raise InvalidType, message, [], cause: nil
         end
 
         ##
-        # This method validates the arguments passed to the post processor. Must be implemented by subclasses.
+        # Validates presence and types from the strategy's +OPTIONS+ contract.
+        #
+        # @param context [Selectors::StepEnv]
+        # @return [void]
+        # @raise [MissingOption] if a required option key is absent
+        # @raise [InvalidType] if a present option has the wrong type
+        def self.validate_options!(context)
+          specs = option_specs
+          return if specs.empty?
+
+          step_config = context.step_config
+          cache = validated_step_configs
+          return if cache[step_config]
+
+          specs.each { |spec| validate_option_spec!(spec, step_config, context) }
+          cache[step_config] = true
+        end
+
+        # rubocop:disable-next ThreadSafety/ClassInstanceVariable
+        def self.option_specs
+          @option_specs ||= OptionSpec.for(self)
+        end
+        private_class_method :option_specs
+
+        # rubocop:disable-next ThreadSafety/ClassInstanceVariable
+        def self.validated_step_configs
+          @validated_step_configs ||= {}.compare_by_identity
+        end
+        private_class_method :validated_step_configs
+
+        ##
+        # @param spec [Selectors::OptionSpec]
+        # @param step_config [Hash] post-processor YAML step configuration
+        # @param context [Selectors::StepEnv]
+        # @return [void]
+        def self.validate_option_spec!(spec, step_config, context)
+          unless step_config.key?(spec.name)
+            return unless spec.required
+
+            raise MissingOption, "The `#{spec.name}` option is missing in: #{step_config.inspect}", [],
+                  cause: nil
+          end
+
+          value = step_config[spec.name]
+          return if value.nil? && !spec.required
+
+          assert_type(value, spec.type, spec.name, context:)
+        end
+
+        ##
+        # Semantic / value checks beyond +VALUE_TYPE+ and the +OPTIONS+ contract.
+        # Override in subclasses that need non-type checks (non-empty, item_env, …).
         #
         # @param _value [Object] extracted selector value
-        # @param _context [Selectors::Context] post-processor execution context
+        # @param _context [Selectors::StepEnv] post-processor execution context
         # @return [void]
         def self.validate_args!(_value, _context)
-          raise NotImplementedError, 'You must implement the `validate_args!` method in the post processor'
+          # no-op default
         end
 
         # Initializes the post processor
         #
         # @param value [Object] the value to be processed
-        # @param context [Selectors::Context] runtime selector context and options
-        def initialize(value, context)
+        # @param context [Selectors::StepEnv] runtime selector context and options
+        # @raise [InvalidType] if +context+ is not a {Selectors::StepEnv} or +VALUE_TYPE+ mismatches
+        def initialize(value, context) # rubocop:disable Metrics/MethodLength -- type guard + VALUE_TYPE + validate_args
+          unless context.is_a?(Selectors::StepEnv)
+            raise InvalidType,
+                  "The type of `context` must be #{Selectors::StepEnv}, but is: #{context.class}",
+                  [], cause: nil
+          end
+
           klass = self.class
-          klass.assert_type(context, Selectors::Context, 'context', context:)
+          klass.validate_options!(context)
+          klass.assert_type(value, klass::VALUE_TYPE, :value, context:) if klass::VALUE_TYPE
           klass.validate_args!(value, context)
 
           @value = value
@@ -71,8 +119,8 @@ module Html2rss
         #
         # @return [Object] transformed value
         # @raise [NotImplementedError] if not implemented in subclass
-        def get
-          raise NotImplementedError, 'You must implement the `get` method in the post processor'
+        def call
+          raise NotImplementedError, 'You must implement the `call` method in the post processor'
         end
       end
     end

data/lib/html2rss/selectors/post_processors/gsub.rb

@@ -29,15 +29,26 @@ module Html2rss
       #
       # See the doc on [String#gsub](https://ruby-doc.org/core/String.html#method-i-gsub) for more info.
       class Gsub < Base
-        # Required config field types (validator introspection via +Options+).
-        OPTION_TYPES = { pattern: String, replacement: String }.freeze
+        # Expected Ruby class for the extracted value before this post-processor runs.
+        VALUE_TYPE = String
 
-        # Config fields required by this post-processor (validator / schema introspection).
-        Options = Struct.new(*OPTION_TYPES.keys, keyword_init: true)
+        # Config-facing options contract (validator / SchemaExport / Base.validate_options!).
+        OPTIONS = [
+          OptionSpec.new(name: :pattern, type: String),
+          OptionSpec.new(name: :replacement, type: [String, Hash])
+        ].freeze
+
+        # Character cap for the authored +pattern+ string (before slash stripping).
+        # Same unit as JSON Schema +maxLength+.
+        MAX_PATTERN_LENGTH = 256
 
-        # JSON Schema description exported via +schema_doc+.
-        DESCRIPTION = 'Replace matches of `pattern` in the extracted string with `replacement` ' \
-                      '(Ruby String#gsub; pattern may be a regexp-like string).'
+        # JSON Schema description exported via +schema_export+.
+        DESCRIPTION = format(
+          'Replace matches of `pattern` in the extracted string with `replacement` ' \
+          '(Ruby String#gsub; pattern may be a regexp-like string). ' \
+          'Patterns over %d characters as written, or with nested quantifiers, are rejected.',
+          MAX_PATTERN_LENGTH
+        ).freeze
 
         # Example post-process objects for JSON Schema +examples+.
         EXAMPLES = [
@@ -45,32 +56,101 @@ module Html2rss
         ].freeze
 
         # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
-        def self.schema_doc = SchemaDoc.for_post_processor(name: :gsub, klass: self)
-
-        # @param value [String] extracted selector value
-        # @param context [Selectors::Context] post-processor context
-        # @return [void]
-        def self.validate_args!(value, context)
-          assert_type value, String, :value, context:
-          expect_options(%i[replacement pattern], context)
-          assert_type context.dig(:options, :replacement), [String, Hash], :replacement, context:
+        def self.schema_export
+          fragment = SchemaExport.for_post_processor(name: :gsub, klass: self)
+          fragment.fetch(:properties).fetch(:pattern)[:maxLength] = MAX_PATTERN_LENGTH
+          fragment
+        end
+
+        ##
+        # Compiles and memoizes a gsub +pattern+ string as a Regexp.
+        #
+        # Raises before memoizing when the authored pattern exceeds
+        # {MAX_PATTERN_LENGTH} characters, its star-height is greater than 1, or it does not parse.
+        #
+        # @param string [String]
+        # @return [Regexp]
+        # @raise [ArgumentError] when +string+ is not a String, exceeds the character cap,
+        #   contains nested quantifiers, or does not parse
+        def self.compiled_pattern(string)
+          compiled_patterns[string] ||= compile_regexp_string(string)
+        end
+
+        # rubocop:disable-next ThreadSafety/ClassInstanceVariable
+        def self.compiled_patterns
+          @compiled_patterns ||= {}
+        end
+        private_class_method :compiled_patterns
+
+        # Parses the given String and builds a Regexp out of it.
+        #
+        # It will remove one pair of surrounding slashes ('/') from the String
+        # to maintain backwards compatibility before building the Regexp.
+        # Length is the character count of the authored string (the JSON Schema
+        # +maxLength+ unit), then the AST is walked for nested quantifiers,
+        # then +to_re+. Parser errors are re-raised as ArgumentError so
+        # validation and execution share one path.
+        #
+        # @param string [String]
+        # @return [Regexp]
+        # @raise [ArgumentError] when +string+ is not a String, exceeds {MAX_PATTERN_LENGTH},
+        #   contains nested quantifiers, or does not parse
+        def self.compile_regexp_string(string)
+          raise ArgumentError, 'must be a string!' unless string.is_a?(String)
+          raise ArgumentError, "pattern exceeds #{MAX_PATTERN_LENGTH} characters" if string.length > MAX_PATTERN_LENGTH
+
+          source = regexp_source(string)
+
+          expression = Regexp::Parser.parse(source, options: ::Regexp::EXTENDED | ::Regexp::IGNORECASE)
+          raise ArgumentError, 'pattern contains nested quantifiers' if nested_quantifiers?(expression)
+
+          expression.to_re
+        rescue Regexp::Parser::Error => error
+          raise ArgumentError, error.message, [], cause: nil
+        end
+        private_class_method :compile_regexp_string
+
+        # Strips one pair of surrounding slashes when the source is long enough.
+        #
+        # @param string [String]
+        # @return [String]
+        def self.regexp_source(string)
+          return string unless string.length >= 3 && string.start_with?('/') && string.end_with?('/')
+
+          string[1..-2]
+        end
+        private_class_method :regexp_source
+
+        # Star-height greater than 1: a quantified node whose subtree contains another.
+        #
+        # +each_expression+ yields +(node, index)+. Do not pass +&:quantified?+:
+        # +Symbol#to_proc+ would forward the index into +quantified?+ (arity 0).
+        #
+        # @param expression [Regexp::Expression::Subexpression]
+        # @return [Boolean]
+        # rubocop:disable-next Style/SymbolProc
+        def self.nested_quantifiers?(expression)
+          expression.each_expression.any? do |node|
+            node.quantified? && !node.terminal? && node.each_expression.any? { |child| child.quantified? }
+          end
         end
+        private_class_method :nested_quantifiers?
 
         ##
         # @param value [String]
-        # @param context [Selectors::Context]
+        # @param context [Selectors::StepEnv]
         def initialize(value, context)
           super
 
-          options = context[:options]
+          step_config = context.step_config
 
-          @replacement = options[:replacement]
-          @pattern = options[:pattern]
+          @replacement = step_config[:replacement]
+          @pattern = step_config[:pattern]
         end
 
         ##
         # @return [String]
-        def get
+        def call
           value.to_s.gsub(pattern, replacement)
         end
 
@@ -81,24 +161,8 @@ module Html2rss
         ##
         # @return [Regexp]
         def pattern
-          @pattern.is_a?(String) ? parse_regexp_string(@pattern) : @pattern
-        end
-
-        ##
-        # Parses the given String and builds a Regexp out of it.
-        #
-        # It will remove one pair of surrounding slashes ('/') from the String
-        # to maintain backwards compatibility before building the Regexp.
-        #
-        # @param string [String]
-        # @return [Regexp]
-        def parse_regexp_string(string)
-          raise ArgumentError, 'must be a string!' unless string.is_a?(String)
-
-          # Only remove surrounding slashes if the string has at least 3 characters
-          # to avoid issues with single character strings like "/"
-          string = string[1..-2] if string.length >= 3 && string.start_with?('/') && string.end_with?('/')
-          Regexp::Parser.parse(string, options: ::Regexp::EXTENDED | ::Regexp::IGNORECASE).to_re
+          raw = @pattern
+          raw.is_a?(String) ? self.class.compiled_pattern(raw) : raw
         end
       end
     end

data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb

@@ -17,9 +17,9 @@ module Html2rss
             'video' => :src # Video player is visible
           }.freeze
 
-          # @param channel_url [String, Html2rss::Url] base URL used to resolve relative links
-          def initialize(channel_url)
-            @channel_url = channel_url
+          # @param base_url [String, Html2rss::Url] base URL used to resolve relative links
+          def initialize(base_url)
+            @base_url = base_url
           end
 
           ##
@@ -35,7 +35,7 @@ module Html2rss
 
             url_attribute = URL_ELEMENTS_WITH_URL_ATTRIBUTE[node_name]
             url = node[url_attribute]
-            node[url_attribute] = Url.from_relative(url, @channel_url).to_s
+            node[url_attribute] = Url.from_relative(url, @base_url).to_s
           end
         end
       end

data/lib/html2rss/selectors/post_processors/markdown_to_html.rb

@@ -33,7 +33,13 @@ module Html2rss
       #
       #    <p>Price: 12.34</p>
       class MarkdownToHtml < Base
-        # JSON Schema description exported via +schema_doc+.
+        # Expected Ruby class for the extracted value before this post-processor runs.
+        VALUE_TYPE = String
+
+        # Config-facing options contract (none).
+        OPTIONS = [].freeze
+
+        # JSON Schema description exported via +schema_export+.
         DESCRIPTION = 'Convert Markdown to HTML (Kramdown) and sanitize the result. ' \
                       'Often chained after `template`.'
 
@@ -43,22 +49,15 @@ module Html2rss
         ].freeze
 
         # @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
-        def self.schema_doc = SchemaDoc.for_post_processor(name: :markdown_to_html, klass: self)
-
-        # @param value [String] extracted selector value
-        # @param context [Selectors::Context] post-processor context
-        # @return [void]
-        def self.validate_args!(value, context)
-          assert_type value, String, :value, context:
-        end
+        def self.schema_export = SchemaExport.for_post_processor(name: :markdown_to_html, klass: self)
 
         ##
         # Converts Markdown to sanitized HTML.
         #
         # @return [String] Sanitized HTML content
-        def get
+        def call
           html_content = Kramdown::Document.new(value).to_html
-          SanitizeHtml.new(html_content, context).get
+          SanitizeHtml.new(html_content, context).call
         end
       end
     end