html2rss 0.32.4 → 0.33.0

data/lib/html2rss/selectors/json_xml.rb

@@ -14,48 +14,47 @@ module Html2rss
         array: ['<array>', '</array>']
       }.freeze
 
-      module_function
-
-      ##
-      # Converts the object to XML format.
-      #
-      # @param object [Object] any Ruby object (Hash, Array, String, Symbol, etc.)
-      # @return [String] representing the object in XML
-      def call(object)
-        object_to_xml(object).tap do |converted|
-          Html2rss::Log.debug("#{self}: converted object to XML (#{converted.bytesize} bytes)")
+      class << self
+        ##
+        # Converts the object to XML format.
+        #
+        # @param object [Object] any Ruby object (Hash, Array, String, Symbol, etc.)
+        # @return [String] representing the object in XML
+        def call(object)
+          object_to_xml(object).tap do |converted|
+            Html2rss::Log.debug("#{self}: converted object to XML (#{converted.bytesize} bytes)")
+          end
         end
-      end
 
-      def object_to_xml(object)
-        case object
-        when Hash
-          hash_to_xml(object)
-        when Array
-          array_to_xml(object)
-        else
-          CGI.escapeHTML(object.to_s)
+        private
+
+        def object_to_xml(object)
+          case object
+          when Hash
+            hash_to_xml(object)
+          when Array
+            array_to_xml(object)
+          else
+            CGI.escapeHTML(object.to_s)
+          end
         end
-      end
-      private_class_method :object_to_xml
 
-      def hash_to_xml(object)
-        prefix, suffix = OBJECT_TO_XML_TAGS[:hash]
-        inner_xml = object.each_with_object(+'') do |(key, value), str|
-          str << "<#{key}>#{object_to_xml(value)}</#{key}>"
-        end
+        def hash_to_xml(object)
+          prefix, suffix = OBJECT_TO_XML_TAGS[:hash]
+          inner_xml = object.each_with_object(+'') do |(key, value), str|
+            str << "<#{key}>#{object_to_xml(value)}</#{key}>"
+          end
 
-        "#{prefix}#{inner_xml}#{suffix}"
-      end
-      private_class_method :hash_to_xml
+          "#{prefix}#{inner_xml}#{suffix}"
+        end
 
-      def array_to_xml(object)
-        prefix, suffix = OBJECT_TO_XML_TAGS[:array]
-        inner_xml = object.each_with_object(+'') { |value, str| str << object_to_xml(value) }
+        def array_to_xml(object)
+          prefix, suffix = OBJECT_TO_XML_TAGS[:array]
+          inner_xml = object.each_with_object(+'') { |value, str| str << object_to_xml(value) }
 
-        "#{prefix}#{inner_xml}#{suffix}"
+          "#{prefix}#{inner_xml}#{suffix}"
+        end
       end
-      private_class_method :array_to_xml
     end
   end
 end

data/lib/html2rss/selectors/post_processors/gsub.rb

@@ -101,7 +101,7 @@ module Html2rss
 
           source = regexp_source(string)
 
-          expression = Regexp::Parser.parse(source, options: ::Regexp::EXTENDED | ::Regexp::IGNORECASE)
+          expression = Regexp::Parser.parse(source)
           raise ArgumentError, 'pattern contains nested quantifiers' if nested_quantifiers?(expression)
 
           expression.to_re
@@ -161,8 +161,7 @@ module Html2rss
         ##
         # @return [Regexp]
         def pattern
-          raw = @pattern
-          raw.is_a?(String) ? self.class.compiled_pattern(raw) : raw
+          self.class.compiled_pattern(@pattern)
         end
       end
     end

data/lib/html2rss/selectors/post_processors/parse_time.rb

@@ -67,11 +67,10 @@ module Html2rss
         def self.validate_args!(_value, context)
           time_zone_value = context.time_zone
 
-          if time_zone_value.nil? || time_zone_value.empty?
-            raise ArgumentError, 'time_zone cannot be nil or empty', [], cause: nil
-          end
+          raise ArgumentError, 'time_zone cannot be nil or empty', [], cause: nil if time_zone_value.nil?
 
           assert_type(time_zone_value, String, :time_zone, context:)
+          raise ArgumentError, 'time_zone cannot be nil or empty', [], cause: nil if time_zone_value.empty?
         end
 
         ##
@@ -89,7 +88,7 @@ module Html2rss
         private
 
         def parse_time(string, time_zone)
-          components = Date._parse(string, false)
+          components = Date._parse(string)
           if components[:zone] || components.key?(:offset)
             Time.parse(string)
           else
@@ -104,7 +103,7 @@ module Html2rss
             components[:mday],
             components[:hour] || 0,
             components[:min] || 0,
-            (components[:sec] || 0) + (components[:sec_fraction] || 0)
+            components[:sec] || 0
           )
         end
       end

data/lib/html2rss/selectors/schema_export.rb

@@ -8,72 +8,72 @@ module Html2rss
     # Each registry class owns +DESCRIPTION+, +EXAMPLES+, and optional +OPTIONS+;
     # this module is the export adapter only.
     module SchemaExport
-      module_function
-
-      ##
-      # @param name [Symbol, String] registry key
-      # @param klass [Class] post-processor class
-      # @return [Hash{Symbol => Object}] JSON Schema object for one post-processor
-      def for_post_processor(name:, klass:)
-        name = name.to_s
-        {
-          type: 'object',
-          title: name,
-          description: klass::DESCRIPTION,
-          examples: klass::EXAMPLES,
-          properties: post_processor_properties(name, klass),
-          required: post_processor_required(klass),
-          additionalProperties: true
-        }
-      end
+      class << self
+        ##
+        # @param name [Symbol, String] registry key
+        # @param klass [Class] post-processor class
+        # @return [Hash{Symbol => Object}] JSON Schema object for one post-processor
+        def for_post_processor(name:, klass:)
+          name = name.to_s
+          {
+            type: 'object',
+            title: name,
+            description: klass::DESCRIPTION,
+            examples: klass::EXAMPLES,
+            properties: post_processor_properties(name, klass),
+            required: post_processor_required(klass),
+            additionalProperties: true
+          }
+        end
 
-      ##
-      # @param name [Symbol, String] registry key
-      # @param klass [Class] extractor class
-      # @return [Hash{Symbol => Object}] JSON Schema for one extractor name
-      def for_extractor(name:, klass:)
-        name = name.to_s
-        {
-          type: 'string',
-          const: name,
-          title: name,
-          description: klass::DESCRIPTION,
-          examples: klass::EXAMPLES
-        }
-      end
+        ##
+        # @param name [Symbol, String] registry key
+        # @param klass [Class] extractor class
+        # @return [Hash{Symbol => Object}] JSON Schema for one extractor name
+        def for_extractor(name:, klass:)
+          name = name.to_s
+          {
+            type: 'string',
+            const: name,
+            title: name,
+            description: klass::DESCRIPTION,
+            examples: klass::EXAMPLES
+          }
+        end
 
-      ##
-      # @param name [String] registry key
-      # @param klass [Class]
-      # @return [Hash{Symbol => Hash}]
-      def post_processor_properties(name, klass)
-        properties = { name: { type: 'string', const: name } }
-        OptionSpec.for(klass).each do |spec|
-          properties[spec.name] = property_schema_for(spec.type)
+        ##
+        # @param name [String] registry key
+        # @param klass [Class]
+        # @return [Hash{Symbol => Hash}]
+        def post_processor_properties(name, klass)
+          properties = { name: { type: 'string', const: name } }
+          OptionSpec.for(klass).each do |spec|
+            properties[spec.name] = property_schema_for(spec.type)
+          end
+          properties
         end
-        properties
-      end
 
-      ##
-      # @param klass [Class]
-      # @return [Array<String>]
-      def post_processor_required(klass)
-        required = ['name']
-        required + OptionSpec.for(klass).select(&:required).map { |spec| spec.name.to_s }
-      end
+        ##
+        # @param klass [Class]
+        # @return [Array<String>]
+        def post_processor_required(klass)
+          required = ['name']
+          required + OptionSpec.for(klass).select(&:required).map { |spec| spec.name.to_s }
+        end
 
-      ##
-      # @param ruby_type [Class, Array<Class>]
-      # @return [Hash{Symbol => String, Array<String>}]
-      def property_schema_for(ruby_type)
-        { type: json_type_for(ruby_type) }
-      end
+        ##
+        # @param ruby_type [Class, Array<Class>]
+        # @return [Hash{Symbol => String, Array<String>}]
+        def property_schema_for(ruby_type)
+          { type: json_type_for(ruby_type) }
+        end
 
-      ##
-      # @param ruby_type [Class, Array<Class>]
-      # @return [String, Array<String>]
-      def json_type_for(ruby_type)
-        OptionSpec.json_type_for(ruby_type)
+        ##
+        # @param ruby_type [Class, Array<Class>]
+        # @return [String, Array<String>]
+        def json_type_for(ruby_type)
+          OptionSpec.json_type_for(ruby_type)
+        end
       end
     end
   end

data/lib/html2rss/selectors.rb

@@ -132,7 +132,7 @@ module Html2rss
 
       field = @compiled_fields[name]
       return select_compiled(field, scope) if field
-      return select_field(name, scope:, config: nil) if name == :url
+      return select_field(name, scope:) if name == :url
 
       raise InvalidSelectorName, "Selector for '#{name}' is not defined."
     end
@@ -225,15 +225,11 @@ module Html2rss
       value.is_a?(Array) ? value : [value]
     end
 
-    def select_field(selector_key, scope:, config:)
-      return default_item_url(scope.item, scope.base_url) if fallback_url_selector?(selector_key, config, scope.item)
-      raise InvalidSelectorName, "Selector for '#{selector_key}' is not defined." if config.nil?
+    # Only reached for +:url+ without a configured selector, so the item must be an anchor.
+    def select_field(selector_key, scope:)
+      return default_item_url(scope.item, scope.base_url) if anchor_element?(scope.item)
 
-      if SPECIAL_ATTRIBUTES.member?(selector_key)
-        select_special(selector_key, scope:, config:)
-      else
-        select_regular(selector_key, scope:, config:)
-      end
+      raise InvalidSelectorName, "Selector for '#{selector_key}' is not defined."
     end
 
     def select_compiled(field, scope)
@@ -327,6 +323,7 @@ module Html2rss
     end
 
     def post_process(scope, value, post_process_steps)
+      value = step_input(value)
       post_process_steps.each do |step_config|
         value = PostProcessors.call(step_config[:name], value, scope.context_for(step_config:))
       end
@@ -334,9 +331,15 @@ module Html2rss
       value
     end
 
+    # Post-processors work on text, so a URL value (from the href extractor) enters them as a String.
+    def step_input(value)
+      value.is_a?(Url) ? value.to_s : value
+    end
+
     def apply_compiled_post_process(scope, value, steps)
       return value unless value && steps
 
+      value = step_input(value)
       steps.each do |klass, step_config|
         value = klass.new(value, scope.context_for(step_config:)).call
       end
@@ -375,15 +378,8 @@ module Html2rss
     end
 
     def extract_category_text(category)
-      text = case category
-             when Nokogiri::XML::Node, Nokogiri::XML::NodeSet
-               Html2rss::Html::Navigator.extract_visible_text(category)
-             else
-               category&.to_s
-             end
-
-      stripped = text&.strip
-      stripped unless stripped.nil? || stripped.empty?
+      stripped = category.to_s.strip
+      stripped unless stripped.empty?
     end
 
     def node_set_with_multiple_elements?(nodes)

data/lib/html2rss/sst/node.rb

@@ -2,7 +2,7 @@
 
 module Html2rss
   module SST
-    Node = Data.define(:name, :attrs, :own_text, :children, :tag_path)
+    Node = Data.define(:name, :attrs, :own_text, :tails, :children, :tag_path)
 
     ##
     # Immutable simplified semantic tree node. Predicates and traversal live on
@@ -11,12 +11,13 @@ module Html2rss
       ##
       # @param name [Symbol, String]
       # @param attrs [Attrs, nil]
-      # @param own_text [String, nil]
+      # @param own_text [String, nil] all direct text, in document order
+      # @param tails [Array<String>, nil] direct text following each child, parallel to +children+
       # @param children [Array<Node>, nil]
       # @param tag_path [String, nil] index-free ancestry path for list clustering
       # @return [Node]
       # @raise [ArgumentError] on invalid name/attrs/children
-      def self.build(name:, attrs: nil, own_text: nil, children: nil, tag_path: nil)
+      def self.build(name:, attrs: nil, own_text: nil, children: nil, tag_path: nil, tails: nil) # rubocop:disable Metrics/ParameterLists
         attrs_value = coerce_attrs(attrs)
         kids = coerce_children(children)
 
@@ -24,6 +25,7 @@ module Html2rss
           name: coerce_name(name),
           attrs: attrs_value,
           own_text: own_text.to_s.freeze,
+          tails: Array(tails).map { _1.to_s.freeze }.freeze,
           children: kids,
           tag_path: tag_path.to_s.freeze
         )
@@ -52,6 +54,10 @@ module Html2rss
       end
       private_class_method :coerce_children
 
+      ##
+      # @return [String] direct text before the first child
+      def lead_text = own_text.delete_suffix(tails.join)
+
       ##
       # @return [Boolean]
       def link?

data/lib/html2rss/sst/normalizer.rb

@@ -8,9 +8,6 @@ module Html2rss
     # Sole Nokogiri consumer on the heuristic auto-source path. Builds an
     # immutable SST::Document with parent/depth/chrome indices.
     class Normalizer # rubocop:disable Metrics/ClassLength
-      # Raised when no SST nodes survive normalization for the chosen root.
-      class EmptyTree < ArgumentError; end
-
       # Hard ceiling for SST node allocations; beyond this we degrade.
       MAX_NODES = 5_000
 
@@ -64,10 +61,10 @@ module Html2rss
         ##
         # @param input [String, Nokogiri::HTML::Document, Nokogiri::XML::Node]
         #   HTML string or already-parsed node (String is parsed once here)
-        # @return [Document]
-        # @raise [ArgumentError] when input is nil or unsupported
+        # @return [Document, nil] +nil+ when no SST nodes survive normalization
+        # @raise [InvalidOption] when input is nil or unsupported
         def call(input)
-          raise ArgumentError, 'input is required' if input.nil?
+          raise InvalidOption, 'input is required' if input.nil?
 
           new(coerce(input)).call
         end
@@ -81,7 +78,7 @@ module Html2rss
           when String then Nokogiri::HTML(input)
           when Nokogiri::XML::Node then input
           else
-            raise ArgumentError, "expected String or Nokogiri::XML::Node, got #{input.class}"
+            raise InvalidOption, "expected String or Nokogiri::XML::Node, got #{input.class}"
           end
         end
       end
@@ -97,11 +94,11 @@ module Html2rss
       end
 
       ##
-      # @return [Document]
+      # @return [Document, nil] +nil+ when no SST nodes survive normalization
       def call
         root_nk = resolve_root_nk
-        root = normalize_element(root_nk, parent: nil, depth: 0, path: '', chrome: false)
-        raise EmptyTree, 'SST Normalizer produced an empty tree' unless root
+        root = normalize_element(root_nk, depth: 0, path: '', chrome: false)
+        return unless root
 
         index = Index.new(root:, parents: @parents, depths: @depths, ignored_chrome: @ignored_chrome)
         Document.build(root:, index:, degraded: @degraded, node_count: @node_count)
@@ -150,15 +147,15 @@ module Html2rss
       end
 
       # rubocop:disable-next Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
-      def normalize_element(nk_node, parent:, depth:, path:, chrome:)
+      def normalize_element(nk_node, depth:, path:, chrome:)
         return unless nk_node.element?
 
-        tag = Html2rss::Html::Probe.tag(nk_node)
+        tag = Html::Probe.tag(nk_node)
         return if STRIPPED_TAGS.include?(tag)
 
         if @node_count >= MAX_NODES && !@degraded
           @degraded = true
-          Html2rss::Log.warn({
+          Log.warn({
             event_name: 'sst.normalizer.max_nodes',
             max_nodes: MAX_NODES,
             message: "sst.normalizer MAX_NODES=#{MAX_NODES} breached; degrading to semantic-tag-only"
@@ -168,21 +165,15 @@ module Html2rss
         return if @degraded && !SEMANTIC_DEGRADE_TAGS.include?(tag)
 
         @node_count += 1
-        tag_path = path.empty? ? "/#{tag}" : "#{path}/#{tag}"
+        tag_path = "#{path}/#{tag}"
         attrs = extract_attrs(nk_node)
-        own_text = direct_text(nk_node)
         chrome_here = chrome || IGNORED_CONTAINER_TAGS.include?(tag)
 
-        children = nk_node.element_children.filter_map do |child|
-          normalize_element(child, parent: :pending, depth: depth + 1, path: tag_path, chrome: chrome_here)
-        end.freeze
+        children, lead, tails = normalize_children(nk_node, depth:, path: tag_path, chrome: chrome_here)
 
-        node = Node.build(name: tag, attrs:, own_text:, children:, tag_path:)
-        @parents[node] = parent == :pending ? nil : parent
+        node = Node.build(name: tag, attrs:, own_text: [lead, *tails].join, children:, tag_path:, tails:)
         @depths[node] = depth
         @ignored_chrome[node] = chrome_here
-
-        # Fix parent pointers now that node exists.
         children.each { |child| @parents[child] = node }
 
         node
@@ -213,8 +204,18 @@ module Html2rss
         end
       end
 
-      def direct_text(nk_node)
-        nk_node.children.select(&:text?).map(&:text).join
+      # Walks child nodes once, keeping direct text in the slot of the preceding kept child
+      # (slot 0 is the lead text) so text order survives the element-only tree.
+      def normalize_children(nk_node, depth:, path:, chrome:)
+        children = []
+        slots = [+'']
+        nk_node.children.each do |child|
+          next slots.last << child.text if child.text?
+
+          node = normalize_element(child, depth: depth + 1, path:, chrome:)
+          (children << node) && (slots << +'') if node
+        end
+        [children, slots.first, slots.drop(1)]
       end
     end
   end

data/lib/html2rss/sst/text.rb

@@ -5,77 +5,73 @@ module Html2rss
     ##
     # Visible-text extraction for SST nodes (port of Navigator::TextExtractor).
     module Text
-      module_function
+      class << self
+        ##
+        # @param node [Node]
+        # @param separator [String]
+        # @param exclude [Array<Node>, nil]
+        # @return [String, nil]
+        def extract(node, separator: ' ', exclude: nil)
+          raise ArgumentError, 'node is required' unless node
 
-      ##
-      # @param node [Node]
-      # @param separator [String]
-      # @param exclude [Array<Node>, nil]
-      # @return [String, nil]
-      def extract(node, separator: ' ', exclude: nil) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
-        raise ArgumentError, 'node is required' unless node
-
-        excluded = exclude&.each_with_object({}.compare_by_identity) { |n, h| h[n] = true }
-        if node.children.empty?
-          text = node.own_text.to_s.gsub(/\s+/, ' ').strip
-          return text.empty? ? nil : text
+          excluded = exclude&.each_with_object({}.compare_by_identity) { |n, h| h[n] = true }
+          join_fragments(fragments(node, separator, excluded), separator)
         end
 
-        parts = collect_parts(node, separator, excluded)
-        own = node.own_text.to_s.gsub(/\s+/, ' ').strip
-        parts = [own, *parts] unless own.empty?
-        return if parts.empty?
-
-        parts.join.squeeze(' ').strip
-      end
+        private
 
-      def collect_parts(node, separator, excluded)
-        last_block = false
-        node.children.each_with_object([]) do |child, parts|
-          next if excluded&.[](child)
-          next unless visible_child?(child)
+        # Ordered [text, block?] pairs: lead text, then each child followed by the text after it.
+        def fragments(node, separator, excluded)
+          pairs = [[squish(node.lead_text), false]]
+          node.children.zip(node.tails).each do |child, tail|
+            pairs << child_text_and_block(child, separator, excluded) if shown?(child, excluded)
+            pairs << [squish(tail), false]
+          end
+          pairs
+        end
 
-          text, block = child_text_and_block(child, separator, excluded)
-          next if text.empty?
+        def join_fragments(pairs, separator)
+          parts = []
+          last_block = false
+          pairs.each do |text, block|
+            next if text.empty?
 
-          append_separator!(parts, separator, block, last_block)
-          parts << text
-          last_block = block
+            append_separator!(parts, separator, block, last_block)
+            parts << text
+            last_block = block
+          end
+          parts.join unless parts.empty?
         end
-      end
-      module_function :collect_parts
-      private_class_method :collect_parts
 
-      def child_text_and_block(child, separator, excluded)
-        text = if child.children.empty?
-                 child.own_text.gsub(/\s+/, ' ').strip
-               else
-                 extract(child, separator:, exclude: excluded&.keys).to_s.strip
-               end
-        text = "- #{text}" if child.name == :li && !text.empty?
-        [text, Tags::BLOCK_NAMES.include?(child.name)]
-      end
-      module_function :child_text_and_block
-      private_class_method :child_text_and_block
+        def shown?(child, excluded) = !excluded&.[](child) && visible_child?(child)
 
-      def append_separator!(parts, separator, is_block, last_was_block)
-        return if parts.empty?
+        def squish(text) = text.to_s.gsub(/\s+/, ' ').strip
 
-        parts << if is_block || last_was_block
-                   (separator == ' ' ? "\n" : separator)
+        def child_text_and_block(child, separator, excluded)
+          text = if child.children.empty?
+                   squish(child.own_text)
                  else
-                   ' '
+                   extract(child, separator:, exclude: excluded&.keys).to_s.strip
                  end
-      end
-      module_function :append_separator!
-      private_class_method :append_separator!
+          text = "- #{text}" if child.name == :li && !text.empty?
+          [text, Tags::BLOCK_NAMES.include?(child.name)]
+        end
+
+        def append_separator!(parts, separator, is_block, last_was_block)
+          return if parts.empty?
 
-      def visible_child?(node)
-        !Tags::INVISIBLE_NAMES.include?(node.name) &&
-          !(node.name == :a && node.attrs.href&.start_with?('#'))
+          parts << if is_block || last_was_block
+                     (separator == ' ' ? "\n" : separator)
+                   else
+                     ' '
+                   end
+        end
+
+        def visible_child?(node)
+          !Tags::INVISIBLE_NAMES.include?(node.name) &&
+            !(node.name == :a && node.attrs.href&.start_with?('#'))
+        end
       end
-      module_function :visible_child?
-      private_class_method :visible_child?
     end
   end
 end