xml_node_stream 2.0.0 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c48b75cdd974d95227eaccd1748c73d6e04d0793d6d9fe6e5fd0a2c9dfecfb32
-  data.tar.gz: 47c1effbbe39895d2cac07f545904a27760c0606406daf0b2abfaf5139f2f4ae
+  metadata.gz: abb1e46f305cb09b383415a0fab348290c2058c98282360ceef41b73f0e6a0c6
+  data.tar.gz: fa5503fb2b14409bae9c92eadbf0cf525ce683e137709723b85f7622a599a831
 SHA512:
-  metadata.gz: e50b6c273e787053f8020731c9e11830a19f3e047ce230e417fa927ae51cd274ca64b8921489a98831ba77aeef34d050fd309496c1018ee482fcf59abca671f7
-  data.tar.gz: d8e2040f6d4117b29054813eb7741bd15567d374736df80a1cbaeff25fe9692848b2de7e44f0bfec34234d9e6bb5229bfb120023c8a5f7b057a614a0b8dc9fc9
+  metadata.gz: c6059c96f122e4e2312b7283b49a11a27c621833df66e3c3434bd330db838c13264c514b7523863d48f5ea364e9c63b0c7a4b03eeec936e7e15fb034fbb2edaa
+  data.tar.gz: 91b44394a4983f916f11ea4366493ed5f04edbbdc6a1ee3ded867d717973592f64a23343fd3cfb88c7fb607ea68f9c0a9b00003ae5fe6ac6d586685a17eb02ed

data/CHANGELOG.md

@@ -4,6 +4,25 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 2.0.1
+
+### Fixed
+
+- Added missing `stringio` require which caused a `NameError` when parsing an XML string on Ruby versions where the standard library no longer loads it implicitly.
+- XPath selectors no longer drop nested elements with the same name (e.g. `.//a/a` now matches `a` elements nested inside other `a` elements).
+- XPath selectors with a leading `//` now correctly search all descendants of the document root instead of only children of the root element.
+- XPath selectors ending a `//` step with `*` (e.g. `book//*`) now match all descendants instead of skipping the first level.
+- The `text()` selector no longer removes duplicate values when different nodes contain identical text.
+- HTTP responses are now streamed incrementally instead of being read entirely into memory before parsing.
+
+### Changed
+
+- HTTP requests that return an error status now raise `XmlNodeStream::HttpError` instead of parsing the error response body as XML.
+
+### Added
+
+- HTTP redirects are now followed (up to 5) when parsing from a URL.
+
 ## 2.0.0
 
 ### Changed

data/VERSION

@@ -1 +1 @@
-2.0.0
+2.0.1

data/lib/xml_node_stream/http_stream.rb

@@ -1,14 +1,30 @@
 # frozen_string_literal: true
 
 require "net/http"
+require "fiber"
 
 module XmlNodeStream
+  # Error raised when an HTTP request returns an unsuccessful response.
+  class HttpError < StandardError
+    attr_reader :response
+
+    # @param message [String] the error message
+    # @param response [Net::HTTPResponse, nil] the HTTP response that caused the error
+    def initialize(message, response = nil)
+      super(message)
+      @response = response
+    end
+  end
+
   # IO-like wrapper for HTTP responses that allows streaming
   class HttpStream
     # Default timeout values in seconds
     DEFAULT_OPEN_TIMEOUT = 10
     DEFAULT_READ_TIMEOUT = 60
 
+    # Maximum number of redirects to follow
+    MAX_REDIRECTS = 5
+
     # Create a new HttpStream.
     #
     # @param uri [URI] the URI to stream from
@@ -16,15 +32,13 @@ module XmlNodeStream
     # @param read_timeout [Integer] read timeout in seconds (default 60)
     def initialize(uri, open_timeout: DEFAULT_OPEN_TIMEOUT, read_timeout: DEFAULT_READ_TIMEOUT)
       @uri = uri
-      @http = Net::HTTP.new(uri.host, uri.port)
-      @http.use_ssl = (uri.scheme == "https")
-      @http.open_timeout = open_timeout
-      @http.read_timeout = read_timeout
-      @request = Net::HTTP::Get.new(uri.request_uri)
+      @open_timeout = open_timeout
+      @read_timeout = read_timeout
+      @http = nil
       @buffer = +""
       @eof = false
       @response = nil
-      @body_reader = nil
+      @fiber = nil
     end
 
     # Read data from the stream.
@@ -108,7 +122,7 @@ module XmlNodeStream
       return nil if @buffer.empty?
 
       line = @buffer
-      @buffer = ""
+      @buffer = +""
       line
     end
 
@@ -148,32 +162,67 @@ module XmlNodeStream
     private
 
     def ensure_response_started
-      return if @response
+      return if @fiber
+
+      # The request runs inside a fiber so that the response body can be consumed incrementally.
+      # Net::HTTP#request with a block streams the body via read_body; each chunk is yielded out
+      # of the fiber and pulled on demand by read_chunk without buffering the whole response.
+      @fiber = Fiber.new do
+        uri = @uri
+        redirects = 0
+
+        loop do
+          connect(uri)
+          request = Net::HTTP::Get.new(uri.request_uri)
+          redirect_location = nil
+
+          @http.request(request) do |response|
+            @response = response
+
+            case response
+            when Net::HTTPRedirection
+              redirects += 1
+              if redirects > MAX_REDIRECTS
+                raise HttpError.new("Too many redirects requesting #{@uri}", response)
+              end
+              redirect_location = response["Location"]
+              unless redirect_location
+                raise HttpError.new("Redirect without Location header from #{uri}", response)
+              end
+            when Net::HTTPSuccess
+              response.read_body do |chunk|
+                Fiber.yield(chunk) unless chunk.empty?
+              end
+            else
+              raise HttpError.new("HTTP error #{response.code} requesting #{uri}", response)
+            end
+          end
+
+          break unless redirect_location
+
+          uri = URI.join(uri.to_s, redirect_location)
+        end
 
-      @http.start unless @http.started?
-      @response = @http.request(@request)
-      @body_reader = @response.read_body
+        nil
+      end
+    end
+
+    def connect(uri)
+      close
+
+      @http = Net::HTTP.new(uri.host, uri.port)
+      @http.use_ssl = (uri.scheme == "https")
+      @http.open_timeout = @open_timeout
+      @http.read_timeout = @read_timeout
+      @http.start
     end
 
     def read_chunk
       return nil if @eof
 
-      if @body_reader.is_a?(String)
-        # Entire body was read at once
-        if @body_reader.empty?
-          @eof = true
-          return nil
-        end
-        # Simulate chunking for consistency
-        chunk = @body_reader.byteslice(0, 8192) || +""
-        @body_reader = @body_reader.byteslice(8192..-1) || +""
-        @eof = true if @body_reader.empty?
-        chunk
-      else
-        # Should not happen with webmock but handling for real HTTP
-        @eof = true
-        nil
-      end
+      chunk = (@fiber.alive? ? @fiber.resume : nil)
+      @eof = true if chunk.nil?
+      chunk
     end
   end
 end

data/lib/xml_node_stream/parser/libxml_parser.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 begin
-  require "libxml"
+  require "libxml-ruby"
 
   module XmlNodeStream
     class Parser

data/lib/xml_node_stream/parser.rb

@@ -9,8 +9,6 @@ module XmlNodeStream
   class Parser
     SUPPORTED_PARSERS = [:nokogiri, :libxml, :rexml]
 
-    @parser = :rexml
-
     class << self
       # Set the parser implementation. The parser argument should be one of :nokogiri, :libxml, or :rexml. If this method
       # is not called, it will default to :rexml which is the slowest choice possible. If you set the parser to one of the
@@ -84,7 +82,7 @@ module XmlNodeStream
         @loaded_parsers ||= {}
         klass = @loaded_parsers[class_symbol]
         unless klass
-          require File.expand_path(File.join(File.dirname(__FILE__), "parser", "#{class_symbol}_parser"))
+          require_relative "parser/#{class_symbol}_parser"
           class_name = "#{class_symbol.to_s.capitalize}Parser"
           klass = const_get(class_name)
           @loaded_parsers[class_symbol] = klass

data/lib/xml_node_stream/selector.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "set"
-
 module XmlNodeStream
   # Partial implementation of XPath selectors. Only abbreviated paths and the text() function are supported. The rest of XPath
   # is unecessary in the context of a Ruby application since XPath is also a programming language. If you really need an XPath
@@ -38,11 +36,10 @@ module XmlNodeStream
       matched = [node]
       @parts.each do |part_matchers|
         context = matched
-        context_set = context.to_set
         matched = []
 
         part_matchers.each do |matcher|
-          matched.concat(matcher.select(context, context_set))
+          matched.concat(matcher.select(context))
         end
 
         break if matched.empty?
@@ -66,15 +63,9 @@ module XmlNodeStream
       path_length = path.length
 
       while i < path_length
-        # Skip leading slash for absolute paths
-        if i == 0 && path[i] == "/"
-          parts << [Matcher.new("")]
-          i += 1
-          next
-        end
-
-        # Look for // (descendant operator)
+        # Look for // (descendant operator); a leading // searches from the document root
         if i < path_length - 1 && path[i] == "/" && path[i + 1] == "/"
+          parts << [Matcher.new("")] if i == 0
           i += 2
           # Check if there's a name after //
           name_match = path[i..].match(/\A([a-zA-Z_][\w-]*)/)
@@ -84,12 +75,23 @@ module XmlNodeStream
           elsif i >= path_length
             # // at end of path is invalid
             raise ArgumentError, "Invalid XPath pattern: #{path} (// cannot be at end)"
+          elsif path[i] == "*" && (i + 1 >= path_length || path[i + 1] == "/")
+            # //* selects all descendants
+            parts << [Matcher.new("%")]
+            i += 1
           else
             parts << [Matcher.new("%")]
           end
           next
         end
 
+        # Skip leading slash for absolute paths
+        if i == 0 && path[i] == "/"
+          parts << [Matcher.new("")]
+          i += 1
+          next
+        end
+
         # Regular path segment
         if path[i] == "/"
           i += 1
@@ -139,43 +141,40 @@ module XmlNodeStream
       # @param path [String] the path pattern to match
       def initialize(path)
         @path = path
+        @text = (path == "text()")
         @extractor = case path
         when "text()"
-          lambda { |node, context_set| node.value unless node.value.nil? || node.value.empty? }
+          lambda { |node| node.value unless node.value.nil? || node.value.empty? }
         when "%"
-          lambda { |node, context_set| node.descendants }
+          lambda { |node| node.descendants }
         when "*"
-          lambda { |node, context_set| node.children }
+          lambda { |node| node.children }
         when "."
-          lambda { |node, context_set| node }
+          lambda { |node| node }
         when ".."
-          lambda { |node, context_set| node.parent || [] }
+          lambda { |node| node.parent || [] }
         when ""
-          lambda { |node, context_set|
+          lambda { |node|
             root = Node.new(nil)
             root.children << node.root
             root
           }
         when /^%(.+)$/  # descendants with name filter: %name
           name = $1
-          lambda { |node, context_set| node.descendants.select { |d| d.name == name } }
+          lambda { |node| node.descendants.select { |d| d.name == name } }
         else
-          lambda { |node, context_set|
-            # Only return children matching the name
-            # Don't include children that are already in the context
-            node.children.select { |child| child.name == @path && !context_set&.include?(child) }
-          }
+          lambda { |node| node.children.select { |child| child.name == @path } }
         end
       end
 
       # Select all nodes that match a partial path.
       #
       # @param context_nodes [Array<Node>] the nodes to select from
-      # @param context_set [Set<Node>, nil] optional set version of context_nodes for performance
       # @return [Array<Node>] the matching nodes
-      def select(context_nodes, context_set = nil)
-        context_set ||= context_nodes.to_set
-        context_nodes.collect { |node| @extractor.call(node, context_set) if node.is_a?(Node) }.flatten.compact.uniq
+      def select(context_nodes)
+        results = context_nodes.collect { |node| @extractor.call(node) if node.is_a?(Node) }.flatten.compact
+        # Text values from different nodes can legitimately be equal, so only dedupe nodes.
+        @text ? results : results.uniq
       end
     end
   end

data/lib/xml_node_stream.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "pathname"
+require "stringio"
 require "uri"
 
 require_relative "xml_node_stream/node"

data/xml_node_stream.gemspec

@@ -35,5 +35,5 @@ Gem::Specification.new do |spec|
 
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler"
+  spec.required_ruby_version = ">= 2.7"
 end

metadata

@@ -1,28 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: xml_node_stream
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.0.1
 platform: ruby
 authors:
 - Brian Durand
 bindir: bin
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
-dependencies:
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
+dependencies: []
 email:
 - bbdurand@gmail.com
 executables: []
@@ -57,7 +43,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="