rexml 3.3.2 → 3.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 70ccd1465a05dba3d53dcfc4a98e76dec865a4f6ac833b954aff4234bce6c255
-  data.tar.gz: 53f43fab8f531e0ba7461ce091e5eae6bec27b12e9139450c7b3e748b4eeacdc
+  metadata.gz: 5e5e2317fb4a12cc855de221be85a9d62c2966c4997ead5a4ede3600561d5ede
+  data.tar.gz: a2b8f326e706211d00a9a8446b84ebd658c9cb82a4f7c98e5760ed2b10d8866c
 SHA512:
-  metadata.gz: b46818d79ae57075c4e0bd620802e82c6958dddc7da1b182504c3fdc16685c887ac0ddd6a4838a080483abba330839e9ef4b2db22cc81b9eae3eac71ac14c965
-  data.tar.gz: 1e5205905eb435c02038dd0539de22472f5364ffc47635f13a1752cb79a423dcca558fb47394ac5d624b358e779b07cbcafedfd06b99742026856f9988109976
+  metadata.gz: 2d26167dc282f9ff928b263927a9f003bddb6591a938b43dfddcd8a2fe2c1ddb4f931f09ec52dd3bf1912953365dcaafafb359bdd6dba1f9ca33a55bbc62ec5b
+  data.tar.gz: b3216114c5978079b102a6492cd0d8afde5eaf0af5ebc803873dc7a9ad4dc9afa785000c923f296b88c3b5c663a543348f65a3734801149f792518a1bcb5844c

data/NEWS.md

@@ -1,5 +1,39 @@
 # News
 
+## 3.3.3 - 2024-08-01 {#version-3-3-3}
+
+### Improvements
+
+  * Added support for detecting invalid XML that has unsupported
+    content before root element
+    * GH-184
+    * Patch by NAITOH Jun.
+
+  * Added support for `REXML::Security.entity_expansion_limit=` and
+    `REXML::Security.entity_expansion_text_limit=` in SAX2 and pull
+    parsers
+    * GH-187
+    * Patch by NAITOH Jun.
+
+  * Added more tests for invalid XMLs.
+    * GH-183
+    * Patch by Watson.
+
+  * Added more performance tests.
+    * Patch by Watson.
+
+  * Improved parse performance.
+    * GH-186
+    * Patch by tomoya ishida.
+
+### Thanks
+
+  * NAITOH Jun
+
+  * Watson
+
+  * tomoya ishida
+
 ## 3.3.2 - 2024-07-16 {#version-3-3-2}
 
 ### Improvements
@@ -15,6 +49,9 @@
     * GH-172
     * GH-173
     * GH-174
+    * GH-175
+    * GH-176
+    * GH-177
     * Patch by Watson.
 
   * Added support for raising a parse exception when an XML has extra

data/lib/rexml/parsers/baseparser.rb

@@ -124,19 +124,10 @@ module REXML
       }
 
       module Private
-        # Terminal requires two or more letters.
-        INSTRUCTION_TERM = "?>"
-        COMMENT_TERM = "-->"
-        CDATA_TERM = "]]>"
-        DOCTYPE_TERM = "]>"
-        # Read to the end of DOCTYPE because there is no proper ENTITY termination
-        ENTITY_TERM = DOCTYPE_TERM
-
-        INSTRUCTION_END = /#{NAME}(\s+.*?)?\?>/um
         TAG_PATTERN = /((?>#{QNAME_STR}))\s*/um
         CLOSE_PATTERN = /(#{QNAME_STR})\s*>/um
         ATTLISTDECL_END = /\s+#{NAME}(?:#{ATTDEF})*\s*>/um
-        NAME_PATTERN = /\s*#{NAME}/um
+        NAME_PATTERN = /#{NAME}/um
         GEDECL_PATTERN = "\\s+#{NAME}\\s+#{ENTITYDEF}\\s*>"
         PEDECL_PATTERN = "\\s+(%)\\s+#{NAME}\\s+#{PEDEF}\\s*>"
         ENTITYDECL_PATTERN = /(?:#{GEDECL_PATTERN})|(?:#{PEDECL_PATTERN})/um
@@ -154,6 +145,7 @@ module REXML
         self.stream = source
         @listeners = []
         @prefixes = Set.new
+        @entity_expansion_count = 0
       end
 
       def add_listener( listener )
@@ -161,6 +153,7 @@ module REXML
       end
 
       attr_reader :source
+      attr_reader :entity_expansion_count
 
       def stream=( source )
         @source = SourceFactory.create_from( source )
@@ -248,10 +241,10 @@ module REXML
         if @document_status == nil
           start_position = @source.position
           if @source.match("<?", true)
-            return process_instruction(start_position)
+            return process_instruction
           elsif @source.match("<!", true)
             if @source.match("--", true)
-              md = @source.match(/(.*?)-->/um, true, term: Private::COMMENT_TERM)
+              md = @source.match(/(.*?)-->/um, true)
               if md.nil?
                 raise REXML::ParseException.new("Unclosed comment", @source)
               end
@@ -318,7 +311,11 @@ module REXML
               raise REXML::ParseException.new( "Bad ELEMENT declaration!", @source ) if md.nil?
               return [ :elementdecl, "<!ELEMENT" + md[1] ]
             elsif @source.match("ENTITY", true)
-              match = [:entitydecl, *@source.match(Private::ENTITYDECL_PATTERN, true, term: Private::ENTITY_TERM).captures.compact]
+              match_data = @source.match(Private::ENTITYDECL_PATTERN, true)
+              unless match_data
+                raise REXML::ParseException.new("Malformed entity declaration", @source)
+              end
+              match = [:entitydecl, *match_data.captures.compact]
               ref = false
               if match[1] == '%'
                 ref = true
@@ -383,14 +380,14 @@ module REXML
                 raise REXML::ParseException.new(message, @source)
               end
               return [:notationdecl, name, *id]
-            elsif md = @source.match(/--(.*?)-->/um, true, term: Private::COMMENT_TERM)
+            elsif md = @source.match(/--(.*?)-->/um, true)
               case md[1]
               when /--/, /-\z/
                 raise REXML::ParseException.new("Malformed comment", @source)
               end
               return [ :comment, md[1] ] if md
             end
-          elsif match = @source.match(/(%.*?;)\s*/um, true, term: Private::DOCTYPE_TERM)
+          elsif match = @source.match(/(%.*?;)\s*/um, true)
             return [ :externalentity, match[1] ]
           elsif @source.match(/\]\s*>/um, true)
             @document_status = :after_doctype
@@ -430,7 +427,7 @@ module REXML
               #STDERR.puts "SOURCE BUFFER = #{source.buffer}, #{source.buffer.size}"
               raise REXML::ParseException.new("Malformed node", @source) unless md
               if md[0][0] == ?-
-                md = @source.match(/--(.*?)-->/um, true, term: Private::COMMENT_TERM)
+                md = @source.match(/--(.*?)-->/um, true)
 
                 if md.nil? || /--|-\z/.match?(md[1])
                   raise REXML::ParseException.new("Malformed comment", @source)
@@ -438,13 +435,13 @@ module REXML
 
                 return [ :comment, md[1] ]
               else
-                md = @source.match(/\[CDATA\[(.*?)\]\]>/um, true, term: Private::CDATA_TERM)
+                md = @source.match(/\[CDATA\[(.*?)\]\]>/um, true)
                 return [ :cdata, md[1] ] if md
               end
               raise REXML::ParseException.new( "Declarations can only occur "+
                 "in the doctype declaration.", @source)
             elsif @source.match("?", true)
-              return process_instruction(start_position)
+              return process_instruction
             else
               # Get the next tag
               md = @source.match(Private::TAG_PATTERN, true)
@@ -482,11 +479,15 @@ module REXML
             if text.chomp!("<")
               @source.position -= "<".bytesize
             end
-            if @tags.empty? and @have_root
+            if @tags.empty?
               unless /\A\s*\z/.match?(text)
-                raise ParseException.new("Malformed XML: Extra content at the end of the document (got '#{text}')", @source)
+                if @have_root
+                  raise ParseException.new("Malformed XML: Extra content at the end of the document (got '#{text}')", @source)
+                else
+                  raise ParseException.new("Malformed XML: Content at the start of the document (got '#{text}')", @source)
+                end
               end
-              return pull_event
+              return pull_event if @have_root
             end
             return [ :text, text ]
           end
@@ -505,7 +506,9 @@ module REXML
       def entity( reference, entities )
         value = nil
         value = entities[ reference ] if entities
-        if not value
+        if value
+          record_entity_expansion
+        else
           value = DEFAULT_ENTITIES[ reference ]
           value = value[2] if value
         end
@@ -544,12 +547,17 @@ module REXML
         }
         matches.collect!{|x|x[0]}.compact!
         if matches.size > 0
+          sum = 0
           matches.each do |entity_reference|
             unless filter and filter.include?(entity_reference)
               entity_value = entity( entity_reference, entities )
               if entity_value
                 re = Private::DEFAULT_ENTITIES_PATTERNS[entity_reference] || /&#{entity_reference};/
                 rv.gsub!( re, entity_value )
+                sum += rv.bytesize
+                if sum > Security.entity_expansion_text_limit
+                  raise "entity expansion has grown too large"
+                end
               else
                 er = DEFAULT_ENTITIES[entity_reference]
                 rv.gsub!( er[0], er[2] ) if er
@@ -562,6 +570,14 @@ module REXML
       end
 
       private
+
+      def record_entity_expansion
+        @entity_expansion_count += 1
+        if @entity_expansion_count > Security.entity_expansion_limit
+          raise "number of entity expansions exceeded, processing aborted."
+        end
+      end
+
       def need_source_encoding_update?(xml_declaration_encoding)
         return false if xml_declaration_encoding.nil?
         return false if /\AUTF-16\z/i =~ xml_declaration_encoding
@@ -571,14 +587,14 @@ module REXML
       def parse_name(base_error_message)
         md = @source.match(Private::NAME_PATTERN, true)
         unless md
-          if @source.match(/\s*\S/um)
+          if @source.match(/\S/um)
             message = "#{base_error_message}: invalid name"
           else
             message = "#{base_error_message}: name is missing"
           end
           raise REXML::ParseException.new(message, @source)
         end
-        md[1]
+        md[0]
       end
 
       def parse_id(base_error_message,
@@ -647,18 +663,24 @@ module REXML
         end
       end
 
-      def process_instruction(start_position)
-        match_data = @source.match(Private::INSTRUCTION_END, true, term: Private::INSTRUCTION_TERM)
-        unless match_data
-          message = "Invalid processing instruction node"
-          @source.position = start_position
-          raise REXML::ParseException.new(message, @source)
+      def process_instruction
+        name = parse_name("Malformed XML: Invalid processing instruction node")
+        if @source.match(/\s+/um, true)
+          match_data = @source.match(/(.*?)\?>/um, true)
+          unless match_data
+            raise ParseException.new("Malformed XML: Unclosed processing instruction", @source)
+          end
+          content = match_data[1]
+        else
+          content = nil
+          unless @source.match("?>", true)
+            raise ParseException.new("Malformed XML: Unclosed processing instruction", @source)
+          end
         end
-        if match_data[1] == "xml"
+        if name == "xml"
           if @document_status
             raise ParseException.new("Malformed XML: XML declaration is not at the start", @source)
           end
-          content = match_data[2]
           version = VERSION.match(content)
           version = version[1] unless version.nil?
           encoding = ENCODING.match(content)
@@ -673,7 +695,7 @@ module REXML
           standalone = standalone[1] unless standalone.nil?
           return [ :xmldecl, version, encoding, standalone ]
         end
-        [:processing_instruction, match_data[1], match_data[2]]
+        [:processing_instruction, name, content]
       end
 
       def parse_attributes(prefixes, curr_ns)

data/lib/rexml/parsers/pullparser.rb

@@ -47,6 +47,10 @@ module REXML
         @listeners << listener
       end
 
+      def entity_expansion_count
+        @parser.entity_expansion_count
+      end
+
       def each
         while has_next?
           yield self.pull

data/lib/rexml/parsers/sax2parser.rb

@@ -22,6 +22,10 @@ module REXML
         @parser.source
       end
 
+      def entity_expansion_count
+        @parser.entity_expansion_count
+      end
+
       def add_listener( listener )
         @parser.add_listener( listener )
       end

data/lib/rexml/rexml.rb

@@ -31,7 +31,7 @@
 module REXML
   COPYRIGHT = "Copyright © 2001-2008 Sean Russell <ser@germane-software.com>"
   DATE = "2008/019"
-  VERSION = "3.3.2"
+  VERSION = "3.3.3"
   REVISION = ""
 
   Copyright = COPYRIGHT

data/lib/rexml/source.rb

@@ -117,7 +117,7 @@ module REXML
     def ensure_buffer
     end
 
-    def match(pattern, cons=false, term: nil)
+    def match(pattern, cons=false)
       if cons
         @scanner.scan(pattern).nil? ? nil : @scanner
       else
@@ -204,10 +204,20 @@ module REXML
       end
     end
 
-    def read(term = nil)
+    def read(term = nil, min_bytes = 1)
       term = encode(term) if term
       begin
-        @scanner << readline(term)
+        str = readline(term)
+        @scanner << str
+        read_bytes = str.bytesize
+        begin
+          while read_bytes < min_bytes
+            str = readline(term)
+            @scanner << str
+            read_bytes += str.bytesize
+          end
+        rescue IOError
+        end
         true
       rescue Exception, NameError
         @source = nil
@@ -237,10 +247,9 @@ module REXML
       read if @scanner.eos? && @source
     end
 
-    # Note: When specifying a string for 'pattern', it must not include '>' except in the following formats:
-    # - ">"
-    # - "XXX>" (X is any string excluding '>')
-    def match( pattern, cons=false, term: nil )
+    def match( pattern, cons=false )
+      # To avoid performance issue, we need to increase bytes to read per scan
+      min_bytes = 1
       while true
         if cons
           md = @scanner.scan(pattern)
@@ -250,7 +259,8 @@ module REXML
         break if md
         return nil if pattern.is_a?(String)
         return nil if @source.nil?
-        return nil unless read(term)
+        return nil unless read(nil, min_bytes)
+        min_bytes *= 2
       end
 
       md.nil? ? nil : @scanner

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: rexml
 version: !ruby/object:Gem::Version
-  version: 3.3.2
+  version: 3.3.3
 platform: ruby
 authors:
 - Kouhei Sutou
 bindir: bin
 cert_chain: []
-date: 2024-07-16 00:00:00.000000000 Z
+date: 2024-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: strscan
@@ -116,7 +116,7 @@ homepage: https://github.com/ruby/rexml
 licenses:
 - BSD-2-Clause
 metadata:
-  changelog_uri: https://github.com/ruby/rexml/releases/tag/v3.3.2
+  changelog_uri: https://github.com/ruby/rexml/releases/tag/v3.3.3
 rdoc_options:
 - "--main"
 - README.md