rexml 3.3.1 → 3.3.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: afaa8e7d5241253a1c36a218f94eeff525cc19378d2ed104f738abfc01693889
-  data.tar.gz: 665e18c0db75cce5e3db16c674c02e986ff9141df54fd7ff3da704b4403a928d
+  metadata.gz: 4b79c22060286dad847e18d30b4b336bda21d2772ccb35413fb9ba51a0012ed2
+  data.tar.gz: feb56a4a3071541e983acd33b8baa6b9052f8d67d871102cfe6e69773a0cfcfe
 SHA512:
-  metadata.gz: 86ea7a0ce4847b320f297b1eb03158003c2931847c07ea118f0a7413f476660dcf40baec8b59a92a2e7096eb665ace359b04c5d8e82617b7162305465472c88d
-  data.tar.gz: ae248f28516ab6c76170623bcc5e5a30389596823133fd0a13cb74235d6101dd469235bab8b1e15bcbd7a7795f04b44e4674dfdcb1712109dce58001cea01648
+  metadata.gz: b615c95f8624212e151443ad03ba9b64f39aee8a200ea212150a10116340157cfda1bf974ab3d03161c0fb37d866e8c1c69ccc6a9549a13398452b32166af2d8
+  data.tar.gz: db7dcac658e1f51f30575c24d6f36dc256349331fa1951c8fdfaf214baf97a5a446a1fcc411358a76d2c6fc36388ec8b1178adeacc3225d16d5d95ac53a8c4b3

data/NEWS.md

@@ -1,5 +1,160 @@
 # News
 
+## 3.3.6 - 2024-08-22 {#version-3-3-6}
+
+### Improvements
+
+  * Removed duplicated entity expansions for performance.
+    * GH-194
+    * Patch by Viktor Ivarsson.
+
+  * Improved namespace conflicted attribute check performance. It was
+    too slow for deep elements.
+    * Reported by l33thaxor.
+
+### Fixes
+
+  * Fixed a bug that default entity expansions are counted for
+    security check. Default entity expansions should not be counted
+    because they don't have a security risk.
+    * GH-198
+    * GH-199
+    * Patch Viktor Ivarsson
+
+  * Fixed a parser bug that parameter entity references in internal
+    subsets are expanded. It's not allowed in the XML specification.
+    * GH-191
+    * Patch by NAITOH Jun.
+
+  * Fixed a stream parser bug that user-defined entity references in
+    text aren't expanded.
+    * GH-200
+    * Patch by NAITOH Jun.
+
+### Thanks
+
+  * Viktor Ivarsson
+
+  * NAITOH Jun
+
+  * l33thaxor
+
+## 3.3.5 - 2024-08-12 {#version-3-3-5}
+
+### Fixes
+
+  * Fixed a bug that `REXML::Security.entity_expansion_text_limit`
+    check has wrong text size calculation in SAX and pull parsers.
+    * GH-193
+    * GH-195
+    * Reported by Viktor Ivarsson.
+    * Patch by NAITOH Jun.
+
+### Thanks
+
+  * Viktor Ivarsson
+
+  * NAITOH Jun
+
+## 3.3.4 - 2024-08-01 {#version-3-3-4}
+
+### Fixes
+
+  * Fixed a bug that `REXML::Security` isn't defined when
+    `REXML::Parsers::StreamParser` is used and
+    `rexml/parsers/streamparser` is only required.
+    * GH-189
+    * Patch by takuya kodama.
+
+### Thanks
+
+  * takuya kodama
+
+## 3.3.3 - 2024-08-01 {#version-3-3-3}
+
+### Improvements
+
+  * Added support for detecting invalid XML that has unsupported
+    content before root element
+    * GH-184
+    * Patch by NAITOH Jun.
+
+  * Added support for `REXML::Security.entity_expansion_limit=` and
+    `REXML::Security.entity_expansion_text_limit=` in SAX2 and pull
+    parsers
+    * GH-187
+    * Patch by NAITOH Jun.
+
+  * Added more tests for invalid XMLs.
+    * GH-183
+    * Patch by Watson.
+
+  * Added more performance tests.
+    * Patch by Watson.
+
+  * Improved parse performance.
+    * GH-186
+    * Patch by tomoya ishida.
+
+### Thanks
+
+  * NAITOH Jun
+
+  * Watson
+
+  * tomoya ishida
+
+## 3.3.2 - 2024-07-16 {#version-3-3-2}
+
+### Improvements
+
+  * Improved parse performance.
+    * GH-160
+    * Patch by NAITOH Jun.
+
+  * Improved parse performance.
+    * GH-169
+    * GH-170
+    * GH-171
+    * GH-172
+    * GH-173
+    * GH-174
+    * GH-175
+    * GH-176
+    * GH-177
+    * Patch by Watson.
+
+  * Added support for raising a parse exception when an XML has extra
+    content after the root element.
+    * GH-161
+    * Patch by NAITOH Jun.
+
+  * Added support for raising a parse exception when an XML
+    declaration exists in wrong position.
+    * GH-162
+    * Patch by NAITOH Jun.
+
+  * Removed needless a space after XML declaration in pretty print mode.
+    * GH-164
+    * Patch by NAITOH Jun.
+
+  * Stopped to emit `:text` event after the root element.
+    * GH-167
+    * Patch by NAITOH Jun.
+
+### Fixes
+
+  * Fixed a bug that SAX2 parser doesn't expand predefined entities for
+    `characters` callback.
+    * GH-168
+    * Patch by NAITOH Jun.
+
+### Thanks
+
+  * NAITOH Jun
+
+  * Watson
+
 ## 3.3.1 - 2024-06-25 {#version-3-3-1}
 
 ### Improvements

data/lib/rexml/element.rb

@@ -441,9 +441,14 @@ module REXML
     # Related: #root_node, #document.
     #
     def root
-      return elements[1] if self.kind_of? Document
-      return self if parent.kind_of? Document or parent.nil?
-      return parent.root
+      target = self
+      while target
+        return target.elements[1] if target.kind_of? Document
+        parent = target.parent
+        return target if parent.kind_of? Document or parent.nil?
+        target = parent
+      end
+      nil
     end
 
     # :call-seq:
@@ -619,8 +624,12 @@ module REXML
       else
         prefix = "xmlns:#{prefix}" unless prefix[0,5] == 'xmlns'
       end
-      ns = attributes[ prefix ]
-      ns = parent.namespace(prefix) if ns.nil? and parent
+      ns = nil
+      target = self
+      while ns.nil? and target
+        ns = target.attributes[prefix]
+        target = target.parent
+      end
       ns = '' if ns.nil? and prefix == 'xmlns'
       return ns
     end
@@ -2375,17 +2384,6 @@ module REXML
       elsif old_attr.kind_of? Hash
         old_attr[value.prefix] = value
       elsif old_attr.prefix != value.prefix
-        # Check for conflicting namespaces
-        if value.prefix != "xmlns" and old_attr.prefix != "xmlns"
-          old_namespace = old_attr.namespace
-          new_namespace = value.namespace
-          if old_namespace == new_namespace
-            raise ParseException.new(
-                    "Namespace conflict in adding attribute \"#{value.name}\": "+
-                    "Prefix \"#{old_attr.prefix}\" = \"#{old_namespace}\" and "+
-                    "prefix \"#{value.prefix}\" = \"#{new_namespace}\"")
-          end
-        end
         store value.name, {old_attr.prefix => old_attr,
                            value.prefix    => value}
       else

data/lib/rexml/entity.rb

@@ -12,6 +12,7 @@ module REXML
     EXTERNALID = "(?:(?:(SYSTEM)\\s+#{SYSTEMLITERAL})|(?:(PUBLIC)\\s+#{PUBIDLITERAL}\\s+#{SYSTEMLITERAL}))"
     NDATADECL = "\\s+NDATA\\s+#{NAME}"
     PEREFERENCE = "%#{NAME};"
+    PEREFERENCE_RE = /#{PEREFERENCE}/um
     ENTITYVALUE = %Q{((?:"(?:[^%&"]|#{PEREFERENCE}|#{REFERENCE})*")|(?:'([^%&']|#{PEREFERENCE}|#{REFERENCE})*'))}
     PEDEF = "(?:#{ENTITYVALUE}|#{EXTERNALID})"
     ENTITYDEF = "(?:#{ENTITYVALUE}|(?:#{EXTERNALID}(#{NDATADECL})?))"
@@ -19,7 +20,7 @@ module REXML
     GEDECL = "<!ENTITY\\s+#{NAME}\\s+#{ENTITYDEF}\\s*>"
     ENTITYDECL = /\s*(?:#{GEDECL})|(?:#{PEDECL})/um
 
-    attr_reader :name, :external, :ref, :ndata, :pubid
+    attr_reader :name, :external, :ref, :ndata, :pubid, :value
 
     # Create a new entity.  Simple entities can be constructed by passing a
     # name, value to the constructor; this creates a generic, plain entity
@@ -68,14 +69,11 @@ module REXML
     end
 
     # Evaluates to the unnormalized value of this entity; that is, replacing
-    # all entities -- both %ent; and &ent; entities.  This differs from
-    # +value()+ in that +value+ only replaces %ent; entities.
+    # &ent; entities.
     def unnormalized
       document.record_entity_expansion unless document.nil?
-      v = value()
-      return nil if v.nil?
-      @unnormalized = Text::unnormalize(v, parent)
-      @unnormalized
+      return nil if @value.nil?
+      @unnormalized = Text::unnormalize(@value, parent)
     end
 
     #once :unnormalized
@@ -121,46 +119,6 @@ module REXML
       write rv
       rv
     end
-
-    PEREFERENCE_RE = /#{PEREFERENCE}/um
-    # Returns the value of this entity.  At the moment, only internal entities
-    # are processed.  If the value contains internal references (IE,
-    # %blah;), those are replaced with their values.  IE, if the doctype
-    # contains:
-    #  <!ENTITY % foo "bar">
-    #  <!ENTITY yada "nanoo %foo; nanoo>
-    # then:
-    #  doctype.entity('yada').value   #-> "nanoo bar nanoo"
-    def value
-      @resolved_value ||= resolve_value
-    end
-
-    def parent=(other)
-      @resolved_value = nil
-      super
-    end
-
-    private
-    def resolve_value
-      return nil if @value.nil?
-      return @value unless @value.match?(PEREFERENCE_RE)
-
-      matches = @value.scan(PEREFERENCE_RE)
-      rv = @value.clone
-      if @parent
-        sum = 0
-        matches.each do |entity_reference|
-          entity_value = @parent.entity( entity_reference[0] )
-          if sum + entity_value.bytesize > Security.entity_expansion_text_limit
-            raise "entity expansion has grown too large"
-          else
-            sum += entity_value.bytesize
-          end
-          rv.gsub!( /%#{entity_reference.join};/um, entity_value )
-        end
-      end
-      rv
-    end
   end
 
   # This is a set of entity constants -- the ones defined in the XML

data/lib/rexml/formatters/pretty.rb

@@ -111,7 +111,7 @@ module REXML
         # itself, then we don't need a carriage return... which makes this
         # logic more complex.
         node.children.each { |child|
-          next if child == node.children[-1] and child.instance_of?(Text)
+          next if child.instance_of?(Text)
           unless child == node.children[0] or child.instance_of?(Text) or
             (child == node.children[1] and !node.children[0].writethis)
             output << "\n"

data/lib/rexml/parsers/baseparser.rb

@@ -1,12 +1,29 @@
 # frozen_string_literal: true
 require_relative '../parseexception'
 require_relative '../undefinednamespaceexception'
+require_relative '../security'
 require_relative '../source'
 require 'set'
 require "strscan"
 
 module REXML
   module Parsers
+    unless [].respond_to?(:tally)
+      module EnumerableTally
+        refine Enumerable do
+          def tally
+            counts = {}
+            each do |item|
+              counts[item] ||= 0
+              counts[item] += 1
+            end
+            counts
+          end
+        end
+      end
+      using EnumerableTally
+    end
+
     if StringScanner::Version < "3.0.8"
       module StringScannerCaptures
         refine StringScanner do
@@ -124,11 +141,11 @@ module REXML
       }
 
       module Private
-        INSTRUCTION_END = /#{NAME}(\s+.*?)?\?>/um
+        PEREFERENCE_PATTERN = /#{PEREFERENCE}/um
         TAG_PATTERN = /((?>#{QNAME_STR}))\s*/um
         CLOSE_PATTERN = /(#{QNAME_STR})\s*>/um
         ATTLISTDECL_END = /\s+#{NAME}(?:#{ATTDEF})*\s*>/um
-        NAME_PATTERN = /\s*#{NAME}/um
+        NAME_PATTERN = /#{NAME}/um
         GEDECL_PATTERN = "\\s+#{NAME}\\s+#{ENTITYDEF}\\s*>"
         PEDECL_PATTERN = "\\s+(%)\\s+#{NAME}\\s+#{PEDEF}\\s*>"
         ENTITYDECL_PATTERN = /(?:#{GEDECL_PATTERN})|(?:#{PEDECL_PATTERN})/um
@@ -146,6 +163,7 @@ module REXML
         self.stream = source
         @listeners = []
         @prefixes = Set.new
+        @entity_expansion_count = 0
       end
 
       def add_listener( listener )
@@ -153,15 +171,18 @@ module REXML
       end
 
       attr_reader :source
+      attr_reader :entity_expansion_count
 
       def stream=( source )
         @source = SourceFactory.create_from( source )
         @closed = nil
+        @have_root = false
         @document_status = nil
         @tags = []
         @stack = []
         @entities = []
-        @nsstack = []
+        @namespaces = {}
+        @namespaces_restore_stack = []
       end
 
       def position
@@ -229,6 +250,10 @@ module REXML
           if @document_status == :in_doctype
             raise ParseException.new("Malformed DOCTYPE: unclosed", @source)
           end
+          unless @tags.empty?
+            path = "/" + @tags.join("/")
+            raise ParseException.new("Missing end tag for '#{path}'", @source)
+          end
           return [ :end_document ]
         end
         return @stack.shift if @stack.size > 0
@@ -239,7 +264,7 @@ module REXML
         if @document_status == nil
           start_position = @source.position
           if @source.match("<?", true)
-            return process_instruction(start_position)
+            return process_instruction
           elsif @source.match("<!", true)
             if @source.match("--", true)
               md = @source.match(/(.*?)-->/um, true)
@@ -261,7 +286,6 @@ module REXML
                 @source.position = start_position
                 raise REXML::ParseException.new(message, @source)
               end
-              @nsstack.unshift(Set.new)
               name = parse_name(base_error_message)
               if @source.match(/\s*\[/um, true)
                 id = [nil, nil, nil]
@@ -309,7 +333,11 @@ module REXML
               raise REXML::ParseException.new( "Bad ELEMENT declaration!", @source ) if md.nil?
               return [ :elementdecl, "<!ELEMENT" + md[1] ]
             elsif @source.match("ENTITY", true)
-              match = [:entitydecl, *@source.match(Private::ENTITYDECL_PATTERN, true).captures.compact]
+              match_data = @source.match(Private::ENTITYDECL_PATTERN, true)
+              unless match_data
+                raise REXML::ParseException.new("Malformed entity declaration", @source)
+              end
+              match = [:entitydecl, *match_data.captures.compact]
               ref = false
               if match[1] == '%'
                 ref = true
@@ -327,6 +355,8 @@ module REXML
                 match[4] = match[4][1..-2] # HREF
                 match.delete_at(5) if match.size > 5 # Chop out NDATA decl
                 # match is [ :entity, name, PUBLIC, pubid, href(, ndata)? ]
+              elsif Private::PEREFERENCE_PATTERN.match?(match[2])
+                raise REXML::ParseException.new("Parameter entity references forbidden in internal subset: #{match[2]}", @source)
               else
                 match[2] = match[2][1..-2]
                 match.pop if match.size == 4
@@ -341,7 +371,7 @@ module REXML
               contents = md[0]
 
               pairs = {}
-              values = md[0].scan( ATTDEF_RE )
+              values = md[0].strip.scan( ATTDEF_RE )
               values.each do |attdef|
                 unless attdef[3] == "#IMPLIED"
                   attdef.compact!
@@ -349,7 +379,7 @@ module REXML
                   val = attdef[4] if val == "#FIXED "
                   pairs[attdef[0]] = val
                   if attdef[0] =~ /^xmlns:(.*)/
-                    @nsstack[0] << $1
+                    @namespaces[$1] = val
                   end
                 end
               end
@@ -402,7 +432,7 @@ module REXML
             # here explicitly.
             @source.ensure_buffer
             if @source.match("/", true)
-              @nsstack.shift
+              @namespaces_restore_stack.pop
               last_tag = @tags.pop
               md = @source.match(Private::CLOSE_PATTERN, true)
               if md and !last_tag
@@ -435,7 +465,7 @@ module REXML
               raise REXML::ParseException.new( "Declarations can only occur "+
                 "in the doctype declaration.", @source)
             elsif @source.match("?", true)
-              return process_instruction(start_position)
+              return process_instruction
             else
               # Get the next tag
               md = @source.match(Private::TAG_PATTERN, true)
@@ -447,21 +477,25 @@ module REXML
               @document_status = :in_element
               @prefixes.clear
               @prefixes << md[2] if md[2]
-              @nsstack.unshift(curr_ns=Set.new)
-              attributes, closed = parse_attributes(@prefixes, curr_ns)
+              push_namespaces_restore
+              attributes, closed = parse_attributes(@prefixes)
               # Verify that all of the prefixes have been defined
               for prefix in @prefixes
-                unless @nsstack.find{|k| k.member?(prefix)}
+                unless @namespaces.key?(prefix)
                   raise UndefinedNamespaceException.new(prefix,@source,self)
                 end
               end
 
               if closed
                 @closed = tag
-                @nsstack.shift
+                pop_namespaces_restore
               else
+                if @tags.empty? and @have_root
+                  raise ParseException.new("Malformed XML: Extra tag at the end of the document (got '<#{tag}')", @source)
+                end
                 @tags.push( tag )
               end
+              @have_root = true
               return [ :start_element, tag, attributes ]
             end
           else
@@ -469,6 +503,16 @@ module REXML
             if text.chomp!("<")
               @source.position -= "<".bytesize
             end
+            if @tags.empty?
+              unless /\A\s*\z/.match?(text)
+                if @have_root
+                  raise ParseException.new("Malformed XML: Extra content at the end of the document (got '#{text}')", @source)
+                else
+                  raise ParseException.new("Malformed XML: Content at the start of the document (got '#{text}')", @source)
+                end
+              end
+              return pull_event if @have_root
+            end
             return [ :text, text ]
           end
         rescue REXML::UndefinedNamespaceException
@@ -484,13 +528,13 @@ module REXML
       private :pull_event
 
       def entity( reference, entities )
-        value = nil
-        value = entities[ reference ] if entities
-        if not value
-          value = DEFAULT_ENTITIES[ reference ]
-          value = value[2] if value
-        end
-        unnormalize( value, entities ) if value
+        return unless entities
+
+        value = entities[ reference ]
+        return if value.nil?
+
+        record_entity_expansion
+        unnormalize( value, entities )
       end
 
       # Escapes all possible entities
@@ -511,7 +555,11 @@ module REXML
 
       # Unescapes all possible entities
       def unnormalize( string, entities=nil, filter=nil )
-        rv = string.gsub( Private::CARRIAGE_RETURN_NEWLINE_PATTERN, "\n" )
+        if string.include?("\r")
+          rv = string.gsub( Private::CARRIAGE_RETURN_NEWLINE_PATTERN, "\n" )
+        else
+          rv = string.dup
+        end
         matches = rv.scan( REFERENCE_RE )
         return rv if matches.size == 0
         rv.gsub!( Private::CHARACTER_REFERENCES ) {
@@ -520,17 +568,29 @@ module REXML
           [Integer(m)].pack('U*')
         }
         matches.collect!{|x|x[0]}.compact!
+        if filter
+          matches.reject! do |entity_reference|
+            filter.include?(entity_reference)
+          end
+        end
         if matches.size > 0
-          matches.each do |entity_reference|
-            unless filter and filter.include?(entity_reference)
-              entity_value = entity( entity_reference, entities )
-              if entity_value
-                re = Private::DEFAULT_ENTITIES_PATTERNS[entity_reference] || /&#{entity_reference};/
-                rv.gsub!( re, entity_value )
-              else
-                er = DEFAULT_ENTITIES[entity_reference]
-                rv.gsub!( er[0], er[2] ) if er
+          matches.tally.each do |entity_reference, n|
+            entity_expansion_count_before = @entity_expansion_count
+            entity_value = entity( entity_reference, entities )
+            if entity_value
+              if n > 1
+                entity_expansion_count_delta =
+                  @entity_expansion_count - entity_expansion_count_before
+                record_entity_expansion(entity_expansion_count_delta * (n - 1))
               end
+              re = Private::DEFAULT_ENTITIES_PATTERNS[entity_reference] || /&#{entity_reference};/
+              rv.gsub!( re, entity_value )
+              if rv.bytesize > Security.entity_expansion_text_limit
+                raise "entity expansion has grown too large"
+              end
+            else
+              er = DEFAULT_ENTITIES[entity_reference]
+              rv.gsub!( er[0], er[2] ) if er
             end
           end
           rv.gsub!( Private::DEFAULT_ENTITIES_PATTERNS['amp'], '&' )
@@ -539,6 +599,39 @@ module REXML
       end
 
       private
+      def add_namespace(prefix, uri)
+        @namespaces_restore_stack.last[prefix] = @namespaces[prefix]
+        if uri.nil?
+          @namespaces.delete(prefix)
+        else
+          @namespaces[prefix] = uri
+        end
+      end
+
+      def push_namespaces_restore
+        namespaces_restore = {}
+        @namespaces_restore_stack.push(namespaces_restore)
+        namespaces_restore
+      end
+
+      def pop_namespaces_restore
+        namespaces_restore = @namespaces_restore_stack.pop
+        namespaces_restore.each do |prefix, uri|
+          if uri.nil?
+            @namespaces.delete(prefix)
+          else
+            @namespaces[prefix] = uri
+          end
+        end
+      end
+
+      def record_entity_expansion(delta=1)
+        @entity_expansion_count += delta
+        if @entity_expansion_count > Security.entity_expansion_limit
+          raise "number of entity expansions exceeded, processing aborted."
+        end
+      end
+
       def need_source_encoding_update?(xml_declaration_encoding)
         return false if xml_declaration_encoding.nil?
         return false if /\AUTF-16\z/i =~ xml_declaration_encoding
@@ -548,14 +641,14 @@ module REXML
       def parse_name(base_error_message)
         md = @source.match(Private::NAME_PATTERN, true)
         unless md
-          if @source.match(/\s*\S/um)
+          if @source.match(/\S/um)
             message = "#{base_error_message}: invalid name"
           else
             message = "#{base_error_message}: name is missing"
           end
           raise REXML::ParseException.new(message, @source)
         end
-        md[1]
+        md[0]
       end
 
       def parse_id(base_error_message,
@@ -624,15 +717,24 @@ module REXML
         end
       end
 
-      def process_instruction(start_position)
-        match_data = @source.match(Private::INSTRUCTION_END, true)
-        unless match_data
-          message = "Invalid processing instruction node"
-          @source.position = start_position
-          raise REXML::ParseException.new(message, @source)
+      def process_instruction
+        name = parse_name("Malformed XML: Invalid processing instruction node")
+        if @source.match(/\s+/um, true)
+          match_data = @source.match(/(.*?)\?>/um, true)
+          unless match_data
+            raise ParseException.new("Malformed XML: Unclosed processing instruction", @source)
+          end
+          content = match_data[1]
+        else
+          content = nil
+          unless @source.match("?>", true)
+            raise ParseException.new("Malformed XML: Unclosed processing instruction", @source)
+          end
         end
-        if @document_status.nil? and match_data[1] == "xml"
-          content = match_data[2]
+        if name == "xml"
+          if @document_status
+            raise ParseException.new("Malformed XML: XML declaration is not at the start", @source)
+          end
           version = VERSION.match(content)
           version = version[1] unless version.nil?
           encoding = ENCODING.match(content)
@@ -647,11 +749,12 @@ module REXML
           standalone = standalone[1] unless standalone.nil?
           return [ :xmldecl, version, encoding, standalone ]
         end
-        [:processing_instruction, match_data[1], match_data[2]]
+        [:processing_instruction, name, content]
       end
 
-      def parse_attributes(prefixes, curr_ns)
+      def parse_attributes(prefixes)
         attributes = {}
+        expanded_names = {}
         closed = false
         while true
           if @source.match(">", true)
@@ -693,7 +796,7 @@ module REXML
                   "(http://www.w3.org/TR/REC-xml-names/#ns-decl)"
                 raise REXML::ParseException.new( msg, @source, self)
               end
-              curr_ns << local_part
+              add_namespace(local_part, value)
             elsif prefix
               prefixes << prefix unless prefix == "xml"
             end
@@ -703,6 +806,20 @@ module REXML
               raise REXML::ParseException.new(msg, @source, self)
             end
 
+            unless prefix == "xmlns"
+              uri = @namespaces[prefix]
+              expanded_name = [uri, local_part]
+              existing_prefix = expanded_names[expanded_name]
+              if existing_prefix
+                message = "Namespace conflict in adding attribute " +
+                          "\"#{local_part}\": " +
+                          "Prefix \"#{existing_prefix}\" = \"#{uri}\" and " +
+                          "prefix \"#{prefix}\" = \"#{uri}\""
+                raise REXML::ParseException.new(message, @source, self)
+              end
+              expanded_names[expanded_name] = prefix
+            end
+
             attributes[name] = value
           else
             message = "Invalid attribute name: <#{@source.buffer.split(%r{[/>\s]}).first}>"

data/lib/rexml/parsers/pullparser.rb

@@ -47,6 +47,10 @@ module REXML
         @listeners << listener
       end
 
+      def entity_expansion_count
+        @parser.entity_expansion_count
+      end
+
       def each
         while has_next?
           yield self.pull

data/lib/rexml/parsers/sax2parser.rb

@@ -22,6 +22,10 @@ module REXML
         @parser.source
       end
 
+      def entity_expansion_count
+        @parser.entity_expansion_count
+      end
+
       def add_listener( listener )
         @parser.add_listener( listener )
       end
@@ -157,25 +161,8 @@ module REXML
               end
             end
           when :text
-            #normalized = @parser.normalize( event[1] )
-            #handle( :characters, normalized )
-            copy = event[1].clone
-
-            esub = proc { |match|
-              if @entities.has_key?($1)
-                @entities[$1].gsub(Text::REFERENCE, &esub)
-              else
-                match
-              end
-            }
-
-            copy.gsub!( Text::REFERENCE, &esub )
-            copy.gsub!( Text::NUMERICENTITY ) {|m|
-              m=$1
-              m = "0#{m}" if m[0] == ?x
-              [Integer(m)].pack('U*')
-            }
-            handle( :characters, copy )
+            unnormalized = @parser.unnormalize( event[1], @entities )
+            handle( :characters, unnormalized )
           when :entitydecl
             handle_entitydecl( event )
           when :processing_instruction, :comment, :attlistdecl,

data/lib/rexml/parsers/streamparser.rb

@@ -7,37 +7,34 @@ module REXML
       def initialize source, listener
         @listener = listener
         @parser = BaseParser.new( source )
-        @tag_stack = []
+        @entities = {}
       end
 
       def add_listener( listener )
         @parser.add_listener( listener )
       end
 
+      def entity_expansion_count
+        @parser.entity_expansion_count
+      end
+
       def parse
         # entity string
         while true
           event = @parser.pull
           case event[0]
           when :end_document
-            unless @tag_stack.empty?
-              tag_path = "/" + @tag_stack.join("/")
-              raise ParseException.new("Missing end tag for '#{tag_path}'",
-                                       @parser.source)
-            end
             return
           when :start_element
-            @tag_stack << event[1]
             attrs = event[2].each do |n, v|
               event[2][n] = @parser.unnormalize( v )
             end
             @listener.tag_start( event[1], attrs )
           when :end_element
             @listener.tag_end( event[1] )
-            @tag_stack.pop
           when :text
-            normalized = @parser.unnormalize( event[1] )
-            @listener.text( normalized )
+            unnormalized = @parser.unnormalize( event[1], @entities )
+            @listener.text( unnormalized )
           when :processing_instruction
             @listener.instruction( *event[1,2] )
           when :start_doctype
@@ -48,6 +45,7 @@ module REXML
           when :comment, :attlistdecl, :cdata, :xmldecl, :elementdecl
             @listener.send( event[0].to_s, *event[1..-1] )
           when :entitydecl, :notationdecl
+            @entities[ event[1] ] = event[2] if event.size == 3
             @listener.send( event[0].to_s, event[1..-1] )
           when :externalentity
             entity_reference = event[1]

data/lib/rexml/parsers/treeparser.rb

@@ -15,7 +15,6 @@ module REXML
       end
 
       def parse
-        tag_stack = []
         entities = nil
         begin
           while true
@@ -23,19 +22,13 @@ module REXML
             #STDERR.puts "TREEPARSER GOT #{event.inspect}"
             case event[0]
             when :end_document
-              unless tag_stack.empty?
-                raise ParseException.new("No close tag for #{@build_context.xpath}",
-                                         @parser.source, @parser)
-              end
               return
             when :start_element
-              tag_stack.push(event[1])
               el = @build_context = @build_context.add_element( event[1] )
               event[2].each do |key, value|
                 el.attributes[key]=Attribute.new(key,value,self)
               end
             when :end_element
-              tag_stack.pop
               @build_context = @build_context.parent
             when :text
               if @build_context[-1].instance_of? Text

data/lib/rexml/rexml.rb

@@ -31,7 +31,7 @@
 module REXML
   COPYRIGHT = "Copyright © 2001-2008 Sean Russell <ser@germane-software.com>"
   DATE = "2008/019"
-  VERSION = "3.3.1"
+  VERSION = "3.3.6"
   REVISION = ""
 
   Copyright = COPYRIGHT

data/lib/rexml/source.rb

@@ -204,10 +204,20 @@ module REXML
       end
     end
 
-    def read(term = nil)
+    def read(term = nil, min_bytes = 1)
       term = encode(term) if term
       begin
-        @scanner << readline(term)
+        str = readline(term)
+        @scanner << str
+        read_bytes = str.bytesize
+        begin
+          while read_bytes < min_bytes
+            str = readline(term)
+            @scanner << str
+            read_bytes += str.bytesize
+          end
+        rescue IOError
+        end
         true
       rescue Exception, NameError
         @source = nil
@@ -237,10 +247,9 @@ module REXML
       read if @scanner.eos? && @source
     end
 
-    # Note: When specifying a string for 'pattern', it must not include '>' except in the following formats:
-    # - ">"
-    # - "XXX>" (X is any string excluding '>')
     def match( pattern, cons=false )
+      # To avoid performance issue, we need to increase bytes to read per scan
+      min_bytes = 1
       while true
         if cons
           md = @scanner.scan(pattern)
@@ -250,7 +259,8 @@ module REXML
         break if md
         return nil if pattern.is_a?(String)
         return nil if @source.nil?
-        return nil unless read
+        return nil unless read(nil, min_bytes)
+        min_bytes *= 2
       end
 
       md.nil? ? nil : @scanner

data/lib/rexml/text.rb

@@ -151,25 +151,45 @@ module REXML
         end
       end
 
-      # context sensitive
-      string.scan(pattern) do
-        if $1[-1] != ?;
-          raise "Illegal character #{$1.inspect} in raw string #{string.inspect}"
-        elsif $1[0] == ?&
-          if $5 and $5[0] == ?#
-            case ($5[1] == ?x ? $5[2..-1].to_i(16) : $5[1..-1].to_i)
-            when *VALID_CHAR
+      pos = 0
+      while (index = string.index(/<|&/, pos))
+        if string[index] == "<"
+          raise "Illegal character \"#{string[index]}\" in raw string #{string.inspect}"
+        end
+
+        unless (end_index = string.index(/[^\s];/, index + 1))
+          raise "Illegal character \"#{string[index]}\" in raw string #{string.inspect}"
+        end
+
+        value = string[(index + 1)..end_index]
+        if /\s/.match?(value)
+          raise "Illegal character \"#{string[index]}\" in raw string #{string.inspect}"
+        end
+
+        if value[0] == "#"
+          character_reference = value[1..-1]
+
+          unless (/\A(\d+|x[0-9a-fA-F]+)\z/.match?(character_reference))
+            if character_reference[0] == "x" || character_reference[-1] == "x"
+              raise "Illegal character \"#{string[index]}\" in raw string #{string.inspect}"
             else
-              raise "Illegal character #{$1.inspect} in raw string #{string.inspect}"
+              raise "Illegal character #{string.inspect} in raw string #{string.inspect}"
             end
-          # FIXME: below can't work but this needs API change.
-          # elsif @parent and $3 and !SUBSTITUTES.include?($1)
-          #   if !doctype or !doctype.entities.has_key?($3)
-          #     raise "Undeclared entity '#{$1}' in raw string \"#{string}\""
-          #   end
           end
+
+          case (character_reference[0] == "x" ? character_reference[1..-1].to_i(16) : character_reference[0..-1].to_i)
+          when *VALID_CHAR
+          else
+            raise "Illegal character #{string.inspect} in raw string #{string.inspect}"
+          end
+        elsif !(/\A#{Entity::NAME}\z/um.match?(value))
+          raise "Illegal character \"#{string[index]}\" in raw string #{string.inspect}"
         end
+
+        pos = end_index + 1
       end
+
+      string
     end
 
     def node_type

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: rexml
 version: !ruby/object:Gem::Version
-  version: 3.3.1
+  version: 3.3.6
 platform: ruby
 authors:
 - Kouhei Sutou
 bindir: bin
 cert_chain: []
-date: 2024-06-25 00:00:00.000000000 Z
+date: 2024-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: strscan
@@ -116,7 +116,7 @@ homepage: https://github.com/ruby/rexml
 licenses:
 - BSD-2-Clause
 metadata:
-  changelog_uri: https://github.com/ruby/rexml/releases/tag/v3.3.1
+  changelog_uri: https://github.com/ruby/rexml/releases/tag/v3.3.6
 rdoc_options:
 - "--main"
 - README.md