rexml 3.2.9 → 3.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ed57404d6f519cb196d671fd629380f5b08a50cf649ae99a71432edceaf15014
-  data.tar.gz: 9097f235049d98aa743998da10724fd6ff5f6cd0cb4f230c59290af4a75e2134
+  metadata.gz: e47ba1209ca1ca2ae0584348378fcefe05de5dc277273d434a37d62e04c676b3
+  data.tar.gz: 867f9e01423f83063aac7c59e07670c88c20f527f676e28cdf9d098248293c56
 SHA512:
-  metadata.gz: 0d7a0be04c12fcd88c64dd2962d4db49f57ed02c08e3f4628e8e546aea135c672839a19a950423afa72d0cf25af0554558f2261b6b5f2b23637f4ca47d43bd73
-  data.tar.gz: 28479b3b11de58e84f57dc88057f185eb2365998f7fbc9f0df78d5899bb9b241c6bb93d26a57ca1b54d8a058b743daa78a8df52c4ddbfbb5f7c94aeb97724808
+  metadata.gz: d87d9cd9384218f3a9bd65870cef99e057022c83bae434318daeab781444378ea830ce46ae20879954f2ae54a7a00cc54eac2839784b989612315ddef909c809
+  data.tar.gz: 1e61927c65b9a058626d0ab19c7f5af0d49169d896e76402e0152476cc772dabf41b8f7a135040b12f5c46eac933de8e60d21fdea8388ed7342be8cc6f9114e9

data/NEWS.md

@@ -1,12 +1,170 @@
 # News
 
-## 3.2.9 - 2024-06-19 {#version-3-2-9}
+## 3.3.4 - 2024-08-01 {#version-3-3-4}
+
+### Fixes
+
+  * Fixed a bug that `REXML::Security` isn't defined when
+    `REXML::Parsers::StreamParser` is used and
+    `rexml/parsers/streamparser` is only required.
+    * GH-189
+    * Patch by takuya kodama.
+
+### Thanks
+
+  * takuya kodama
+
+## 3.3.3 - 2024-08-01 {#version-3-3-3}
+
+### Improvements
+
+  * Added support for detecting invalid XML that has unsupported
+    content before root element
+    * GH-184
+    * Patch by NAITOH Jun.
+
+  * Added support for `REXML::Security.entity_expansion_limit=` and
+    `REXML::Security.entity_expansion_text_limit=` in SAX2 and pull
+    parsers
+    * GH-187
+    * Patch by NAITOH Jun.
+
+  * Added more tests for invalid XMLs.
+    * GH-183
+    * Patch by Watson.
+
+  * Added more performance tests.
+    * Patch by Watson.
+
+  * Improved parse performance.
+    * GH-186
+    * Patch by tomoya ishida.
+
+### Thanks
+
+  * NAITOH Jun
+
+  * Watson
+
+  * tomoya ishida
+
+## 3.3.2 - 2024-07-16 {#version-3-3-2}
+
+### Improvements
+
+  * Improved parse performance.
+    * GH-160
+    * Patch by NAITOH Jun.
+
+  * Improved parse performance.
+    * GH-169
+    * GH-170
+    * GH-171
+    * GH-172
+    * GH-173
+    * GH-174
+    * GH-175
+    * GH-176
+    * GH-177
+    * Patch by Watson.
+
+  * Added support for raising a parse exception when an XML has extra
+    content after the root element.
+    * GH-161
+    * Patch by NAITOH Jun.
+
+  * Added support for raising a parse exception when an XML
+    declaration exists in wrong position.
+    * GH-162
+    * Patch by NAITOH Jun.
+
+  * Removed needless a space after XML declaration in pretty print mode.
+    * GH-164
+    * Patch by NAITOH Jun.
+
+  * Stopped to emit `:text` event after the root element.
+    * GH-167
+    * Patch by NAITOH Jun.
+
+### Fixes
+
+  * Fixed a bug that SAX2 parser doesn't expand predefined entities for
+    `characters` callback.
+    * GH-168
+    * Patch by NAITOH Jun.
+
+### Thanks
+
+  * NAITOH Jun
+
+  * Watson
+
+## 3.3.1 - 2024-06-25 {#version-3-3-1}
+
+### Improvements
+
+  * Added support for detecting malformed top-level comments.
+    * GH-145
+    * Patch by Hiroya Fujinami.
+
+  * Improved `REXML::Element#attribute` performance.
+    * GH-146
+    * Patch by Hiroya Fujinami.
+
+  * Added support for detecting malformed `<!-->` comments.
+    * GH-147
+    * Patch by Hiroya Fujinami.
+
+  * Added support for detecting unclosed `DOCTYPE`.
+    * GH-152
+    * Patch by Hiroya Fujinami.
+
+  * Added `changlog_uri` metadata to gemspec.
+    * GH-156
+    * Patch by fynsta.
+
+  * Improved parse performance.
+    * GH-157
+    * GH-158
+    * Patch by NAITOH Jun.
+
+### Fixes
+
+  * Fixed a bug that large XML can't be parsed.
+    * GH-154
+    * Patch by NAITOH Jun.
+
+  * Fixed a bug that private constants are visible.
+    * GH-155
+    * Patch by NAITOH Jun.
+
+### Thanks
+
+  * Hiroya Fujinami
+
+  * NAITOH Jun
+
+  * fynsta
+
+## 3.3.0 - 2024-06-11 {#version-3-3-0}
+
+### Improvements
+
+  * Added support for strscan 0.7.0 installed with Ruby 2.6.
+    * GH-142
+    * Reported by Fernando Trigoso.
+
+### Thanks
+
+  * Fernando Trigoso
+
+## 3.2.9 - 2024-06-09 {#version-3-2-9}
 
 ### Improvements
 
   * Added support for old strscan.
     * GH-132
-    * Reported by Adam
+    * Reported by Adam.
 
   * Improved attribute value parse performance.
     * GH-135

data/lib/rexml/element.rb

@@ -7,14 +7,6 @@ require_relative "xpath"
 require_relative "parseexception"
 
 module REXML
-  # An implementation note about namespaces:
-  # As we parse, when we find namespaces we put them in a hash and assign
-  # them a unique ID.  We then convert the namespace prefix for the node
-  # to the unique ID.  This makes namespace lookup much faster for the
-  # cost of extra memory use.  We save the namespace prefix for the
-  # context node and convert it back when we write it.
-  @@namespaces = {}
-
   # An \REXML::Element object represents an XML element.
   #
   # An element:
@@ -1284,16 +1276,11 @@ module REXML
     #   document.root.attribute("x", "a") # => a:x='a:x'
     #
     def attribute( name, namespace=nil )
-      prefix = nil
-      if namespaces.respond_to? :key
-        prefix = namespaces.key(namespace) if namespace
-      else
-        prefix = namespaces.index(namespace) if namespace
-      end
+      prefix = namespaces.key(namespace) if namespace
       prefix = nil if prefix == 'xmlns'
 
       ret_val =
-        attributes.get_attribute( "#{prefix ? prefix + ':' : ''}#{name}" )
+        attributes.get_attribute( prefix ? "#{prefix}:#{name}" : name )
 
       return ret_val unless ret_val.nil?
       return nil if prefix.nil?

data/lib/rexml/formatters/pretty.rb

@@ -111,7 +111,7 @@ module REXML
         # itself, then we don't need a carriage return... which makes this
         # logic more complex.
         node.children.each { |child|
-          next if child == node.children[-1] and child.instance_of?(Text)
+          next if child.instance_of?(Text)
           unless child == node.children[0] or child.instance_of?(Text) or
             (child == node.children[1] and !node.children[0].writethis)
             output << "\n"

data/lib/rexml/parsers/baseparser.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 require_relative '../parseexception'
 require_relative '../undefinednamespaceexception'
+require_relative '../security'
 require_relative '../source'
 require 'set'
 require "strscan"
@@ -124,21 +125,28 @@ module REXML
       }
 
       module Private
-        INSTRUCTION_END = /#{NAME}(\s+.*?)?\?>/um
         TAG_PATTERN = /((?>#{QNAME_STR}))\s*/um
         CLOSE_PATTERN = /(#{QNAME_STR})\s*>/um
         ATTLISTDECL_END = /\s+#{NAME}(?:#{ATTDEF})*\s*>/um
-        NAME_PATTERN = /\s*#{NAME}/um
+        NAME_PATTERN = /#{NAME}/um
         GEDECL_PATTERN = "\\s+#{NAME}\\s+#{ENTITYDEF}\\s*>"
         PEDECL_PATTERN = "\\s+(%)\\s+#{NAME}\\s+#{PEDEF}\\s*>"
         ENTITYDECL_PATTERN = /(?:#{GEDECL_PATTERN})|(?:#{PEDECL_PATTERN})/um
+        CARRIAGE_RETURN_NEWLINE_PATTERN = /\r\n?/
+        CHARACTER_REFERENCES = /&#0*((?:\d+)|(?:x[a-fA-F0-9]+));/
+        DEFAULT_ENTITIES_PATTERNS = {}
+        default_entities = ['gt', 'lt', 'quot', 'apos', 'amp']
+        default_entities.each do |term|
+          DEFAULT_ENTITIES_PATTERNS[term] = /&#{term};/
+        end
       end
       private_constant :Private
-      include Private
 
       def initialize( source )
         self.stream = source
         @listeners = []
+        @prefixes = Set.new
+        @entity_expansion_count = 0
       end
 
       def add_listener( listener )
@@ -146,10 +154,12 @@ module REXML
       end
 
       attr_reader :source
+      attr_reader :entity_expansion_count
 
       def stream=( source )
         @source = SourceFactory.create_from( source )
         @closed = nil
+        @have_root = false
         @document_status = nil
         @tags = []
         @stack = []
@@ -204,6 +214,8 @@ module REXML
 
       # Returns the next event.  This is a +PullEvent+ object.
       def pull
+        @source.drop_parsed_content
+
         pull_event.tap do |event|
           @listeners.each do |listener|
             listener.receive event
@@ -216,7 +228,12 @@ module REXML
           x, @closed = @closed, nil
           return [ :end_element, x ]
         end
-        return [ :end_document ] if empty?
+        if empty?
+          if @document_status == :in_doctype
+            raise ParseException.new("Malformed DOCTYPE: unclosed", @source)
+          end
+          return [ :end_document ]
+        end
         return @stack.shift if @stack.size > 0
         #STDERR.puts @source.encoding
         #STDERR.puts "BUFFER = #{@source.buffer.inspect}"
@@ -225,10 +242,17 @@ module REXML
         if @document_status == nil
           start_position = @source.position
           if @source.match("<?", true)
-            return process_instruction(start_position)
+            return process_instruction
           elsif @source.match("<!", true)
             if @source.match("--", true)
-              return [ :comment, @source.match(/(.*?)-->/um, true)[1] ]
+              md = @source.match(/(.*?)-->/um, true)
+              if md.nil?
+                raise REXML::ParseException.new("Unclosed comment", @source)
+              end
+              if /--|-\z/.match?(md[1])
+                raise REXML::ParseException.new("Malformed comment", @source)
+              end
+              return [ :comment, md[1] ]
             elsif @source.match("DOCTYPE", true)
               base_error_message = "Malformed DOCTYPE"
               unless @source.match(/\s+/um, true)
@@ -240,7 +264,7 @@ module REXML
                 @source.position = start_position
                 raise REXML::ParseException.new(message, @source)
               end
-              @nsstack.unshift(curr_ns=Set.new)
+              @nsstack.unshift(Set.new)
               name = parse_name(base_error_message)
               if @source.match(/\s*\[/um, true)
                 id = [nil, nil, nil]
@@ -288,7 +312,11 @@ module REXML
               raise REXML::ParseException.new( "Bad ELEMENT declaration!", @source ) if md.nil?
               return [ :elementdecl, "<!ELEMENT" + md[1] ]
             elsif @source.match("ENTITY", true)
-              match = [:entitydecl, *@source.match(ENTITYDECL_PATTERN, true).captures.compact]
+              match_data = @source.match(Private::ENTITYDECL_PATTERN, true)
+              unless match_data
+                raise REXML::ParseException.new("Malformed entity declaration", @source)
+              end
+              match = [:entitydecl, *match_data.captures.compact]
               ref = false
               if match[1] == '%'
                 ref = true
@@ -314,13 +342,13 @@ module REXML
               match << '%' if ref
               return match
             elsif @source.match("ATTLIST", true)
-              md = @source.match(ATTLISTDECL_END, true)
+              md = @source.match(Private::ATTLISTDECL_END, true)
               raise REXML::ParseException.new( "Bad ATTLIST declaration!", @source ) if md.nil?
               element = md[1]
               contents = md[0]
 
               pairs = {}
-              values = md[0].scan( ATTDEF_RE )
+              values = md[0].strip.scan( ATTDEF_RE )
               values.each do |attdef|
                 unless attdef[3] == "#IMPLIED"
                   attdef.compact!
@@ -366,6 +394,9 @@ module REXML
             @document_status = :after_doctype
             return [ :end_doctype ]
           end
+          if @document_status == :in_doctype
+            raise ParseException.new("Malformed DOCTYPE: invalid declaration", @source)
+          end
         end
         if @document_status == :after_doctype
           @source.match(/\s*/um, true)
@@ -380,7 +411,7 @@ module REXML
             if @source.match("/", true)
               @nsstack.shift
               last_tag = @tags.pop
-              md = @source.match(CLOSE_PATTERN, true)
+              md = @source.match(Private::CLOSE_PATTERN, true)
               if md and !last_tag
                 message = "Unexpected top-level end tag (got '#{md[1]}')"
                 raise REXML::ParseException.new(message, @source)
@@ -399,12 +430,11 @@ module REXML
               if md[0][0] == ?-
                 md = @source.match(/--(.*?)-->/um, true)
 
-                case md[1]
-                when /--/, /-\z/
+                if md.nil? || /--|-\z/.match?(md[1])
                   raise REXML::ParseException.new("Malformed comment", @source)
                 end
 
-                return [ :comment, md[1] ] if md
+                return [ :comment, md[1] ]
               else
                 md = @source.match(/\[CDATA\[(.*?)\]\]>/um, true)
                 return [ :cdata, md[1] ] if md
@@ -412,22 +442,22 @@ module REXML
               raise REXML::ParseException.new( "Declarations can only occur "+
                 "in the doctype declaration.", @source)
             elsif @source.match("?", true)
-              return process_instruction(start_position)
+              return process_instruction
             else
               # Get the next tag
-              md = @source.match(TAG_PATTERN, true)
+              md = @source.match(Private::TAG_PATTERN, true)
               unless md
                 @source.position = start_position
                 raise REXML::ParseException.new("malformed XML: missing tag start", @source)
               end
               tag = md[1]
               @document_status = :in_element
-              prefixes = Set.new
-              prefixes << md[2] if md[2]
+              @prefixes.clear
+              @prefixes << md[2] if md[2]
               @nsstack.unshift(curr_ns=Set.new)
-              attributes, closed = parse_attributes(prefixes, curr_ns)
+              attributes, closed = parse_attributes(@prefixes, curr_ns)
               # Verify that all of the prefixes have been defined
-              for prefix in prefixes
+              for prefix in @prefixes
                 unless @nsstack.find{|k| k.member?(prefix)}
                   raise UndefinedNamespaceException.new(prefix,@source,self)
                 end
@@ -437,8 +467,12 @@ module REXML
                 @closed = tag
                 @nsstack.shift
               else
+                if @tags.empty? and @have_root
+                  raise ParseException.new("Malformed XML: Extra tag at the end of the document (got '<#{tag}')", @source)
+                end
                 @tags.push( tag )
               end
+              @have_root = true
               return [ :start_element, tag, attributes ]
             end
           else
@@ -446,6 +480,16 @@ module REXML
             if text.chomp!("<")
               @source.position -= "<".bytesize
             end
+            if @tags.empty?
+              unless /\A\s*\z/.match?(text)
+                if @have_root
+                  raise ParseException.new("Malformed XML: Extra content at the end of the document (got '#{text}')", @source)
+                else
+                  raise ParseException.new("Malformed XML: Content at the start of the document (got '#{text}')", @source)
+                end
+              end
+              return pull_event if @have_root
+            end
             return [ :text, text ]
           end
         rescue REXML::UndefinedNamespaceException
@@ -463,7 +507,9 @@ module REXML
       def entity( reference, entities )
         value = nil
         value = entities[ reference ] if entities
-        if not value
+        if value
+          record_entity_expansion
+        else
           value = DEFAULT_ENTITIES[ reference ]
           value = value[2] if value
         end
@@ -488,34 +534,51 @@ module REXML
 
       # Unescapes all possible entities
       def unnormalize( string, entities=nil, filter=nil )
-        rv = string.gsub( /\r\n?/, "\n" )
+        if string.include?("\r")
+          rv = string.gsub( Private::CARRIAGE_RETURN_NEWLINE_PATTERN, "\n" )
+        else
+          rv = string.dup
+        end
         matches = rv.scan( REFERENCE_RE )
         return rv if matches.size == 0
-        rv.gsub!( /&#0*((?:\d+)|(?:x[a-fA-F0-9]+));/ ) {
+        rv.gsub!( Private::CHARACTER_REFERENCES ) {
           m=$1
           m = "0#{m}" if m[0] == ?x
           [Integer(m)].pack('U*')
         }
         matches.collect!{|x|x[0]}.compact!
         if matches.size > 0
+          sum = 0
           matches.each do |entity_reference|
             unless filter and filter.include?(entity_reference)
               entity_value = entity( entity_reference, entities )
               if entity_value
-                re = /&#{entity_reference};/
+                re = Private::DEFAULT_ENTITIES_PATTERNS[entity_reference] || /&#{entity_reference};/
                 rv.gsub!( re, entity_value )
+                sum += rv.bytesize
+                if sum > Security.entity_expansion_text_limit
+                  raise "entity expansion has grown too large"
+                end
               else
                 er = DEFAULT_ENTITIES[entity_reference]
                 rv.gsub!( er[0], er[2] ) if er
               end
             end
           end
-          rv.gsub!( /&amp;/, '&' )
+          rv.gsub!( Private::DEFAULT_ENTITIES_PATTERNS['amp'], '&' )
         end
         rv
       end
 
       private
+
+      def record_entity_expansion
+        @entity_expansion_count += 1
+        if @entity_expansion_count > Security.entity_expansion_limit
+          raise "number of entity expansions exceeded, processing aborted."
+        end
+      end
+
       def need_source_encoding_update?(xml_declaration_encoding)
         return false if xml_declaration_encoding.nil?
         return false if /\AUTF-16\z/i =~ xml_declaration_encoding
@@ -523,16 +586,16 @@ module REXML
       end
 
       def parse_name(base_error_message)
-        md = @source.match(NAME_PATTERN, true)
+        md = @source.match(Private::NAME_PATTERN, true)
         unless md
-          if @source.match(/\s*\S/um)
+          if @source.match(/\S/um)
             message = "#{base_error_message}: invalid name"
           else
             message = "#{base_error_message}: name is missing"
           end
           raise REXML::ParseException.new(message, @source)
         end
-        md[1]
+        md[0]
       end
 
       def parse_id(base_error_message,
@@ -601,15 +664,24 @@ module REXML
         end
       end
 
-      def process_instruction(start_position)
-        match_data = @source.match(INSTRUCTION_END, true)
-        unless match_data
-          message = "Invalid processing instruction node"
-          @source.position = start_position
-          raise REXML::ParseException.new(message, @source)
+      def process_instruction
+        name = parse_name("Malformed XML: Invalid processing instruction node")
+        if @source.match(/\s+/um, true)
+          match_data = @source.match(/(.*?)\?>/um, true)
+          unless match_data
+            raise ParseException.new("Malformed XML: Unclosed processing instruction", @source)
+          end
+          content = match_data[1]
+        else
+          content = nil
+          unless @source.match("?>", true)
+            raise ParseException.new("Malformed XML: Unclosed processing instruction", @source)
+          end
         end
-        if @document_status.nil? and match_data[1] == "xml"
-          content = match_data[2]
+        if name == "xml"
+          if @document_status
+            raise ParseException.new("Malformed XML: XML declaration is not at the start", @source)
+          end
           version = VERSION.match(content)
           version = version[1] unless version.nil?
           encoding = ENCODING.match(content)
@@ -624,7 +696,7 @@ module REXML
           standalone = standalone[1] unless standalone.nil?
           return [ :xmldecl, version, encoding, standalone ]
         end
-        [:processing_instruction, match_data[1], match_data[2]]
+        [:processing_instruction, name, content]
       end
 
       def parse_attributes(prefixes, curr_ns)

data/lib/rexml/parsers/pullparser.rb

@@ -47,6 +47,10 @@ module REXML
         @listeners << listener
       end
 
+      def entity_expansion_count
+        @parser.entity_expansion_count
+      end
+
       def each
         while has_next?
           yield self.pull

data/lib/rexml/parsers/sax2parser.rb

@@ -22,6 +22,10 @@ module REXML
         @parser.source
       end
 
+      def entity_expansion_count
+        @parser.entity_expansion_count
+      end
+
       def add_listener( listener )
         @parser.add_listener( listener )
       end
@@ -157,25 +161,8 @@ module REXML
               end
             end
           when :text
-            #normalized = @parser.normalize( event[1] )
-            #handle( :characters, normalized )
-            copy = event[1].clone
-
-            esub = proc { |match|
-              if @entities.has_key?($1)
-                @entities[$1].gsub(Text::REFERENCE, &esub)
-              else
-                match
-              end
-            }
-
-            copy.gsub!( Text::REFERENCE, &esub )
-            copy.gsub!( Text::NUMERICENTITY ) {|m|
-              m=$1
-              m = "0#{m}" if m[0] == ?x
-              [Integer(m)].pack('U*')
-            }
-            handle( :characters, copy )
+            unnormalized = @parser.unnormalize( event[1], @entities )
+            handle( :characters, unnormalized )
           when :entitydecl
             handle_entitydecl( event )
           when :processing_instruction, :comment, :attlistdecl,

data/lib/rexml/parsers/streamparser.rb

@@ -36,8 +36,8 @@ module REXML
             @listener.tag_end( event[1] )
             @tag_stack.pop
           when :text
-            normalized = @parser.unnormalize( event[1] )
-            @listener.text( normalized )
+            unnormalized = @parser.unnormalize( event[1] )
+            @listener.text( unnormalized )
           when :processing_instruction
             @listener.instruction( *event[1,2] )
           when :start_doctype

data/lib/rexml/parsers/treeparser.rb

@@ -16,7 +16,6 @@ module REXML
 
       def parse
         tag_stack = []
-        in_doctype = false
         entities = nil
         begin
           while true
@@ -39,17 +38,15 @@ module REXML
               tag_stack.pop
               @build_context = @build_context.parent
             when :text
-              if not in_doctype
-                if @build_context[-1].instance_of? Text
-                  @build_context[-1] << event[1]
-                else
-                  @build_context.add(
-                    Text.new(event[1], @build_context.whitespace, nil, true)
-                  ) unless (
-                    @build_context.ignore_whitespace_nodes and
-                    event[1].strip.size==0
-                  )
-                end
+              if @build_context[-1].instance_of? Text
+                @build_context[-1] << event[1]
+              else
+                @build_context.add(
+                  Text.new(event[1], @build_context.whitespace, nil, true)
+                ) unless (
+                  @build_context.ignore_whitespace_nodes and
+                  event[1].strip.size==0
+                )
               end
             when :comment
               c = Comment.new( event[1] )
@@ -60,14 +57,12 @@ module REXML
             when :processing_instruction
               @build_context.add( Instruction.new( event[1], event[2] ) )
             when :end_doctype
-              in_doctype = false
               entities.each { |k,v| entities[k] = @build_context.entities[k].value }
               @build_context = @build_context.parent
             when :start_doctype
               doctype = DocType.new( event[1..-1], @build_context )
               @build_context = doctype
               entities = {}
-              in_doctype = true
             when :attlistdecl
               n = AttlistDecl.new( event[1..-1] )
               @build_context.add( n )

data/lib/rexml/rexml.rb

@@ -31,7 +31,7 @@
 module REXML
   COPYRIGHT = "Copyright © 2001-2008 Sean Russell <ser@germane-software.com>"
   DATE = "2008/019"
-  VERSION = "3.2.9"
+  VERSION = "3.3.4"
   REVISION = ""
 
   Copyright = COPYRIGHT

data/lib/rexml/source.rb

@@ -1,8 +1,28 @@
 # coding: US-ASCII
 # frozen_string_literal: false
+
+require "strscan"
+
 require_relative 'encoding'
 
 module REXML
+  if StringScanner::Version < "1.0.0"
+    module StringScannerCheckScanString
+      refine StringScanner do
+        def check(pattern)
+          pattern = /#{Regexp.escape(pattern)}/ if pattern.is_a?(String)
+          super(pattern)
+        end
+
+        def scan(pattern)
+          pattern = /#{Regexp.escape(pattern)}/ if pattern.is_a?(String)
+          super(pattern)
+        end
+      end
+    end
+    using StringScannerCheckScanString
+  end
+
   # Generates Source-s.  USE THIS CLASS.
   class SourceFactory
     # Generates a Source object
@@ -35,6 +55,7 @@ module REXML
     attr_reader :encoding
 
     module Private
+      SCANNER_RESET_SIZE = 100000
       PRE_DEFINED_TERM_PATTERNS = {}
       pre_defined_terms = ["'", '"', "<"]
       pre_defined_terms.each do |term|
@@ -42,7 +63,6 @@ module REXML
       end
     end
     private_constant :Private
-    include Private
 
     # Constructor
     # @param arg must be a String, and should be a valid XML document
@@ -64,6 +84,12 @@ module REXML
       @scanner.rest
     end
 
+    def drop_parsed_content
+      if @scanner.pos > Private::SCANNER_RESET_SIZE
+        @scanner.string = @scanner.rest
+      end
+    end
+
     def buffer_encoding=(encoding)
       @scanner.string.force_encoding(encoding)
     end
@@ -178,10 +204,20 @@ module REXML
       end
     end
 
-    def read(term = nil)
+    def read(term = nil, min_bytes = 1)
       term = encode(term) if term
       begin
-        @scanner << readline(term)
+        str = readline(term)
+        @scanner << str
+        read_bytes = str.bytesize
+        begin
+          while read_bytes < min_bytes
+            str = readline(term)
+            @scanner << str
+            read_bytes += str.bytesize
+          end
+        rescue IOError
+        end
         true
       rescue Exception, NameError
         @source = nil
@@ -211,10 +247,9 @@ module REXML
       read if @scanner.eos? && @source
     end
 
-    # Note: When specifying a string for 'pattern', it must not include '>' except in the following formats:
-    # - ">"
-    # - "XXX>" (X is any string excluding '>')
     def match( pattern, cons=false )
+      # To avoid performance issue, we need to increase bytes to read per scan
+      min_bytes = 1
       while true
         if cons
           md = @scanner.scan(pattern)
@@ -224,7 +259,8 @@ module REXML
         break if md
         return nil if pattern.is_a?(String)
         return nil if @source.nil?
-        return nil unless read
+        return nil unless read(nil, min_bytes)
+        min_bytes *= 2
       end
 
       md.nil? ? nil : @scanner

data/lib/rexml/text.rb

@@ -151,25 +151,45 @@ module REXML
         end
       end
 
-      # context sensitive
-      string.scan(pattern) do
-        if $1[-1] != ?;
-          raise "Illegal character #{$1.inspect} in raw string #{string.inspect}"
-        elsif $1[0] == ?&
-          if $5 and $5[0] == ?#
-            case ($5[1] == ?x ? $5[2..-1].to_i(16) : $5[1..-1].to_i)
-            when *VALID_CHAR
+      pos = 0
+      while (index = string.index(/<|&/, pos))
+        if string[index] == "<"
+          raise "Illegal character \"#{string[index]}\" in raw string #{string.inspect}"
+        end
+
+        unless (end_index = string.index(/[^\s];/, index + 1))
+          raise "Illegal character \"#{string[index]}\" in raw string #{string.inspect}"
+        end
+
+        value = string[(index + 1)..end_index]
+        if /\s/.match?(value)
+          raise "Illegal character \"#{string[index]}\" in raw string #{string.inspect}"
+        end
+
+        if value[0] == "#"
+          character_reference = value[1..-1]
+
+          unless (/\A(\d+|x[0-9a-fA-F]+)\z/.match?(character_reference))
+            if character_reference[0] == "x" || character_reference[-1] == "x"
+              raise "Illegal character \"#{string[index]}\" in raw string #{string.inspect}"
             else
-              raise "Illegal character #{$1.inspect} in raw string #{string.inspect}"
+              raise "Illegal character #{string.inspect} in raw string #{string.inspect}"
             end
-          # FIXME: below can't work but this needs API change.
-          # elsif @parent and $3 and !SUBSTITUTES.include?($1)
-          #   if !doctype or !doctype.entities.has_key?($3)
-          #     raise "Undeclared entity '#{$1}' in raw string \"#{string}\""
-          #   end
           end
+
+          case (character_reference[0] == "x" ? character_reference[1..-1].to_i(16) : character_reference[0..-1].to_i)
+          when *VALID_CHAR
+          else
+            raise "Illegal character #{string.inspect} in raw string #{string.inspect}"
+          end
+        elsif !(/\A#{Entity::NAME}\z/um.match?(value))
+          raise "Illegal character \"#{string[index]}\" in raw string #{string.inspect}"
         end
+
+        pos = end_index + 1
       end
+
+      string
     end
 
     def node_type

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: rexml
 version: !ruby/object:Gem::Version
-  version: 3.2.9
+  version: 3.3.4
 platform: ruby
 authors:
 - Kouhei Sutou
 bindir: bin
 cert_chain: []
-date: 2024-06-09 00:00:00.000000000 Z
+date: 2024-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: strscan
@@ -115,7 +115,8 @@ files:
 homepage: https://github.com/ruby/rexml
 licenses:
 - BSD-2-Clause
-metadata: {}
+metadata:
+  changelog_uri: https://github.com/ruby/rexml/releases/tag/v3.3.4
 rdoc_options:
 - "--main"
 - README.md