rexml 3.3.7 → 3.4.4

data/lib/rexml/parsers/baseparser.rb

@@ -144,18 +144,20 @@ module REXML
         PEREFERENCE_PATTERN = /#{PEREFERENCE}/um
         TAG_PATTERN = /((?>#{QNAME_STR}))\s*/um
         CLOSE_PATTERN = /(#{QNAME_STR})\s*>/um
+        EQUAL_PATTERN = /\s*=\s*/um
         ATTLISTDECL_END = /\s+#{NAME}(?:#{ATTDEF})*\s*>/um
         NAME_PATTERN = /#{NAME}/um
         GEDECL_PATTERN = "\\s+#{NAME}\\s+#{ENTITYDEF}\\s*>"
         PEDECL_PATTERN = "\\s+(%)\\s+#{NAME}\\s+#{PEDEF}\\s*>"
         ENTITYDECL_PATTERN = /(?:#{GEDECL_PATTERN})|(?:#{PEDECL_PATTERN})/um
         CARRIAGE_RETURN_NEWLINE_PATTERN = /\r\n?/
-        CHARACTER_REFERENCES = /&#0*((?:\d+)|(?:x[a-fA-F0-9]+));/
+        CHARACTER_REFERENCES = /&#((?:\d+)|(?:x[a-fA-F0-9]+));/
         DEFAULT_ENTITIES_PATTERNS = {}
         default_entities = ['gt', 'lt', 'quot', 'apos', 'amp']
         default_entities.each do |term|
           DEFAULT_ENTITIES_PATTERNS[term] = /&#{term};/
         end
+        XML_PREFIXED_NAMESPACE = "http://www.w3.org/XML/1998/namespace"
       end
       private_constant :Private
 
@@ -166,6 +168,8 @@ module REXML
         @entity_expansion_count = 0
         @entity_expansion_limit = Security.entity_expansion_limit
         @entity_expansion_text_limit = Security.entity_expansion_text_limit
+        @source.ensure_buffer
+        @version = nil
       end
 
       def add_listener( listener )
@@ -179,13 +183,17 @@ module REXML
 
       def stream=( source )
         @source = SourceFactory.create_from( source )
+        reset
+      end
+
+      def reset
         @closed = nil
         @have_root = false
         @document_status = nil
         @tags = []
         @stack = []
         @entities = []
-        @namespaces = {}
+        @namespaces = {"xml" => Private::XML_PREFIXED_NAMESPACE}
         @namespaces_restore_stack = []
       end
 
@@ -200,12 +208,12 @@ module REXML
 
       # Returns true if there are no more events
       def empty?
-        return (@source.empty? and @stack.empty?)
+        (@source.empty? and @stack.empty?)
       end
 
       # Returns true if there are more events.  Synonymous with !empty?
       def has_next?
-        return !(@source.empty? and @stack.empty?)
+        !(@source.empty? and @stack.empty?)
       end
 
       # Push an event back on the head of the stream.  This method
@@ -258,6 +266,11 @@ module REXML
             path = "/" + @tags.join("/")
             raise ParseException.new("Missing end tag for '#{path}'", @source)
           end
+
+          unless @document_status == :in_element
+            raise ParseException.new("Malformed XML: No root element", @source)
+          end
+
           return [ :end_document ]
         end
         return @stack.shift if @stack.size > 0
@@ -267,22 +280,15 @@ module REXML
         @source.ensure_buffer
         if @document_status == nil
           start_position = @source.position
-          if @source.match("<?", true)
+          if @source.match?("<?", true)
             return process_instruction
-          elsif @source.match("<!", true)
-            if @source.match("--", true)
-              md = @source.match(/(.*?)-->/um, true)
-              if md.nil?
-                raise REXML::ParseException.new("Unclosed comment", @source)
-              end
-              if /--|-\z/.match?(md[1])
-                raise REXML::ParseException.new("Malformed comment", @source)
-              end
-              return [ :comment, md[1] ]
-            elsif @source.match("DOCTYPE", true)
+          elsif @source.match?("<!", true)
+            if @source.match?("--", true)
+              return [ :comment, process_comment ]
+            elsif @source.match?("DOCTYPE", true)
               base_error_message = "Malformed DOCTYPE"
-              unless @source.match(/\s+/um, true)
-                if @source.match(">")
+              unless @source.skip_spaces
+                if @source.match?(">")
                   message = "#{base_error_message}: name is missing"
                 else
                   message = "#{base_error_message}: invalid name"
@@ -291,10 +297,11 @@ module REXML
                 raise REXML::ParseException.new(message, @source)
               end
               name = parse_name(base_error_message)
-              if @source.match(/\s*\[/um, true)
+              @source.skip_spaces
+              if @source.match?("[", true)
                 id = [nil, nil, nil]
                 @document_status = :in_doctype
-              elsif @source.match(/\s*>/um, true)
+              elsif @source.match?(">", true)
                 id = [nil, nil, nil]
                 @document_status = :after_doctype
                 @source.ensure_buffer
@@ -306,9 +313,10 @@ module REXML
                   # For backward compatibility
                   id[1], id[2] = id[2], nil
                 end
-                if @source.match(/\s*\[/um, true)
+                @source.skip_spaces
+                if @source.match?("[", true)
                   @document_status = :in_doctype
-                elsif @source.match(/\s*>/um, true)
+                elsif @source.match?(">", true)
                   @document_status = :after_doctype
                   @source.ensure_buffer
                 else
@@ -318,7 +326,7 @@ module REXML
               end
               args = [:start_doctype, name, *id]
               if @document_status == :after_doctype
-                @source.match(/\s*/um, true)
+                @source.skip_spaces
                 @stack << [ :end_doctype ]
               end
               return args
@@ -329,14 +337,14 @@ module REXML
           end
         end
         if @document_status == :in_doctype
-          @source.match(/\s*/um, true) # skip spaces
+          @source.skip_spaces
           start_position = @source.position
-          if @source.match("<!", true)
-            if @source.match("ELEMENT", true)
+          if @source.match?("<!", true)
+            if @source.match?("ELEMENT", true)
               md = @source.match(/(.*?)>/um, true)
               raise REXML::ParseException.new( "Bad ELEMENT declaration!", @source ) if md.nil?
               return [ :elementdecl, "<!ELEMENT" + md[1] ]
-            elsif @source.match("ENTITY", true)
+            elsif @source.match?("ENTITY", true)
               match_data = @source.match(Private::ENTITYDECL_PATTERN, true)
               unless match_data
                 raise REXML::ParseException.new("Malformed entity declaration", @source)
@@ -368,11 +376,11 @@ module REXML
               end
               match << '%' if ref
               return match
-            elsif @source.match("ATTLIST", true)
+            elsif @source.match?("ATTLIST", true)
               md = @source.match(Private::ATTLISTDECL_END, true)
               raise REXML::ParseException.new( "Bad ATTLIST declaration!", @source ) if md.nil?
               element = md[1]
-              contents = md[0]
+              contents = "<!ATTLIST" + md[0]
 
               pairs = {}
               values = md[0].strip.scan( ATTDEF_RE )
@@ -388,10 +396,10 @@ module REXML
                 end
               end
               return [ :attlistdecl, element, pairs, contents ]
-            elsif @source.match("NOTATION", true)
+            elsif @source.match?("NOTATION", true)
               base_error_message = "Malformed notation declaration"
-              unless @source.match(/\s+/um, true)
-                if @source.match(">")
+              unless @source.skip_spaces
+                if @source.match?(">")
                   message = "#{base_error_message}: name is missing"
                 else
                   message = "#{base_error_message}: invalid name"
@@ -403,39 +411,37 @@ module REXML
               id = parse_id(base_error_message,
                             accept_external_id: true,
                             accept_public_id: true)
-              unless @source.match(/\s*>/um, true)
+              @source.skip_spaces
+              unless @source.match?(">", true)
                 message = "#{base_error_message}: garbage before end >"
                 raise REXML::ParseException.new(message, @source)
               end
               return [:notationdecl, name, *id]
-            elsif md = @source.match(/--(.*?)-->/um, true)
-              case md[1]
-              when /--/, /-\z/
-                raise REXML::ParseException.new("Malformed comment", @source)
-              end
-              return [ :comment, md[1] ] if md
+            elsif @source.match?("--", true)
+              return [ :comment, process_comment ]
+            else
+              raise REXML::ParseException.new("Malformed node: Started with '<!' but not a comment nor ELEMENT,ENTITY,ATTLIST,NOTATION", @source)
             end
           elsif match = @source.match(/(%.*?;)\s*/um, true)
             return [ :externalentity, match[1] ]
-          elsif @source.match(/\]\s*>/um, true)
+          elsif @source.match?(/\]\s*>/um, true)
             @document_status = :after_doctype
             return [ :end_doctype ]
-          end
-          if @document_status == :in_doctype
+          else
             raise ParseException.new("Malformed DOCTYPE: invalid declaration", @source)
           end
         end
         if @document_status == :after_doctype
-          @source.match(/\s*/um, true)
+          @source.skip_spaces
         end
         begin
           start_position = @source.position
-          if @source.match("<", true)
+          if @source.match?("<", true)
             # :text's read_until may remain only "<" in buffer. In the
             # case, buffer is empty here. So we need to fill buffer
             # here explicitly.
             @source.ensure_buffer
-            if @source.match("/", true)
+            if @source.match?("/", true)
               @namespaces_restore_stack.pop
               last_tag = @tags.pop
               md = @source.match(Private::CLOSE_PATTERN, true)
@@ -450,25 +456,21 @@ module REXML
                 raise REXML::ParseException.new(message, @source)
               end
               return [ :end_element, last_tag ]
-            elsif @source.match("!", true)
-              md = @source.match(/([^>]*>)/um)
+            elsif @source.match?("!", true)
               #STDERR.puts "SOURCE BUFFER = #{source.buffer}, #{source.buffer.size}"
-              raise REXML::ParseException.new("Malformed node", @source) unless md
-              if md[0][0] == ?-
-                md = @source.match(/--(.*?)-->/um, true)
-
-                if md.nil? || /--|-\z/.match?(md[1])
-                  raise REXML::ParseException.new("Malformed comment", @source)
+              if @source.match?("--", true)
+                return [ :comment, process_comment ]
+              elsif @source.match?("[CDATA[", true)
+                text = @source.read_until("]]>")
+                if text.chomp!("]]>")
+                  return [ :cdata, text ]
+                else
+                  raise REXML::ParseException.new("Malformed CDATA: Missing end ']]>'", @source)
                 end
-
-                return [ :comment, md[1] ]
               else
-                md = @source.match(/\[CDATA\[(.*?)\]\]>/um, true)
-                return [ :cdata, md[1] ] if md
+                raise REXML::ParseException.new("Malformed node: Started with '<!' but not a comment nor CDATA", @source)
               end
-              raise REXML::ParseException.new( "Declarations can only occur "+
-                "in the doctype declaration.", @source)
-            elsif @source.match("?", true)
+            elsif @source.match?("?", true)
               return process_instruction
             else
               # Get the next tag
@@ -527,7 +529,8 @@ module REXML
           raise REXML::ParseException.new( "Exception parsing",
             @source, self, (error ? error : $!) )
         end
-        return [ :dummy ]
+        # NOTE: The end of the method never runs, because it is unreachable.
+        #       All branches of code above have explicit unconditional return or raise statements.
       end
       private :pull_event
 
@@ -568,8 +571,12 @@ module REXML
         return rv if matches.size == 0
         rv.gsub!( Private::CHARACTER_REFERENCES ) {
           m=$1
-          m = "0#{m}" if m[0] == ?x
-          [Integer(m)].pack('U*')
+          if m.start_with?("x")
+            code_point = Integer(m[1..-1], 16)
+          else
+            code_point = Integer(m, 10)
+          end
+          [code_point].pack('U*')
         }
         matches.collect!{|x|x[0]}.compact!
         if filter
@@ -642,10 +649,14 @@ module REXML
         true
       end
 
+      def normalize_xml_declaration_encoding(xml_declaration_encoding)
+        /\AUTF-16(?:BE|LE)\z/i.match?(xml_declaration_encoding) ? "UTF-16" : nil
+      end
+
       def parse_name(base_error_message)
         md = @source.match(Private::NAME_PATTERN, true)
         unless md
-          if @source.match(/\S/um)
+          if @source.match?(/\S/um)
             message = "#{base_error_message}: invalid name"
           else
             message = "#{base_error_message}: name is missing"
@@ -687,73 +698,171 @@ module REXML
                                    accept_public_id:)
         public = /\A\s*PUBLIC/um
         system = /\A\s*SYSTEM/um
-        if (accept_external_id or accept_public_id) and @source.match(/#{public}/um)
-          if @source.match(/#{public}(?:\s+[^'"]|\s*[\[>])/um)
+        if (accept_external_id or accept_public_id) and @source.match?(/#{public}/um)
+          if @source.match?(/#{public}(?:\s+[^'"]|\s*[\[>])/um)
             return "public ID literal is missing"
           end
-          unless @source.match(/#{public}\s+#{PUBIDLITERAL}/um)
+          unless @source.match?(/#{public}\s+#{PUBIDLITERAL}/um)
             return "invalid public ID literal"
           end
           if accept_public_id
-            if @source.match(/#{public}\s+#{PUBIDLITERAL}\s+[^'"]/um)
+            if @source.match?(/#{public}\s+#{PUBIDLITERAL}\s+[^'"]/um)
               return "system ID literal is missing"
             end
-            unless @source.match(/#{public}\s+#{PUBIDLITERAL}\s+#{SYSTEMLITERAL}/um)
+            unless @source.match?(/#{public}\s+#{PUBIDLITERAL}\s+#{SYSTEMLITERAL}/um)
               return "invalid system literal"
             end
             "garbage after system literal"
           else
             "garbage after public ID literal"
           end
-        elsif accept_external_id and @source.match(/#{system}/um)
-          if @source.match(/#{system}(?:\s+[^'"]|\s*[\[>])/um)
+        elsif accept_external_id and @source.match?(/#{system}/um)
+          if @source.match?(/#{system}(?:\s+[^'"]|\s*[\[>])/um)
             return "system literal is missing"
           end
-          unless @source.match(/#{system}\s+#{SYSTEMLITERAL}/um)
+          unless @source.match?(/#{system}\s+#{SYSTEMLITERAL}/um)
             return "invalid system literal"
           end
           "garbage after system literal"
         else
-          unless @source.match(/\A\s*(?:PUBLIC|SYSTEM)\s/um)
+          unless @source.match?(/\A\s*(?:PUBLIC|SYSTEM)\s/um)
             return "invalid ID type"
           end
           "ID type is missing"
         end
       end
 
+      def process_comment
+        text = @source.read_until("-->")
+        unless text.chomp!("-->")
+          raise REXML::ParseException.new("Unclosed comment: Missing end '-->'", @source)
+        end
+
+        if text.include? "--" or text.end_with?("-")
+          raise REXML::ParseException.new("Malformed comment", @source)
+        end
+        text
+      end
+
       def process_instruction
         name = parse_name("Malformed XML: Invalid processing instruction node")
-        if @source.match(/\s+/um, true)
-          match_data = @source.match(/(.*?)\?>/um, true)
-          unless match_data
-            raise ParseException.new("Malformed XML: Unclosed processing instruction", @source)
+        if name == "xml"
+          xml_declaration
+        else # PITarget
+          if @source.skip_spaces # e.g. <?name content?>
+            start_position = @source.position
+            content = @source.read_until("?>")
+            unless content.chomp!("?>")
+              @source.position = start_position
+              raise ParseException.new("Malformed XML: Unclosed processing instruction: <#{name}>", @source)
+            end
+          else # e.g. <?name?>
+            content = nil
+            unless @source.match?("?>", true)
+              raise ParseException.new("Malformed XML: Unclosed processing instruction: <#{name}>", @source)
+            end
           end
-          content = match_data[1]
-        else
-          content = nil
-          unless @source.match("?>", true)
-            raise ParseException.new("Malformed XML: Unclosed processing instruction", @source)
+          [:processing_instruction, name, content]
+        end
+      end
+
+      def xml_declaration
+        unless @version.nil?
+          raise ParseException.new("Malformed XML: XML declaration is duplicated", @source)
+        end
+        if @document_status
+          raise ParseException.new("Malformed XML: XML declaration is not at the start", @source)
+        end
+        unless @source.skip_spaces
+          raise ParseException.new("Malformed XML: XML declaration misses spaces before version", @source)
+        end
+        unless @source.match?("version", true)
+          raise ParseException.new("Malformed XML: XML declaration misses version", @source)
+        end
+        @version = parse_attribute_value_with_equal("xml")
+        unless @source.skip_spaces
+          unless @source.match?("?>", true)
+            raise ParseException.new("Malformed XML: Unclosed XML declaration", @source)
           end
+          encoding = normalize_xml_declaration_encoding(@source.encoding)
+          return [ :xmldecl, @version, encoding, nil ] # e.g. <?xml version="1.0"?>
         end
-        if name == "xml"
-          if @document_status
-            raise ParseException.new("Malformed XML: XML declaration is not at the start", @source)
+
+        if @source.match?("encoding", true)
+          encoding = parse_attribute_value_with_equal("xml")
+          unless @source.skip_spaces
+            unless @source.match?("?>", true)
+              raise ParseException.new("Malformed XML: Unclosed XML declaration", @source)
+            end
+            if need_source_encoding_update?(encoding)
+              @source.encoding = encoding
+            end
+            encoding ||= normalize_xml_declaration_encoding(@source.encoding)
+            return [ :xmldecl, @version, encoding, nil ] # e.g. <?xml version="1.1" encoding="UTF-8"?>
           end
-          version = VERSION.match(content)
-          version = version[1] unless version.nil?
-          encoding = ENCODING.match(content)
-          encoding = encoding[1] unless encoding.nil?
-          if need_source_encoding_update?(encoding)
-            @source.encoding = encoding
+        end
+
+        if @source.match?("standalone", true)
+          standalone = parse_attribute_value_with_equal("xml")
+          case standalone
+          when "yes", "no"
+          else
+            raise ParseException.new("Malformed XML: XML declaration standalone is not yes or no : <#{standalone}>", @source)
           end
-          if encoding.nil? and /\AUTF-16(?:BE|LE)\z/i =~ @source.encoding
-            encoding = "UTF-16"
+        end
+        @source.skip_spaces
+        unless @source.match?("?>", true)
+          raise ParseException.new("Malformed XML: Unclosed XML declaration", @source)
+        end
+
+        if need_source_encoding_update?(encoding)
+          @source.encoding = encoding
+        end
+        encoding ||= normalize_xml_declaration_encoding(@source.encoding)
+
+        # e.g. <?xml version="1.0" ?>
+        #      <?xml version="1.1" encoding="UTF-8" ?>
+        #      <?xml version="1.1" standalone="yes"?>
+        #      <?xml version="1.1" encoding="UTF-8" standalone="yes" ?>
+        [ :xmldecl, @version, encoding, standalone ]
+      end
+
+      if StringScanner::Version < "3.1.1"
+        def scan_quote
+          @source.match(/(['"])/, true)&.[](1)
+        end
+      else
+        def scan_quote
+          case @source.peek_byte
+          when 34 # '"'.ord
+            @source.scan_byte
+            '"'
+          when 39 # "'".ord
+            @source.scan_byte
+            "'"
+          else
+            nil
           end
-          standalone = STANDALONE.match(content)
-          standalone = standalone[1] unless standalone.nil?
-          return [ :xmldecl, version, encoding, standalone ]
         end
-        [:processing_instruction, name, content]
+      end
+
+      def parse_attribute_value_with_equal(name)
+        unless @source.match?(Private::EQUAL_PATTERN, true)
+          message = "Missing attribute equal: <#{name}>"
+          raise REXML::ParseException.new(message, @source)
+        end
+        unless quote = scan_quote
+          message = "Missing attribute value start quote: <#{name}>"
+          raise REXML::ParseException.new(message, @source)
+        end
+        start_position = @source.position
+        value = @source.read_until(quote)
+        unless value.chomp!(quote)
+          @source.position = start_position
+          message = "Missing attribute value end quote: <#{name}>: <#{quote}>"
+          raise REXML::ParseException.new(message, @source)
+        end
+        value
       end
 
       def parse_attributes(prefixes)
@@ -761,36 +870,20 @@ module REXML
         expanded_names = {}
         closed = false
         while true
-          if @source.match(">", true)
+          if @source.match?(">", true)
             return attributes, closed
-          elsif @source.match("/>", true)
+          elsif @source.match?("/>", true)
             closed = true
             return attributes, closed
           elsif match = @source.match(QNAME, true)
             name = match[1]
             prefix = match[2]
             local_part = match[3]
-
-            unless @source.match(/\s*=\s*/um, true)
-              message = "Missing attribute equal: <#{name}>"
-              raise REXML::ParseException.new(message, @source)
-            end
-            unless match = @source.match(/(['"])/, true)
-              message = "Missing attribute value start quote: <#{name}>"
-              raise REXML::ParseException.new(message, @source)
-            end
-            quote = match[1]
-            start_position = @source.position
-            value = @source.read_until(quote)
-            unless value.chomp!(quote)
-              @source.position = start_position
-              message = "Missing attribute value end quote: <#{name}>: <#{quote}>"
-              raise REXML::ParseException.new(message, @source)
-            end
-            @source.match(/\s*/um, true)
+            value = parse_attribute_value_with_equal(name)
+            @source.skip_spaces
             if prefix == "xmlns"
               if local_part == "xml"
-                if value != "http://www.w3.org/XML/1998/namespace"
+                if value != Private::XML_PREFIXED_NAMESPACE
                   msg = "The 'xml' prefix must not be bound to any other namespace "+
                     "(http://www.w3.org/TR/REC-xml-names/#ns-decl)"
                   raise REXML::ParseException.new( msg, @source, self )

data/lib/rexml/parsers/pullparser.rb

@@ -93,6 +93,10 @@ module REXML
       def unshift token
         @my_stack.unshift token
       end
+
+      def reset
+        @parser.reset
+      end
     end
 
     # A parsing event.  The contents of the event are accessed as an +Array?,

data/lib/rexml/parsers/sax2parser.rb

@@ -259,6 +259,8 @@ module REXML
       end
 
       def get_namespace( prefix )
+        return nil if @namespace_stack.empty?
+
         uris = (@namespace_stack.find_all { |ns| not ns[prefix].nil? }) ||
           (@namespace_stack.find { |ns| not ns[nil].nil? })
         uris[-1][prefix] unless uris.nil? or 0 == uris.size

data/lib/rexml/parsers/xpathparser.rb

@@ -215,7 +215,7 @@ module REXML
         else
           path << yield( parsed )
         end
-        return path.squeeze(" ")
+        path.squeeze(" ")
       end
       # For backward compatibility
       alias_method :preciate_to_string, :predicate_to_path
@@ -252,7 +252,7 @@ module REXML
             path = path[1..-1]
           end
         end
-        return RelativeLocationPath( path, parsed ) if path.size > 0
+        RelativeLocationPath( path, parsed ) if path.size > 0
       end
 
       #RelativeLocationPath
@@ -388,7 +388,7 @@ module REXML
         else
           path = original_path
         end
-        return path
+        path
       end
 
       # Filters the supplied nodeset on the predicate(s)
@@ -600,7 +600,7 @@ module REXML
         end
         rest = LocationPath(rest, n) if rest =~ /\A[\/\.\@\[\w*]/
         parsed.concat(n)
-        return rest
+        rest
       end
 
       #| FilterExpr Predicate