sdl4r 0.9.6 → 0.9.7

data/lib/sdl4r/parser/reader.rb

@@ -29,16 +29,24 @@ module SDL4R
 
       # +io+  an open IO from which the characters are read.
       def initialize(io)
-        raise ArgumentError, "io == nil" if io.nil?
+        raise ArgumentError, "io == nil" unless io
 
         @io = io
         @line = nil
         @line_chars = nil
-        @line_no = 0
+        @line_no = -1
         @pos = 0
       end
 
-      attr_reader :line_no, :pos, :line;
+      # The current line no (zero-based)
+      attr_reader :line_no
+
+      # The position of the char currently pointed by this reader in the current line.
+      # This is not necessarily the position of the char you just got from a method, it might be
+      # the position of the next char, for instance.
+      attr_reader :pos
+      
+      attr_reader :line
 
       def line_length
         return @line_chars.nil? ? 0 : @line_chars.length
@@ -65,22 +73,22 @@ module SDL4R
       end
 
       # Indicates whether the end of file has been reached.
-      def end_of_file?
+      def eof?
         return @line.nil?
       end
 
-      # Indicates whether there are more characters in the current line
+      # Indicates whether there are more characters in the current line after the current char
       def more_chars_in_line?
         return @pos < line_length - 1
       end
 
       # Returns whether the end of the current +line+ as been reached.
-      def end_of_line?
+      def eol?
         return @pos >= line_length
       end
 
       # Skips the current line by going just after its end.
-      def skip_line
+      def skip_to_eol
         @pos = line_length
       end
 
@@ -107,24 +115,27 @@ module SDL4R
         return get_line_char(@pos)
       end
 
-      # Go to the next character in the stream.
+      # Go to the next character in the line. This method doesn't skip to the next line once the
+      # reached eol.
       def skip_char
         @pos += 1 if @pos < line_length
       end
 
-      # Go to the next character and returns it (or nil if end-of-line or -file has been reached).
+      # Returns the current char and go to the next.
+      # Returns nil if end-of-line or -file has been reached.
       def read_char
+        c = current_char
         skip_char()
-        return current_char
+        c
       end
 
       # Returns to the previous char if possible.
       def previous_char
-        @pos -= 1 if @pos >= -1
+        @pos -= 1 if @pos >= 1
       end
 
-      # Returns the next index of the expression (string, regexp, fixnum) in the current line, starting
-      # from after the current position if no position is specified.
+      # Returns the next index of the expression (string, regexp, fixnum) in the current line,
+      # starting from after the current position if no position is specified.
       def find_next_in_line(searched, start_pos = nil)
         start_pos = @pos + 1 unless start_pos
         return @line.index(searched, start_pos)
@@ -149,12 +160,8 @@ module SDL4R
       def read_raw_line
         @line = @io.gets()
 
-        # We ensure that only \n is used as an end-of-line by replacing \r and \r\n.
-        if @line
-          if not @line.gsub!(/\r\n/m, "\n")
-            @line.gsub!(/\r/m, "\n")
-          end
-        end
+        # Remove a possible \r at the end of line
+        @line.gsub!(/\r+$/, "") if @line
 
         @pos = 0;
         @line_chars = nil

data/lib/sdl4r/parser/token.rb

@@ -33,7 +33,7 @@ module SDL4R
       def initialize(text, line = -1, position = -1)
         @text = text
         @line = line
-        @pos = position
+        @position = position
         @size = text.length
 
         begin
@@ -92,7 +92,7 @@ module SDL4R
           end
 
         rescue ArgumentError
-          raise SdlParseError.new($!.message, @line, @pos)
+          raise SdlParseError.new($!.message, @line, @position)
         end
 
         @type = :IDENTIFIER if @type.nil? # if all hope is lost, it's an identifier
@@ -116,7 +116,7 @@ module SDL4R
       end
 
       def to_s
-        @type.to_s + " " + @text + " pos:" + @pos.to_s
+        @type.to_s + " " + @text + " pos:" + @position.to_s
       end
 
       # This special parse method is used only by the Token class for

data/lib/sdl4r/parser/tokenizer.rb

@@ -45,9 +45,9 @@ module SDL4R
 
         @reader = Parser::Reader.new(io)
         @token_start = 0
-        @startEscapedQuoteLine = false
+        @starting_escaped_quote_line = false
         @tokens = nil
-        @tokenText = nil
+        @token_text = nil
       end
 
       # Closes this Tokenizer and its underlying +Reader+.
@@ -55,19 +55,12 @@ module SDL4R
         @reader.close
       end
 
-      # Close the reader and throw a SdlParseError.
+      # Raises a SdlParseError.
       def parse_error(description, line_no = nil, position = nil)
-        begin
-          @reader.close()
-        rescue IOError
-          # no recourse
-        end
+        line_no = @reader.line_no unless line_no
+        position = @reader.pos - 1 unless position # @reader.pos points to the *next* position
 
-        line_no = @reader.line_no if line_no.nil?
-        position = @reader.pos if position.nil?
-
-        # We add one because editors typically start with line 1 and position 1
-        # rather than 0...
+        # We add one because editors typically start with line 1 rather than 0...
         raise SdlParseError.new(description, line_no + 1, position + 1, @reader.line)
       end
 
@@ -111,26 +104,31 @@ module SDL4R
       #
       # Returns a logical line as a list of Tokens.
       # Returns an empty array if the line was empty.
+      # Returns +nil+ if the end of the stream has been reached.
       #
       def read_line_tokens_even_if_empty
+        # Reset of the token-related fields
         @tokens = nil
-        @tokenText = nil
+        @token_text = nil
         @token_start = nil
 
-        @reader.read_line() if @reader.end_of_line?
-        return @tokens unless @reader.line
+        if @reader.eol? and not @reader.read_line()
+          return nil
+        end
 
         @tokens = []
         @token_start = @reader.pos
 
-        while not @reader.end_of_line?
-          if @tokenText
-            @tokens << Token.new(@tokenText, @reader.line_no, @token_start)
-            @tokenText = nil
+        until @reader.eol?
+          if @token_text
+            @tokens << Token.new(@token_text, @reader.line_no, @token_start)
+            @token_text = nil
+            @token_start = @reader.pos
           end
 
           c = @reader.current_char
           next_c = @reader.get_line_char(@reader.pos + 1)
+          
           case c
           when "\""
             # handle "" style strings including line continuations
@@ -142,7 +140,7 @@ module SDL4R
           when "{", "}", "=", ":", ";"
             # handle punctuation
             punctuation_token = Token.new(c, @reader.line_no, @reader.pos)
-            @tokenText = nil
+            @token_text = nil
 
             if punctuation_token.type == :SEMICOLON
               @reader.skip_char()
@@ -153,13 +151,13 @@ module SDL4R
 
           when "#"
             # skip : hash comment at end of line
-            @reader.skip_line()
+            @reader.skip_to_eol()
 
           when "/"
             # handle // and /**/ style comments
             if next_c == "/"
               # skip : // comment
-              @reader.skip_line()
+              @reader.skip_to_eol()
             else
               handle_slash_comment()
             end
@@ -177,12 +175,13 @@ module SDL4R
 
           when "\\"
             # line continuations (outside a string literal)
-            handle_line_continuation();
+            handle_line_continuation()
+            next # otherwise 1st char of the continuation line is skipped by skip_char()
 
           when /^[0-9\-\.]$/
             if c == "-" and next_c == "-"
               # -- comments : ignore
-              @reader.skip_line()
+              @reader.skip_to_eol()
             else
               # handle numbers, dates, and time spans
               handle_number_date_or_time_span()
@@ -194,9 +193,8 @@ module SDL4R
             # So, we implement a subset of these characters.
             handle_identifier()
 
-          when "\n", "\r"
-            # end of line
-            @reader.skip_line()
+          when "\n", "\r" # end of line
+            @reader.skip_to_eol()
 
           else
             parse_error("Unexpected character '#{c}'")
@@ -205,45 +203,44 @@ module SDL4R
           @reader.skip_char()
         end
 
-        if @tokenText
-          @tokens << Token.new(@tokenText, @reader.line_no, @token_start)
+        if @token_text
+          @tokens << Token.new(@token_text, @reader.line_no, @token_start)
         end
 
         return @tokens
       end
 
-      # Adds the current escaped character (represented by ((|c|))) to @tokenText.
+      # Adds the current escaped character (represented by ((|c|))) to @token_text.
       # This method assumes the previous char was a backslash.
       #
       def add_escaped_char_in_string(c)
         case c
         when "\\", "\""
-          @tokenText << c
+          @token_text << c
         when "n"
-          @tokenText << ?\n
+          @token_text << ?\n
         when "r"
-          @tokenText << ?\r
+          @token_text << ?\r
         when "t"
-          @tokenText << ?\t
+          @token_text << ?\t
         else
-          parse_error("Illegal escape character in string literal: '#{c.chr}'.")
+          parse_error("Illegal escape character in string literal: '#{c}'.")
         end
       end
 
       def handle_double_quote_string
         escaped = false
-        @startEscapedQuoteLine = false
+        @starting_escaped_quote_line = false
 
-        @tokenText = "\""
-        @reader.skip_char()
+        @token_text = @reader.read_char()
 
-        while not @reader.end_of_line?
+        until @reader.eol?
           c = @reader.current_char
 
-          if "\s\t".include?(c) and @startEscapedQuoteLine
-            # we continue
+          if "\s\t".include?(c) and @starting_escaped_quote_line
+            # skip the heading spaces (indentation) of a continued line
           else
-            @startEscapedQuoteLine = false;
+            @starting_escaped_quote_line = false
 
             if escaped
               add_escaped_char_in_string(c)
@@ -255,15 +252,15 @@ module SDL4R
                 handle_escaped_double_quoted_string()
                 next # as we are at the beginning of a new line
               else
-                escaped = true;
+                escaped = true
               end
 
             else
-              @tokenText << c
-              if c == "\""
+              @token_text << c
+              if c == '"'
                 # end of double-quoted string detected
-                @tokens << Token.new(@tokenText, @reader.line_no, @token_start)
-                @tokenText = nil
+                @tokens << Token.new(@token_text, @reader.line_no, @token_start)
+                @token_text = nil
                 return
               end
             end
@@ -273,21 +270,24 @@ module SDL4R
         end
 
         # detection of ill-terminated literals
-        if @tokenText =~ /^".*[^"]$/
+        if @token_text =~ /^".*[^"]$/
           parse_error(
-            "String literal \"#{@tokenText}\" not terminated by end quote.", @reader.line_no, @reader.line_length);
-        elsif @tokenText == "\""
-          parse_error("Orphan quote (unterminated string)", @reader.line_no, @reader.line_length);
+            "String literal \"#{@token_text}\" not terminated by end quote.",
+            @reader.line_no,
+            @reader.line_length)
+        else#if @token_text == '"'
+          parse_error("Orphan quote (unterminated string)", @reader.line_no, @reader.line_length)
         end
+puts "end of chars"
       end
 
       def handle_escaped_double_quoted_string
         # '\' can be followed by whitespaces
         if @reader.rest_of_line =~ /^\\\s*$/
           @reader.read_line()
-          parse_error("Escape at end of file.") if @reader.end_of_file?
+          parse_error("Escape at end of file.") if @reader.eof?
 
-          @startEscapedQuoteLine = true
+          @starting_escaped_quote_line = true
 
         else
           parse_error(
@@ -297,22 +297,18 @@ module SDL4R
       end
 
       def handle_character_literal
-        if not @reader.more_chars_in_line?
-          parse_error("Got ' at end of line")
-        end
+        @reader.skip_char # skip the starting quote
+
+        parse_error("Got ' at end of line") if @reader.eol?
 
         c2 = @reader.read_char()
 
         if c2 == "\\"
-          if @reader.end_of_line?
-            parse_error("Got '\\ at end of line")
-          end
+          parse_error("Got '\\ at end of line") if @reader.eol?
 
           c3 = @reader.read_char()
 
-          if not @reader.more_chars_in_line?
-            parse_error("Got '\\#{c3} at end of line")
-          end
+          parse_error("Got '\\#{c3} at end of line") if @reader.eol?
 
           case c3
           when "\\"
@@ -329,17 +325,16 @@ module SDL4R
             parse_error("Illegal escape character #{@reader.current_char}")
           end
 
-          @reader.skip_char()
-          if @reader.current_char != "'"
+          if @reader.read_char != "'"
             expecting_but_got("single quote (')", "\"#{@reader.current_char}\"")
           end
+
         else
           @tokens << Token.new("'#{c2}'", @reader.line_no, @reader.pos)
-          if not @reader.more_chars_in_line?
-            parse_error("Got '#{c2} at end of line")
-          end
-          @reader.skip_char()
-          if @reader.current_char != "'"
+
+          parse_error("Got '#{c2} at end of line") if @reader.eol?
+          
+          if @reader.read_char != "'"
             expecting_but_got(
               "quote (')", "\"#{@reader.current_char}\"", @reader.line_no, @reader.pos)
           end
@@ -360,7 +355,7 @@ module SDL4R
             # handle multiline comments
             loop do
               @reader.read_raw_line()
-              if @reader.end_of_file?
+              if @reader.eof?
                 parse_error("/* comment not terminated.", @reader.line_no, -2)
               end
 
@@ -383,31 +378,31 @@ module SDL4R
         if end_index
           # handle end quote on same line
           @tokens << Token.new(@reader.substring(@reader.pos, end_index), @reader.line_no, @reader.pos)
-          @tokenText = nil
+          @token_text = nil
           @reader.skip_to(end_index)
 
         else
-          @tokenText = @reader.rest_of_line
+          @token_text = @reader.rest_of_line
           @token_start = @reader.pos
           # handle multiline quotes
           loop do
             @reader.read_raw_line()
-            if @reader.end_of_file?
+            if @reader.eof?
               parse_error("` quote not terminated.", @reader.line_no, -2)
             end
 
             end_index = @reader.find_next_in_line("`", 0)
             if end_index
-              @tokenText << @reader.substring(0, end_index)
+              @token_text << @reader.substring(0, end_index)
               @reader.skip_to(end_index)
               break
             else
-              @tokenText << @reader.line
+              @token_text << @reader.line
             end
           end
 
-          @tokens << Token.new(@tokenText, @reader.line_no, @token_start)
-          @tokenText = nil
+          @tokens << Token.new(@token_text, @reader.line_no, @token_start)
+          @token_text = nil
         end
       end
 
@@ -417,30 +412,30 @@ module SDL4R
         if end_index
           # handle end quote on same line
           @tokens << Token.new(@reader.substring(@reader.pos, end_index), @reader.line_no, @reader.pos)
-          @tokenText = nil
+          @token_text = nil
           @reader.skip_to(end_index)
         else
-          @tokenText = @reader.substring(@reader.pos)
+          @token_text = @reader.substring(@reader.pos)
           @token_start = @reader.pos
           # handle multiline quotes
           loop do
             @reader.read_raw_line()
-            if @reader.end_of_file?
+            if @reader.eof?
               parse_error("[base64] binary literal not terminated.", @reader.line_no, -2)
             end
 
             end_index = @reader.find_next_in_line("]", 0)
             if end_index
-              @tokenText << @reader.substring(0, end_index)
+              @token_text << @reader.substring(0, end_index)
               @reader.skip_to(end_index)
               break
             else
-              @tokenText << @reader.line
+              @token_text << @reader.line
             end
           end
 
-          @tokens << Token.new(@tokenText, @reader.line_no, @token_start)
-          @tokenText = nil
+          @tokens << Token.new(@token_text, @reader.line_no, @token_start)
+          @token_text = nil
         end
       end
 
@@ -452,7 +447,7 @@ module SDL4R
           parse_error("Line continuation (\\) before end of line")
         else
           @line = @reader.read_line()
-          if @line.nil?
+          unless @line
             parse_error("Line continuation at end of file.", @reader.line_no, @reader.pos)
           end
         end
@@ -460,15 +455,15 @@ module SDL4R
 
       def handle_number_date_or_time_span
         @token_start = @reader.pos
-        @tokenText = ""
+        @token_text = ""
 
-        while not @reader.end_of_line?
+        until @reader.eol?
           c = @reader.current_char
 
           if c =~ /[\w\.\-+:]/
-            @tokenText << c
+            @token_text << c
           elsif c == "/" and not @reader.get_line_char(@reader.pos + 1) == "*"
-            @tokenText << c
+            @token_text << c
           else
             @reader.previous_char()
             break
@@ -477,21 +472,21 @@ module SDL4R
           @reader.skip_char()
         end
 
-        @tokens << Token.new(@tokenText, @reader.line_no, @token_start)
-        @tokenText = nil
+        @tokens << Token.new(@token_text, @reader.line_no, @token_start)
+        @token_text = nil
       end
 
       def handle_identifier
-        @token_start = @reader.pos;
-        @tokenText = ""
+        @token_start = @reader.pos
+        @token_text = ""
 
-        while not @reader.end_of_line?
+        until @reader.eol?
           c = @reader.current_char
 
           # FIXME here we are stricter than the Java version because there is no
           # easy way to implement Character.isJavaIdentifierPart() in Ruby :)
           if c =~ /[\w_$-\.]/
-            @tokenText << c
+            @token_text << c
           else
             @reader.previous_char()
             break
@@ -500,8 +495,8 @@ module SDL4R
           @reader.skip_char()
         end
 
-        @tokens << Token.new(@tokenText, @reader.line_no, @token_start)
-        @tokenText = nil
+        @tokens << Token.new(@token_text, @reader.line_no, @token_start)
+        @token_text = nil
       end
 
     end