parser 1.4.2 → 2.0.0.beta1

data/lib/parser/lexer.rl

@@ -88,17 +88,20 @@ class Parser::Lexer
 
   attr_accessor :cond, :cmdarg
 
-  attr_reader   :comments
+  attr_accessor :tokens, :comments
 
   def initialize(version)
     @version    = version
     @static_env = nil
 
+    @tokens     = nil
+    @comments   = nil
+
     reset
   end
 
   def reset(reset_state=true)
-    # Ragel-related variables:
+    # Ragel state:
     if reset_state
       # Unit tests set state prior to resetting lexer.
       @cs     = self.class.lex_en_line_begin
@@ -119,7 +122,8 @@ class Parser::Lexer
     @token_queue   = []
     @literal_stack = []
 
-    @comments      = ""  # collected comments
+    @eq_begin_s    = nil # location of last encountered =begin
+    @sharp_s       = nil # location of last encountered #
 
     @newline_s     = nil # location of last encountered newline
 
@@ -150,7 +154,7 @@ class Parser::Lexer
       #
       # Patches accepted.
       #
-      @source = @source_buffer.source + "\0\0\0"
+      @source = @source_buffer.source.gsub(/\r\n/, "\n") + "\0\0\0"
     else
       @source = nil
     end
@@ -212,14 +216,6 @@ class Parser::Lexer
     end
   end
 
-  # Return the current collected comment block and clear the storage.
-  def clear_comments
-    comments  = @comments
-    @comments = ""
-
-    comments
-  end
-
   protected
 
   def eof_char?(char)
@@ -244,7 +240,13 @@ class Parser::Lexer
   end
 
   def emit(type, value = tok, s = @ts, e = @te)
-    @token_queue << [ type, [ value, range(s, e) ] ]
+    token = [ type, [ value, range(s, e) ] ]
+
+    @token_queue.push(token)
+
+    @tokens.push(token) if @tokens
+
+    token
   end
 
   def emit_table(table, s = @ts, e = @te)
@@ -253,6 +255,18 @@ class Parser::Lexer
     emit(table[value], value, s, e)
   end
 
+  def emit_comment(s = @ts, e = @te)
+    if @comments
+      @comments.push(Parser::Source::Comment.new(range(s, e)))
+    end
+
+    if @tokens
+      @tokens.push([ :tCOMMENT, [ tok(s, e), range(s, e) ] ])
+    end
+
+    nil
+  end
+
   def diagnostic(type, message, location=range, highlights=[])
     @diagnostics.process(
         Parser::Diagnostic.new(type, message, location, highlights))
@@ -363,7 +377,7 @@ class Parser::Lexer
     @newline_s = p
   }
 
-  c_nl       = '\r'? '\n' $ do_nl;
+  c_nl       = '\n' $ do_nl;
   c_space    = [ \t\r\f\v];
   c_space_nl = c_space | c_nl;
   c_eof      = 0x04 | 0x1a | 0; # ^D, ^Z, EOF
@@ -486,21 +500,23 @@ class Parser::Lexer
   action unicode_points {
     @escape = ""
 
-    codepoints = tok(@escape_s + 2, p - 1)
+    codepoints  = tok(@escape_s + 2, p - 1)
+    codepoint_s = @escape_s + 2
+
     codepoints.split(/[ \t]/).each do |codepoint_str|
       codepoint = codepoint_str.to_i(16)
 
       if codepoint >= 0x110000
         @escape = lambda do
-          # TODO better location reporting
           diagnostic :error, Parser::ERRORS[:unicode_point_too_large],
-                     range(@escape_s, p)
+                     range(codepoint_s, codepoint_s + codepoint_str.length)
         end
 
         break
       end
 
       @escape += codepoint.chr(Encoding::UTF_8)
+      codepoint_s += codepoint_str.length + 1
     end
   }
 
@@ -886,6 +902,44 @@ class Parser::Lexer
       };
   *|;
 
+  #
+  # === WHITESPACE HANDLING ===
+  #
+
+  # Various contexts in Ruby allow various kinds of whitespace
+  # to be used. They are grouped to clarify the lexing machines
+  # and ease collection of comments.
+
+  # A line of code with inline #comment at end is always equivalent
+  # to a line of code ending with just a newline, so an inline
+  # comment is deemed equivalent to non-newline whitespace
+  # (c_space character class).
+
+  w_space =
+      c_space+
+    | '\\' e_heredoc_nl
+    | '#' %{ @sharp_s = p - 1 } c_line* %{ emit_comment(@sharp_s, p) }
+    ;
+
+  # A newline in non-literal context always interoperates with
+  # here document logic and can always be escaped by a backslash,
+  # still interoperating with here document logic in the same way,
+  # yet being invisible to anything else.
+  #
+  # To demonstrate:
+  #
+  #     foo = <<FOO \
+  #     bar
+  #     FOO
+  #      + 2
+  #
+  # is equivalent to `foo = "bar\n" + 2`.
+
+  w_newline = e_heredoc_nl;
+
+  w_space_newline = w_space | w_newline;
+
+
   #
   # === EXPRESSION PARSING ===
   #
@@ -1019,9 +1073,7 @@ class Parser::Lexer
       ':'
       => { fhold; fgoto expr_beg; };
 
-      # TODO whitespace rule
-      c_space;
-      e_heredoc_nl;
+      w_space_newline;
 
       c_any
       => { fhold; fgoto expr_end; };
@@ -1037,10 +1089,9 @@ class Parser::Lexer
   expr_endfn := |*
       bareword ':'
       => { emit(:tLABEL, tok(@ts, @te - 1))
-           fnext expr_end; fbreak; };
+           fnext expr_beg; fbreak; };
 
-      # TODO whitespace rule
-      c_space;
+      w_space;
 
       c_any
       => { fhold; fgoto expr_end; };
@@ -1072,11 +1123,7 @@ class Parser::Lexer
       => { emit_table(PUNCTUATION)
            fnext expr_arg; fbreak; };
 
-      # TODO whitespace rule
-      c_space;
-      e_heredoc_nl;
-      '\\' e_heredoc_nl;
-      '#' c_line*;
+      w_space_newline;
 
       c_any
       => { fhold; fgoto expr_end; };
@@ -1194,7 +1241,7 @@ class Parser::Lexer
         fgoto expr_end;
       };
 
-      c_space* ( '#' c_line* )? c_nl
+      w_space? w_newline
       => { fhold; fgoto expr_end; };
 
       c_any
@@ -1227,7 +1274,7 @@ class Parser::Lexer
       => { emit(:kDO_BLOCK)
            fnext expr_value; };
 
-      c_space;
+      w_space;
 
       c_any
       => { fhold; fgoto expr_end; };
@@ -1245,11 +1292,8 @@ class Parser::Lexer
       => { emit_table(KEYWORDS)
            fnext expr_beg; fbreak; };
 
-      # TODO whitespace rule
-      c_space;
-      '#' c_line*;
-
-      e_heredoc_nl
+      w_space;
+      w_newline
       => { fhold; fgoto expr_end; };
 
       c_any
@@ -1487,14 +1531,7 @@ class Parser::Lexer
       # WHITESPACE
       #
 
-      # TODO whitespace rule
-      c_space;
-      e_heredoc_nl;
-      '\\' e_heredoc_nl;
-
-      '#' c_line* c_eol
-      => { @comments << tok
-           fhold; };
+      w_space_newline;
 
       e_heredoc_nl '=begin' ( c_space | c_eol )
       => { p = @ts - 1
@@ -1523,8 +1560,7 @@ class Parser::Lexer
       => { p = @ts - 1
            fgoto expr_end; };
 
-      # TODO whitespace rule
-      c_space;
+      w_space;
 
       c_any
       => { fhold; fgoto expr_beg; };
@@ -1779,8 +1815,13 @@ class Parser::Lexer
       # WHITESPACE
       #
 
-      # TODO whitespace rule
-      '\\' e_heredoc_nl;
+      w_space;
+      w_newline
+      => { fgoto leading_dot; };
+
+      ';'
+      => { emit_table(PUNCTUATION)
+           fnext expr_value; fbreak; };
 
       '\\' c_line {
         diagnostic :error, Parser::ERRORS[:bare_backslash],
@@ -1788,18 +1829,6 @@ class Parser::Lexer
         fhold;
       };
 
-      c_space+;
-
-      '#' c_line*
-      => { @comments << tok(@ts, @te + 1) };
-
-      e_heredoc_nl
-      => { fgoto leading_dot; };
-
-      ';'
-      => { emit_table(PUNCTUATION)
-           fnext expr_value; fbreak; };
-
       c_any
       => {
         message = Parser::ERRORS[:unexpected] % { :character => tok.inspect[1..-2] }
@@ -1827,30 +1856,26 @@ class Parser::Lexer
   #
 
   line_comment := |*
-      '=end' c_line* c_nl
-      => { @comments << tok
-           fgoto line_begin; };
+      '=end' c_line* c_nl?
+      => {
+        emit_comment(@eq_begin_s, @te)
+        fgoto line_begin;
+      };
 
-      c_line* c_nl
-      => { @comments << tok };
+      c_any;
 
       c_eof
       => {
-        # TODO better location information here
-        diagnostic :fatal, Parser::ERRORS[:embedded_document], range(p - 1, p)
+        diagnostic :fatal, Parser::ERRORS[:embedded_document],
+                   range(@eq_begin_s, @eq_begin_s + '=begin'.length)
       };
   *|;
 
   line_begin := |*
-      # TODO whitespace rule
-      c_space_nl+;
-
-      '#' c_line* c_eol
-      => { @comments << tok
-           fhold; };
+      w_space_newline;
 
       '=begin' ( c_space | c_eol )
-      => { @comments << tok(@ts, @te)
+      => { @eq_begin_s = @ts
            fgoto line_comment; };
 
       '__END__' c_eol

data/lib/parser/lexer/explanation.rb

@@ -2,19 +2,10 @@ module Parser
 
   module Lexer::Explanation
 
-    def self.included(klass)
-      klass.class_exec do
-        alias_method :state_before_explanation=,  :state=
-        alias_method :advance_before_explanation, :advance
-
-        remove_method :state=, :advance
-      end
-    end
-
     # Like #advance, but also pretty-print the token and its position
     # in the stream to `stdout`.
     def advance
-      type, (val, range) = advance_before_explanation
+      type, (val, range) = super
 
       puts decorate(range,
                     "\e[0;32m#{type} #{val.inspect}\e[0m",
@@ -27,7 +18,7 @@ module Parser
       puts "  \e[1;33m>>> STATE SET <<<\e[0m " +
            "#{new_state.to_s.ljust(12)} #{@cond} #{@cmdarg}".rjust(66)
 
-      self.state_before_explanation = new_state
+      super
     end
 
     private

data/lib/parser/rewriter.rb

@@ -11,8 +11,7 @@ module Parser
     private
 
     def assignment?(node)
-      [:lvasgn, :ivasgn, :gvasgn,
-       :cvasgn, :cvdecl, :cdecl].include?(node.type)
+      [:lvasgn, :ivasgn, :gvasgn, :cvasgn, :casgn].include?(node.type)
     end
 
     def remove(range)

data/lib/parser/ruby18.y

@@ -1155,7 +1155,6 @@ rule
                         diagnostic(:error, :class_in_def, val[0])
                       end
 
-                      @comments.push @lexer.clear_comments
                       @static_env.extend_static
                     }
                     bodystmt kEND
@@ -1166,7 +1165,6 @@ rule
                                                   val[4], val[5])
 
                       @static_env.unextend
-                      @lexer.clear_comments
                     }
                 | kCLASS tLSHFT expr term
                     {
@@ -1181,7 +1179,6 @@ rule
                                                    val[5], val[6])
 
                       @static_env.unextend
-                      @lexer.clear_comments
 
                       @def_level = val[4]
                     }
@@ -1191,7 +1188,6 @@ rule
                         diagnostic(:error, :module_in_def, val[0])
                       end
 
-                      @comments.push @lexer.clear_comments
                       @static_env.extend_static
                     }
                     bodystmt kEND
@@ -1200,26 +1196,22 @@ rule
                                                    val[3], val[4])
 
                       @static_env.unextend
-                      @lexer.clear_comments
                     }
                 | kDEF fname
                     {
-                      @comments.push @lexer.clear_comments
                       @def_level += 1
                       @static_env.extend_static
                     }
                     f_arglist bodystmt kEND
                     {
                       result = @builder.def_method(val[0], val[1],
-                                  val[3], val[4], val[5], @comments.pop)
+                                  val[3], val[4], val[5])
 
                       @static_env.unextend
                       @def_level -= 1
-                      @lexer.clear_comments
                     }
                 | kDEF singleton dot_or_colon
                     {
-                      @comments.push @lexer.clear_comments
                       @lexer.state = :expr_fname
                     }
                     fname
@@ -1230,11 +1222,10 @@ rule
                     f_arglist bodystmt kEND
                     {
                       result = @builder.def_singleton(val[0], val[1], val[2],
-                                  val[4], val[6], val[7], val[8], @comments.pop)
+                                  val[4], val[6], val[7], val[8])
 
                       @static_env.unextend
                       @def_level -= 1
-                      @lexer.clear_comments
                     }
                 | kBREAK
                     {

data/lib/parser/ruby19.y

@@ -1112,7 +1112,6 @@ rule
                         diagnostic(:error, :class_in_def, val[0])
                       end
 
-                      @comments.push @lexer.clear_comments
                       @static_env.extend_static
                     }
                     bodystmt kEND
@@ -1123,7 +1122,6 @@ rule
                                                   val[4], val[5])
 
                       @static_env.unextend
-                      @lexer.clear_comments
                     }
                 | kCLASS tLSHFT expr term
                     {
@@ -1138,7 +1136,6 @@ rule
                                                    val[5], val[6])
 
                       @static_env.unextend
-                      @lexer.clear_comments
 
                       @def_level = val[4]
                     }
@@ -1148,7 +1145,6 @@ rule
                         diagnostic(:error, :module_in_def, val[0])
                       end
 
-                      @comments.push @lexer.clear_comments
                       @static_env.extend_static
                     }
                     bodystmt kEND
@@ -1157,26 +1153,22 @@ rule
                                                    val[3], val[4])
 
                       @static_env.unextend
-                      @lexer.clear_comments
                     }
                 | kDEF fname
                     {
-                      @comments.push @lexer.clear_comments
                       @def_level += 1
                       @static_env.extend_static
                     }
                     f_arglist bodystmt kEND
                     {
                       result = @builder.def_method(val[0], val[1],
-                                  val[3], val[4], val[5], @comments.pop)
+                                  val[3], val[4], val[5])
 
                       @static_env.unextend
                       @def_level -= 1
-                      @lexer.clear_comments
                     }
                 | kDEF singleton dot_or_colon
                     {
-                      @comments.push @lexer.clear_comments
                       @lexer.state = :expr_fname
                     }
                     fname
@@ -1187,11 +1179,10 @@ rule
                     f_arglist bodystmt kEND
                     {
                       result = @builder.def_singleton(val[0], val[1], val[2],
-                                  val[4], val[6], val[7], val[8], @comments.pop)
+                                  val[4], val[6], val[7], val[8])
 
                       @static_env.unextend
                       @def_level -= 1
-                      @lexer.clear_comments
                     }
                 | kBREAK
                     {
@@ -2103,9 +2094,7 @@ keyword_variable: kNIL
                     }
                 | tLABEL arg_value
                     {
-                      # TODO: Extract colon
-                      key = @builder.symbol(val[0])
-                      result = @builder.pair(key, nil, val[1])
+                      result = @builder.pair_keyword(val[0], val[1])
                     }
 
        operation: tIDENTIFIER | tCONSTANT | tFID