parser 1.1.0 → 1.2.0

data/lib/parser/compatibility/slop.rb

@@ -0,0 +1,12 @@
+# https://github.com/injekt/slop/pull/110
+class Slop
+  def extract_long_flag(objects, config)
+    flag = objects.first.to_s
+    if flag =~ /\A(?:--?)?[a-zA-Z0-9][a-zA-Z0-9_.-]+\=?\??\z/
+      config[:argument] ||= true if flag.end_with?('=')
+      config[:optional_argument] = true if flag.end_with?('=?')
+      objects.shift
+      clean(flag).sub(/\=\??\z/, '')
+    end
+  end
+end

data/lib/parser/lexer.rl

@@ -992,6 +992,9 @@ class Parser::Lexer
       => { emit(KEYWORDS[tok]);
            fnext expr_endfn; fbreak; };
 
+      constant
+      => { emit(:tCONSTANT) };
+
       bareword [?=!]?
       => { emit(:tIDENTIFIER)
            fnext expr_endfn; fbreak; };
@@ -1009,6 +1012,9 @@ class Parser::Lexer
       => { emit_table(PUNCTUATION)
            fnext expr_endfn; fbreak; };
 
+      '::'
+      => { fhold; fhold; fgoto expr_end; };
+
       ':'
       => { fhold; fgoto expr_beg; };
 
@@ -1266,7 +1272,7 @@ class Parser::Lexer
       => {
         fhold;
         if tok.start_with? '-'
-          emit(:tUMINUS_NUM, '-')
+          emit(:tUMINUS_NUM, '-', @ts, @ts + 1)
           fnext expr_end; fbreak;
         end
       };
@@ -1635,13 +1641,18 @@ class Parser::Lexer
                ( '_' digit+ )* digit* '_'?
       | '0'       %{ @num_base = 8;  @num_digits_s = @ts }
                ( '_' digit+ )* digit* '_'?
-      )
+      ) %{ tm = p } c_alpha?
       => {
-        digits = tok(@num_digits_s)
+        unless (char = tok(tm, @te)).empty?
+          diagnostic :fatal, Parser::ERRORS[:unexpected] % { :character => char },
+                     range(tm, tm + 1)
+        end
+
+        digits = tok(@num_digits_s, tm)
 
         if digits.end_with? '_'
           diagnostic :error, Parser::ERRORS[:trailing_underscore],
-                     range(@te - 1, @te)
+                     range(tm - 1, tm)
         elsif digits.empty? && @num_base == 8 && version?(18)
           # 1.8 did not raise an error on 0o.
           digits = "0"
@@ -1653,7 +1664,8 @@ class Parser::Lexer
                      range(invalid_s, invalid_s + 1)
         end
 
-        emit(:tINTEGER, digits.to_i(@num_base))
+        emit(:tINTEGER, digits.to_i(@num_base), @ts, tm)
+        p = tm - 1
         fbreak;
       };
 
@@ -1665,18 +1677,26 @@ class Parser::Lexer
       (
           '.' ( digit+ '_' )* digit+ |
         ( '.' ( digit+ '_' )* digit+ )? [eE] [+\-]? ( digit+ '_' )* digit+
-      )
+      ) %{ tm = p } c_alpha?
       => {
-        if tok.start_with? '.'
+        unless (char = tok(tm, @te)).empty?
+          diagnostic :fatal, Parser::ERRORS[:unexpected] % { :character => char },
+                     range(tm, tm + 1)
+        end
+
+        digits = tok(@ts, tm)
+
+        if digits.start_with? '.'
           diagnostic :error, Parser::ERRORS[:no_dot_digit_literal]
-        elsif tok =~ /^[eE]/
+        elsif digits =~ /^[eE]/
           # The rule above allows to specify floats as just `e10', which is
           # certainly not a float. Send a patch if you can do this better.
-          emit(:tIDENTIFIER, tok)
+          emit(:tIDENTIFIER, digits, @ts, tm)
           fbreak;
         end
 
-        emit(:tFLOAT, tok.to_f)
+        emit(:tFLOAT, digits.to_f, @ts, tm)
+        p = tm - 1
         fbreak;
       };
 

data/lib/parser/lexer/explanation.rb

@@ -17,7 +17,7 @@ module Parser
     private
 
     def decorate(range, token, info)
-      from, to = range.begin_column, range.end_column
+      from, to = range.begin.column, range.end.column
 
       line = range.source_line
       line[from..to] = "\e[4m#{line[from..to]}\e[0m"

data/lib/parser/lexer/literal.rb

@@ -2,7 +2,7 @@ module Parser
 
   class Lexer::Literal
     DELIMITERS = { '(' => ')', '[' => ']', '{' => '}', '<' => '>' }
-    MONOLITHIC = { :tSTRING_BEG => :tSTRING, :tSYMBEG => :tSYMBOL }
+    MONOLITHIC = { :tSTRING_BEG => :tSTRING }
 
     TYPES = {
     # type      start token     interpolate?
@@ -64,12 +64,11 @@ module Parser
 
       # Monolithic strings are glued into a single token, e.g.
       # tSTRING_BEG tSTRING_CONTENT tSTRING_END -> tSTRING.
-      @monolithic  = (
-          [:tSTRING_BEG, :tSYMBEG].include?(type) &&
-          !heredoc?
-      )
+      @monolithic  = (@start_tok == :tSTRING_BEG  &&
+                      %w(' ").include?(str_type) &&
+                      !heredoc?)
 
-      # Also capture delimiter in %w() style literals
+      # Capture opening delimiter in percent-literals.
       unless @heredoc_e || @str_type.end_with?(delimiter)
         @str_type << delimiter
       end

data/lib/parser/ruby18.y

@@ -51,17 +51,17 @@ rule
         bodystmt: compstmt opt_rescue opt_else opt_ensure
                     {
                       rescue_bodies     = val[1]
-                      else_,   t_else   = val[2]
-                      ensure_, t_ensure = val[3]
+                      else_t,   else_   = val[2]
+                      ensure_t, ensure_ = val[3]
 
                       if rescue_bodies.empty? && !else_.nil?
-                        diagnostic :warning, :useless_else, t_else
+                        diagnostic :warning, :useless_else, else_t
                       end
 
                       result = @builder.begin_body(val[0],
                                   rescue_bodies,
-                                  else_,   t_else,
-                                  ensure_, t_ensure)
+                                  else_t,   else_,
+                                  ensure_t, ensure_)
                     }
 
         compstmt: stmts opt_terms
@@ -126,11 +126,11 @@ rule
                     }
                 | stmt kWHILE_MOD expr_value
                     {
-                      result = @builder.loop_mod(val[0], val[1], val[2])
+                      result = @builder.loop_mod(:while, val[0], val[1], val[2])
                     }
                 | stmt kUNTIL_MOD expr_value
                     {
-                      result = @builder.loop_mod(val[0], val[1], val[2])
+                      result = @builder.loop_mod(:until, val[0], val[1], val[2])
                     }
                 | stmt kRESCUE_MOD stmt
                     {
@@ -351,7 +351,7 @@ rule
                     }
                 | tLPAREN mlhs_entry tRPAREN
                     {
-                      result = @builder.parenthesize(val[0], val[1], val[2])
+                      result = @builder.begin(val[0], val[1], val[2])
                     }
 
       mlhs_entry: mlhs_basic
@@ -391,7 +391,7 @@ rule
        mlhs_item: mlhs_node
                 | tLPAREN mlhs_entry tRPAREN
                     {
-                      result = @builder.parenthesize(val[0], val[1], val[2])
+                      result = @builder.begin(val[0], val[1], val[2])
                     }
 
        mlhs_head: mlhs_item tCOMMA
@@ -1009,11 +1009,11 @@ rule
                     }
                     opt_nl tRPAREN
                     {
-                      result = @builder.parenthesize(val[0], val[1], val[3])
+                      result = @builder.begin(val[0], val[1], val[4])
                     }
                 | tLPAREN compstmt tRPAREN
                     {
-                      result = @builder.parenthesize(val[0], val[1], val[2])
+                      result = @builder.begin(val[0], val[1], val[2])
                     }
                 | primary_value tCOLON2 tCONSTANT
                     {
@@ -1095,7 +1095,7 @@ rule
                     }
                     compstmt kEND
                     {
-                      result = @builder.loop(val[0], val[2], val[3],
+                      result = @builder.loop(:while, val[0], val[2], val[3],
                                              val[5], val[6])
                     }
                 | kUNTIL
@@ -1108,20 +1108,32 @@ rule
                     }
                     compstmt kEND
                     {
-                      result = @builder.loop(val[0], val[2], val[3],
+                      result = @builder.loop(:until, val[0], val[2], val[3],
                                              val[5], val[6])
                     }
                 | kCASE expr_value opt_terms case_body kEND
                     {
-                      result = @builder.case(val[0], val[1], val[3], val[4])
+                      when_bodies       = val[3][0..-2]
+                      else_t, else_body = val[3][-1]
+
+                      result = @builder.case(val[0], val[1],
+                                             when_bodies, else_t, else_body,
+                                             val[4])
                     }
                 | kCASE            opt_terms case_body kEND
                     {
-                      result = @builder.case(val[0], nil, val[2], val[3])
+                      when_bodies       = val[2][0..-2]
+                      else_t, else_body = val[2][-1]
+
+                      result = @builder.case(val[0], nil,
+                                             when_bodies, else_t, else_body,
+                                             val[3])
                     }
                 | kCASE opt_terms kELSE compstmt kEND
                     {
-                      result = @builder.case(val[0], nil, [ val[3] ], val[4])
+                      result = @builder.case(val[0], nil,
+                                             [], val[2], val[3],
+                                             val[4])
                     }
                 | kFOR for_var kIN
                     {
@@ -1456,8 +1468,7 @@ rule
 
            cases: opt_else
                     {
-                      else_t, else_ = val[0]
-                      result = [ else_ ]
+                      result = [ val[0] ]
                     }
                 | case_body
 
@@ -1488,13 +1499,13 @@ rule
 
          exc_var: tASSOC lhs
                     {
-                      result = val
+                      result = [ val[0], val[1] ]
                     }
                 | none
 
       opt_ensure: kENSURE compstmt
                     {
-                      result = val
+                      result = [ val[0], val[1] ]
                     }
                 | none
 
@@ -1629,10 +1640,6 @@ xstring_contents: # nothing
 
           symbol: tSYMBOL
                     {
-                      if val[0][0].empty?
-                        diagnostic(:error, :empty_symbol, val[0])
-                      end
-
                       result = @builder.symbol(val[0])
                     }
 

data/lib/parser/ruby19.y

@@ -77,17 +77,17 @@ rule
         bodystmt: compstmt opt_rescue opt_else opt_ensure
                     {
                       rescue_bodies     = val[1]
-                      else_,   t_else   = val[2]
-                      ensure_, t_ensure = val[3]
+                      else_t,   else_   = val[2]
+                      ensure_t, ensure_ = val[3]
 
                       if rescue_bodies.empty? && !else_.nil?
-                        diagnostic :warning, :useless_else, t_else
+                        diagnostic :warning, :useless_else, else_t
                       end
 
                       result = @builder.begin_body(val[0],
                                   rescue_bodies,
-                                  else_,   t_else,
-                                  ensure_, t_ensure)
+                                  else_t,   else_,
+                                  ensure_t, ensure_)
                     }
 
         compstmt: stmts opt_terms
@@ -152,11 +152,11 @@ rule
                     }
                 | stmt kWHILE_MOD expr_value
                     {
-                      result = @builder.loop_mod(val[0], val[1], val[2])
+                      result = @builder.loop_mod(:while, val[0], val[1], val[2])
                     }
                 | stmt kUNTIL_MOD expr_value
                     {
-                      result = @builder.loop_mod(val[0], val[1], val[2])
+                      result = @builder.loop_mod(:until, val[0], val[1], val[2])
                     }
                 | stmt kRESCUE_MOD stmt
                     {
@@ -371,7 +371,7 @@ rule
                     }
                 | tLPAREN mlhs_inner rparen
                     {
-                      result = @builder.parenthesize(val[0], val[1], val[2])
+                      result = @builder.begin(val[0], val[1], val[2])
                     }
 
       mlhs_inner: mlhs_basic
@@ -433,7 +433,7 @@ rule
        mlhs_item: mlhs_node
                 | tLPAREN mlhs_inner rparen
                     {
-                      result = @builder.parenthesize(val[0], val[1], val[2])
+                      result = @builder.begin(val[0], val[1], val[2])
                     }
 
        mlhs_head: mlhs_item tCOMMA
@@ -962,11 +962,11 @@ rule
                     }
                     opt_nl tRPAREN
                     {
-                      result = @builder.parenthesize(val[0], val[1], val[4])
+                      result = @builder.begin(val[0], val[1], val[4])
                     }
                 | tLPAREN compstmt tRPAREN
                     {
-                      result = @builder.parenthesize(val[0], val[1], val[2])
+                      result = @builder.begin(val[0], val[1], val[2])
                     }
                 | primary_value tCOLON2 tCONSTANT
                     {
@@ -1060,7 +1060,7 @@ rule
                     }
                     compstmt kEND
                     {
-                      result = @builder.loop(val[0], val[2], val[3],
+                      result = @builder.loop(:while, val[0], val[2], val[3],
                                              val[5], val[6])
                     }
                 | kUNTIL
@@ -1073,16 +1073,24 @@ rule
                     }
                     compstmt kEND
                     {
-                      result = @builder.loop(val[0], val[2], val[3],
+                      result = @builder.loop(:until, val[0], val[2], val[3],
                                              val[5], val[6])
                     }
                 | kCASE expr_value opt_terms case_body kEND
                     {
-                      result = @builder.case(val[0], val[1], val[3], val[4])
+                      *when_bodies, (else_t, else_body) = *val[3]
+
+                      result = @builder.case(val[0], val[1],
+                                             when_bodies, else_t, else_body,
+                                             val[4])
                     }
                 | kCASE            opt_terms case_body kEND
                     {
-                      result = @builder.case(val[0], nil, val[2], val[3])
+                      *when_bodies, (else_t, else_body) = *val[2]
+
+                      result = @builder.case(val[0], nil,
+                                             when_bodies, else_t, else_body,
+                                             val[3])
                     }
                 | kFOR for_var kIN
                     {
@@ -1567,8 +1575,7 @@ rule
 
            cases: opt_else
                     {
-                      else_t, else_ = val[0]
-                      result = [ else_ ]
+                      result = [ val[0] ]
                     }
                 | case_body
 
@@ -1599,13 +1606,13 @@ rule
 
          exc_var: tASSOC lhs
                     {
-                      result = val
+                      result = [ val[0], val[1] ]
                     }
                 | none
 
       opt_ensure: kENSURE compstmt
                     {
-                      result = val
+                      result = [ val[0], val[1] ]
                     }
                 | none
 

data/lib/parser/ruby20.y

@@ -78,17 +78,17 @@ rule
         bodystmt: compstmt opt_rescue opt_else opt_ensure
                     {
                       rescue_bodies     = val[1]
-                      else_,   t_else   = val[2]
-                      ensure_, t_ensure = val[3]
+                      else_t,   else_   = val[2]
+                      ensure_t, ensure_ = val[3]
 
                       if rescue_bodies.empty? && !else_.nil?
-                        diagnostic :warning, :useless_else, t_else
+                        diagnostic :warning, :useless_else, else_t
                       end
 
                       result = @builder.begin_body(val[0],
                                   rescue_bodies,
-                                  else_,   t_else,
-                                  ensure_, t_ensure)
+                                  else_t,   else_,
+                                  ensure_t, ensure_)
                     }
 
         compstmt: stmts opt_terms
@@ -163,11 +163,11 @@ rule
                     }
                 | stmt kWHILE_MOD expr_value
                     {
-                      result = @builder.loop_mod(val[0], val[1], val[2])
+                      result = @builder.loop_mod(:while, val[0], val[1], val[2])
                     }
                 | stmt kUNTIL_MOD expr_value
                     {
-                      result = @builder.loop_mod(val[0], val[1], val[2])
+                      result = @builder.loop_mod(:until, val[0], val[1], val[2])
                     }
                 | stmt kRESCUE_MOD stmt
                     {
@@ -379,7 +379,7 @@ rule
                     }
                 | tLPAREN mlhs_inner rparen
                     {
-                      result = @builder.parenthesize(val[0], val[1], val[2])
+                      result = @builder.begin(val[0], val[1], val[2])
                     }
 
       mlhs_inner: mlhs_basic
@@ -441,7 +441,7 @@ rule
        mlhs_item: mlhs_node
                 | tLPAREN mlhs_inner rparen
                     {
-                      result = @builder.parenthesize(val[0], val[1], val[2])
+                      result = @builder.begin(val[0], val[1], val[2])
                     }
 
        mlhs_head: mlhs_item tCOMMA
@@ -991,7 +991,7 @@ rule
                     }
                     opt_nl tRPAREN
                     {
-                      result = @builder.parenthesize(val[0], val[1], val[4])
+                      result = @builder.begin(val[0], val[1], val[4])
                     }
                 | tLPAREN_ARG
                     {
@@ -999,11 +999,11 @@ rule
                     }
                     opt_nl tRPAREN
                     {
-                      result = @builder.parenthesize(val[0], nil, val[3])
+                      result = @builder.begin(val[0], nil, val[3])
                     }
                 | tLPAREN compstmt tRPAREN
                     {
-                      result = @builder.parenthesize(val[0], val[1], val[2])
+                      result = @builder.begin(val[0], val[1], val[2])
                     }
                 | primary_value tCOLON2 tCONSTANT
                     {
@@ -1097,7 +1097,7 @@ rule
                     }
                     compstmt kEND
                     {
-                      result = @builder.loop(val[0], val[2], val[3],
+                      result = @builder.loop(:while, val[0], val[2], val[3],
                                              val[5], val[6])
                     }
                 | kUNTIL
@@ -1110,16 +1110,24 @@ rule
                     }
                     compstmt kEND
                     {
-                      result = @builder.loop(val[0], val[2], val[3],
+                      result = @builder.loop(:until, val[0], val[2], val[3],
                                              val[5], val[6])
                     }
                 | kCASE expr_value opt_terms case_body kEND
                     {
-                      result = @builder.case(val[0], val[1], val[3], val[4])
+                      *when_bodies, (else_t, else_body) = *val[3]
+
+                      result = @builder.case(val[0], val[1],
+                                             when_bodies, else_t, else_body,
+                                             val[4])
                     }
                 | kCASE            opt_terms case_body kEND
                     {
-                      result = @builder.case(val[0], nil, val[2], val[3])
+                      *when_bodies, (else_t, else_body) = *val[2]
+
+                      result = @builder.case(val[0], nil,
+                                             when_bodies, else_t, else_body,
+                                             val[3])
                     }
                 | kFOR for_var kIN
                     {
@@ -1641,8 +1649,7 @@ opt_block_args_tail:
 
            cases: opt_else
                     {
-                      else_t, else_ = val[0]
-                      result = [ else_ ]
+                      result = [ val[0] ]
                     }
                 | case_body
 
@@ -1673,13 +1680,13 @@ opt_block_args_tail:
 
          exc_var: tASSOC lhs
                     {
-                      result = val
+                      result = [ val[0], val[1] ]
                     }
                 | none
 
       opt_ensure: kENSURE compstmt
                     {
-                      result = val
+                      result = [ val[0], val[1] ]
                     }
                 | none