parser 2.3.0.2 → 2.3.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3cb2802760d71d558e9ab344e63deb18525be110
-  data.tar.gz: 32edc10c308de0d00030d4b9fbac6bd40bfff36f
+  metadata.gz: 32b8e79e47ae5063bd6a4451c47b337f43d5a2a2
+  data.tar.gz: c8d08375dca4b4089fa0dade5d77f13284498852
 SHA512:
-  metadata.gz: 365c674e6cf6b3fa43f81903a7f46ce787075eb927e4aecce9b59ff05d81b5c47d4304f063f2f31c528f8846faec1983736f7bbd169697e19950e03346b0e52a
-  data.tar.gz: 1504d14896c1cb0dde7a8bc46aa60b04036e03310b1f097e84e5f14587699f18b086184292fbb5fb1aaff4d201d172cd9c424d1b8bc11de725d2c64567893928
+  metadata.gz: 039b018cd53937074f6a0eb1b366c5d83c0de6ed104a3ca63597b265c695f010e2ae85fdee74bc9c00418510e386e9fc7e2ff44dd9dc9ead2ddef5cc3ef0ec1e
+  data.tar.gz: 4b8b8b3497d1034e4662aa8ad3cf85c57cd65cb73e038bf7737c63db832d719714aa2bafd2fef5dcdb85d2ee4342ef3c4c8ceb5d26f3abc5cfed67e4c6902aff

data/CHANGELOG.md

@@ -1,6 +1,12 @@
 Changelog
 =========
 
+v2.3.0.3 (2016-02-06)
+---------------------
+
+API modifications:
+ * lexer.rl: "a?=b": lex via tCHARACTER (closes #255). (whitequark)
+
 v2.3.0.2 (2016-01-24)
 ---------------------
 
@@ -8,7 +14,7 @@ Bugs fixed:
  * Add :csend to Parser::Meta::NODE_TYPES (Markus Schirp)
  * lexer/dedenter: "\<\<x\n  y\\n  z\nx": don't dedent after escaped newline. (whitequark)
 
-v2.3.0.2 (2016-01-16)
+v2.3.0.3 (2016-01-16)
 ---------------------
 
 v2.3.0.1 (2016-01-14)

data/doc/AST_FORMAT.md

@@ -795,7 +795,7 @@ Format:
 Format:
 
 ~~~
-(defs (self) (args) nil)
+(defs (self) :foo (args) nil)
 "def self.foo; end"
  ~~~ keyword
           ~~~ name

data/lib/parser/lexer.rl

@@ -82,15 +82,12 @@ class Parser::Lexer
   # %
 
   ESCAPES = {
-    'a' => "\a", 'b'  => "\b", 'e'  => "\e", 'f' => "\f",
-    'n' => "\n", 'r'  => "\r", 's'  => "\s", 't' => "\t",
-    'v' => "\v", '\\' => "\\"
-  }
+    ?a.ord => "\a", ?b.ord  => "\b", ?e.ord => "\e", ?f.ord => "\f",
+    ?n.ord => "\n", ?r.ord  => "\r", ?s.ord => "\s", ?t.ord => "\t",
+    ?v.ord => "\v", ?\\.ord => "\\"
+  }.freeze
 
-  BLANK_STRING = ''.freeze
-  ESCAPED_NEXT_LINE = "\\\n".freeze
   REGEXP_META_CHARACTERS = Regexp.union(*"\\$()*+.<>?[]^{|}".chars).freeze
-  UNDERSCORE_STRING = '_'.freeze
 
   RBRACE_OR_RBRACK = %w"} ]".freeze
 
@@ -192,9 +189,7 @@ class Parser::Lexer
       @source = @source_buffer.source
       @need_encode = false
 
-      if @has_encode
-        @encoding   = @source.encoding
-      end
+      @encoding = @source.encoding if @has_encode
 
       if @has_encode && @source.encoding == Encoding::UTF_8
         @source_pts = @source.unpack('U*')
@@ -315,10 +310,10 @@ class Parser::Lexer
     if @token_queue.any?
       @token_queue.shift
     elsif @cs == klass.lex_error
-      [ false, [ '$error', range(p - 1, p) ] ]
+      [ false, [ '$error'.freeze, range(p - 1, p) ] ]
     else
       eof = @source_pts.size + 1
-      [ false, [ '$eof',   range(eof, eof) ] ]
+      [ false, [ '$eof'.freeze,   range(eof, eof) ] ]
     end
   end
 
@@ -332,11 +327,6 @@ class Parser::Lexer
     versions.include?(@version)
   end
 
-  def stack_pop
-    @top -= 1
-    @stack[@top]
-  end
-
   if "".respond_to?(:encode)
     def encode_escape(ord)
       ord.chr.force_encoding(@encoding)
@@ -379,11 +369,11 @@ class Parser::Lexer
 
   def emit_do(do_block=false)
     if @cond.active?
-      emit(:kDO_COND)
+      emit(:kDO_COND, 'do'.freeze)
     elsif @cmdarg.active? || do_block
-      emit(:kDO_BLOCK)
+      emit(:kDO_BLOCK, 'do'.freeze)
     else
-      emit(:kDO)
+      emit(:kDO, 'do'.freeze)
     end
   end
 
@@ -699,8 +689,8 @@ class Parser::Lexer
   }
 
   action unescape_char {
-    char = @source[p - 1].chr
-    @escape = ESCAPES.fetch(char, char)
+    codepoint = @source_pts[p - 1]
+    @escape = ESCAPES[codepoint] || encode_escape(codepoint)
   }
 
   action invalid_complex_escape {
@@ -847,7 +837,6 @@ class Parser::Lexer
 
   action extend_string {
     string = tok
-    string = string.encode(@encoding) if @need_encode
 
     # tLABEL_END is only possible in non-cond context on >= 2.2
     if @version >= 22 && !@cond.active?
@@ -899,7 +888,7 @@ class Parser::Lexer
       if current_literal.regexp?
         # Regular expressions should include escape sequences in their
         # escaped form. On the other hand, escaped newlines are removed.
-        current_literal.extend_string(tok.gsub(ESCAPED_NEXT_LINE, BLANK_STRING), @ts, @te)
+        current_literal.extend_string(tok.gsub("\\\n".freeze, ''.freeze), @ts, @te)
       else
         current_literal.extend_string(@escape || tok, @ts, @te)
       end
@@ -917,11 +906,11 @@ class Parser::Lexer
     end
 
     if current_literal.heredoc?
-      line = tok(@herebody_s, @ts).gsub(/\r+$/, BLANK_STRING)
+      line = tok(@herebody_s, @ts).gsub(/\r+$/, ''.freeze)
 
       if version?(18, 19, 20)
         # See ruby:c48b4209c
-        line = line.gsub(/\r.*$/, BLANK_STRING)
+        line = line.gsub(/\r.*$/, ''.freeze)
       end
 
       # Try ending the heredoc with the complete most recently
@@ -1023,9 +1012,9 @@ class Parser::Lexer
     if current_literal
       if current_literal.end_interp_brace_and_try_closing
         if version?(18, 19)
-          emit(:tRCURLY, '}', p - 1, p)
+          emit(:tRCURLY, '}'.freeze, p - 1, p)
         else
-          emit(:tSTRING_DEND, '}', p - 1, p)
+          emit(:tSTRING_DEND, '}'.freeze, p - 1, p)
         end
 
         if current_literal.saved_herebody_s
@@ -1033,7 +1022,7 @@ class Parser::Lexer
         end
 
         fhold;
-        fnext *stack_pop;
+        fret;
         fbreak;
       end
     end
@@ -1044,7 +1033,7 @@ class Parser::Lexer
     current_literal.flush_string
     current_literal.extend_content
 
-    emit(:tSTRING_DBEG, '#{')
+    emit(:tSTRING_DBEG, '#{'.freeze)
 
     if current_literal.heredoc?
       current_literal.saved_herebody_s = @herebody_s
@@ -1183,28 +1172,28 @@ class Parser::Lexer
   # The default longest-match scanning does not work here due
   # to sheer ambiguity.
 
-  ambiguous_fid_suffix =       # actual    parsed
-      [?!]  %{ tm = p }      | # a?        a?
-      '!='  %{ tm = p - 2 }    # a!=b      a != b
+  ambiguous_fid_suffix =         # actual    parsed
+      [?!]    %{ tm = p }      | # a?        a?
+      [?!]'=' %{ tm = p - 2 }    # a!=b      a != b
   ;
 
-  ambiguous_ident_suffix =     # actual    parsed
-      ambiguous_fid_suffix   |
-      '='   %{ tm = p }      | # a=        a=
-      '=='  %{ tm = p - 2 }  | # a==b      a == b
-      '=~'  %{ tm = p - 2 }  | # a=~b      a =~ b
-      '=>'  %{ tm = p - 2 }  | # a=>b      a => b
-      '===' %{ tm = p - 3 }    # a===b     a === b
+  ambiguous_ident_suffix =       # actual    parsed
+      ambiguous_fid_suffix     |
+      '='     %{ tm = p }      | # a=        a=
+      '=='    %{ tm = p - 2 }  | # a==b      a == b
+      '=~'    %{ tm = p - 2 }  | # a=~b      a =~ b
+      '=>'    %{ tm = p - 2 }  | # a=>b      a => b
+      '==='   %{ tm = p - 3 }    # a===b     a === b
   ;
 
-  ambiguous_symbol_suffix =    # actual    parsed
+  ambiguous_symbol_suffix =      # actual    parsed
       ambiguous_ident_suffix |
-      '==>' %{ tm = p - 2 }    # :a==>b    :a= => b
+      '==>'   %{ tm = p - 2 }    # :a==>b    :a= => b
   ;
 
   # Ambiguous with 1.9 hash labels.
-  ambiguous_const_suffix =     # actual    parsed
-      '::'  %{ tm = p - 2 }    # A::B      A :: B
+  ambiguous_const_suffix =       # actual    parsed
+      '::'    %{ tm = p - 2 }    # A::B      A :: B
   ;
 
   # Resolving kDO/kDO_COND/kDO_BLOCK ambiguity requires embedding
@@ -1252,7 +1241,7 @@ class Parser::Lexer
           emit(:tGVAR)
         end
 
-        fnext *stack_pop; fbreak;
+        fret; fbreak;
       };
 
       class_var_v
@@ -1262,7 +1251,7 @@ class Parser::Lexer
         end
 
         emit(:tCVAR)
-        fnext *stack_pop; fbreak;
+        fret; fbreak;
       };
 
       instance_var_v
@@ -1272,7 +1261,7 @@ class Parser::Lexer
         end
 
         emit(:tIVAR)
-        fnext *stack_pop; fbreak;
+        fret; fbreak;
       };
   *|;
 
@@ -1388,10 +1377,10 @@ class Parser::Lexer
       w_space+ e_lparen
       => {
         if version?(18)
-          emit(:tLPAREN2, '(', @te - 1, @te)
+          emit(:tLPAREN2, '('.freeze, @te - 1, @te)
           fnext expr_value; fbreak;
         else
-          emit(:tLPAREN_ARG, '(', @te - 1, @te)
+          emit(:tLPAREN_ARG, '('.freeze, @te - 1, @te)
           fnext expr_beg; fbreak;
         end
       };
@@ -1399,13 +1388,13 @@ class Parser::Lexer
       # meth(1 + 2)
       # Regular method call.
       e_lparen
-      => { emit(:tLPAREN2)
+      => { emit(:tLPAREN2, '('.freeze)
            fnext expr_beg; fbreak; };
 
       # meth [...]
       # Array argument. Compare with indexing `meth[...]`.
       w_space+ e_lbrack
-      => { emit(:tLBRACK, '[', @te - 1, @te)
+      => { emit(:tLBRACK, '['.freeze, @te - 1, @te)
            fnext expr_beg; fbreak; };
 
       # cmd {}
@@ -1416,7 +1405,7 @@ class Parser::Lexer
           p = @ts - 1
           fgoto expr_end;
         else
-          emit(:tLCURLY, '{', @te - 1, @te)
+          emit(:tLCURLY, '{'.freeze, @te - 1, @te)
           fnext expr_value; fbreak;
         end
       };
@@ -1447,7 +1436,7 @@ class Parser::Lexer
       | '<<'
       )
       => {
-        if tok(tm, tm + 1) == '/'
+        if tok(tm, tm + 1) == '/'.freeze
           # Ambiguous regexp literal.
           diagnostic :warning, :ambiguous_literal, nil, range(tm, tm + 1)
         end
@@ -1524,7 +1513,7 @@ class Parser::Lexer
   expr_cmdarg := |*
       w_space+ e_lparen
       => {
-        emit(:tLPAREN_ARG, '(', @te - 1, @te)
+        emit(:tLPAREN_ARG, '('.freeze, @te - 1, @te)
         if version?(18)
           fnext expr_value; fbreak;
         else
@@ -1535,9 +1524,9 @@ class Parser::Lexer
       w_space* 'do'
       => {
         if @cond.active?
-          emit(:kDO_COND, 'do', @te - 2, @te)
+          emit(:kDO_COND, 'do'.freeze, @te - 2, @te)
         else
-          emit(:kDO, 'do', @te - 2, @te)
+          emit(:kDO, 'do'.freeze, @te - 2, @te)
         end
         fnext expr_value; fbreak;
       };
@@ -1569,7 +1558,7 @@ class Parser::Lexer
   # `do` (as `kDO_BLOCK` in `expr_beg`).
   expr_endarg := |*
       e_lbrace
-      => { emit(:tLBRACE_ARG)
+      => { emit(:tLBRACE_ARG, '{'.freeze)
            fnext expr_value; };
 
       'do'
@@ -1622,15 +1611,15 @@ class Parser::Lexer
       [+\-][0-9]
       => {
         fhold;
-        if tok.start_with? '-'
-          emit(:tUMINUS_NUM, '-', @ts, @ts + 1)
+        if tok.start_with? '-'.freeze
+          emit(:tUMINUS_NUM, '-'.freeze, @ts, @ts + 1)
           fnext expr_end; fbreak;
         end
       };
 
       # splat *a
       '*'
-      => { emit(:tSTAR)
+      => { emit(:tSTAR, '*'.freeze)
            fbreak; };
 
       #
@@ -1641,7 +1630,7 @@ class Parser::Lexer
       # /=/ (disambiguation with /=)
       '/' c_any
       => {
-        type = delimiter = @source[@ts].chr
+        type = delimiter = tok[0].chr
         fhold; fgoto *push_literal(type, delimiter, @ts);
       };
 
@@ -1655,7 +1644,7 @@ class Parser::Lexer
       # %w(we are the people)
       '%' [A-Za-z]+ c_any
       => {
-        type, delimiter = @source[@ts...(@te - 1)], @source[@te - 1].chr
+        type, delimiter = tok[0..-2], tok[-1].chr
         fgoto *push_literal(type, delimiter, @ts);
       };
 
@@ -1672,27 +1661,24 @@ class Parser::Lexer
         ( '"' ( c_line - '"' )* '"'
         | "'" ( c_line - "'" )* "'"
         | "`" ( c_line - "`" )* "`"
-        | bareword ) % { heredoc_e     = p }
+        | bareword ) % { heredoc_e      = p }
         c_line* c_nl % { new_herebody_s = p }
       => {
         tok(@ts, heredoc_e) =~ /^<<(-?)(~?)(["'`]?)(.*)\3$/
 
         indent      = !$1.empty? || !$2.empty?
         dedent_body = !$2.empty?
-        type        =  '<<' + ($3.empty? ? '"' : $3)
+        type        =  $3.empty? ? '<<"'.freeze : ('<<'.freeze + $3)
         delimiter   =  $4
 
         if dedent_body && version?(18, 19, 20, 21, 22)
-          emit(:tLSHFT, '<<', @ts, @ts + 2)
+          emit(:tLSHFT, '<<'.freeze, @ts, @ts + 2)
           p = @ts + 1
           fnext expr_beg; fbreak;
         else
           fnext *push_literal(type, delimiter, @ts, heredoc_e, indent, dedent_body);
 
-          if @herebody_s.nil?
-            @herebody_s = new_herebody_s
-          end
-
+          @herebody_s ||= new_herebody_s
           p = @herebody_s - 1
         end
       };
@@ -1704,7 +1690,7 @@ class Parser::Lexer
       # :"bar", :'baz'
       ':' ['"] # '
       => {
-        type, delimiter = tok, @source[@te - 1].chr
+        type, delimiter = tok, tok[-1].chr
         fgoto *push_literal(type, delimiter, @ts);
       };
 
@@ -1774,21 +1760,21 @@ class Parser::Lexer
       => {
         if @lambda_stack.last == @paren_nest
           @lambda_stack.pop
-          emit(:tLAMBEG)
+          emit(:tLAMBEG, '{'.freeze)
         else
-          emit(:tLBRACE)
+          emit(:tLBRACE, '{'.freeze)
         end
         fbreak;
       };
 
       # a([1, 2])
       e_lbrack
-      => { emit(:tLBRACK)
+      => { emit(:tLBRACK, '['.freeze)
            fbreak; };
 
       # a()
       e_lparen
-      => { emit(:tLPAREN)
+      => { emit(:tLPAREN, '('.freeze)
            fbreak; };
 
       # a(+b)
@@ -1799,7 +1785,7 @@ class Parser::Lexer
       # rescue Exception => e: Block rescue.
       # Special because it should transition to expr_mid.
       'rescue' %{ tm = p } '=>'?
-      => { emit(:kRESCUE, tok(@ts, tm), @ts, tm)
+      => { emit(:kRESCUE, 'rescue'.freeze, @ts, tm)
            p = tm - 1
            fnext expr_mid; fbreak; };
 
@@ -1929,7 +1915,7 @@ class Parser::Lexer
 
       '->'
       => {
-        emit(:tLAMBDA, tok(@ts, @ts + 2), @ts, @ts + 2)
+        emit(:tLAMBDA, '->'.freeze, @ts, @ts + 2)
 
         @lambda_stack.push @paren_nest
         fnext expr_endfn; fbreak;
@@ -1940,14 +1926,14 @@ class Parser::Lexer
         if @lambda_stack.last == @paren_nest
           @lambda_stack.pop
 
-          if tok == '{'
-            emit(:tLAMBEG)
+          if tok == '{'.freeze
+            emit(:tLAMBEG, '{'.freeze)
           else # 'do'
-            emit(:kDO_LAMBDA)
+            emit(:kDO_LAMBDA, 'do'.freeze)
           end
         else
-          if tok == '{'
-            emit(:tLCURLY)
+          if tok == '{'.freeze
+            emit(:tLCURLY, '{'.freeze)
           else # 'do'
             emit_do
           end
@@ -1965,8 +1951,8 @@ class Parser::Lexer
            fnext expr_fname; fbreak; };
 
       'class' w_any* '<<'
-      => { emit(:kCLASS, 'class', @ts, @ts + 5)
-           emit(:tLSHFT, '<<',    @te - 2, @te)
+      => { emit(:kCLASS, 'class'.freeze, @ts, @ts + 5)
+           emit(:tLSHFT, '<<'.freeze,    @te - 2, @te)
            fnext expr_value; fbreak; };
 
       # a if b:c: Syntax error.
@@ -1987,7 +1973,7 @@ class Parser::Lexer
       => {
         emit_table(KEYWORDS)
 
-        if version?(18) && tok == 'not'
+        if version?(18) && tok == 'not'.freeze
           fnext expr_beg; fbreak;
         else
           fnext expr_arg; fbreak;
@@ -2003,7 +1989,7 @@ class Parser::Lexer
             fnext *arg_or_cmdarg;
           end
         else
-          emit(:k__ENCODING__)
+          emit(:k__ENCODING__, '__ENCODING__'.freeze)
         end
         fbreak;
       };
@@ -2026,12 +2012,12 @@ class Parser::Lexer
       => {
         digits = tok(@num_digits_s, @num_suffix_s)
 
-        if digits.end_with? UNDERSCORE_STRING
-          diagnostic :error, :trailing_in_number, { :character => UNDERSCORE_STRING },
+        if digits.end_with? '_'.freeze
+          diagnostic :error, :trailing_in_number, { :character => '_'.freeze },
                      range(@te - 1, @te)
         elsif digits.empty? && @num_base == 8 && version?(18)
           # 1.8 did not raise an error on 0o.
-          digits = "0"
+          digits = '0'.freeze
         elsif digits.empty?
           diagnostic :error, :empty_numeric
         elsif @num_base == 8 && (invalid_idx = digits.index(/[89]/))
@@ -2041,7 +2027,7 @@ class Parser::Lexer
         end
 
         if version?(18, 19, 20)
-          emit(:tINTEGER, digits.to_i(@num_base))
+          emit(:tINTEGER, digits.to_i(@num_base), @ts, @num_suffix_s)
           p = @num_suffix_s - 1
         else
           @num_xfrm.call(digits.to_i(@num_base))
@@ -2061,7 +2047,7 @@ class Parser::Lexer
                      :trailing_in_number, { :character => tok(@te - 1, @te) },
                      range(@te - 1, @te)
         else
-          emit(:tINTEGER, tok(@ts, @te - 1).to_i)
+          emit(:tINTEGER, tok(@ts, @te - 1).to_i, @ts, @te - 1)
           fhold; fbreak;
         end
       };
@@ -2073,7 +2059,7 @@ class Parser::Lexer
                      :trailing_in_number, { :character => tok(@te - 1, @te) },
                      range(@te - 1, @te)
         else
-          emit(:tFLOAT, tok(@ts, @te - 1).to_f)
+          emit(:tFLOAT, tok(@ts, @te - 1).to_f, @ts, @te - 1)
           fhold; fbreak;
         end
       };
@@ -2086,7 +2072,7 @@ class Parser::Lexer
         digits = tok(@ts, @num_suffix_s)
 
         if version?(18, 19, 20)
-          emit(:tFLOAT, Float(digits))
+          emit(:tFLOAT, Float(digits), @ts, @num_suffix_s)
           p = @num_suffix_s - 1
         else
           @num_xfrm.call(digits)
@@ -2101,7 +2087,7 @@ class Parser::Lexer
       # `echo foo`, "bar", 'baz'
       '`' | ['"] # '
       => {
-        type, delimiter = tok, @source[@te - 1].chr
+        type, delimiter = tok, tok[-1].chr
         fgoto *push_literal(type, delimiter, @ts, nil, false, false, true);
       };
 
@@ -2174,11 +2160,11 @@ class Parser::Lexer
            fnext expr_beg; fbreak; };
 
       '?'
-      => { emit(:tEH)
+      => { emit(:tEH, '?'.freeze)
            fnext expr_value; fbreak; };
 
       e_lbrack
-      => { emit(:tLBRACK2)
+      => { emit(:tLBRACK2, '['.freeze)
            fnext expr_beg; fbreak; };
 
       punctuation_end
@@ -2195,7 +2181,7 @@ class Parser::Lexer
       => { fgoto leading_dot; };
 
       ';'
-      => { emit(:tSEMI)
+      => { emit(:tSEMI, ';'.freeze)
            fnext expr_value; fbreak; };
 
       '\\' c_line {