parser 2.0.0 → 2.1.0.pre1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bf0b6a5c97a393fd82aae713e9a10d4aa2108a0f
-  data.tar.gz: cf126c874ffc8d2430ccec47583b9dfca2afc7a1
+  metadata.gz: 85ff1f420d063d29c4d333d54a120569fd0429e9
+  data.tar.gz: f7fbd9186d41a3e8ca2561cbb590f8bde8067a6e
 SHA512:
-  metadata.gz: 07a91a9884e3f0a22010c7cf60089de7b4bd4e0285df85771b9f0b52317d3b6930ba6bea44fa57c4c168723a92a3ccf4602b354a9e9e55ed7461e1009fa8431b
-  data.tar.gz: c95ab5caef72d3c0f936def4dd0dcb28298fbc335c9558d9d642edcfb9398a7f2b6479983cd5406ca62512f07c864b1884f4d0269bb34fb7fe75182a5086984c
+  metadata.gz: 40f6351913140cd1b73bf9b2d2ada48dd56ce29dad4e7c08bb347762b5d36a9bac95aa5b2b59c9f46155d35fa2db7e6e3400da72ceabe6d847381d932886e030
+  data.tar.gz: d1f1581bffe9ebecbd6fa02eaac2125db6ce329203d2d28061361588101b8442ace659f1b53e9a09189a6430506bc3662f323d7bebadef3cf41c2b98eb43634c

data/CHANGELOG.md

@@ -1,6 +1,17 @@
 Changelog
 =========
 
+v2.1.0.pre1 (2013-11-12)
+------------------------
+
+API modifications:
+ * lexer.rl: correctly handle __END__ with non-whitespace after it (Peter Zotov)
+ * lexer.rl: handle \r in middle of a line as mere whitespace (Peter Zotov)
+ * ruby{18,19,20,21}.y, builders/default: precisely point to tUMINUS_NUM. (Peter Zotov)
+
+Features implemented:
+ * lexer.rl, ruby21.y, builders/default: rational/complex literals. (Peter Zotov)
+
 v2.0.0 (2013-10-06)
 -------------------
 
@@ -26,6 +37,7 @@ v2.0.0.pre7 (2013-09-10)
 
 Features implemented:
  * Parser::Base: add #parse_with_comments, #parse_file_with_comments. (Trent Ogren)
+ * lexer.rl (Ruby 2.1): "1end": lex non-exponent `e' separate from number. (Peter Zotov)
 
 Bugs fixed:
  * lexer.rl: "->*{}": tLAMBEG at expr_beg (fixes #103). (Peter Zotov)

data/README.md

@@ -15,7 +15,7 @@ Sponsored by [Evil Martians](http://evilmartians.com).
 
 ## Installation
 
-    $ gem install parser -v=2.0
+    $ gem install parser
 
 ## Usage
 
@@ -31,11 +31,11 @@ Parse a chunk of code:
 Access the AST's source map:
 
     p Parser::CurrentRuby.parse("2 + 2").loc
-    # #<Parser::Source::Map::Send:0x007fe5a1ac2388 
-    #   @dot=nil, 
-    #   @begin=nil, 
-    #   @end=nil, 
-    #   @selector=#<Source::Range (string) 2...3>, 
+    # #<Parser::Source::Map::Send:0x007fe5a1ac2388
+    #   @dot=nil,
+    #   @begin=nil,
+    #   @end=nil,
+    #   @selector=#<Source::Range (string) 2...3>,
     #   @expression=#<Source::Range (string) 0...5>>
 
     p Parser::CurrentRuby.parse("2 + 2").loc.selector.source
@@ -98,6 +98,7 @@ with Parser:
 * A simple interface and a powerful, tweakable one.
 * Parses 1.8, 1.9, 2.0 and 2.1 (preliminary) syntax with backwards-compatible
   AST formats.
+* [Rewriting][rewriting] support.
 * Parsing error recovery.
 * Improved [clang-like][] diagnostic messages with location information.
 * Written in pure Ruby, runs on MRI 1.8.7 or >=1.9.2, JRuby and Rubinius in 1.8
@@ -111,6 +112,7 @@ with Parser:
 [ast]: http://rubygems.org/gems/ast
 [slop]: http://rubygems.org/gems/slop
 [insane-lexer]: http://whitequark.org/blog/2013/04/01/ruby-hacking-guide-ch-11-finite-state-lexer/
+[rewriting]: http://whitequark.org/blog/2013/04/26/lets-play-with-ruby-code/
 
 ## Documentation
 
@@ -224,7 +226,7 @@ As of 2013-07-25, affected gems are: activerdf, activerdf_net7, fastreader, gkel
 
 ### Invalid Unicode escape sequences
 
-Ruby MRI 1.9+ permits to specify invalid UTF-8 sequences in Unicode escape sequences, such as `\u{d800}`.
+Ruby MRI 1.9+ permits to specify invalid Unicode codepoints in Unicode escape sequences, such as `\u{d800}`.
 
 As of 2013-07-25, affected gems are: aws_cloud_search.
 

data/Rakefile

@@ -65,7 +65,7 @@ task :changelog do
     end
   end
 
-  IO.popen("git log --pretty='#{format}' HEAD", 'r') do |io|
+  IO.popen("git log --pretty='#{format}' 2.0 master", 'r') do |io|
     current_version = nil
 
     io.each_line do |line|
@@ -102,7 +102,7 @@ task :changelog do
         next unless commits[sigil].any?
 
         io.puts description
-        commits[sigil].each do |commit|
+        commits[sigil].uniq.each do |commit|
           io.puts " * #{commit.gsub('<', '\<').lstrip}"
         end
         io.puts

data/doc/AST_FORMAT.md

@@ -29,6 +29,11 @@ Format:
 (int 123)
 "123"
  ~~~ expression
+
+(int -123)
+"-123"
+ ^ operator
+ ~~~ expression
 ~~~
 
 ### Float
@@ -39,6 +44,11 @@ Format:
 (float 1.0)
 "1.0"
  ~~~ expression
+
+(float -1.0)
+"-1.0"
+ ^ operator
+ ~~~~ expression
 ~~~
 
 ### String

data/lib/parser.rb

@@ -17,8 +17,10 @@ end
 module Parser
   require 'parser/version'
 
-  require 'parser/ast/node'
-  require 'parser/ast/processor'
+  module AST
+    require 'parser/ast/node'
+    require 'parser/ast/processor'
+  end
 
   module Source
     require 'parser/source/buffer'

data/lib/parser/ast/processor.rb

@@ -21,8 +21,6 @@ module Parser
       alias on_erange   process_regular_node
 
       def on_var(node)
-        name, = *node
-
         node
       end
 

data/lib/parser/builders/default.rb

@@ -53,25 +53,42 @@ module Parser
 
     # Numerics
 
-    def integer(integer_t, negate=false)
-      val = value(integer_t)
-      val = -val if negate
+    def integer(integer_t)
+      numeric(:int, integer_t)
+    end
 
-      n(:int, [ val ],
-        numeric_map(integer_t, negate))
+    def float(float_t)
+      numeric(:float, float_t)
     end
 
-    def __LINE__(__LINE__t)
-      n0(:__LINE__,
-        token_map(__LINE__t))
+    def rational(rational_t)
+      numeric(:rational, rational_t)
     end
 
-    def float(float_t, negate=false)
-      val = value(float_t)
-      val = -val if negate
+    def complex(complex_t)
+      numeric(:complex, complex_t)
+    end
+
+    def numeric(kind, token)
+      n(kind, [ value(token) ],
+        Source::Map::Operator.new(nil, loc(token)))
+    end
+    private :numeric
+
+    def negate(uminus_t, numeric)
+      value, = *numeric
+      operator_loc = loc(uminus_t)
+
+      numeric.updated(nil, [ -value ],
+        :location =>
+          Source::Map::Operator.new(
+            operator_loc,
+            operator_loc.join(numeric.loc.expression)))
+    end
 
-      n(:float, [ val ],
-        numeric_map(float_t, negate))
+    def __LINE__(__LINE__t)
+      n0(:__LINE__,
+        token_map(__LINE__t))
     end
 
     # Strings
@@ -1017,19 +1034,6 @@ module Parser
       Source::Map.new(loc(token))
     end
 
-    def numeric_map(num_t, negate)
-      if negate
-        num_range = loc(num_t)
-        range = Source::Range.new(num_range.source_buffer,
-                                  num_range.begin_pos - 1,
-                                  num_range.end_pos)
-
-        Source::Map.new(range)
-      else
-        token_map(num_t)
-      end
-    end
-
     def delimited_string_map(string_t)
       str_range = loc(string_t)
 

data/lib/parser/current.rb

@@ -28,6 +28,10 @@ module Parser
     require 'parser/ruby20'
     CurrentRuby = Ruby20
 
+  when /^2\.1\./
+    require 'parser/ruby21'
+    CurrentRuby = Ruby21
+
   else # :nocov:
     raise NotImplementedError, "Parser does not support parsing Ruby #{RUBY_VERSION}"
   end

data/lib/parser/lexer.rl

@@ -134,6 +134,8 @@ class Parser::Lexer
 
     @num_base      = nil # last numeric base
     @num_digits_s  = nil # starting position of numeric digits
+    @num_suffix_s  = nil # starting position of numeric suffix
+    @num_xfrm      = nil # numeric suffix-induced transformation
 
     @escape_s      = nil # starting position of current sequence
     @escape        = nil # last escaped sequence, as string
@@ -500,8 +502,7 @@ class Parser::Lexer
 
   # A list of operators which can occur within an assignment shortcut (+ → +=).
   operator_arithmetic = '&'  | '|'   | '&&' | '||' | '^'  | '+'   | '-'  |
-                        '*'  | '/'   | '**' | '~'  | '**' | '<<'  | '>>' |
-                        '%'  ;
+                        '*'  | '/'   | '**' | '~'  | '<<' | '>>'  | '%'  ;
 
   # A list of all user-definable operators not covered by groups above.
   operator_rest       = '=~' | '!~' | '==' | '!=' | '!'   | '===' |
@@ -570,6 +571,33 @@ class Parser::Lexer
 
   label          = bareword [?!]? ':';
 
+  #
+  # === NUMERIC PARSING ===
+  #
+
+  int_hex  = ( xdigit+ '_' )* xdigit* '_'? ;
+  int_dec  = ( digit+ '_' )* digit* '_'? ;
+  int_bin  = ( [01]+ '_' )* [01]* '_'? ;
+
+  flo_int  = [1-9] [0-9]* ( '_' digit+ )* | '0';
+  flo_frac = '.' ( digit+ '_' )* digit+;
+  flo_pow  = [eE] [+\-]? ( digit+ '_' )* digit+;
+
+  int_suffix =
+    ''   % { @num_xfrm = lambda { |value|  emit(:tINTEGER,   value) } }
+  | 'r'  % { @num_xfrm = lambda { |value|  emit(:tRATIONAL,  Rational(value)) } }
+  | 'i'  % { @num_xfrm = lambda { |value|  emit(:tIMAGINARY, Complex(0, value)) } }
+  | 'ri' % { @num_xfrm = lambda { |value|  emit(:tIMAGINARY, Complex(0, Rational(value))) } };
+
+  flo_pow_suffix =
+    ''   % { @num_xfrm = lambda { |digits| emit(:tFLOAT,     Float(digits)) } }
+  | 'i'  % { @num_xfrm = lambda { |digits| emit(:tIMAGINARY, Complex(0, Float(digits))) } };
+
+  flo_suffix =
+    flo_pow_suffix
+  | 'r'  % { @num_xfrm = lambda { |digits| emit(:tRATIONAL,  Rational(digits)) } }
+  | 'ri' % { @num_xfrm = lambda { |digits| emit(:tIMAGINARY, Complex(0, Rational(digits))) } };
+
   #
   # === ESCAPE SEQUENCE PARSING ===
   #
@@ -835,6 +863,11 @@ class Parser::Lexer
     if literal.heredoc?
       line = tok(@herebody_s, @ts).gsub(/\r+$/, '')
 
+      if version?(18, 19, 20)
+        # See ruby:c48b4209c
+        line = line.gsub(/\r.*$/, '')
+      end
+
       # Try ending the heredoc with the complete most recently
       # scanned line. @herebody_s always refers to the start of such line.
       if literal.nest_and_try_closing(line, @herebody_s, @ts)
@@ -1870,23 +1903,15 @@ class Parser::Lexer
       # NUMERIC LITERALS
       #
 
-      ( '0' [Xx]  %{ @num_base = 16; @num_digits_s = p }
-               ( xdigit+ '_' )* xdigit* '_'?
-      | '0' [Dd]  %{ @num_base = 10; @num_digits_s = p }
-               ( digit+ '_' )* digit* '_'?
-      | '0' [Oo]  %{ @num_base = 8;  @num_digits_s = p }
-               ( digit+ '_' )* digit* '_'?
-      | '0' [Bb]  %{ @num_base = 2;  @num_digits_s = p }
-               ( [01]+ '_' )* [01]* '_'?
-      | [1-9] digit*
-                  %{ @num_base = 10; @num_digits_s = @ts }
-               ( '_' digit+ )* digit* '_'?
-      | '0' digit*
-                  %{ @num_base = 8;  @num_digits_s = @ts }
-               ( '_' digit+ )* digit* '_'?
-      )
+      ( '0' [Xx] %{ @num_base = 16; @num_digits_s = p } int_hex
+      | '0' [Dd] %{ @num_base = 10; @num_digits_s = p } int_dec
+      | '0' [Oo] %{ @num_base = 8;  @num_digits_s = p } int_dec
+      | '0' [Bb] %{ @num_base = 2;  @num_digits_s = p } int_bin
+      | [1-9] digit* '_'? %{ @num_base = 10; @num_digits_s = @ts } int_dec
+      | '0'   digit* '_'? %{ @num_base = 8;  @num_digits_s = @ts } int_dec
+      ) %{ @num_suffix_s = p } int_suffix
       => {
-        digits = tok(@num_digits_s)
+        digits = tok(@num_digits_s, @num_suffix_s)
 
         if digits.end_with? '_'
           diagnostic :error, Parser::ERRORS[:trailing_in_number] % { :character => '_' },
@@ -1902,27 +1927,57 @@ class Parser::Lexer
                      range(invalid_s, invalid_s + 1)
         end
 
-        emit(:tINTEGER, digits.to_i(@num_base))
+        if version?(18, 19, 20)
+          emit(:tINTEGER, digits.to_i(@num_base))
+          p = @num_suffix_s - 1
+        else
+          @num_xfrm.call(digits.to_i(@num_base))
+        end
         fbreak;
       };
 
-      '.' ( digit+ '_' )* digit+
+      flo_frac flo_pow?
       => {
         diagnostic :error, Parser::ERRORS[:no_dot_digit_literal]
       };
 
-      (
-        ( [1-9] [0-9]* ( '_' digit+ )* | '0' )
-        ( '.' ( digit+ '_' )* digit+ )?
-        ( [eE] [+\-]? ( digit+ '_' )* digit* )?
-      )
+      flo_int [eE]
       => {
-        if tok.end_with? 'e'
-          diagnostic :error, Parser::ERRORS[:trailing_in_number] % { :character => 'e' },
+        if version?(18, 19, 20)
+          diagnostic :error,
+                     Parser::ERRORS[:trailing_in_number] % { :character => tok(@te - 1, @te) },
                      range(@te - 1, @te)
+        else
+          emit(:tINTEGER, tok(@ts, @te - 1).to_i)
+          fhold; fbreak;
         end
+      };
 
-        emit(:tFLOAT, tok.to_f)
+      flo_int flo_frac [eE]
+      => {
+        if version?(18, 19, 20)
+          diagnostic :error,
+                     Parser::ERRORS[:trailing_in_number] % { :character => tok(@te - 1, @te) },
+                     range(@te - 1, @te)
+        else
+          emit(:tFLOAT, tok(@ts, @te - 1).to_f)
+          fhold; fbreak;
+        end
+      };
+
+      flo_int
+      ( flo_frac? flo_pow %{ @num_suffix_s = p } flo_pow_suffix
+      | flo_frac          %{ @num_suffix_s = p } flo_suffix
+      )
+      => {
+        digits = tok(@ts, @num_suffix_s)
+
+        if version?(18, 19, 20)
+          emit(:tFLOAT, Float(digits))
+          p = @num_suffix_s - 1
+        else
+          @num_xfrm.call(digits)
+        end
         fbreak;
       };
 
@@ -2077,8 +2132,8 @@ class Parser::Lexer
       => { @eq_begin_s = @ts
            fgoto line_comment; };
 
-      '__END__' c_nl_zlen
-      => { p = pe - 1 };
+      '__END__' ( c_eol - zlen )
+      => { p = pe - 3 };
 
       c_any
       => { fhold; fgoto expr_value; };

data/lib/parser/ruby18.y

@@ -1619,11 +1619,13 @@ xstring_contents: # nothing
                     }
                 | tUMINUS_NUM tINTEGER =tLOWEST
                     {
-                      result = @builder.integer(val[1], true)
+                      result = @builder.negate(val[0],
+                                  @builder.integer(val[1]))
                     }
                 | tUMINUS_NUM tFLOAT   =tLOWEST
                     {
-                      result = @builder.float(val[1], true)
+                      result = @builder.negate(val[0],
+                                  @builder.float(val[1]))
                     }
 
         variable: tIDENTIFIER

data/lib/parser/ruby19.y

@@ -1749,11 +1749,13 @@ regexp_contents: # nothing
                     }
                 | tUMINUS_NUM tINTEGER =tLOWEST
                     {
-                      result = @builder.integer(val[1], true)
+                      result = @builder.negate(val[0],
+                                  @builder.integer(val[1]))
                     }
                 | tUMINUS_NUM tFLOAT   =tLOWEST
                     {
-                      result = @builder.float(val[1], true)
+                      result = @builder.negate(val[0],
+                                  @builder.float(val[1]))
                     }
 
    user_variable: tIDENTIFIER

data/lib/parser/ruby20.y

@@ -1842,11 +1842,13 @@ regexp_contents: # nothing
                     }
                 | tUMINUS_NUM tINTEGER =tLOWEST
                     {
-                      result = @builder.integer(val[1], true)
+                      result = @builder.negate(val[0],
+                                  @builder.integer(val[1]))
                     }
                 | tUMINUS_NUM tFLOAT   =tLOWEST
                     {
-                      result = @builder.float(val[1], true)
+                      result = @builder.negate(val[0],
+                                  @builder.float(val[1]))
                     }
 
    user_variable: tIDENTIFIER

data/lib/parser/ruby21.y

@@ -17,6 +17,7 @@ token kCLASS kMODULE kDEF kUNDEF kBEGIN kRESCUE kENSURE kEND kIF kUNLESS
       tWORDS_BEG tQWORDS_BEG tSYMBOLS_BEG tQSYMBOLS_BEG tSTRING_DBEG
       tSTRING_DVAR tSTRING_END tSTRING_DEND tSTRING tSYMBOL
       tNL tEH tCOLON tCOMMA tSPACE tSEMI tLAMBDA tLAMBEG tCHARACTER
+      tRATIONAL tIMAGINARY
 
 prechigh
   right    tBANG tTILDE tUPLUS
@@ -689,19 +690,11 @@ rule
                     {
                       result = @builder.binary_op(val[0], val[1], val[2])
                     }
-                | tUMINUS_NUM tINTEGER tPOW arg
+                | tUMINUS_NUM simple_numeric tPOW arg
                     {
                       result = @builder.unary_op(val[0],
                                   @builder.binary_op(
-                                    @builder.integer(val[1]),
-                                      val[2], val[3]))
-                    }
-                | tUMINUS_NUM tFLOAT tPOW arg
-                    {
-                      result = @builder.unary_op(val[0],
-                                  @builder.binary_op(
-                                    @builder.float(val[1]),
-                                      val[2], val[3]))
+                                    val[1], val[2], val[3]))
                     }
                 | tUPLUS arg
                     {
@@ -1235,8 +1228,6 @@ rule
 
           f_marg: f_norm_arg
                     {
-                      @static_env.declare val[0][0]
-
                       result = @builder.arg(val[0])
                     }
                 | tLPAREN f_margs rparen
@@ -1256,15 +1247,11 @@ rule
          f_margs: f_marg_list
                 | f_marg_list tCOMMA tSTAR f_norm_arg
                     {
-                      @static_env.declare val[3][0]
-
                       result = val[0].
                                   push(@builder.restarg(val[2], val[3]))
                     }
                 | f_marg_list tCOMMA tSTAR f_norm_arg tCOMMA f_marg_list
                     {
-                      @static_env.declare val[3][0]
-
                       result = val[0].
                                   push(@builder.restarg(val[2], val[3])).
                                   concat(val[5])
@@ -1282,14 +1269,10 @@ rule
                     }
                 |                    tSTAR f_norm_arg
                     {
-                      @static_env.declare val[1][0]
-
                       result = [ @builder.restarg(val[0], val[1]) ]
                     }
                 |                    tSTAR f_norm_arg tCOMMA f_marg_list
                     {
-                      @static_env.declare val[1][0]
-
                       result = [ @builder.restarg(val[0], val[1]),
                                  *val[3] ]
                     }
@@ -1829,7 +1812,16 @@ regexp_contents: # nothing
                       result = @builder.symbol_compose(val[0], val[1], val[2])
                     }
 
-         numeric: tINTEGER
+         numeric: simple_numeric
+                    {
+                      result = val[0]
+                    }
+                | tUMINUS_NUM simple_numeric =tLOWEST
+                    {
+                      result = @builder.negate(val[0], val[1])
+                    }
+
+  simple_numeric: tINTEGER
                     {
                       result = @builder.integer(val[0])
                     }
@@ -1837,13 +1829,13 @@ regexp_contents: # nothing
                     {
                       result = @builder.float(val[0])
                     }
-                | tUMINUS_NUM tINTEGER =tLOWEST
+                | tRATIONAL
                     {
-                      result = @builder.integer(val[1], true)
+                      result = @builder.rational(val[0])
                     }
-                | tUMINUS_NUM tFLOAT   =tLOWEST
+                | tIMAGINARY
                     {
-                      result = @builder.float(val[1], true)
+                      result = @builder.complex(val[0])
                     }
 
    user_variable: tIDENTIFIER
@@ -2087,11 +2079,14 @@ keyword_variable: kNIL
 
       f_norm_arg: f_bad_arg
                 | tIDENTIFIER
-
-      f_arg_item: f_norm_arg
                     {
                       @static_env.declare val[0][0]
 
+                      result = val[0]
+                    }
+
+      f_arg_item: f_norm_arg
+                    {
                       result = @builder.arg(val[0])
                     }
                 | tLPAREN f_margs rparen
@@ -2108,37 +2103,30 @@ keyword_variable: kNIL
                       result = val[0] << val[2]
                     }
 
-            f_kw: tLABEL arg_value
+         f_label: tLABEL
                     {
                       check_kwarg_name(val[0])
 
                       @static_env.declare val[0][0]
 
+                      result = val[0]
+                    }
+
+            f_kw: f_label arg_value
+                    {
                       result = @builder.kwoptarg(val[0], val[1])
                     }
-                | tLABEL
+                | f_label
                     {
-                      check_kwarg_name(val[0])
-
-                      @static_env.declare val[0][0]
-
                       result = @builder.kwarg(val[0])
                     }
 
-      f_block_kw: tLABEL primary_value
+      f_block_kw: f_label primary_value
                     {
-                      check_kwarg_name(val[0])
-
-                      @static_env.declare val[0][0]
-
                       result = @builder.kwoptarg(val[0], val[1])
                     }
-                | tLABEL
+                | f_label
                     {
-                      check_kwarg_name(val[0])
-
-                      @static_env.declare val[0][0]
-
                       result = @builder.kwarg(val[0])
                     }
 
@@ -2173,17 +2161,13 @@ keyword_variable: kNIL
                       result = [ @builder.kwrestarg(val[0]) ]
                     }
 
-           f_opt: tIDENTIFIER tEQL arg_value
+           f_opt: f_norm_arg tEQL arg_value
                     {
-                      @static_env.declare val[0][0]
-
                       result = @builder.optarg(val[0], val[1], val[2])
                     }
 
-     f_block_opt: tIDENTIFIER tEQL primary_value
+     f_block_opt: f_norm_arg tEQL primary_value
                     {
-                      @static_env.declare val[0][0]
-
                       result = @builder.optarg(val[0], val[1], val[2])
                     }
 

data/lib/parser/version.rb

@@ -1,3 +1,3 @@
 module Parser
-  VERSION = '2.0.0'
+  VERSION = '2.1.0.pre1'
 end

data/parser.gemspec

@@ -29,18 +29,19 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler',   '~> 1.2'
   spec.add_development_dependency 'rake',      '~> 0.9'
-  spec.add_development_dependency 'racc'
+  spec.add_development_dependency 'racc',      '= 1.4.9' # update to 1.4.11 when it's done
+  spec.add_development_dependency 'cliver',    '~> 0.3.0'
+
+  spec.add_development_dependency 'yard'
+  spec.add_development_dependency 'kramdown'
 
   spec.add_development_dependency 'minitest',  '~> 5.0'
   spec.add_development_dependency 'simplecov', '~> 0.7'
   spec.add_development_dependency 'coveralls'
   spec.add_development_dependency 'json_pure' # for coveralls on 1.9.2
-  spec.add_development_dependency 'cliver',    '~> 0.2.2' # executable version detection
+  spec.add_development_dependency 'mime-types', '~> 1.25' # for coveralls on 1.8.7
 
   spec.add_development_dependency 'simplecov-sublime-ruby-coverage'
 
-  spec.add_development_dependency 'yard'
-  spec.add_development_dependency 'kramdown'
-
   spec.add_development_dependency 'gauntlet'
 end

data/test/test_current.rb

@@ -10,6 +10,8 @@ class TestCurrent < Minitest::Test
       assert_equal Parser::Ruby19, Parser::CurrentRuby
     when '2.0.0'
       assert_equal Parser::Ruby20, Parser::CurrentRuby
+    when '2.1.0'
+      assert_equal Parser::Ruby21, Parser::CurrentRuby
     else
       flunk "Update test_parser_current for #{RUBY_VERSION}"
     end

data/test/test_lexer.rb

@@ -1,6 +1,7 @@
 # encoding: ascii-8bit
 
 require 'helper'
+require 'complex'
 
 class TestLexer < Minitest::Test
   def setup_lexer(version)
@@ -86,7 +87,7 @@ class TestLexer < Minitest::Test
 
       lex_token, (lex_value, *) = @lex.advance
       assert lex_token, 'no more tokens'
-      assert_equal [token, value], [lex_token, lex_value], input
+      assert_operator [lex_token, lex_value], :eql?, [token, value], input
     end
 
     lex_token, (lex_value, *) = @lex.advance
@@ -646,6 +647,24 @@ class TestLexer < Minitest::Test
     assert_scanned "0e0", :tFLOAT, 0e0
   end
 
+  def test_float_e_nothing
+    [18, 19, 20].each do |version|
+      setup_lexer version
+
+      refute_scanned "1end"
+      refute_scanned "1.1end"
+    end
+
+    setup_lexer 21
+
+    assert_scanned("1end",
+                   :tINTEGER, 1,
+                   :kEND,     'end')
+    assert_scanned("1.1end",
+                   :tFLOAT,   1.1,
+                   :kEND,     'end')
+  end
+
   def test_float_neg
     assert_scanned("-1.0",
                    :tUMINUS_NUM, "-",
@@ -2109,13 +2128,14 @@ class TestLexer < Minitest::Test
   end
 
   def test_underscore_end
-    source_buffer = Parser::Source::Buffer.new('(underscore_end)')
-    source_buffer.source = "__END__\n"
-
-    @lex.source_buffer = source_buffer
-
-    tok, = @lex.advance
-    refute tok
+    assert_scanned("__END__\n")
+    assert_scanned("__END__")
+    assert_scanned("__END__ foo",
+                   :tIDENTIFIER, '__END__',
+                   :tIDENTIFIER, 'foo')
+    assert_scanned("__END__\rfoo",
+                   :tIDENTIFIER, '__END__',
+                   :tIDENTIFIER, 'foo')
   end
 
   def test_uplus
@@ -2163,6 +2183,71 @@ class TestLexer < Minitest::Test
                    :tRBRACK,     "]")
   end
 
+  def test_int_suffix
+    [18, 19, 20].each do |version|
+      setup_lexer version
+
+      assert_scanned("42r",
+                     :tINTEGER,    42,
+                     :tIDENTIFIER, 'r')
+
+      assert_scanned("42if",
+                     :tINTEGER,    42,
+                     :kIF_MOD,     'if')
+    end
+
+    setup_lexer 21
+
+    assert_scanned("42r",  :tRATIONAL,  Rational(42))
+    assert_scanned("42i",  :tIMAGINARY, Complex(0, 42))
+    assert_scanned("42ri", :tIMAGINARY, Complex(0, Rational(42)))
+  end
+
+  def test_float_suffix
+    [18, 19, 20].each do |version|
+      setup_lexer version
+
+      assert_scanned("42.1r",
+                     :tFLOAT,      42.1,
+                     :tIDENTIFIER, 'r')
+
+      assert_scanned("42.1if",
+                     :tFLOAT,      42.1,
+                     :kIF_MOD,     'if')
+
+      assert_scanned("1e1r",
+                     :tFLOAT,      1e1,
+                     :tIDENTIFIER, 'r')
+    end
+
+    begin
+      # Feature-check.
+      Rational("10")
+
+      setup_lexer 21
+
+      assert_scanned("42.1r",  :tRATIONAL,  Rational(421, 10))
+      assert_scanned("42.1i",  :tIMAGINARY, Complex(0, 42.1))
+      assert_scanned("42.1ri", :tIMAGINARY, Complex(0, Rational(421, 10)))
+      assert_scanned("42.1ir",
+                     :tIMAGINARY,  Complex(0, 42.1),
+                     :tIDENTIFIER, 'r')
+
+      assert_scanned("1e1i",   :tIMAGINARY, Complex(0, 1e1))
+      assert_scanned("1e1r",
+                     :tFLOAT,      1e1,
+                     :tIDENTIFIER, 'r')
+      assert_scanned("1e1ri",
+                     :tFLOAT,      1e1,
+                     :tIDENTIFIER, 'ri')
+      assert_scanned("1e1ir",
+                     :tIMAGINARY,  Complex(0, 1e1),
+                     :tIDENTIFIER, 'r')
+    rescue NoMethodError
+      # Ruby not modern enough
+    end
+  end
+
   #
   # Tests for whitespace.
   #
@@ -2407,6 +2492,19 @@ class TestLexer < Minitest::Test
                    :tINTEGER, 1)
   end
 
+  def test_whitespace_cr
+    setup_lexer(20)
+    assert_scanned("<<E\nfoo\nE\rO",
+                   :tSTRING_BEG,     '"',
+                   :tSTRING_CONTENT, "foo\n",
+                   :tSTRING_END,     'E',
+                   :tNL,             nil)
+
+    setup_lexer(21)
+    refute_scanned("<<E\nfoo\nE\rO",
+                   :tSTRING_BEG,     '"',
+                   :tSTRING_CONTENT, "foo\n")
+  end
 
   #
   # Tests for bugs.

data/test/test_parser.rb

@@ -92,7 +92,8 @@ class TestParser < Minitest::Test
     assert_parses(
       s(:int, -42),
       %q{-42},
-      %q{~~~ expression})
+      %q{^ operator
+        |~~~ expression})
   end
 
   def test_int___LINE__
@@ -111,7 +112,48 @@ class TestParser < Minitest::Test
     assert_parses(
       s(:float, -1.33),
       %q{-1.33},
-      %q{~~~~~ expression})
+      %q{^ operator
+        |~~~~~ expression})
+  end
+
+  def test_rational
+    assert_parses(
+      s(:rational, Rational(42)),
+      %q{42r},
+      %q{~~~ expression},
+      ALL_VERSIONS - %w(1.8 1.9 2.0))
+
+    assert_parses(
+      s(:rational, Rational(421, 10)),
+      %q{42.1r},
+      %q{~~~~~ expression},
+      ALL_VERSIONS - %w(1.8 1.9 2.0))
+  end
+
+  def test_complex
+    assert_parses(
+      s(:complex, Complex(0, 42)),
+      %q{42i},
+      %q{~~~ expression},
+      ALL_VERSIONS - %w(1.8 1.9 2.0))
+
+    assert_parses(
+      s(:complex, Complex(0, Rational(42))),
+      %q{42ri},
+      %q{~~~~ expression},
+      ALL_VERSIONS - %w(1.8 1.9 2.0))
+
+    assert_parses(
+      s(:complex, Complex(0, 42.1)),
+      %q{42.1i},
+      %q{~~~~~ expression},
+      ALL_VERSIONS - %w(1.8 1.9 2.0))
+
+    assert_parses(
+      s(:complex, Complex(0, Rational(421, 10))),
+      %q{42.1ri},
+      %q{~~~~~~ expression},
+      ALL_VERSIONS - %w(1.8 1.9 2.0))
   end
 
   # Strings

metadata

@@ -1,215 +1,229 @@
 --- !ruby/object:Gem::Specification
 name: parser
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.1.0.pre1
 platform: ruby
 authors:
 - Peter Zotov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-05 00:00:00.000000000 Z
+date: 2013-11-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ast
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
 - !ruby/object:Gem::Dependency
   name: slop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.4'
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 3.4.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.4'
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 3.4.5
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.2'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.9'
 - !ruby/object:Gem::Dependency
   name: racc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.4.9
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.4.9
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: cliver
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: 0.3.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: 0.3.0
 - !ruby/object:Gem::Dependency
-  name: simplecov
+  name: yard
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: coveralls
+  name: kramdown
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: json_pure
+  name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '5.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '5.0'
 - !ruby/object:Gem::Dependency
-  name: cliver
+  name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.2
+        version: '0.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.2
+        version: '0.7'
 - !ruby/object:Gem::Dependency
-  name: simplecov-sublime-ruby-coverage
+  name: coveralls
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: yard
+  name: json_pure
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: kramdown
+  name: mime-types
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.25'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.25'
+- !ruby/object:Gem::Dependency
+  name: simplecov-sublime-ruby-coverage
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: gauntlet
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: A Ruby parser written in pure Ruby.
@@ -221,10 +235,10 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .rubocop.yml
-- .travis.yml
-- .yardopts
+- ".gitignore"
+- ".rubocop.yml"
+- ".travis.yml"
+- ".yardopts"
 - CHANGELOG.md
 - CONTRIBUTING.md
 - Gemfile
@@ -250,14 +264,19 @@ files:
 - lib/parser/current.rb
 - lib/parser/diagnostic.rb
 - lib/parser/diagnostic/engine.rb
+- lib/parser/lexer.rb
 - lib/parser/lexer.rl
 - lib/parser/lexer/explanation.rb
 - lib/parser/lexer/literal.rb
 - lib/parser/lexer/stack_state.rb
 - lib/parser/rewriter.rb
+- lib/parser/ruby18.rb
 - lib/parser/ruby18.y
+- lib/parser/ruby19.rb
 - lib/parser/ruby19.y
+- lib/parser/ruby20.rb
 - lib/parser/ruby20.y
+- lib/parser/ruby21.rb
 - lib/parser/ruby21.y
 - lib/parser/runner.rb
 - lib/parser/runner/ruby_parse.rb
@@ -302,11 +321,6 @@ files:
 - test/test_source_rewriter.rb
 - test/test_source_rewriter_action.rb
 - test/test_static_environment.rb
-- lib/parser/lexer.rb
-- lib/parser/ruby18.rb
-- lib/parser/ruby19.rb
-- lib/parser/ruby20.rb
-- lib/parser/ruby21.rb
 homepage: http://github.com/whitequark/parser
 licenses:
 - MIT
@@ -317,17 +331,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.0
+rubygems_version: 2.2.0.preview.1
 signing_key: 
 specification_version: 4
 summary: A Ruby parser written in pure Ruby.