parser 2.0.0.beta8 → 2.0.0.beta9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ccb29ca45008c480f7d06bebc486b5a6c4137a83
-  data.tar.gz: ad0be62764cf0e9ee5ad18e3d353adaac06e0951
+  metadata.gz: db31925f63c3bfa1d7dcd31e2da5a4d9093956cb
+  data.tar.gz: c320c20f73ae66ecd8264b69fbdbebe200e203ef
 SHA512:
-  metadata.gz: b305ef3b909c5aabf17957dd55a72441c47482b280bc6ab3253dd732be9643366f3012c11c95286ec3ea7de10ab69972e80f607b9c8afa7bd3c37732817060b0
-  data.tar.gz: 1d1d3b29136fa4ba6faf37fa09a98f396294055cc14cf296ee9cfb6c0ca77f490a9cb306721ea0e311913bd66a43cc2cc371fc558f8c4a222809d072a4a752f8
+  metadata.gz: b76ad89767945e8f83931e9968ff2a7f980498e29d0aba637ca6d1c6abf186deb2d75d50525ffba120c2fbd252a9b8fec3f988c17575414430d46f901bf1c515
+  data.tar.gz: 21b368d7bb373c2ee197b711ed80d659041a7220b79586d6a295889761604f2d50bc2558aba57325bd24e39fe37e2f8ea3df31a2c83e261f6eaf9f5fbfba34ee

data/CHANGELOG.md

@@ -1,6 +1,25 @@
 Changelog
 =========
 
+v2.0.0.beta9 (2013-06-28)
+-------------------------
+
+API modifications:
+ * ruby{18,19,20,21}.y: removed obsolete warnings and linting. (Peter Zotov)
+
+Features implemented:
+ * builders/default: add keyword source range for BEGIN/END (fixes #85). (Peter Zotov)
+
+Bugs fixed:
+ * lexer.rl: "t=1;(a)?t:T" context sensitivity in expr_value (fixes #87). (Peter Zotov)
+ * lexer.rl: keywords as labels, e.g. "unless:" (fixes #83, #84). (Peter Zotov)
+ * lexer.rl: rectify usage of c_space/c_space_nl (fixes #81). (Peter Zotov)
+ * ruby{18,19,20,21}.y: fix warnings for class/module in method body. (Peter Zotov)
+ * lexer.rl: fix warning for ?\s. (Peter Zotov)
+ * lexer.rl: expr_fname emits expr_beg-like keywords (fixes #82). (Peter Zotov)
+ * lexer.rl: get rid of harmful nondeterminism in w_space (fixes #80). (Peter Zotov)
+ * lexer/explanation: 1.8, 1.9 compatibility (fixes #76). (Peter Zotov)
+
 v2.0.0.beta8 (2013-06-24)
 -------------------------
 

data/README.md

@@ -11,8 +11,10 @@ ruby_parser.
 
 ## Installation
 
+Most recent version of Parser is 2.0; however, per [release schedule](https://github.com/whitequark/parser/issues/51), it stays in the beta status for a while. However, it handles much more input than stable 1.x branch, and for new work it is advisable to use the beta versions.
+
 ~~~
-$ gem install parser
+$ gem install parser --pre
 ~~~
 
 ## Usage
@@ -117,6 +119,14 @@ $ ruby-parse -E -e "2+2"
 
 Documentation for parser is available online on [rdoc.info](http://rdoc.info/github/whitequark/parser).
 
+## Known issues
+
+### Void value expressions
+
+So-called "void value expressions" are not handled by Parser. For a description of what a void value expression is, see [this gist](https://gist.github.com/JoshCheek/5625007) and [this Parser issue](https://github.com/whitequark/parser/issues/72).
+
+It is not clear which rules this piece of static analysis follows, or which problem does it solve. It is not implemented because there is no clear specification allowing us to verify the behavior.
+
 ## Contributors
 
  * Peter Zotov ([whitequark][])

data/doc/AST_FORMAT.md

@@ -1,5 +1,5 @@
-AST and Source Location RFC
-===========================
+AST and Source Location
+=======================
 
 ## Literals
 

data/doc/CUSTOMIZATION.md

@@ -27,8 +27,8 @@ To use a custom node class you have to override the method
 `Parser::Builders::Default#n`:
 
     class MyBuilder < Parser::Builders::Default
-      def n(type, children, source_map)
-        return MyNodeClass.new(type, children, :source_map => source_map)
+      def n(type, children, location)
+        return MyNodeClass.new(type, children, :location => location)
       end
     end
 

data/lib/parser.rb

@@ -59,7 +59,6 @@ module Parser
     # Lexer errors
     :unicode_point_too_large => 'invalid Unicode codepoint (too large)',
     :invalid_escape          => 'invalid escape character syntax',
-    :invalid_escape_use      => 'invalid character syntax; use ?%{escape}',
     :incomplete_escape       => 'incomplete character syntax',
     :invalid_hex_escape      => 'invalid hex escape',
     :invalid_unicode_escape  => 'invalid Unicode escape',
@@ -78,6 +77,7 @@ module Parser
     :embedded_document       => 'embedded document meats end of file (and they embark on a romantic journey)',
 
     # Lexer warnings
+    :invalid_escape_use      => 'invalid character syntax; use ?%{escape}',
     :ambiguous_literal       => 'ambiguous first argument; parenthesize arguments or add whitespace to the right',
     :ambiguous_prefix        => "`%{prefix}' interpreted as argument prefix",
 
@@ -104,8 +104,6 @@ module Parser
     :masgn_as_condition      => 'multiple assignment in conditional context',
 
     # Parser warnings
-    :end_in_method           => 'END in method; use at_exit',
-    :space_before_lparen     => "don't put space before argument parentheses",
     :useless_else            => 'else without rescue is useless',
   }.freeze
 

data/lib/parser/builders/default.rb

@@ -743,12 +743,12 @@ module Parser
 
     def preexe(preexe_t, lbrace_t, compstmt, rbrace_t)
       n(:preexe, [ compstmt ],
-        block_map(loc(preexe_t), lbrace_t, rbrace_t))
+        keyword_map(preexe_t, lbrace_t, [], rbrace_t))
     end
 
     def postexe(postexe_t, lbrace_t, compstmt, rbrace_t)
       n(:postexe, [ compstmt ],
-        block_map(loc(postexe_t), lbrace_t, rbrace_t))
+        keyword_map(postexe_t, lbrace_t, [], rbrace_t))
     end
 
     # Exception handling

data/lib/parser/lexer.rl

@@ -344,18 +344,18 @@ class Parser::Lexer
     'BEGIN'  => :klBEGIN,      'END'      => :klEND,
   }
 
-  %w(class module def undef begin end then elsif else ensure case when
-     for break next redo retry in do return yield super self nil true
-     false and or not alias __FILE__ __LINE__ __ENCODING__).each do |keyword|
-    KEYWORDS[keyword] = :"k#{keyword.upcase}"
-  end
-
   KEYWORDS_BEGIN = {
     'if'     => :kIF,          'unless' => :kUNLESS,
     'while'  => :kWHILE,       'until'  => :kUNTIL,
     'rescue' => :kRESCUE
   }
 
+  %w(class module def undef begin end then elsif else ensure case when
+     for break next redo retry in do return yield super self nil true
+     false and or not alias __FILE__ __LINE__ __ENCODING__).each do |keyword|
+    KEYWORDS_BEGIN[keyword] = KEYWORDS[keyword] = :"k#{keyword.upcase}"
+  end
+
   %%{
   # %
 
@@ -370,7 +370,8 @@ class Parser::Lexer
   # for scanners.
 
   action do_nl {
-    # Record position of a newline for precise line and column reporting.
+    # Record position of a newline for precise location reporting on tNL
+    # tokens.
     #
     # This action is embedded directly into c_nl, as it is idempotent and
     # there are no cases when we need to skip it.
@@ -484,6 +485,8 @@ class Parser::Lexer
   class_var_v    = '@@' c_alnum+;
   instance_var_v = '@' c_alnum+;
 
+  label          = bareword [?!]? ':';
+
   #
   # === ESCAPE SEQUENCE PARSING ===
   #
@@ -918,7 +921,15 @@ class Parser::Lexer
   w_space =
       c_space+
     | '\\' e_heredoc_nl
-    | '#' %{ @sharp_s = p - 1 } c_line* %{ emit_comment(@sharp_s, p) }
+    ;
+
+  w_comment =
+      '#' %{ @sharp_s = p - 1 } c_line* %{ emit_comment(@sharp_s, p) }
+    ;
+
+  w_space_comment =
+      w_space
+    | w_comment
     ;
 
   # A newline in non-literal context always interoperates with
@@ -935,9 +946,14 @@ class Parser::Lexer
   #
   # is equivalent to `foo = "bar\n" + 2`.
 
-  w_newline = e_heredoc_nl;
+  w_newline =
+      e_heredoc_nl;
 
-  w_space_newline = w_space | w_newline;
+  w_any =
+      w_space
+    | w_comment
+    | w_newline
+    ;
 
 
   #
@@ -992,6 +1008,18 @@ class Parser::Lexer
     @paren_nest -= 1
   };
 
+  # Ruby >=1.9.2 is context-sensitive wrt/ local identifiers.
+  action local_ident {
+    emit(:tIDENTIFIER)
+
+    if !version?(18) &&
+          !@static_env.nil? && @static_env.declared?(tok)
+      fnext expr_end; fbreak;
+    else
+      fnext expr_arg; fbreak;
+    end
+  }
+
   # Variable lexing code is accessed from both expressions and
   # string interpolation related code.
   #
@@ -1043,7 +1071,7 @@ class Parser::Lexer
   #
   expr_fname := |*
       keyword
-      => { emit(KEYWORDS[tok]);
+      => { emit(KEYWORDS_BEGIN[tok]);
            fnext expr_endfn; fbreak; };
 
       constant
@@ -1073,7 +1101,7 @@ class Parser::Lexer
       ':'
       => { fhold; fgoto expr_beg; };
 
-      w_space_newline;
+      w_any;
 
       c_any
       => { fhold; fgoto expr_end; };
@@ -1087,11 +1115,11 @@ class Parser::Lexer
   # Transitions to `expr_end` afterwards.
   #
   expr_endfn := |*
-      bareword ':'
+      label
       => { emit(:tLABEL, tok(@ts, @te - 1))
            fnext expr_beg; fbreak; };
 
-      w_space;
+      w_space_comment;
 
       c_any
       => { fhold; fgoto expr_end; };
@@ -1123,7 +1151,7 @@ class Parser::Lexer
       => { emit_table(PUNCTUATION)
            fnext expr_arg; fbreak; };
 
-      w_space_newline;
+      w_any;
 
       c_any
       => { fhold; fgoto expr_end; };
@@ -1141,7 +1169,7 @@ class Parser::Lexer
 
       # cmd (1 + 2)
       # See below the rationale about expr_endarg.
-      c_space+ e_lparen
+      w_space+ e_lparen
       => { emit(:tLPAREN_ARG, '(', @te - 1, @te)
            fnext expr_beg; fbreak; };
 
@@ -1153,13 +1181,13 @@ class Parser::Lexer
 
       # meth [...]
       # Array argument. Compare with indexing `meth[...]`.
-      c_space+ e_lbrack
+      w_space+ e_lbrack
       => { emit(:tLBRACK, '[', @te - 1, @te)
            fnext expr_beg; fbreak; };
 
       # cmd {}
       # Command: method call without parentheses.
-      c_space* e_lbrace
+      w_space* e_lbrace
       => {
         if @lambda_stack.last == @paren_nest
           p = @ts - 1
@@ -1176,11 +1204,11 @@ class Parser::Lexer
 
       # a ?b
       # Character literal.
-      c_space+ '?'
+      w_space+ '?'
       => { fhold; fgoto expr_beg; };
 
                # a %{1}, a %[1] (but not "a %=1=" or "a % foo")
-      c_space+ ( '%' [^= ]
+      w_space+ ( '%' [^= ]
                # a /foo/ (but not "a / foo" or "a /=foo")
                | '/' ( c_any - c_space_nl - '=' )
                # a <<HEREDOC
@@ -1190,7 +1218,7 @@ class Parser::Lexer
 
       # x /1
       # Ambiguous regexp literal.
-      c_space+ '/'
+      w_space+ '/'
       => {
         diagnostic :warning, Parser::ERRORS[:ambiguous_literal],
                    range(@te - 1, @te)
@@ -1200,7 +1228,7 @@ class Parser::Lexer
 
       # x *1
       # Ambiguous splat, kwsplat or block-pass.
-      c_space+ %{ tm = p } ( '+' | '-' | '*' | '&' | '**' )
+      w_space+ %{ tm = p } ( '+' | '-' | '*' | '&' | '**' )
       => {
         message = Parser::ERRORS[:ambiguous_prefix] % { :prefix => tok(tm, @te) }
         diagnostic :warning, message,
@@ -1212,36 +1240,44 @@ class Parser::Lexer
 
       # x ::Foo
       # Ambiguous toplevel constant access.
-      c_space+ '::'
+      w_space+ '::'
       => { fhold; fhold; fgoto expr_beg; };
 
       # x:b
       # Symbol.
-      c_space* ':'
+      w_space* ':'
       => { fhold; fgoto expr_beg; };
 
+      w_space+ label
+      => { p = @ts - 1; fgoto expr_beg; };
+
       #
       # AMBIGUOUS TOKENS RESOLVED VIA EXPR_END
       #
 
       # a ? b
       # Ternary operator.
-      c_space+ '?' c_space_nl
+      w_space+ '?' c_space_nl
       => { fhold; fhold; fgoto expr_end; };
 
       # x + 1: Binary operator or operator-assignment.
-      c_space* operator_arithmetic
+      w_space* operator_arithmetic
                   ( '=' | c_space_nl )?    |
       # x rescue y: Modifier keyword.
-      c_space+ keyword_modifier            |
+      w_space+ keyword_modifier            |
       # Miscellanea.
-      c_space* punctuation_end
+      w_space* punctuation_end
       => {
         p = @ts - 1
         fgoto expr_end;
       };
 
-      w_space* w_newline
+      w_space;
+
+      w_comment
+      => { fgoto expr_end; };
+
+      w_newline
       => { fhold; fgoto expr_end; };
 
       c_any
@@ -1274,7 +1310,7 @@ class Parser::Lexer
       => { emit(:kDO_BLOCK)
            fnext expr_value; };
 
-      w_space;
+      w_space_comment;
 
       c_any
       => { fhold; fgoto expr_end; };
@@ -1292,7 +1328,8 @@ class Parser::Lexer
       => { emit_table(KEYWORDS)
            fnext expr_beg; fbreak; };
 
-      w_space;
+      w_space_comment;
+
       w_newline
       => { fhold; fgoto expr_end; };
 
@@ -1433,10 +1470,9 @@ class Parser::Lexer
       '?' c_space_nl
       => {
         escape = { " "  => '\s', "\r" => '\r', "\n" => '\n', "\t" => '\t',
-                   "\v" => '\v', "\f" => '\f' }[tok[@ts + 1]]
+                   "\v" => '\v', "\f" => '\f' }[tok[1]]
         message = Parser::ERRORS[:invalid_escape_use] % { :escape => escape }
-        diagnostic :warning, message,
-                   range(@ts, @ts + 1)
+        diagnostic :warning, message, range
 
         p = @ts - 1
         fgoto expr_end;
@@ -1488,7 +1524,7 @@ class Parser::Lexer
       # RUBY 1.9 HASH LABELS
       #
 
-      bareword [?!]? ':' ( c_any - ':' )
+      label ( any - ':' )
       => {
         fhold;
 
@@ -1515,23 +1551,13 @@ class Parser::Lexer
 
       # a = 42;     a [42]: Indexing.
       # def a; end; a [42]: Array argument.
-      call_or_var
-      => {
-        emit(:tIDENTIFIER)
-
-        if !version?(18) &&
-              !@static_env.nil? && @static_env.declared?(tok)
-          fnext expr_end; fbreak;
-        else
-          fnext expr_arg; fbreak;
-        end
-      };
+      call_or_var => local_ident;
 
       #
       # WHITESPACE
       #
 
-      w_space_newline;
+      w_any;
 
       e_heredoc_nl '=begin' ( c_space | c_eol )
       => { p = @ts - 1
@@ -1556,11 +1582,11 @@ class Parser::Lexer
   #
   expr_value := |*
       # a:b: a(:b), a::B, A::B
-      bareword ':' (c_any - ':')
+      label (any - ':')
       => { p = @ts - 1
            fgoto expr_end; };
 
-      w_space;
+      w_space_comment;
 
       c_any
       => { fhold; fgoto expr_beg; };
@@ -1573,7 +1599,7 @@ class Parser::Lexer
       # STABBY LAMBDA
       #
 
-      '->' c_space*
+      '->'
       => {
         emit_table(PUNCTUATION, @ts, @ts + 2)
 
@@ -1616,7 +1642,7 @@ class Parser::Lexer
       => { emit_table(KEYWORDS)
            fnext expr_fname; fbreak; };
 
-      'class' c_space_nl* '<<'
+      'class' w_any* '<<'
       => { emit(:kCLASS, 'class', @ts, @ts + 5)
            emit(:tLSHFT, '<<',    @te - 2, @te)
            fnext expr_value; fbreak; };
@@ -1770,9 +1796,7 @@ class Parser::Lexer
       => { emit_table(PUNCTUATION)
            fnext expr_dot; fbreak; };
 
-      call_or_var
-      => { emit(:tIDENTIFIER)
-           fnext expr_arg; fbreak; };
+      call_or_var => local_ident;
 
       call_or_var ambiguous_fid_suffix
       => { emit(:tFID, tok(@ts, tm), @ts, tm)
@@ -1783,11 +1807,11 @@ class Parser::Lexer
       # OPERATORS
       #
 
-      ( e_lparen            |
-        operator_arithmetic |
-        operator_rest
-      ) %{ tm = p } c_space_nl*
-      => { emit_table(PUNCTUATION, @ts, tm)
+      ( e_lparen
+      | operator_arithmetic
+      | operator_rest
+      )
+      => { emit_table(PUNCTUATION)
            fnext expr_beg; fbreak; };
 
       e_rbrace | e_rparen | ']'
@@ -1815,7 +1839,8 @@ class Parser::Lexer
       # WHITESPACE
       #
 
-      w_space;
+      w_space_comment;
+
       w_newline
       => { fgoto leading_dot; };
 
@@ -1848,7 +1873,7 @@ class Parser::Lexer
 
       any
       => { emit(:tNL, nil, @newline_s, @newline_s + 1)
-           fnext line_begin; fhold; fbreak; };
+           fhold; fnext line_begin; fbreak; };
   *|;
 
   #
@@ -1872,7 +1897,7 @@ class Parser::Lexer
   *|;
 
   line_begin := |*
-      w_space_newline;
+      w_any;
 
       '=begin' ( c_space | c_eol )
       => { @eq_begin_s = @ts

data/lib/parser/lexer/explanation.rb

@@ -2,10 +2,19 @@ module Parser
 
   module Lexer::Explanation
 
+    def self.included(klass)
+      klass.class_exec do
+        alias_method :state_before_explanation=,  :state=
+        alias_method :advance_before_explanation, :advance
+
+        remove_method :state=, :advance
+      end
+    end
+
     # Like #advance, but also pretty-print the token and its position
     # in the stream to `stdout`.
     def advance
-      type, (val, range) = super
+      type, (val, range) = advance_before_explanation
 
       puts decorate(range,
                     "\e[0;32m#{type} #{val.inspect}\e[0m",
@@ -18,7 +27,7 @@ module Parser
       puts "  \e[1;33m>>> STATE SET <<<\e[0m " +
            "#{new_state.to_s.ljust(12)} #{@cond} #{@cmdarg}".rjust(66)
 
-      super
+      self.state_before_explanation = new_state
     end
 
     private

data/lib/parser/ruby18.y

@@ -150,10 +150,6 @@ rule
                     }
                 | klEND tLCURLY compstmt tRCURLY
                     {
-                      if in_def?
-                        diagnostic(:warning, :end_in_method, val[0])
-                      end
-
                       result = @builder.postexe(val[0], val[1], val[2], val[3])
                     }
                 | lhs tEQL command_call
@@ -930,8 +926,6 @@ rule
                     }
                     tRPAREN
                     {
-                      diagnostic(:warning, :space_before_lparen, val[0])
-
                       result = [ val[0], [], val[2] ]
                     }
                 | tLPAREN_ARG call_args2
@@ -940,8 +934,6 @@ rule
                     }
                     tRPAREN
                     {
-                      diagnostic(:warning, :space_before_lparen, val[0])
-
                       result = [ val[0], val[1], val[3] ]
                     }
 
@@ -1145,14 +1137,14 @@ rule
                     }
                 | kCLASS cpath superclass
                     {
-                      if in_def?
-                        diagnostic(:error, :class_in_def, val[0])
-                      end
-
                       @static_env.extend_static
                     }
                     bodystmt kEND
                     {
+                      if in_def?
+                        diagnostic(:error, :class_in_def, val[0])
+                      end
+
                       lt_t, superclass = val[2]
                       result = @builder.def_class(val[0], val[1],
                                                   lt_t, superclass,
@@ -1178,14 +1170,14 @@ rule
                     }
                 | kMODULE cpath
                     {
-                      if in_def?
-                        diagnostic(:error, :module_in_def, val[0])
-                      end
-
                       @static_env.extend_static
                     }
                     bodystmt kEND
                     {
+                      if in_def?
+                        diagnostic(:error, :module_in_def, val[0])
+                      end
+
                       result = @builder.def_module(val[0], val[1],
                                                    val[3], val[4])
 

data/lib/parser/ruby19.y

@@ -168,10 +168,6 @@ rule
                     }
                 | klEND tLCURLY compstmt tRCURLY
                     {
-                      if in_def?
-                        diagnostic(:warning, :end_in_method, val[0])
-                      end
-
                       result = @builder.postexe(val[0], val[1], val[2], val[3])
                     }
                 | command_asgn
@@ -1102,14 +1098,14 @@ rule
                     }
                 | kCLASS cpath superclass
                     {
-                      if in_def?
-                        diagnostic(:error, :class_in_def, val[0])
-                      end
-
                       @static_env.extend_static
                     }
                     bodystmt kEND
                     {
+                      if in_def?
+                        diagnostic(:error, :class_in_def, val[0])
+                      end
+
                       lt_t, superclass = val[2]
                       result = @builder.def_class(val[0], val[1],
                                                   lt_t, superclass,
@@ -1135,14 +1131,14 @@ rule
                     }
                 | kMODULE cpath
                     {
-                      if in_def?
-                        diagnostic(:error, :module_in_def, val[0])
-                      end
-
                       @static_env.extend_static
                     }
                     bodystmt kEND
                     {
+                      if in_def?
+                        diagnostic(:error, :module_in_def, val[0])
+                      end
+
                       result = @builder.def_module(val[0], val[1],
                                                    val[3], val[4])
 

data/lib/parser/ruby20.y

@@ -179,10 +179,6 @@ rule
                     }
                 | klEND tLCURLY compstmt tRCURLY
                     {
-                      if in_def?
-                        diagnostic(:warning, :end_in_method, val[0])
-                      end
-
                       result = @builder.postexe(val[0], val[1], val[2], val[3])
                     }
                 | command_asgn
@@ -1139,14 +1135,14 @@ rule
                     }
                 | kCLASS cpath superclass
                     {
-                      if in_def?
-                        diagnostic(:error, :class_in_def, val[0])
-                      end
-
                       @static_env.extend_static
                     }
                     bodystmt kEND
                     {
+                      if in_def?
+                        diagnostic(:error, :class_in_def, val[0])
+                      end
+
                       lt_t, superclass = val[2]
                       result = @builder.def_class(val[0], val[1],
                                                   lt_t, superclass,
@@ -1172,14 +1168,14 @@ rule
                     }
                 | kMODULE cpath
                     {
-                      if in_def?
-                        diagnostic(:error, :module_in_def, val[0])
-                      end
-
                       @static_env.extend_static
                     }
                     bodystmt kEND
                     {
+                      if in_def?
+                        diagnostic(:error, :module_in_def, val[0])
+                      end
+
                       result = @builder.def_module(val[0], val[1],
                                                    val[3], val[4])
 

data/lib/parser/ruby21.y

@@ -179,10 +179,6 @@ rule
                     }
                 | klEND tLCURLY compstmt tRCURLY
                     {
-                      if in_def?
-                        diagnostic(:warning, :end_in_method, val[0])
-                      end
-
                       result = @builder.postexe(val[0], val[1], val[2], val[3])
                     }
                 | command_asgn
@@ -1140,14 +1136,14 @@ rule
                     }
                 | kCLASS cpath superclass
                     {
-                      if in_def?
-                        diagnostic(:error, :class_in_def, val[0])
-                      end
-
                       @static_env.extend_static
                     }
                     bodystmt kEND
                     {
+                      if in_def?
+                        diagnostic(:error, :class_in_def, val[0])
+                      end
+
                       lt_t, superclass = val[2]
                       result = @builder.def_class(val[0], val[1],
                                                   lt_t, superclass,
@@ -1173,14 +1169,14 @@ rule
                     }
                 | kMODULE cpath
                     {
-                      if in_def?
-                        diagnostic(:error, :module_in_def, val[0])
-                      end
-
                       @static_env.extend_static
                     }
                     bodystmt kEND
                     {
+                      if in_def?
+                        diagnostic(:error, :module_in_def, val[0])
+                      end
+
                       result = @builder.def_module(val[0], val[1],
                                                    val[3], val[4])
 

data/lib/parser/runner/ruby_parse.rb

@@ -103,7 +103,7 @@ module Parser
       @slop.on 'E', 'explain', 'Explain how the source is tokenized' do
         ENV['RACC_DEBUG'] = '1'
 
-        Lexer.send :prepend, Lexer::Explanation
+        Lexer.send :include, Lexer::Explanation
       end
     end
 

data/lib/parser/version.rb

@@ -1,3 +1,3 @@
 module Parser
-  VERSION = '2.0.0.beta8'
+  VERSION = '2.0.0.beta9'
 end

data/parser.gemspec

@@ -25,7 +25,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency             'ast',       '~> 1.1'
-  spec.add_dependency             'slop',      '~> 3.4'
+  spec.add_dependency             'slop',      ['~> 3.4', '>= 3.4.5']
 
   spec.add_development_dependency 'bundler',   '~> 1.2'
   spec.add_development_dependency 'rake',      '~> 0.9'

data/test/parse_helper.rb

@@ -121,12 +121,9 @@ module ParseHelper
         # do nothing; the diagnostic was reported
       end
 
-      # Remove this `if' when no diagnostics fail to render.
-      if @diagnostics.count != 1
-        assert_equal 1, @diagnostics.count,
-                     "(#{version}) emits a single diagnostic, not\n" \
-                     "#{@diagnostics.map(&:render).join("\n")}"
-      end
+      assert_equal 1, @diagnostics.count,
+                   "(#{version}) emits a single diagnostic, not\n" \
+                   "#{@diagnostics.map(&:render).join("\n")}"
 
       emitted_diagnostic = @diagnostics.first
 

data/test/test_lexer.rb

@@ -2173,6 +2173,12 @@ class TestLexer < Minitest::Test
                    :tIDENTIFIER, 'foo',
                    :tCOLON,      ':')
 
+    @lex.state = :expr_endfn
+    util_lex_token("\nfoo: ",
+                   :tNL,         nil,
+                   :tIDENTIFIER, 'foo',
+                   :tCOLON,      ':')
+
     @lex.state = :expr_endfn
     util_lex_token("\\\nfoo:",
                    :tLABEL, 'foo')
@@ -2182,6 +2188,12 @@ class TestLexer < Minitest::Test
                    :tNL,         nil,
                    :tIDENTIFIER, 'foo',
                    :tCOLON,      ':')
+
+    @lex.state = :expr_endfn
+    util_lex_token("#foo\nfoo: ",
+                   :tNL,         nil,
+                   :tIDENTIFIER, 'foo',
+                   :tCOLON,      ':')
   end
 
   def test_whitespace_dot
@@ -2509,6 +2521,18 @@ class TestLexer < Minitest::Test
                    :tINTEGER,   1)
   end
 
+  def test_bug_expr_arg_label
+    setup_lexer 19
+
+    @lex.state = :expr_arg
+    util_lex_token(" unless:",
+                   :tLABEL,     'unless')
+
+    @lex.state = :expr_arg
+    util_lex_token(" unless: ",
+                   :tLABEL,     'unless')
+  end
+
   def test_bug_heredoc_continuation
     @lex.state = :expr_arg
     util_lex_token(" <<EOS\nEOS\nend",
@@ -2573,6 +2597,16 @@ class TestLexer < Minitest::Test
                    :tCONSTANT,   'CLEANINGS')
   end
 
+  def test_bug_sclass_comment_lshft_label
+    util_lex_token("class # foo\n<< a:b;end",
+                   :kCLASS,      'class',
+                   :tLSHFT,      '<<',
+                   :tIDENTIFIER, 'a',
+                   :tSYMBOL,     'b',
+                   :tSEMI,       ';',
+                   :kEND,        'end')
+  end
+
   def test_bug_expr_dot_comment
     util_lex_token("foo. #bar\nbaz",
                    :tIDENTIFIER, 'foo',

data/test/test_parser.rb

@@ -1330,7 +1330,8 @@ class TestParser < Minitest::Test
   def test_module_invalid
     assert_diagnoses(
       [:error, :module_in_def],
-      %q{def a; module Foo; end; end})
+      %q{def a; module Foo; end; end},
+      %q{       ^^^^^^ location})
   end
 
   def test_cpath
@@ -1392,7 +1393,8 @@ class TestParser < Minitest::Test
   def test_class_invalid
     assert_diagnoses(
       [:error, :class_in_def],
-      %q{def a; class Foo; end; end})
+      %q{def a; class Foo; end; end},
+      %q{       ^^^^^ location})
   end
 
   def test_sclass
@@ -1425,6 +1427,10 @@ class TestParser < Minitest::Test
     assert_parses(
       s(:def, :String=, s(:args), nil),
       %q{def String=; end})
+
+    assert_parses(
+      s(:def, :until, s(:args), nil),
+      %q{def until; end})
   end
 
   def test_defs
@@ -3382,12 +3388,6 @@ class TestParser < Minitest::Test
       %q{fun (1, 2)},
       %q{},
       %w(1.8))
-
-    assert_diagnoses(
-      [:warning, :space_before_lparen],
-      %q{fun (1, 2)},
-      %q{    ^ location},
-      %w(1.8))
   end
 
   def test_space_args_none
@@ -3396,12 +3396,6 @@ class TestParser < Minitest::Test
       %q{fun ()},
       %q{},
       %w(1.8))
-
-    assert_diagnoses(
-      [:warning, :space_before_lparen],
-      %q{fun ()},
-      %q{    ^ location},
-      %w(1.8))
   end
 
   def test_space_args_block
@@ -3615,6 +3609,18 @@ class TestParser < Minitest::Test
         |~~~~~~~~~~~ expression})
   end
 
+  def test_ternary_ambiguous_symbol
+    assert_parses(
+      s(:begin,
+        s(:lvasgn, :t, s(:int, 1)),
+        s(:if, s(:begin, s(:lvar, :foo)),
+          s(:lvar, :t),
+          s(:const, nil, :T))),
+      %q{t=1;(foo)?t:T},
+      %q{},
+      ALL_VERSIONS - %w(1.8))
+  end
+
   def test_if_masgn
     assert_diagnoses(
       [:error, :masgn_as_condition],
@@ -4240,7 +4246,8 @@ class TestParser < Minitest::Test
     assert_parses(
       s(:preexe, s(:int, 1)),
       %q{BEGIN { 1 }},
-      %q{      ^ begin
+      %q{~~~~~ keyword
+        |      ^ begin
         |          ^ end
         |~~~~~~~~~~~ expression})
   end
@@ -4258,14 +4265,10 @@ class TestParser < Minitest::Test
     assert_parses(
       s(:postexe, s(:int, 1)),
       %q{END { 1 }},
-      %q{    ^ begin
+      %q{~~~ keyword
+        |    ^ begin
         |        ^ end
         |~~~~~~~~~ expression})
-
-    assert_diagnoses(
-      [:warning, :end_in_method],
-      %q{def f; END{ 1 }; end},
-      %q{       ~~~ location})
   end
 
   #
@@ -4369,26 +4372,38 @@ class TestParser < Minitest::Test
   # Token and comment extraction
   #
 
-  def test_comments
+  def assert_parses_with_comments(ast_pattern, source, comments_pattern)
     with_versions(ALL_VERSIONS) do |_ver, parser|
       source_file = Parser::Source::Buffer.new('(comments)')
-      source_file.source = "1 + # foo\n 2"
+      source_file.source = source
 
-      range = lambda do |from, to|
-        Parser::Source::Range.new(source_file, from, to)
+      comments_pattern_here = comments_pattern.map do |(from, to)|
+        range = Parser::Source::Range.new(source_file, from, to)
+        Parser::Source::Comment.new(range)
       end
 
       ast, comments = parser.parse_with_comments(source_file)
 
-      assert_equal s(:send, s(:int, 1), :+, s(:int, 2)),
-                   ast
+      assert_equal ast_pattern, ast
 
-      assert_equal [
-                     Parser::Source::Comment.new(range.call(4, 9))
-                   ], comments
+      assert_equal comments_pattern_here, comments
     end
   end
 
+  def test_comment_interleaved
+    assert_parses_with_comments(
+      s(:send, s(:int, 1), :+, s(:int, 2)),
+      %Q{1 + # foo\n 2},
+      [ [4, 9]  ])
+  end
+
+  def test_comment_single
+    assert_parses_with_comments(
+      s(:send, nil, :puts),
+      %Q{puts # whatever},
+      [ [5, 15] ])
+  end
+
   def test_tokenize
     with_versions(ALL_VERSIONS) do |_ver, parser|
       source_file = Parser::Source::Buffer.new('(tokenize)')

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parser
 version: !ruby/object:Gem::Version
-  version: 2.0.0.beta8
+  version: 2.0.0.beta9
 platform: ruby
 authors:
 - Peter Zotov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-06-24 00:00:00.000000000 Z
+date: 2013-06-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ast
@@ -31,6 +31,9 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '3.4'
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 3.4.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -38,6 +41,9 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '3.4'
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 3.4.5
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -225,7 +231,6 @@ files:
 - lib/parser/base.rb
 - lib/parser/builders/default.rb
 - lib/parser/compatibility/ruby1_8.rb
-- lib/parser/compatibility/slop.rb
 - lib/parser/current.rb
 - lib/parser/diagnostic.rb
 - lib/parser/diagnostic/engine.rb

data/lib/parser/compatibility/slop.rb

@@ -1,12 +0,0 @@
-# https://github.com/injekt/slop/pull/110
-class Slop
-  def extract_long_flag(objects, config)
-    flag = objects.first.to_s
-    if flag =~ /\A(?:--?)?[a-zA-Z0-9][a-zA-Z0-9_.-]+\=?\??\z/
-      config[:argument] ||= true if flag.end_with?('=')
-      config[:optional_argument] = true if flag.end_with?('=?')
-      objects.shift
-      clean(flag).sub(/\=\??\z/, '')
-    end
-  end
-end