ruby_parser 3.8.4 → 3.9.0

data/lib/ruby_lexer.rb

@@ -3,9 +3,9 @@
 class RubyLexer
 
   # :stopdoc:
-  RUBY19 = "".respond_to? :encoding
+  HAS_ENC = "".respond_to? :encoding
 
-  IDENT_CHAR = if RUBY19 then
+  IDENT_CHAR = if HAS_ENC then
                  /[\w\u0080-\u{10ffff}]/u
                else
                  /[\w\x80-\xFF]/n
@@ -285,7 +285,7 @@ class RubyLexer
   end
 
   def ruby22_label?
-    ruby22? and is_label_possible?
+    ruby22plus? and is_label_possible?
   end
 
   def is_label_possible?
@@ -481,7 +481,7 @@ class RubyLexer
     return if in_lex_state?(:expr_beg, :expr_value, :expr_class,
                             :expr_fname, :expr_dot, :expr_labelarg)
 
-    if scan(/([\ \t\r\f\v]*)\./) then
+    if scan(/([\ \t\r\f\v]*)(\.|&)/) then
       self.space_seen = true unless ss[1].empty?
 
       ss.pos -= 1
@@ -778,7 +778,7 @@ class RubyLexer
       when lpar_beg && lpar_beg == paren_nest then
         self.lpar_beg = nil
         self.paren_nest -= 1
-        result(state, :kDO_LAMBDA, value)
+        expr_result(:kDO_LAMBDA, value)
       when cond.is_in_state then
         result(state, :kDO_COND, value)
       when cmdarg.is_in_state && lex_state != :expr_cmdarg then
@@ -909,11 +909,7 @@ class RubyLexer
   end
 
   def ruby18
-    Ruby18Parser === parser
-  end
-
-  def ruby19
-    Ruby19Parser === parser
+    RubyParser::V18 === parser
   end
 
   def scan re
@@ -1047,7 +1043,7 @@ class RubyLexer
         t = Regexp.escape term
         x = Regexp.escape(paren) if paren && paren != "\000"
         re = if qwords then
-               if RUBY19 then
+               if HAS_ENC then
                  /[^#{t}#{x}\#\0\\\s]+|./ # |. to pick up whatever
                else
                  /[^#{t}#{x}\#\0\\\s\v]+|./ # argh. 1.8's \s doesn't pick up \v
@@ -1098,7 +1094,7 @@ class RubyLexer
         else
           s
         end
-    x.force_encoding "UTF-8" if RUBY19
+    x.force_encoding "UTF-8" if HAS_ENC
     x
   end
 
@@ -1106,9 +1102,8 @@ class RubyLexer
     # do nothing for now
   end
 
-  def ruby22?
-    Ruby22Parser === parser or
-      Ruby23Parser === parser
+  def ruby22plus?
+    parser.class.version >= 22
   end
 
   def process_string # TODO: rewrite / remove
@@ -1120,7 +1115,7 @@ class RubyLexer
 
     token_type, c = token
 
-    if ruby22? && token_type == :tSTRING_END && ["'", '"'].include?(c) then
+    if ruby22plus? && token_type == :tSTRING_END && ["'", '"'].include?(c) then
       if (([:expr_beg, :expr_endfn].include?(lex_state) &&
            !cond.is_in_state) || is_arg?) &&
           is_label_suffix? then

data/lib/ruby_parser.rb

@@ -1,7 +1,86 @@
-require 'ruby18_parser'
-require 'ruby19_parser'
-require 'ruby20_parser'
-require 'ruby21_parser'
-require 'ruby22_parser'
-require 'ruby23_parser'
-require 'ruby_parser_extras'
+require "ruby_parser_extras"
+require "racc/parser"
+
+##
+# RubyParser is a compound parser that uses all known versions to
+# attempt to parse.
+
+class RubyParser
+
+  VERSIONS = []
+
+  class Parser < Racc::Parser
+    include RubyParserStuff
+
+    def self.inherited x
+      RubyParser::VERSIONS << x
+    end
+
+    def self.version
+      Parser > self and self.name[/V(\d+)$/, 1].to_i
+    end
+  end
+
+  class SyntaxError < RuntimeError; end
+
+  def process s, f = "(string)", t = 10
+    e = nil
+    VERSIONS.each do |klass|
+      parser = klass.new
+      begin
+        return parser.process s, f, t
+      rescue Racc::ParseError, RubyParser::SyntaxError => exc
+        e = exc
+      end
+    end
+    raise e
+  end
+
+  alias :parse :process
+
+  def reset
+    # do nothing
+  end
+
+  def self.latest
+    VERSIONS.first.new
+  end
+
+  def self.for_current_ruby
+    name  = "V#{RUBY_VERSION[/^\d+\.\d+/].delete "."}"
+    klass = if const_defined? name then
+              const_get name
+            else
+              latest = VERSIONS.first
+              warn "NOTE: RubyParser::#{name} undefined, using #{latest}."
+              latest
+            end
+
+    klass.new
+  end
+end
+
+##
+# Unfortunately a problem with racc is that it won't let me namespace
+# properly, so instead of RubyParser::V18, I still have to generate
+# the old Ruby23Parser and shove it in as V23.
+
+require "ruby18_parser"
+require "ruby19_parser"
+require "ruby20_parser"
+require "ruby21_parser"
+require "ruby22_parser"
+require "ruby23_parser"
+require "ruby24_parser"
+
+class RubyParser # HACK
+  VERSIONS.clear # also a HACK caused by racc namespace issues
+
+  class V24 < ::Ruby24Parser; end
+  class V23 < ::Ruby23Parser; end
+  class V22 < ::Ruby22Parser; end
+  class V21 < ::Ruby21Parser; end
+  class V20 < ::Ruby20Parser; end
+  class V19 < ::Ruby19Parser; end
+  class V18 < ::Ruby18Parser; end
+end

data/lib/ruby_parser.yy

@@ -1,13 +1,17 @@
 # -*- racc -*-
 
-#if defined(RUBY20)
+#if V==20
 class Ruby20Parser
-#elif defined(RUBY21)
+#elif V==21
 class Ruby21Parser
-#elif defined(RUBY22)
+#elif V == 22
 class Ruby22Parser
-#elif defined(RUBY23)
+#elif V == 23
 class Ruby23Parser
+#elif V == 24
+class Ruby24Parser
+#else
+fail "version not specified or supported on code generation"
 #endif
 
 token kCLASS kMODULE kDEF kUNDEF kBEGIN kRESCUE kENSURE kEND kIF kUNLESS
@@ -27,13 +31,13 @@ token kCLASS kMODULE kDEF kUNDEF kBEGIN kRESCUE kENSURE kEND kIF kUNLESS
       tWORDS_BEG tQWORDS_BEG tSTRING_DBEG tSTRING_DVAR tSTRING_END
       tSTRING tSYMBOL tNL tEH tCOLON tCOMMA tSPACE tSEMI tLAMBDA
       tLAMBEG tDSTAR tCHAR tSYMBOLS_BEG tQSYMBOLS_BEG tSTRING_DEND tUBANG
-#if defined(RUBY21) || defined(RUBY22) || defined(RUBY23)
+#if V >= 21
       tRATIONAL tIMAGINARY
 #endif
-#if defined(RUBY22) || defined(RUBY23)
+#if V >= 22
       tLABEL_END
 #endif
-#if defined(RUBY23)
+#if V >= 23
        tLONELY
 #endif
 
@@ -227,15 +231,15 @@ rule
                     }
                 | lhs tEQL mrhs
                     {
-                      result = self.node_assign val[0], s(:svalue, val[2])
+                      result = new_assign val[0], s(:svalue, val[2])
                     }
-#if defined(RUBY20)
+#if V == 20
                 | mlhs tEQL arg_value
                     {
                       result = new_masgn val[0], val[2], :wrap
                     }
                 | mlhs tEQL mrhs
-#elif defined(RUBY21) || defined(RUBY22 || defined(RUBY23))
+#else
                 | mlhs tEQL mrhs_arg
 #endif
                     {
@@ -245,21 +249,21 @@ rule
 
     command_asgn: lhs tEQL command_call
                     {
-                      result = self.node_assign val[0], val[2]
+                      result = new_assign val[0], val[2]
                     }
                 | lhs tEQL command_asgn
                     {
-                      result = self.node_assign val[0], val[2]
+                      result = new_assign val[0], val[2]
                     }
 
             expr: command_call
                 | expr kAND expr
                     {
-                      result = logop(:and, val[0], val[2])
+                      result = logical_op :and, val[0], val[2]
                     }
                 | expr kOR expr
                     {
-                      result = logop(:or, val[0], val[2])
+                      result = logical_op :or, val[0], val[2]
                     }
                 | kNOT opt_nl expr
                     {
@@ -612,7 +616,7 @@ rule
                 |   tNEQ     | tLSHFT  | tRSHFT   | tPLUS | tMINUS | tSTAR2
                 |   tSTAR    | tDIVIDE | tPERCENT | tPOW  | tDSTAR | tBANG   | tTILDE
                 |   tUPLUS   | tUMINUS | tAREF    | tASET | tBACK_REF2
-#if defined(RUBY20)
+#if V == 20
                 |   tUBANG
 #endif
 
@@ -628,11 +632,11 @@ rule
 
              arg: lhs tEQL arg
                     {
-                      result = self.node_assign val[0], val[2]
+                      result = new_assign val[0], val[2]
                     }
                 | lhs tEQL arg kRESCUE_MOD arg
                     {
-                      result = self.node_assign val[0], s(:rescue, val[2], new_resbody(s(:array), val[4]))
+                      result = new_assign val[0], s(:rescue, val[2], new_resbody(s(:array), val[4]))
                     }
                 | var_lhs tOP_ASGN arg
                     {
@@ -714,18 +718,18 @@ rule
                     {
                       result = new_call val[0], :**, argl(val[2])
                     }
-#if defined(RUBY20)
+#if V == 20
                 | tUMINUS_NUM tINTEGER tPOW arg
                     {
                       result = new_call(new_call(s(:lit, val[1]), :"**", argl(val[3])), :"-@")
                     }
                 | tUMINUS_NUM tFLOAT tPOW arg
-#elif defined(RUBY21) || defined(RUBY22) || defined(RUBY23)
+#else
                 | tUMINUS_NUM simple_numeric tPOW arg
 #endif
                     {
                       result = new_call(new_call(s(:lit, val[1]), :"**", argl(val[3])), :"-@")
-#if defined(RUBY20)
+#if V == 20
                       ## TODO: why is this 2.0 only?
                       debug20 12, val, result
 #endif
@@ -784,11 +788,11 @@ rule
                     }
                 | arg tMATCH arg
                     {
-                      result = self.get_match_node val[0], val[2]
+                      result = new_match val[0], val[2]
                     }
                 | arg tNMATCH arg
                     {
-                      result = s(:not, self.get_match_node(val[0], val[2]))
+                      result = s(:not, new_match(val[0], val[2]))
                     }
                 | tBANG arg
                     {
@@ -812,11 +816,11 @@ rule
                     }
                 | arg tANDOP arg
                     {
-                      result = logop(:and, val[0], val[2])
+                      result = logical_op :and, val[0], val[2]
                     }
                 | arg tOROP arg
                     {
-                      result = logop(:or, val[0], val[2])
+                      result = logical_op :or, val[0], val[2]
                     }
                 | kDEFINED opt_nl arg
                     {
@@ -939,7 +943,7 @@ rule
                       result = self.list_append val[0], s(:splat, val[3])
                     }
 
-#if defined(RUBY21) || defined(RUBY22 || defined(RUBY23))
+#if V >= 21
         mrhs_arg: mrhs
                     {
                       result = new_masgn_arg val[0]
@@ -1022,9 +1026,13 @@ rule
                       result = val[1] || s(:array)
                       result[0] = :array # aref_args is :args
                     }
-                | tLBRACE assoc_list tRCURLY
+                | tLBRACE
+                    {
+                      result = self.lexer.lineno
+                    }
+                    assoc_list tRCURLY
                     {
-                      result = s(:hash, *val[1].values) # TODO: array_to_hash?
+                      result = new_hash val
                     }
                 | kRETURN
                     {
@@ -1332,7 +1340,6 @@ rule
                       args, _, _, _, args2 = val
 
                       result = block_var args, :*, args2
-                      debug20 16, val, result
                     }
                 | tSTAR f_norm_arg
                     {
@@ -1345,12 +1352,10 @@ rule
                       _, splat, _, args = val
 
                       result = block_var :"*#{splat}", args
-                      debug20 17, val, result
                     }
                 | tSTAR
                     {
                       result = block_var :*
-                      debug20 18, val, result
                     }
                 | tSTAR tCOMMA f_marg_list
                     {
@@ -1671,7 +1676,7 @@ opt_block_args_tail: tCOMMA block_args_tail
                       _, klasses, var, _, body, rest = val
 
                       klasses ||= s(:array)
-                      klasses << node_assign(var, s(:gvar, :"$!")) if var
+                      klasses << new_assign(var, s(:gvar, :"$!")) if var
 
                       result = new_resbody(klasses, body)
                       result << rest if rest # UGH, rewritten above
@@ -1886,9 +1891,9 @@ regexp_contents: none
                     }
                     compstmt tRCURLY
                     {
-#if defined(RUBY20)
+#if V == 20
                       # TODO: tRCURLY -> tSTRING_DEND
-#elif defined(RUBY21) || defined(RUBY22 || defined(RUBY23))
+#else
                       # TODO: tRCURLY -> tSTRING_END
 #endif
                       _, memo, stmt, _ = val
@@ -1956,17 +1961,17 @@ regexp_contents: none
                       end
                     }
 
-#if defined(RUBY20)
+#if V == 20
          numeric: tINTEGER
                 | tFLOAT
                 | tUMINUS_NUM tINTEGER =tLOWEST
-#elif defined(RUBY21) || defined(RUBY22) || defined(RUBY23)
+#else
          numeric: simple_numeric
                 | tUMINUS_NUM simple_numeric
 #endif
                     {
                       result = -val[1] # TODO: pt_testcase
-#if defined(RUBY20)
+#if V == 20
                     }
                 | tUMINUS_NUM tFLOAT   =tLOWEST
                     {
@@ -1974,7 +1979,7 @@ regexp_contents: none
 #endif
                     }
 
-#if defined(RUBY21) || defined(RUBY22) || defined(RUBY23)
+#if V >= 21
   simple_numeric: tINTEGER
                 | tFLOAT
                 | tRATIONAL
@@ -2173,7 +2178,7 @@ keyword_variable: kNIL      { result = s(:nil)   }
                       result = identifier
                     }
 
-#if defined(RUBY22) || defined(RUBY23)
+#if V >= 22
       f_arg_asgn: f_norm_arg
 
       f_arg_item: f_arg_asgn
@@ -2215,9 +2220,9 @@ keyword_variable: kNIL      { result = s(:nil)   }
                       result << item
                     }
 
-#if defined(RUBY20)
+#if V == 20
             f_kw: tLABEL arg_value
-#elif defined(RUBY21) || defined(RUBY22) || defined(RUBY23)
+#else
          f_label: tLABEL
 
             f_kw: f_label arg_value
@@ -2230,7 +2235,7 @@ keyword_variable: kNIL      { result = s(:nil)   }
 
                       result = s(:array, s(:kwarg, identifier, val[1]))
                     }
-#if defined(RUBY21) || defined(RUBY22) || defined(RUBY23)
+#if V >= 21
                 | f_label
                     {
                       label, _ = val[0] # TODO: fix lineno?
@@ -2241,9 +2246,9 @@ keyword_variable: kNIL      { result = s(:nil)   }
                     }
 #endif
 
-#if defined(RUBY20)
+#if V == 20
       f_block_kw: tLABEL primary_value
-#elif defined(RUBY21) || defined(RUBY22) || defined(RUBY23)
+#else
       f_block_kw: f_label primary_value
 #endif
                     {
@@ -2254,7 +2259,7 @@ keyword_variable: kNIL      { result = s(:nil)   }
 
                       result = s(:array, s(:kwarg, identifier, val[1]))
                     }
-#if defined(RUBY21) || defined(RUBY22) || defined(RUBY23)
+#if V >= 21
                 | f_label
                     {
                       label, _ = val[0] # TODO: fix lineno?
@@ -2290,11 +2295,11 @@ keyword_variable: kNIL      { result = s(:nil)   }
                       result = :"**"
                     }
 
-#if defined(RUBY20)
+#if V == 20
            f_opt: tIDENTIFIER tEQL arg_value
-#elif defined(RUBY21)
+#elif V == 21
            f_opt: f_norm_arg tEQL arg_value
-#elif defined(RUBY22) || defined(RUBY23)
+#else
            f_opt: f_arg_asgn tEQL arg_value
 #endif
                     {
@@ -2302,11 +2307,11 @@ keyword_variable: kNIL      { result = s(:nil)   }
                       # TODO: detect duplicate names
                     }
 
-#if defined(RUBY20)
+#if V == 20
      f_block_opt: tIDENTIFIER tEQL primary_value
-#elif defined(RUBY21)
+#elif V == 21
      f_block_opt: f_norm_arg tEQL primary_value
-#elif defined(RUBY22) || defined(RUBY23)
+#else
      f_block_opt: f_arg_asgn tEQL primary_value
 #endif
                     {
@@ -2396,7 +2401,6 @@ keyword_variable: kNIL      { result = s(:nil)   }
                       list.push(*more) unless more.empty?
                       result = list
                       result[0] = :hash
-                      # TODO: shouldn't this be a hash?
                     }
 
            assoc: arg_value tASSOC arg_value
@@ -2407,7 +2411,7 @@ keyword_variable: kNIL      { result = s(:nil)   }
                     {
                       result = s(:array, s(:lit, val[0][0].to_sym), val[1])
                     }
-#if defined(RUBY22) || defined(RUBY23)
+#if V >= 22
                 | tSTRING_BEG string_contents tLABEL_END arg_value
                     {
                       _, sym, _, value = val
@@ -2430,7 +2434,7 @@ keyword_variable: kNIL      { result = s(:nil)   }
       operation3: tIDENTIFIER | tFID | op
     dot_or_colon: tDOT | tCOLON2
          call_op: tDOT
-#if defined(RUBY23)
+#if V >= 23
                 | tLONELY
 #endif
        opt_terms:  | terms