ruby_parser 2.3.1 → 3.0.0.a1

data/lib/ruby_parser_extras.rb

@@ -3,10 +3,15 @@ require 'racc/parser'
 require 'sexp'
 require 'strscan'
 
+def d o
+  $stderr.puts o.inspect
+end
+
 # WHY do I have to do this?!?
 class Regexp
-  unless defined? ONCE then
-    ONCE     = 0 # 16 # ?
+  ONCE = 0 unless defined? ONCE # FIX: remove this - it makes no sense
+
+  unless defined? ENC_NONE then
     ENC_NONE = /x/n.options
     ENC_EUC  = /x/e.options
     ENC_SJIS = /x/s.options
@@ -47,11 +52,6 @@ class RPStringScanner < StringScanner
 
   # TODO: once we get rid of these, we can make things like
   # TODO: current_line and lineno much more accurate and easy to do
-  def unread c # TODO: remove this entirely - we should not need it
-    return if c.nil? # UGH
-    warn({:unread => caller[0]}.inspect) if ENV['TALLY']
-    string[pos, 0] = c
-  end
 
   def unread_many str # TODO: remove this entirely - we should not need it
     warn({:unread_many => caller[0]}.inspect) if ENV['TALLY']
@@ -59,14 +59,6 @@ class RPStringScanner < StringScanner
     string[pos, 0] = str
   end
 
-  def begin_of_line?
-    pos == 0 or string[pos-1] == ?\n
-  end
-
-  def was_begin_of_line # TODO: kill me
-    pos <= 2 or string[pos-2] == ?\n
-  end
-
   if ENV['DEBUG'] then
     alias :old_getch :getch
     def getch
@@ -78,7 +70,7 @@ class RPStringScanner < StringScanner
     alias :old_scan :scan
     def scan re
       s = old_scan re
-      p :scan => [s, caller.first] if s
+      d :scan => [s, caller.first] if s
       s
     end
   end
@@ -122,8 +114,8 @@ class RPStringScanner < StringScanner
   # end
 end
 
-class RubyParser < Racc::Parser
-  VERSION = '2.3.1' unless constants.include? "VERSION" # SIGH
+module RubyParserStuff
+  VERSION = '3.0.0.a1' unless constants.include? "VERSION" # SIGH
 
   attr_accessor :lexer, :in_def, :in_single, :file
   attr_reader :env, :comments
@@ -149,7 +141,32 @@ class RubyParser < Racc::Parser
     node1
   end
 
-  def args arg, optarg, rest_arg, block_arg
+  def block_var ary, splat, block
+    ary ||= s(:array)
+
+    if splat then
+      if splat == s(:splat) then
+        ary << splat
+      else
+        ary << s(:splat, splat)
+      end
+    end
+
+    if block then
+      block[-1] = :"&#{block[-1]}"
+      ary << block
+    end
+
+    result = if ary.length > 2 or ary.splat then
+               s(:masgn, ary)
+             else
+               ary.last
+             end
+
+    result
+  end
+
+  def args arg, optarg, rest_arg, block_arg, post_arg = nil
     arg ||= s(:args)
 
     result = arg
@@ -161,15 +178,53 @@ class RubyParser < Racc::Parser
     end
 
     result << rest_arg  if rest_arg
+
     result << :"&#{block_arg.last}" if block_arg
     result << optarg    if optarg # TODO? huh - processed above as well
+    post_arg[1..-1].each {|pa| result << pa } if post_arg
+
+    result
+  end
+
+  def args19 vals # TODO: migrate to args once 1.8 tests pass as well
+    result = s(:args)
+    block  = nil
+
+    vals.each do |val|
+      case val
+      when Sexp then
+        case val.first
+        when :args then
+          val[1..-1].each do |name|
+            result << name
+          end
+        when :block_arg then
+          result << :"&#{val.last}"
+        when :block then
+         block = val
+          val[1..-1].each do |lasgn| # FIX clean sexp iter
+            raise "wtf? #{val.inspect}" unless lasgn[0] == :lasgn
+            result << lasgn[1]
+          end
+        else
+          raise "unhandled sexp: #{val.inspect}"
+        end
+      when Symbol then
+        result << val
+      when ",", nil then
+        # ignore
+      else
+        raise "unhandled val: #{val.inspect} in #{vals.inspect}"
+      end
+    end
+
+    result << block if block
 
     result
   end
 
   def aryset receiver, index
-    index[0] = :arglist if index[0] == :array
-    s(:attrasgn, receiver, :"[]=", index)
+    s(:attrasgn, receiver, :"[]=", *index[1..-1])
   end
 
   def assignable(lhs, value = nil)
@@ -288,7 +343,7 @@ class RubyParser < Racc::Parser
       end
     end
 
-    return s(:call, lhs, :"=~", s(:arglist, rhs)).line(lhs.line)
+    return new_call(lhs, :"=~", argl(rhs)).line(lhs.line)
   end
 
   def gettable(id)
@@ -307,10 +362,8 @@ class RubyParser < Racc::Parser
                type = env[id]
                if type then
                  s(type, id)
-               elsif env.dynamic? and :dvar == env[id] then
-                 s(:lvar, id)
                else
-                 s(:call, nil, id, s(:arglist))
+                 new_call(nil, id)
                end
              end
 
@@ -332,7 +385,9 @@ class RubyParser < Racc::Parser
 
   def initialize(options = {})
     super()
-    self.lexer = RubyLexer.new
+
+    v = self.class.name[/1[89]/]
+    self.lexer = RubyLexer.new v && v.to_i
     self.lexer.parser = self
     @env = Environment.new
     @comments = []
@@ -453,14 +508,37 @@ class RubyParser < Racc::Parser
     return result
   end
 
+  def argl x
+    x = s(:arglist, x) if x and x[0] != :arglist
+    x
+  end
+
+  def backref_assign_error ref
+    # TODO: need a test for this... obviously
+    case ref.first
+    when :nth_ref then
+      raise SyntaxError, "Can't set variable %p" % ref.last
+    when :back_ref then
+      raise SyntaxError, "Can't set back reference %p" % ref.last
+    else
+      raise "Unknown backref type: #{ref.inspect}"
+    end
+  end
+
   def new_call recv, meth, args = nil
     result = s(:call, recv, meth)
     result.line = recv.line if recv
 
+    # TODO: need a test with f(&b) to produce block_pass
+    # TODO: need a test with f(&b) { } to produce warning
+
     args ||= s(:arglist)
     args[0] = :arglist if args.first == :array
     args = s(:arglist, args) unless args.first == :arglist
-    result << args
+
+    # HACK quick hack to make this work quickly... easy to clean up above
+    result.concat args[1..-1]
+
     result
   end
 
@@ -473,6 +551,11 @@ class RubyParser < Racc::Parser
       body = body.delete_at 3
     end
 
+    result[2..-1].each do |node|
+      block = node.block(:delete)
+      node.concat block[1..-1] if block
+    end
+
     # else
     body = nil if body == s(:block)
     result << body
@@ -483,8 +566,17 @@ class RubyParser < Racc::Parser
 
   def new_class val
     line, path, superclass, body = val[1], val[2], val[3], val[5]
-    scope = s(:scope, body).compact
-    result = s(:class, path, superclass, scope)
+
+    result = s(:class, path, superclass)
+
+    if body then
+      if body.first == :block then
+        result.push(*body[1..-1])
+      else
+        result.push body
+      end
+    end
+
     result.line = line
     result.comments = self.comments.pop
     result
@@ -500,10 +592,16 @@ class RubyParser < Racc::Parser
     (_, line), name, args, body = val[0], val[1], val[3], val[4]
     body ||= s(:nil)
 
-    body ||= s(:block)
-    body = s(:block, body) unless body.first == :block
+    result = s(:defn, name.to_sym, args)
+
+    if body then
+      if body.first == :block then
+        result.push(*body[1..-1])
+      else
+        result.push body
+      end
+    end
 
-    result = s(:defn, name.to_sym, args, s(:scope, body))
     result.line = line
     result.comments = self.comments.pop
     result
@@ -512,10 +610,16 @@ class RubyParser < Racc::Parser
   def new_defs val
     recv, name, args, body = val[1], val[4], val[6], val[7]
 
-    body ||= s(:block)
-    body = s(:block, body) unless body.first == :block
+    result = s(:defs, recv, name.to_sym, args)
+
+    if body then
+      if body.first == :block then
+        result.push(*body[1..-1])
+      else
+        result.push body
+      end
+    end
 
-    result = s(:defs, recv, name.to_sym, args, s(:scope, body))
     result.line = recv.line
     result.comments = self.comments.pop
     result
@@ -555,7 +659,7 @@ class RubyParser < Racc::Parser
   def new_module val
     line, path, body = val[1], val[2], val[4]
     body = s(:scope, body).compact
-    result = s(:module, path, body)
+    result = s(:module, path, *body[1..-1])
     result.line = line
     result.comments = self.comments.pop
     result
@@ -574,8 +678,7 @@ class RubyParser < Racc::Parser
                s(:op_asgn_and, self.gettable(name), lhs)
              else
                # TODO: why [2] ?
-               lhs[2] = new_call(self.gettable(name), asgn_op,
-                                 s(:arglist, arg))
+               lhs[2] = new_call(self.gettable(name), asgn_op, argl(arg))
                lhs
              end
     result.line = lhs.line
@@ -627,10 +730,28 @@ class RubyParser < Racc::Parser
     node
   end
 
+  def new_resbody cond, body
+    if body && body.first == :block then
+      body.shift # remove block and splat it in directly
+    else
+      body = [body]
+    end
+    s(:resbody, cond, *body)
+  end
+
   def new_sclass val
     recv, in_def, in_single, body = val[3], val[4], val[6], val[7]
-    scope = s(:scope, body).compact
-    result = s(:sclass, recv, scope)
+
+    result = s(:sclass, recv)
+
+    if body then
+      if body.first == :block then
+        result.push(*body[1..-1])
+      else
+        result.push body
+      end
+    end
+
     result.line = val[2]
     self.in_def = in_def
     self.in_single = in_single
@@ -654,6 +775,10 @@ class RubyParser < Racc::Parser
     end
   end
 
+  def new_until block, expr, pre
+    new_until_or_while :until, block, expr, pre
+  end
+
   def new_until_or_while type, block, expr, pre
     other = type == :until ? :while : :until
     line = [block && block.line, expr.line].compact.min
@@ -671,8 +796,8 @@ class RubyParser < Racc::Parser
     result
   end
 
-  def new_until block, expr, pre
-    new_until_or_while :until, block, expr, pre
+  def new_when cond, body
+    s(:when, cond, body)
   end
 
   def new_while block, expr, pre
@@ -723,12 +848,11 @@ class RubyParser < Racc::Parser
     rhs = value_expr rhs
 
     case lhs[0]
-    when :gasgn, :iasgn, :lasgn, :dasgn, :dasgn_curr,
-      :masgn, :cdecl, :cvdecl, :cvasgn then
+    when :gasgn, :iasgn, :lasgn, :masgn, :cdecl, :cvdecl, :cvasgn then
       lhs << rhs
     when :attrasgn, :call then
       args = lhs.pop unless Symbol === lhs.last
-      lhs << arg_add(args, rhs)
+      lhs.concat arg_add(args, rhs)[1..-1]
     when :const then
       lhs[0] = :cdecl
       lhs << rhs
@@ -743,7 +867,7 @@ class RubyParser < Racc::Parser
     raise "bad val: #{str.inspect}" unless String === str
 
     self.file = file
-    self.lexer.src = str
+    self.lexer.src = str.dup
 
     @yydebug = ENV.has_key? 'DEBUG'
 
@@ -809,10 +933,9 @@ class RubyParser < Racc::Parser
     # do nothing for now
   end
 
-  alias :old_yyerror :yyerror
   def yyerror msg
     # for now do nothing with the msg
-    old_yyerror
+    super
   end
 
   class Keyword
@@ -998,6 +1121,22 @@ class RubyParser < Racc::Parser
   end
 end
 
+class Ruby19Parser < Racc::Parser
+  include RubyParserStuff
+end
+
+class Ruby18Parser < Racc::Parser
+  include RubyParserStuff
+end
+
+class RubyParser < Ruby18Parser
+  def initialize
+    super
+    warn "WA\RNING: Deprecated: RubyParser. Use Ruby18Parser or Ruby19Parser"
+    warn "  from #{caller.first}"
+  end
+end
+
 ############################################################
 # HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK
 
@@ -1023,6 +1162,14 @@ class Sexp
     self.value.to_sym
   end
 
+  def add x
+    raise "no" # TODO: need a test to trigger this
+  end
+
+  def add_all x
+    raise "no" # TODO: need a test to trigger this
+  end
+
   alias :node_type :sexp_type
   alias :values :sexp_body # TODO: retire
 end

data/test/test_ruby_lexer.rb

@@ -1,15 +1,21 @@
 #!/usr/local/bin/ruby
 
 require 'rubygems'
+gem "minitest"
+
 require 'minitest/autorun'
 require 'ruby_lexer'
-require 'ruby_parser'
+require 'ruby18_parser'
 
 class TestRubyLexer < MiniTest::Unit::TestCase
   alias :deny :refute
 
   def setup
-    p = RubyParser.new
+    setup_lexer Ruby18Parser
+  end
+
+  def setup_lexer parser_class
+    p = parser_class.new
     @lex = p.lexer
     @lex.src = "blah blah"
     @lex.lex_state = :expr_beg
@@ -125,6 +131,38 @@ class TestRubyLexer < MiniTest::Unit::TestCase
     util_lex_token "=>", :tASSOC, "=>"
   end
 
+  def test_yylex_label__18
+    util_lex_token("{a:",
+                   :tLBRACE,     "{",
+                   :tIDENTIFIER, "a",
+                   :tSYMBEG,     ":")
+  end
+
+  def test_yylex_label_in_params__18
+    util_lex_token("foo(a:",
+                   :tIDENTIFIER, "foo",
+                   :tLPAREN2,    "(",
+                   :tIDENTIFIER, "a",
+                   :tSYMBEG,     ":")
+  end
+
+  def test_yylex_label__19
+    setup_lexer Ruby19Parser
+
+    util_lex_token("{a:",
+                   :tLBRACE, "{",
+                   :tLABEL,  "a")
+  end
+
+  def test_yylex_label_in_params__19
+    setup_lexer Ruby19Parser
+
+    util_lex_token("foo(a:",
+                   :tIDENTIFIER, "foo",
+                   :tLPAREN2,    "(",
+                   :tLABEL,      "a")
+  end
+
   def test_yylex_back_ref
     util_lex_token("[$&, $`, $', $+]",
                    :tLBRACK,   "[",
@@ -610,7 +648,6 @@ class TestRubyLexer < MiniTest::Unit::TestCase
                    :tEQL,              "=",
                    :tSTRING_BEG,     "\"",
                    :tSTRING_CONTENT, "blah\nblah\n",
-                   :tSTRING_CONTENT, "",
                    :tSTRING_END,     "EOF",
                    :tNL,             nil)
   end
@@ -628,7 +665,6 @@ class TestRubyLexer < MiniTest::Unit::TestCase
                    :tEQL,              "=",
                    :tSTRING_BEG,     "\"",
                    :tSTRING_CONTENT, "blah\nblah\n",
-                   :tSTRING_CONTENT, "",
                    :tSTRING_END,     "EOF",
                    :tNL,             nil)
   end
@@ -805,14 +841,30 @@ class TestRubyLexer < MiniTest::Unit::TestCase
     util_bad_token "0d42__24"
   end
 
-  def test_yylex_integer_eh_a
-    util_lex_token '?a', :tINTEGER, 97
+  def test_yylex_question_eh_a__18
+    @lex = RubyLexer.new 18
+
+    util_lex_token "?a", :tINTEGER, 97
+  end
+
+  def test_yylex_question_eh_a__19
+    @lex = RubyLexer.new 19
+
+    util_lex_token '?a', :tSTRING, "a"
   end
 
-  def test_yylex_integer_eh_escape_M_escape_C
+  def test_yylex_question_eh_escape_M_escape_C__18
+    @lex = RubyLexer.new 18
+
     util_lex_token '?\M-\C-a', :tINTEGER, 129
   end
 
+  def test_yylex_question_eh_escape_M_escape_C__19
+    @lex = RubyLexer.new 19
+
+    util_lex_token '?\M-\C-a', :tSTRING, "\M-\C-a"
+  end
+
   def test_yylex_integer_hex
     util_lex_token "0x2a", :tINTEGER, 42
   end
@@ -1028,7 +1080,7 @@ class TestRubyLexer < MiniTest::Unit::TestCase
   def test_yylex_open_square_bracket_meth
     util_lex_token("m[3]",
                    :tIDENTIFIER, "m",
-                   "[", "[",
+                   :tLBRACK2, "[",
                    :tINTEGER, 3,
                    :tRBRACK, "]")
   end
@@ -1089,10 +1141,18 @@ class TestRubyLexer < MiniTest::Unit::TestCase
                    :tINTEGER, 42)
   end
 
-  def test_yylex_question
+  def test_yylex_question__18
+    @lex = RubyLexer.new 18
+
     util_lex_token "?*", :tINTEGER, 42
   end
 
+  def test_yylex_question__19
+    @lex = RubyLexer.new 19
+
+    util_lex_token "?*", :tSTRING, "*"
+  end
+
   def test_yylex_question_bad_eos
     util_bad_token "?"
   end
@@ -1106,7 +1166,9 @@ class TestRubyLexer < MiniTest::Unit::TestCase
     util_lex_token "?\f", :tEH, "?"
   end
 
-  def test_yylex_question_ws_backslashed
+  def test_yylex_question_ws_backslashed__18
+    @lex = RubyLexer.new 18
+
     @lex.lex_state = :expr_beg
     util_lex_token "?\\ ", :tINTEGER, 32
     @lex.lex_state = :expr_beg
@@ -1121,6 +1183,23 @@ class TestRubyLexer < MiniTest::Unit::TestCase
     util_lex_token "?\\f", :tINTEGER, 12
   end
 
+  def test_yylex_question_ws_backslashed__19
+    @lex = RubyLexer.new 19
+
+    @lex.lex_state = :expr_beg
+    util_lex_token "?\\ ", :tSTRING, " "
+    @lex.lex_state = :expr_beg
+    util_lex_token "?\\n", :tSTRING, "\n"
+    @lex.lex_state = :expr_beg
+    util_lex_token "?\\t", :tSTRING, "\t"
+    @lex.lex_state = :expr_beg
+    util_lex_token "?\\v", :tSTRING, "\v"
+    @lex.lex_state = :expr_beg
+    util_lex_token "?\\r", :tSTRING, "\r"
+    @lex.lex_state = :expr_beg
+    util_lex_token "?\\f", :tSTRING, "\f"
+  end
+
   def test_yylex_rbracket
     util_lex_token "]", :tRBRACK, "]"
   end
@@ -1614,7 +1693,7 @@ class TestRubyLexer < MiniTest::Unit::TestCase
 
   def test_yylex_string_pct_w
     util_bad_token("%w[s1 s2 ",
-                   :tAWORDS_BEG,     "%w[",
+                   :tQWORDS_BEG,     "%w[",
                    :tSTRING_CONTENT, "s1",
                    :tSPACE,              nil,
                    :tSTRING_CONTENT, "s2",
@@ -1623,7 +1702,7 @@ class TestRubyLexer < MiniTest::Unit::TestCase
 
   def test_yylex_string_pct_w_bs_nl
     util_lex_token("%w[s1 \\\ns2]",
-                   :tAWORDS_BEG,     "%w[",
+                   :tQWORDS_BEG,     "%w[",
                    :tSTRING_CONTENT, "s1",
                    :tSPACE,              nil,
                    :tSTRING_CONTENT, "\ns2",
@@ -1633,7 +1712,7 @@ class TestRubyLexer < MiniTest::Unit::TestCase
 
   def test_yylex_string_pct_w_bs_sp
     util_lex_token("%w[s\\ 1 s\\ 2]",
-                   :tAWORDS_BEG,     "%w[",
+                   :tQWORDS_BEG,     "%w[",
                    :tSTRING_CONTENT, "s 1",
                    :tSPACE,              nil,
                    :tSTRING_CONTENT, "s 2",
@@ -1643,7 +1722,7 @@ class TestRubyLexer < MiniTest::Unit::TestCase
 
   def test_yylex_string_pct_w_tab
     util_lex_token("%w[abc\tdef]",
-                   :tAWORDS_BEG,      "%w[",
+                   :tQWORDS_BEG,      "%w[",
                    :tSTRING_CONTENT, "abc\tdef",
                    :tSPACE,              nil,
                    :tSTRING_END,     nil)
@@ -1820,7 +1899,7 @@ class TestRubyLexer < MiniTest::Unit::TestCase
       token = args.shift
       value = args.shift
       assert @lex.advance, "no more tokens"
-      assert_equal [token, value], [@lex.token, [@lex.yacc_value].flatten.first]
+      assert_equal [token, value], [@lex.token, [@lex.yacc_value].flatten.first], input
     end
 
     deny @lex.advance, "must be empty, but had #{[@lex.token, @lex.yacc_value].inspect}"