ruby_parser 3.0.0.a2 → 3.0.0.a3

data/lib/ruby_lexer.rb

@@ -35,6 +35,8 @@ class RubyLexer
   # What handles warnings
   attr_accessor :warnings
 
+  attr_accessor :space_seen
+
   EOF = :eof_haha!
 
   # ruby constants for strings (should this be moved somewhere else?)
@@ -221,9 +223,9 @@ class RubyLexer
 
   def initialize v = 18
     self.version = v
-    self.cond = RubyParser::StackState.new(:cond)
-    self.cmdarg = RubyParser::StackState.new(:cmdarg)
-    self.tern = RubyParser::StackState.new(:tern)
+    self.cond = RubyParserStuff::StackState.new(:cond)
+    self.cmdarg = RubyParserStuff::StackState.new(:cmdarg)
+    self.tern = RubyParserStuff::StackState.new(:tern)
     self.nest = 0
     @comments = []
 
@@ -237,6 +239,7 @@ class RubyLexer
   end
 
   def lex_state= o
+    # warn "wtf lex_state = #{o.inspect}"
     raise "wtf\?" unless Symbol === o
     @lex_state = o
   end
@@ -640,7 +643,7 @@ class RubyLexer
 
   def yylex # 826 lines
     c = ''
-    space_seen = false
+    self.space_seen = false
     command_state = false
     src = self.src
 
@@ -656,7 +659,7 @@ class RubyLexer
 
     loop do # START OF CASE
       if src.scan(/[\ \t\r\f\v]/) then # \s - \n + \v
-        space_seen = true
+        self.space_seen = true
         next
       elsif src.check(/[^a-zA-Z]/) then
         if src.scan(/\n|#/) then
@@ -711,9 +714,9 @@ class RubyLexer
           end
         elsif src.scan(/\(/) then
           result = if ruby18 then
-                     yylex_paren18 space_seen
+                     yylex_paren18
                    else
-                     yylex_paren19 space_seen
+                     yylex_paren19
                    end
 
           self.expr_beg_push "("
@@ -755,10 +758,7 @@ class RubyLexer
 
           return process_token(command_state)
         elsif src.scan(/\:\:/) then
-          if (lex_state == :expr_beg ||
-              lex_state == :expr_mid ||
-              lex_state == :expr_class ||
-              (lex_state.is_argument && space_seen)) then
+          if is_beg? || lex_state == :expr_class || is_space_arg? then
             self.lex_state = :expr_beg
             self.yacc_value = "::"
             return :tCOLON3
@@ -1139,7 +1139,7 @@ class RubyLexer
         elsif src.scan(/\\/) then
           if src.scan(/\n/) then
             self.lineno = nil
-            space_seen = true
+            self.space_seen = true
             next
           end
           rb_compile_error "bare backslash only allowed before newline"
@@ -1233,7 +1233,7 @@ class RubyLexer
     end
   end
 
-  def yylex_paren18 space_seen
+  def yylex_paren18
     self.command_start = true
     result = :tLPAREN2
 
@@ -1253,17 +1253,39 @@ class RubyLexer
     result
   end
 
-  def yylex_paren19 space_seen
-    if (lex_state == :expr_beg || lex_state == :expr_mid ||
-        lex_state == :expr_value || lex_state == :expr_class) then
+  def is_end?
+    (lex_state == :expr_end    ||
+     lex_state == :expr_endarg ||
+     lex_state == :expr_endfn)
+  end
+
+  def is_arg?
+    lex_state == :expr_arg || lex_state == :expr_cmdarg
+  end
+
+  def is_beg?
+    (lex_state == :expr_beg   ||
+     lex_state == :expr_mid   ||
+     lex_state == :expr_value ||
+     lex_state == :expr_class)
+  end
+
+  def is_space_arg? c = "x"
+    is_arg? and space_seen and c !~ /\s/
+  end
+
+  def yylex_paren19
+    if is_beg? then
       result = :tLPAREN
-    elsif ((lex_state == :expr_arg || lex_state == :expr_cmdarg) and
-           space_seen) then
+    elsif is_space_arg? then
       result = :tLPAREN_ARG
     else
       self.tern.push false
       result = :tLPAREN2
     end
+
+    # p :wtf_paren => [lex_state, space_seen, result]
+
     # HACK paren_nest++;
 
     # HACK: this is a mess, but it makes the tests pass, so suck it
@@ -1334,7 +1356,11 @@ class RubyLexer
 
       unless lex_state == :expr_dot then
         # See if it is a reserved word.
-        keyword = RubyParser::Keyword.keyword token
+        keyword = if ruby18 then # REFACTOR need 18/19 lexer subclasses
+                    RubyParserStuff::Keyword.keyword18 token
+                  else
+                    RubyParserStuff::Keyword.keyword19 token
+                  end
 
         if keyword then
           state           = lex_state

data/lib/ruby_parser_extras.rb

@@ -2,6 +2,7 @@ require 'stringio'
 require 'racc/parser'
 require 'sexp'
 require 'strscan'
+require 'ruby_lexer'
 
 def d o
   $stderr.puts o.inspect
@@ -115,7 +116,7 @@ class RPStringScanner < StringScanner
 end
 
 module RubyParserStuff
-  VERSION = '3.0.0.a2' unless constants.include? "VERSION" # SIGH
+  VERSION = '3.0.0.a3' unless constants.include? "VERSION" # SIGH
 
   attr_accessor :lexer, :in_def, :in_single, :file
   attr_reader :env, :comments
@@ -223,6 +224,57 @@ module RubyParserStuff
     result
   end
 
+  def block_args19 val, id
+    # HACK OMG THIS CODE IS SOOO UGLY! CLEAN ME
+    untested = %w[1 2 3 4 7 9 10 11 12 14]
+    raise "no block_args19 #{id}" if untested.include? id
+
+    r = s(:array)
+
+    val.compact.each do |v|
+      next if %w[,].include? v
+      case v
+      when Sexp then
+        case v.first
+        when :args then
+          r.concat v[1..-1].map { |s| s(:lasgn, s) }
+        when :block_arg then
+          r << s(:lasgn, :"&#{v.last}")
+        else
+          raise "block_args19 #{id} unhandled sexp type:: #{v.inspect}"
+        end
+      when Symbol
+        case v.to_s
+        when /^\*(.+)/ then
+          r << s(:splat, s(:lasgn, $1.to_sym))
+        when /^\*/ then
+          r << s(:splat)
+        else
+          raise "block_args19 #{id} unhandled symbol type:: #{v.inspect}"
+        end
+      else
+        raise "block_args19 #{id} unhandled type:: #{v.inspect}"
+      end
+    end
+
+    if r.size > 2 then
+      r = s(:masgn, r)
+    elsif r.size == 2 then
+      case r.last.first
+      when :splat then
+        r = s(:masgn, r)
+      when :lasgn then
+        r = r.last
+      else
+        raise "oh noes!: #{r.inspect}"
+      end
+    else
+      raise "fuck no #{r.inspect}"
+    end
+
+    r
+  end
+
   def aryset receiver, index
     s(:attrasgn, receiver, :"[]=", *index[1..-1])
   end
@@ -389,7 +441,7 @@ module RubyParserStuff
     v = self.class.name[/1[89]/]
     self.lexer = RubyLexer.new v && v.to_i
     self.lexer.parser = self
-    @env = Environment.new
+    @env = RubyParserStuff::Environment.new
     @comments = []
 
     @canonicalize_conditions = true
@@ -583,7 +635,7 @@ module RubyParserStuff
   end
 
   def new_compstmt val
-    result = void_stmts(val[0])
+    result = void_stmts(val.grep(Sexp)[0])
     result = remove_begin(result) if result
     result
   end
@@ -1011,15 +1063,29 @@ module RubyParserStuff
                 ["BEGIN",    [:klBEGIN,   :klBEGIN     ], :expr_end   ],
                 ["while",    [:kWHILE,    :kWHILE_MOD  ], :expr_beg   ],
                 ["alias",    [:kALIAS,    :kALIAS      ], :expr_fname ],
+                ["__ENCODING__", [:k__ENCODING__, :k__ENCODING__], :expr_end],
                ].map { |args| KWtable.new(*args) }
 
     # :startdoc:
 
-    WORDLIST = Hash[*wordlist.map { |o| [o.name, o] }.flatten] unless
-      defined? WORDLIST
+    WORDLIST18 = Hash[*wordlist.map { |o| [o.name, o] }.flatten]
+    WORDLIST19 = Hash[*wordlist.map { |o| [o.name, o] }.flatten]
+
+    %w[and case elsif for if in module or unless until when while].each do |k|
+      WORDLIST19[k] = WORDLIST19[k].dup
+      WORDLIST19[k].state = :expr_value
+    end
+    %w[not].each do |k|
+      WORDLIST19[k] = WORDLIST19[k].dup
+      WORDLIST19[k].state = :expr_arg
+    end
+
+    def self.keyword18 str # REFACTOR
+      WORDLIST18[str]
+    end
 
-    def self.keyword str
-      WORDLIST[str]
+    def self.keyword19 str
+      WORDLIST19[str]
     end
   end
 
@@ -1138,11 +1204,21 @@ class Ruby18Parser < Racc::Parser
   include RubyParserStuff
 end
 
-class RubyParser < Ruby18Parser
+##
+# RubyParser is a compound parser that first attempts to parse using
+# the 1.9 syntax parser and falls back to the 1.8 syntax parser on a
+# parse error.
+
+class RubyParser
   def initialize
-    super
-    warn "WA\RNING: Deprecated: RubyParser. Use Ruby18Parser or Ruby19Parser"
-    warn "  from #{caller.first}"
+    @p18 = Ruby18Parser.new
+    @p19 = Ruby19Parser.new
+  end
+
+  def parse s
+    @p19.parse s
+  rescue Racc::ParseError
+    @p18.parse s
   end
 end
 
@@ -1172,11 +1248,11 @@ class Sexp
   end
 
   def add x
-    raise "no" # TODO: need a test to trigger this
+    concat x
   end
 
   def add_all x
-    raise "no" # TODO: need a test to trigger this
+    raise "no: #{self.inspect}.add_all #{x.inspect}" # TODO: need a test to trigger this
   end
 
   alias :node_type :sexp_type

data/test/test_ruby_parser.rb

@@ -1,28 +1,16 @@
 #!/usr/local/bin/ruby
 
-ENV['VERBOSE'] = "1"
+# ENV['VERBOSE'] = "1"
 
 require 'rubygems'
 gem "minitest"
 require 'minitest/autorun'
 require 'ruby_parser'
 
-$: << File.expand_path('~/Work/p4/zss/src/ParseTree/dev/test')
+$: << File.expand_path('~/Work/p4/zss/src/sexp_processor/dev/lib')
 
 require 'pt_testcase'
 
-class Ruby18Parser # FIX
-  def process input
-    parse input
-  end
-end
-
-class Ruby19Parser
-  def process input
-    parse input
-  end
-end
-
 class RubyParserTestCase < ParseTreeTestCase
   attr_accessor :result, :processor
 
@@ -44,13 +32,21 @@ class RubyParserTestCase < ParseTreeTestCase
     assert_equal pt, result
   end
 
+  def assert_parse_error rb, emsg
+    e = assert_raises Racc::ParseError do
+      processor.parse rb
+    end
+
+    assert_equal emsg, e.message.strip # TODO: why strip?
+  end
+
   def assert_parse_line rb, pt, line
     assert_parse rb, pt
     assert_equal line, result.line,   "call should have line number"
   end
 end
 
-module TestRubyParser
+module TestRubyParserShared
   def test_attrasgn_array_lhs
     rb = '[1, 2, 3, 4][from .. to] = ["a", "b", "c"]'
     pt = s(:attrasgn,
@@ -617,6 +613,103 @@ module TestRubyParser
     assert_equal 3, result.if.return.line
     assert_equal 3, result.if.return.lit.line
   end
+end
+
+class TestRubyParser < MiniTest::Unit::TestCase
+  def test_parse
+    processor = RubyParser.new
+
+    # 1.8 only syntax
+    rb = "while false : 42 end"
+    pt = s(:while, s(:false), s(:lit, 42), true)
+
+    assert_equal pt, processor.parse(rb)
+
+    # 1.9 only syntax
+    rb = "a.()"
+    pt = s(:call, s(:call, nil, :a), :call)
+
+    assert_equal pt, processor.parse(rb)
+
+    # bad syntax
+    e = assert_raises Racc::ParseError do
+      processor.parse "a.("
+    end
+
+    msg = "parse error on value \"(\" (tLPAREN2)"
+    assert_equal msg, e.message.strip
+  end
+end
+
+class TestRuby18Parser < RubyParserTestCase
+  include TestRubyParserShared
+
+  def setup
+    super
+
+    self.processor = Ruby18Parser.new
+  end
+
+  def test_flip2_env_lvar
+    rb = "if a..b then end"
+    pt = s(:if, s(:flip2, s(:call, nil, :a), s(:call, nil, :b)), nil, nil)
+
+    assert_parse rb, pt
+
+    top_env = processor.env.env.first
+
+    assert_kind_of Hash, top_env
+
+    flip = top_env.find { |k,v| k =~ /^flip/ }
+
+    assert flip
+    assert_equal :lvar, flip.last
+  end
+
+  def test_assoc_list_18
+    rb = "{1, 2, 3, 4}"
+    pt = s(:hash, s(:lit, 1), s(:lit, 2), s(:lit, 3), s(:lit, 4))
+
+    assert_parse rb, pt
+  end
+
+  def test_case_then_colon_18
+    rb = "case x; when Fixnum: 42; end"
+    pt = s(:case,
+           s(:call, nil, :x),
+           s(:when, s(:array, s(:const, :Fixnum)), s(:lit, 42)),
+           nil)
+
+    assert_parse rb, pt
+  end
+
+  def test_do_colon_18
+    rb = "while false : 42 end"
+    pt = s(:while, s(:false), s(:lit, 42), true)
+
+    assert_parse rb, pt
+  end
+
+  def test_parse_until_not_canonical
+    rb = "until not var.nil?\n  'foo'\nend"
+
+    pt = s(:while,
+           s(:call, s(:call, nil, :var), :nil?),
+           s(:str, "foo"), true)
+
+    assert_parse rb, pt
+  end
+
+  def test_parse_until_not_noncanonical
+    rb = "until not var.nil?\n  'foo'\nend"
+    pt = s(:until,
+           s(:not, s(:call, s(:call, nil, :var), :nil?)),
+           s(:str, "foo"), true)
+
+    processor.canonicalize_conditions = false
+
+    assert_parse rb, pt
+  end
 
   def test_parse_if_not_canonical
     rb = "if not var.nil? then 'foo' else 'bar'\nend"
@@ -631,8 +724,7 @@ module TestRubyParser
   def test_parse_if_not_noncanonical
     rb = "if not var.nil? then 'foo' else 'bar'\nend"
     pt = s(:if,
-           s(:not,
-             s(:call, s(:call, nil, :var), :nil?)),
+           s(:not, s(:call, s(:call, nil, :var), :nil?)),
            s(:str, "foo"),
            s(:str, "bar"))
 
@@ -653,20 +745,151 @@ module TestRubyParser
   def test_parse_while_not_noncanonical
     rb = "while not var.nil?\n  'foo'\nend"
     pt = s(:while,
-           s(:not,
-             s(:call, s(:call, nil, :var), :nil?)),
+           s(:not, s(:call, s(:call, nil, :var), :nil?)),
            s(:str, "foo"), true)
 
     processor.canonicalize_conditions = false
 
     assert_parse rb, pt
   end
+end
+
+class TestRuby19Parser < RubyParserTestCase
+  include TestRubyParserShared
+
+  def setup
+    super
+
+    self.processor = Ruby19Parser.new
+  end
+
+  def test_mlhs_back_splat
+    rb = "a, b, c, *s = f"
+    pt = s(:masgn,
+           s(:array,
+             s(:lasgn, :a), s(:lasgn, :b), s(:lasgn, :c),
+             s(:splat, s(:lasgn, :s))),
+           s(:to_ary, s(:call, nil, :f)))
+
+    assert_parse rb, pt
+  end
+
+  def test_mlhs_back_anonsplat
+    rb = "a, b, c, * = f"
+    pt = s(:masgn,
+           s(:array,
+             s(:lasgn, :a), s(:lasgn, :b), s(:lasgn, :c),
+             s(:splat)),
+           s(:to_ary, s(:call, nil, :f)))
+
+    assert_parse rb, pt
+  end
+
+  def test_mlhs_mid_splat
+    rb = "a, b, c, *s, x, y, z = f"
+    pt = s(:masgn,
+           s(:array,
+             s(:lasgn, :a), s(:lasgn, :b), s(:lasgn, :c),
+             s(:splat, s(:lasgn, :s)),
+             s(:lasgn, :x), s(:lasgn, :y), s(:lasgn, :z)),
+           s(:to_ary, s(:call, nil, :f)))
+
+    assert_parse rb, pt
+  end
+
+  def test_mlhs_mid_anonsplat
+    rb = "a, b, c, *, x, y, z = f"
+    pt = s(:masgn,
+           s(:array, s(:lasgn, :a), s(:splat), s(:lasgn, :z)),
+           s(:to_ary, s(:call, nil, :f)))
+    pt = s(:masgn,
+           s(:array,
+             s(:lasgn, :a), s(:lasgn, :b), s(:lasgn, :c),
+             s(:splat),
+             s(:lasgn, :x), s(:lasgn, :y), s(:lasgn, :z)),
+           s(:to_ary, s(:call, nil, :f)))
+
+    assert_parse rb, pt
+  end
+
+  def test_mlhs_front_splat
+    rb = "*s, x, y, z = f"
+    pt = s(:masgn,
+           s(:array, s(:splat, s(:lasgn, :s)), s(:lasgn, :z)),
+           s(:to_ary, s(:call, nil, :f)))
+    pt = s(:masgn,
+           s(:array,
+             s(:splat, s(:lasgn, :s)),
+             s(:lasgn, :x), s(:lasgn, :y), s(:lasgn, :z)),
+           s(:to_ary, s(:call, nil, :f)))
+
+    assert_parse rb, pt
+  end
+
+  def test_mlhs_front_anonsplat
+    rb = "*, x, y, z = f"
+    pt = s(:masgn,
+           s(:array,
+             s(:splat),
+             s(:lasgn, :x), s(:lasgn, :y), s(:lasgn, :z)),
+           s(:to_ary, s(:call, nil, :f)))
+
+    assert_parse rb, pt
+  end
+
+  def test_expr_not_bang
+    rb = "! a b"
+    pt = s(:call, s(:call, nil, :a, s(:call, nil, :b)), :"!")
+
+    assert_parse rb, pt
+  end
+
+  def test_encoding
+    rb = '__ENCODING__'
+    pt = s(:str, "Unsupported!")
+
+    assert_parse rb, pt
+  end
+
+  def test_do_colon_19
+    rb = "while false : 42 end"
+
+    assert_parse_error rb, "parse error on value \":\" (tCOLON)"
+  end
+
+  def test_assoc_list_19
+    rb = "{1, 2, 3, 4}"
+
+    assert_parse_error rb, "parse error on value \",\" (tCOMMA)"
+  end
+
+  def test_case_then_colon_19
+    rb = <<-EOM
+      case x
+      when Fixnum : # need the space to not hit new hash arg syntax
+        42
+      end
+    EOM
+
+    assert_parse_error rb, "parse error on value \":\" (tCOLON)"
+  end
+
+  def test_parse_def_xxx1
+    rb = 'def f(a, *b, c = nil) end'
+
+    assert_parse_error rb, 'parse error on value "=" (tEQL)'
+  end
+
+  def test_parse_def_xxx2
+    rb = 'def f(a = nil, *b, c = nil) end'
+
+    assert_parse_error rb, 'parse error on value "=" (tEQL)'
+  end
 
   def test_parse_until_not_canonical
     rb = "until not var.nil?\n  'foo'\nend"
-
-    pt = s(:while,
-           s(:call, s(:call, nil, :var), :nil?),
+    pt = s(:until,
+           s(:call, s(:call, s(:call, nil, :var), :nil?), :"!"),
            s(:str, "foo"), true)
 
     assert_parse rb, pt
@@ -675,49 +898,61 @@ module TestRubyParser
   def test_parse_until_not_noncanonical
     rb = "until not var.nil?\n  'foo'\nend"
     pt = s(:until,
-           s(:not,
-             s(:call, s(:call, nil, :var), :nil?)),
+           s(:call, s(:call, s(:call, nil, :var), :nil?), :"!"),
            s(:str, "foo"), true)
 
     processor.canonicalize_conditions = false
 
     assert_parse rb, pt
   end
-end
 
-class TestRuby18Parser < RubyParserTestCase
-  include TestRubyParser
-
-  def setup
-    super
+  def test_parse_if_not_canonical
+    rb = "if not var.nil? then 'foo' else 'bar'\nend"
+    pt = s(:if,
+           s(:call, s(:call, s(:call, nil, :var), :nil?), :"!"),
+           s(:str, "foo"),
+           s(:str, "bar"))
 
-    self.processor = Ruby18Parser.new
+    assert_parse rb, pt
   end
 
-  def test_flip2_env_lvar
-    rb = "if a..b then end"
-    pt = s(:if, s(:flip2, s(:call, nil, :a), s(:call, nil, :b)), nil, nil)
+  def test_parse_if_not_noncanonical
+    rb = "if not var.nil? then 'foo' else 'bar'\nend"
+    pt = s(:if,
+           s(:call, s(:call, s(:call, nil, :var), :nil?), :"!"),
+           s(:str, "foo"),
+           s(:str, "bar"))
+
+    processor.canonicalize_conditions = false
 
     assert_parse rb, pt
+  end
 
-    top_env = processor.env.env.first
+  def test_parse_while_not_canonical
+    rb = "while not var.nil?\n  'foo'\nend"
+    pt = s(:while,
+           s(:call, s(:call, s(:call, nil, :var), :nil?), :"!"),
+           s(:str, "foo"), true)
 
-    assert_kind_of Hash, top_env
+    assert_parse rb, pt
+  end
 
-    flip = top_env.find { |k,v| k =~ /^flip/ }
+  def test_parse_while_not_noncanonical
+    rb = "while not var.nil?\n  'foo'\nend"
+    pt = s(:while,
+           s(:call, s(:call, s(:call, nil, :var), :nil?), :"!"),
+           s(:str, "foo"), true)
 
-    assert flip
-    assert_equal :lvar, flip.last
-  end
-end
+    processor.canonicalize_conditions = false
 
-class TestRuby19Parser < RubyParserTestCase
-  include TestRubyParser
+    assert_parse rb, pt
+  end
 
-  def setup
-    super
+  def test_parse_opt_call_args_assocs_comma
+    rb = "1[2=>3,]"
+    pt = s(:call, s(:lit, 1), :[], s(:lit, 2), s(:lit, 3))
 
-    self.processor = Ruby19Parser.new
+    assert_parse rb, pt
   end
 
   # HACK: need to figure out the desired structure and get this working