ruby_parser 1.0.0

data/test/test_ruby_lexer.rb

@@ -0,0 +1,398 @@
+#!/usr/local/bin/ruby
+
+require 'test/unit'
+require 'ruby_lexer'
+
+class TestRubyLexer < Test::Unit::TestCase
+  def deny cond, msg = nil
+    assert ! cond, msg
+  end
+
+  def setup
+    @lex = RubyLexer.new
+    @lex.src = StringIO.new("blah blah")
+    @lex.lex_state = :expr_beg # HACK ? I have no idea actually
+  end
+
+  def test_advance
+    assert @lex.advance # blah
+    assert @lex.advance # blah
+    deny   @lex.advance # nada
+  end
+
+  def test_is_next_identchar
+    assert @lex.is_next_identchar
+    @lex.src = StringIO.new(" ")
+    deny @lex.is_next_identchar
+    @lex.src = StringIO.new("-")
+    deny @lex.is_next_identchar
+  end
+
+  def test_is_next_no_case # TODO: worst name evah
+    @lex.src = StringIO.new("123 456")
+    assert @lex.is_next_no_case("123")
+    pos = @lex.src.pos
+    deny @lex.is_next_no_case("begin")
+    assert_equal " 456", @lex.src.read_all, "must put back contents"
+  end
+
+  def test_number_token
+    node = @lex.number_token("42", false, "\0")
+    assert_equal :tINTEGER, node
+    assert_equal 42, @lex.yacc_value
+  end
+
+  def test_parse_number
+    @lex.src = StringIO.new '42'
+    node = @lex.parse_number('1')
+    assert_equal :tINTEGER, node
+    assert_equal 142, @lex.yacc_value
+  end
+
+  def test_parse_quote
+    @lex.src = StringIO.new 'blah)'
+    node = @lex.parse_quote('(')
+    assert_equal :tSTRING_BEG, node
+    assert_equal s(:strterm, RubyLexer::STR_DQUOTE, ")", "("), @lex.lex_strterm
+    assert_equal ["%)"], @lex.yacc_value.args # FIX double check this
+  end
+
+  def test_yylex_integer
+    util_lex_token "42", :tINTEGER, 42
+  end
+
+  def test_yylex_integer_eh_a
+    util_lex_token('?a',
+                   :tINTEGER,     97)
+  end
+
+  def test_yylex_integer_eh_escape_M_escape_C
+    util_lex_token('?\M-\C-a',
+                   :tINTEGER,     129)
+  end
+
+  def test_yylex_float
+    util_lex_token "1.0", :tFLOAT, 1.0
+  end
+
+  def test_yylex_constant
+    util_lex_token("ArgumentError",
+                   :tCONSTANT, t("ArgumentError"))
+  end
+
+  def test_yylex_constant_semi
+    util_lex_token("ArgumentError;",
+                   :tCONSTANT, t("ArgumentError"),
+                   ";", t(";"))
+  end
+
+  def test_yylex_identifier
+    util_lex_token("identifier",
+                   :tIDENTIFIER, t("identifier"))
+  end
+
+  def test_yylex_regexp
+    util_lex_token("/regexp/",
+                   :tREGEXP_BEG,     t("/"),
+                   :tSTRING_CONTENT, s(:str, "regexp"),
+                   :tREGEXP_END,     "")
+  end
+
+  def test_yylex_regexp_nm
+    util_lex_token("/.*/nm",
+                   :tREGEXP_BEG,     t("/"),
+                   :tSTRING_CONTENT, s(:str, ".*"),
+                   :tREGEXP_END,     "nm")
+  end
+
+  def test_yylex_regexp_escapes
+    util_lex_token('/re\tge\nxp/',
+                   :tREGEXP_BEG,     t("/"),
+                   :tSTRING_CONTENT, s(:str, "re\\tge\\nxp"),
+                   :tREGEXP_END,     "")
+  end
+
+  def test_yylex_regexp_escape_oct
+    util_lex_token('/re\tge\101xp/',
+                   :tREGEXP_BEG,     t("/"),
+                   :tSTRING_CONTENT, s(:str, "re\\tge\\101xp"),
+                   :tREGEXP_END,     "")
+  end
+
+  def test_yylex_regexp_escape_hex
+    util_lex_token('/re\tge\x61xp/',
+                   :tREGEXP_BEG,     t("/"),
+                   :tSTRING_CONTENT, s(:str, "re\\tge\\x61xp"),
+                   :tREGEXP_END,     "")
+  end
+
+  def test_yylex_string_double
+    util_lex_token('"string"',
+                   :tSTRING_BEG,     t('"'),
+                   :tSTRING_CONTENT, s(:str, "string"),
+                   :tSTRING_END,     t('"'))
+  end
+
+  def test_yylex_string_double_escapes
+    util_lex_token('"s\tri\ng"',
+                   :tSTRING_BEG,     t('"'),
+                   :tSTRING_CONTENT, s(:str, "s\tri\ng"),
+                   :tSTRING_END,     t('"'))
+  end
+
+  def test_yylex_string_double_escape_M
+    util_lex_token('"\M-g"',
+                   :tSTRING_BEG,     t('"'),
+                   :tSTRING_CONTENT, s(:str, "\347"),
+                   :tSTRING_END,     t('"'))
+  end
+
+  def test_yylex_string_double_escape_octal
+    util_lex_token('"n = \101\102\103"',
+                   :tSTRING_BEG,     t('"'),
+                   :tSTRING_CONTENT, s(:str, "n = ABC"),
+                   :tSTRING_END,     t('"'))
+  end
+
+  def test_yylex_string_double_escape_hex
+    util_lex_token('"n = \x61\x62\x63"',
+                   :tSTRING_BEG,     t('"'),
+                   :tSTRING_CONTENT, s(:str, "n = abc"),
+                   :tSTRING_END,     t('"'))
+  end
+
+  def test_yylex_string_single
+    util_lex_token("'string'",
+                   :tSTRING_BEG,     t("'"),
+                   :tSTRING_CONTENT, s(:str, "string"),
+                   :tSTRING_END,     t("'"))
+  end
+
+  def test_yylex_string_pct_Q
+    util_lex_token("%Q[string]",
+                   :tSTRING_BEG,     t("%Q["),
+                   :tSTRING_CONTENT, s(:str, "string"),
+                   :tSTRING_END,     t("]"))
+  end
+
+  def test_yylex_string_single_escapes
+    util_lex_token("'s\\tri\\ng'",
+                   :tSTRING_BEG,     t("'"),
+                   :tSTRING_CONTENT, s(:str, "s\\tri\\ng"),
+                   :tSTRING_END,     t("'"))
+  end
+
+  def test_yylex_global
+    util_lex_token("$blah",
+                   :tGVAR,     t("$blah"))
+  end
+
+  def test_yylex_global_wierd
+    util_lex_token("$__blah",
+                   :tGVAR,     t("$__blah"))
+  end
+
+  def test_yylex_global_dollar_underscore
+    util_lex_token("$_",
+                   :tGVAR,     t("$_"))
+  end
+
+  def test_yylex_symbol
+    util_lex_token(":symbol",
+                   :tSYMBEG, t(":"),
+                   :tIDENTIFIER, t("symbol"))
+  end
+
+  def test_yylex_comment_begin
+    util_lex_token("=begin\nblah\nblah\n=end\n42",
+                   :tINTEGER, 42)
+  end
+
+  def util_lex_token input, *args
+    @lex.src = StringIO.new input
+
+    until args.empty? do
+      token = args.shift
+      value = args.shift
+      assert @lex.advance, "no more tokens"
+      assert_equal [token, value], [@lex.token, @lex.yacc_value]
+    end
+
+    deny @lex.advance, "must be empty, but had #{[@lex.token, @lex.yacc_value].inspect}"
+  end
+end
+
+class TestStackState < Test::Unit::TestCase
+  def test_stack_state
+    s = StackState.new :test
+    s.push true
+    s.push false
+    s.lexpop
+    assert_equal [false, true], s.stack
+  end
+
+  def test_is_in_state
+    s = StackState.new :test
+    assert_equal false, s.is_in_state
+    s.push false
+    assert_equal false, s.is_in_state
+    s.push true
+    assert_equal true, s.is_in_state
+    s.push false
+    assert_equal false, s.is_in_state
+  end
+
+  def test_lexpop
+    s = StackState.new :test
+    assert_equal [false], s.stack
+    s.push true
+    s.push false
+    assert_equal [false, true, false], s.stack
+    s.lexpop
+    assert_equal [false, true], s.stack
+  end
+
+  def test_pop
+    s = StackState.new :test
+    assert_equal [false], s.stack
+    s.push true
+    assert_equal [false, true], s.stack
+    assert_equal true, s.pop
+    assert_equal [false], s.stack
+  end
+
+  def test_push
+    s = StackState.new :test
+    assert_equal [false], s.stack
+    s.push true
+    s.push false
+    assert_equal [false, true, false], s.stack
+  end
+end
+
+class TestEnvironment < Test::Unit::TestCase
+  def deny t
+    assert ! t
+  end
+
+  def setup
+    @env = Environment.new
+    @env[:blah] = 42
+    assert_equal 42, @env[:blah]
+  end
+
+  def test_use
+    @env.use :blah
+    expected = [{ :blah => true }]
+    assert_equal expected, @env.instance_variable_get(:"@use")
+  end
+
+  def test_use_scoped
+    @env.use :blah
+    @env.extend
+    expected = [{}, { :blah => true }]
+    assert_equal expected, @env.instance_variable_get(:"@use")
+  end
+
+  def test_used_eh
+    @env.extend :dynamic
+    @env[:x] = :dvar
+    @env.use :x
+    assert_equal true, @env.used?(:x)
+  end
+
+  def test_used_eh_none
+    assert_equal nil, @env.used?(:x)
+  end
+
+  def test_used_eh_scoped
+    self.test_used_eh
+    @env.extend :dynamic
+    assert_equal true, @env.used?(:x)
+  end
+
+  def test_var_scope_dynamic
+    @env.extend :dynamic
+    assert_equal 42, @env[:blah]
+    @env.unextend
+    assert_equal 42, @env[:blah]
+  end
+
+  def test_var_scope_static
+    @env.extend
+    assert_equal nil, @env[:blah]
+    @env.unextend
+    assert_equal 42, @env[:blah]
+  end
+
+  def test_dynamic
+    expected1 = {}
+    expected2 = { :x => 42 }
+
+    assert_equal expected1, @env.dynamic
+    begin
+      @env.extend :dynamic
+      assert_equal expected1, @env.dynamic
+
+      @env[:x] = 42
+      assert_equal expected2, @env.dynamic
+
+      begin
+        @env.extend :dynamic
+        assert_equal expected2, @env.dynamic
+        @env.unextend
+      end
+
+      assert_equal expected2, @env.dynamic
+      @env.unextend
+    end
+    assert_equal expected1, @env.dynamic
+  end
+
+  def test_all_dynamic
+    expected = { :blah => 42 }
+
+    @env.extend :dynamic
+    assert_equal expected, @env.all
+    @env.unextend
+    assert_equal expected, @env.all
+  end
+
+  def test_all_static
+    @env.extend
+    expected = { }
+    assert_equal expected, @env.all
+
+    @env.unextend
+    expected = { :blah => 42 }
+    assert_equal expected, @env.all
+  end
+
+  def test_dynamic_eh
+    assert_equal false, @env.dynamic?
+    @env.extend :dynamic
+    assert_equal true, @env.dynamic?
+    @env.extend
+    assert_equal false, @env.dynamic?
+  end
+
+  def test_all_static_deeper
+    expected0 = { :blah => 42 }
+    expected1 = { :blah => 42, :blah2 => 24 }
+    expected2 = { :blah => 27 }
+
+    @env.extend :dynamic
+    @env[:blah2] = 24
+    assert_equal expected1, @env.all
+
+    @env.extend 
+    @env[:blah] = 27
+    assert_equal expected2, @env.all
+
+    @env.unextend
+    assert_equal expected1, @env.all
+
+    @env.unextend
+    assert_equal expected0, @env.all
+  end
+end

data/test/test_ruby_parser.rb

@@ -0,0 +1,326 @@
+#!/usr/local/bin/ruby
+
+require 'test/unit'
+require 'ruby_parser'
+
+$: << File.expand_path('~/Work/p4/zss/src/ParseTree/dev/lib')
+$: << File.expand_path('~/Work/p4/zss/src/ParseTree/dev/test')
+
+require 'pt_testcase'
+
+class TestRubyParser < Test::Unit::TestCase # ParseTreeTestCase
+
+  # Regular ParseTreeTestCase tests
+  eval ParseTreeTestCase.testcases.map { |node, data|
+    next if node.to_s =~ /bmethod|dmethod/
+    next if Array === data['Ruby'] # runtime only crap
+    "def test_#{node}
+       rb = #{data['Ruby'].inspect}
+       pt = #{data['ParseTree'].inspect}
+
+       assert_not_nil rb, \"Ruby for #{node} undefined\"
+       assert_not_nil pt, \"ParseTree for #{node} undefined\"
+
+       assert_equal Sexp.from_array(pt), @processor.parse(rb)
+     end"
+  }.compact.join("\n")
+
+  # Scour the world and compare against ParseTree
+  if ENV['ZOMGPONIES'] or File.exist? 'zomgponies' then
+    require 'parse_tree'
+
+    base = "/usr/lib/ruby"
+    base = "unit"
+
+    files = Dir[File.join(base, "**/*.rb")]
+
+    # these files/patterns cause parse_tree_show to bus error (or just suck):
+    files.reject! { |f| f =~ /environments.environment|rss.maker.base|rails_generator|ferret.browser|rubinius.spec.core.module.(constants|name|remove_const)_spec|tkextlib.tcllib.tablelist/ }
+
+    # these are rejected for dasgn_curr ordering failures... I'll fix them later.
+    # (or mri parse errors--I should have separated them out)
+    files.reject! { |f| f =~ /lib.flog|lib.autotest.notify|lib.analyzer.tools.rails.stat|flog.lib.flog|rakelib.struct.generator|rubinius.kernel.core.array|lib.rbosa.rb|src.rbosa.rb|spec.spec.mocks.mock.spec.rb|dsl.shared.behaviour.spec.rb|spec.spec.dsl.behaviour.spec|lib.hpricot.parse.rb|resolve.rb|parsers.parse.f95|rubinius.shotgun.lib.primitives.ltm|rubinius.lib.bin.compile|rubinius.kernel.core.object|rubinius.kernel.core.file|rubinius.compiler2.garnet.bindingagent|ruby_to_c_test_r2ctestcase|lib.more.like.this|resolv\.rb|test.r2ctestcase/ }
+
+    warn "Generating #{files.size} tests from #{base}"
+
+    eval files.map { |file|
+      name = file[base.size..-1].gsub(/\W+/, '_')
+
+      loc = `wc -l #{file}`.strip.to_i
+
+      "def test#{name}_#{loc}
+         file = #{file.inspect}
+         rb = File.read(file)
+
+         pt = ParseTree.new.parse_tree_for_string rb
+         assert_not_nil pt, \"ParseTree for #{name} undefined\"
+
+         rp = @processor.parse rb
+         assert_equal Sexp.from_array(pt).first, rp, \"RP different from PT\"
+         File.unlink #{file.inspect}
+       end"
+    }.compact.join("\n")
+  end
+
+  def setup
+    super
+
+    # puts self.name
+
+    @processor = RubyParser.new
+  end
+
+  def test_block_append
+    head = s(:args)
+    tail = s(:zsuper)
+    expected = s(:block, s(:args), s(:zsuper))
+    assert_equal expected, @processor.block_append(head, tail)
+  end
+
+  def test_block_append_block
+    head = s(:block, s(:args))
+    tail = s(:zsuper)
+    expected = s(:block, s(:args), s(:zsuper))
+    assert_equal expected, @processor.block_append(head, tail)
+  end
+
+  def test_block_append_tail_block
+    head = s(:vcall, :f1)
+    tail = s(:block, s(:undef, s(:lit, :x)), s(:undef, s(:lit, :y)))
+    expected = s(:block,
+                 s(:vcall, :f1),
+                 s(:block, s(:undef, s(:lit, :x)), s(:undef, s(:lit, :y))))
+    assert_equal expected, @processor.block_append(head, tail)
+  end
+
+  def test_block_append_begin_begin
+    head = s(:begin, s(:args))
+    tail = s(:begin, s(:args))
+    expected = s(:block, s(:args), s(:begin, s(:args)))
+    assert_equal expected, @processor.block_append(head, tail)
+  end
+
+  def test_block_append_nil_head
+    head = nil
+    tail = s(:zsuper)
+    expected = s(:zsuper)
+    assert_equal expected, @processor.block_append(head, tail)
+  end
+
+  def test_block_append_nil_tail
+    head = s(:args)
+    tail = nil
+    expected = s(:args)
+    assert_equal expected, @processor.block_append(head, tail)
+  end
+
+  def test_call_env
+    @processor.env[:a] = :lvar
+    expected = s(:call, s(:lvar, :a), :happy)
+
+    assert_equal expected, @processor.parse('a.happy')
+  end
+
+  def test_do_bug # TODO: rename
+    rb = "a 1\na.b do |c|\n  # do nothing\nend"
+    pt = s(:block,
+           s(:fcall, :a, s(:array, s(:lit, 1))),
+           s(:iter, s(:call, s(:vcall, :a), :b), s(:dasgn_curr, :c)))
+
+    assert_equal pt, @processor.parse(rb)
+  end
+
+  def test_lasgn_env
+    rb = 'a = 42'
+    pt = s(:lasgn, :a, s(:lit, 42))
+    expected_env = { :a => :lvar }
+    
+    assert_equal pt, @processor.parse(rb)
+    assert_equal expected_env, @processor.env.all
+  end
+
+  def test_logop_12
+    lhs = s(:lit, 1)
+    rhs = s(:lit, 2)
+    exp = s(:and, s(:lit, 1), s(:lit, 2))
+
+    assert_equal exp, @processor.logop(:and, lhs, rhs)
+  end
+
+  def test_logop_12_3
+    lhs = s(:and, s(:lit, 1), s(:lit, 2))
+    rhs = s(:lit, 3)
+    exp = s(:and, s(:lit, 1), s(:and, s(:lit, 2), s(:lit, 3)))
+
+    assert_equal exp, @processor.logop(:and, lhs, rhs)
+  end
+
+  def test_logop_123_4
+    lhs = s(:and, s(:lit, 1), s(:and, s(:lit, 2), s(:lit, 3)))
+    rhs = s(:lit, 4)
+    exp = s(:and,
+            s(:lit, 1),
+            s(:and,
+              s(:lit, 2),
+              s(:and,
+                s(:lit, 3),
+                s(:lit, 4))))
+
+    assert_equal exp, @processor.logop(:and, lhs, rhs)
+  end
+
+  def test_logop_1234_5
+    lhs = s(:and, s(:lit, 1), s(:and, s(:lit, 2), s(:and, s(:lit, 3), s(:lit, 4))))
+    rhs = s(:lit, 5)
+    exp = s(:and,
+            s(:lit, 1),
+            s(:and,
+              s(:lit, 2),
+              s(:and,
+                s(:lit, 3),
+                s(:and,
+                  s(:lit, 4),
+                  s(:lit, 5)))))
+
+    assert_equal exp, @processor.logop(:and, lhs, rhs)
+  end
+
+  def test_logop_nested_mix
+    lhs = s(:or, s(:vcall, :a), s(:vcall, :b))
+    rhs = s(:and, s(:vcall, :c), s(:vcall, :d))
+    exp = s(:or,
+            s(:or, s(:vcall, :a), s(:vcall, :b)),
+            s(:and, s(:vcall, :c), s(:vcall, :d)))
+
+    lhs.paren = true
+    rhs.paren = true
+
+    assert_equal exp, @processor.logop(:or, lhs, rhs)
+  end
+
+  def test_literal_concat_str_evstr
+    lhs, rhs = s(:str, ""), s(:evstr, s(:str, "blah"))
+
+    assert_equal s(:str, "blah"), @processor.literal_concat(lhs, rhs)
+  end
+
+  def test_literal_concat_evstr_evstr
+    lhs, rhs = s(:evstr, s(:lit, 1)), s(:evstr, s(:lit, 2))
+    expected = s(:dstr, "", s(:evstr, s(:lit, 1)), s(:evstr, s(:lit, 2)))
+
+    assert_equal expected, @processor.literal_concat(lhs, rhs)
+  end
+
+  def test_literal_concat_dstr_evstr
+    lhs, rhs = s(:dstr, "a"), s(:evstr, s(:vcall, :b))
+    expected = s(:dstr, "a", s(:evstr, s(:vcall, :b)))
+
+    assert_equal expected, @processor.literal_concat(lhs, rhs)
+  end
+
+  def test_literal_concat_dstr_dstr
+    lhs      = s(:dstr, "Failed to download spec ",
+                 s(:evstr, s(:vcall, :spec_name)),
+                 s(:str, " from "),
+                 s(:evstr, s(:vcall, :source_uri)),
+                 s(:str, ":\n"))
+    rhs      = s(:dstr, "\t",
+                 s(:evstr, s(:call, s(:ivar, :@fetch_error), :message)))
+    expected = s(:dstr, "Failed to download spec ",
+                 s(:evstr, s(:vcall, :spec_name)),
+                 s(:str, " from "),
+                 s(:evstr, s(:vcall, :source_uri)),
+                 s(:str, ":\n"),
+                 s(:str, "\t"),
+                 s(:evstr, s(:call, s(:ivar, :@fetch_error), :message)))
+
+    assert_equal expected, @processor.literal_concat(lhs, rhs)
+  end
+
+  def test_str_pct_Q_nested
+    rb = "%Q[before [#\{nest}] after]"
+    pt = s(:dstr, "before [", s(:evstr, s(:vcall, :nest)), s(:str, "] after"))
+
+    assert_equal pt, @processor.parse(rb)
+  end
+
+  def test_str_str
+    rb = "\"a #\{'b'}\""
+    pt = s(:str, "a b")
+
+    assert_equal pt, @processor.parse(rb)
+  end
+
+  def test_str_str_str
+    rb = "\"a #\{'b'} c\""
+    pt = s(:str, "a b c")
+
+    assert_equal pt, @processor.parse(rb)
+  end
+
+  def test_dstr_str
+    rb = "\"#\{'a'} b\""
+    pt = s(:str, "a b")
+
+    assert_equal pt, @processor.parse(rb)
+  end
+
+  def test_str_evstr
+    rb = "\"a #\{b}\""
+    pt = s(:dstr, "a ", s(:evstr, s(:vcall, :b)))
+
+    assert_equal pt, @processor.parse(rb)
+  end
+
+  def test_dstr_evstr
+    rb = "\"#\{'a'}#\{b}\""
+    pt = s(:dstr, "a", s(:evstr, s(:vcall, :b)))
+
+    assert_equal pt, @processor.parse(rb)
+  end
+
+  def test_evstr_str
+    rb = "\"#\{a} b\""
+    pt = s(:dstr, "", s(:evstr, s(:vcall, :a)), s(:str, " b"))
+
+    assert_equal pt, @processor.parse(rb)
+  end
+
+  def test_evstr_evstr
+    rb = "\"#\{a}#\{b}\""
+    pt = s(:dstr, "", s(:evstr, s(:vcall, :a)), s(:evstr, s(:vcall, :b)))
+
+    assert_equal pt, @processor.parse(rb)
+  end
+
+
+  def test_dasgn_icky2
+    rb = "a do\n  v = nil\n  begin\n    yield\n  rescue Exception => v\n    break\n  end\nend"
+    pt = s(:iter,
+           s(:fcall, :a),
+           nil,
+           s(:block,
+             s(:dasgn_curr, :v, s(:nil)),
+             s(:begin,
+               s(:rescue,
+                 s(:yield),
+                 s(:resbody,
+                   s(:array, s(:const, :Exception)),
+                   s(:block, s(:dasgn_curr, :v, s(:gvar, :$!)), s(:break)))))))
+
+    assert_equal pt, @processor.parse(rb)
+  end
+
+  def test_list_append
+    lhs, rhs = s(:array, s(:lit, :iter)), s(:when, s(:const, :BRANCHING), nil)
+    expected = s(:array, s(:lit, :iter), s(:when, s(:const, :BRANCHING), nil))
+
+    assert_equal expected, @processor.list_append(lhs, rhs)
+  end
+end
+
+__END__
+
+# blah18.rb
+
+assert_equal("sub", $_)