ruby_parser 3.0.0 → 3.19.1

data/test/test_ruby_parser_extras.rb

@@ -1,13 +1,10 @@
 # encoding: US-ASCII
 
-require 'rubygems'
-gem "minitest"
-require 'minitest/autorun'
-require 'ruby_parser_extras'
+require "minitest/autorun"
+require "ruby_parser_extras"
+require "ruby_parser"
 
-require 'minitest/unit'
-
-class TestStackState < MiniTest::Unit::TestCase
+class TestStackState < Minitest::Test
   attr_reader :s
 
   def setup
@@ -16,14 +13,14 @@ class TestStackState < MiniTest::Unit::TestCase
 
   def assert_encoding str, default = false
     orig_str = str.dup
-    p = Ruby19Parser.new
+    p = RubyParser.latest
     s = nil
 
-    out, err = capture_io do
+    out, err = capture_io {
       s = p.handle_encoding str
-    end
+    }
 
-    assert_equal orig_str.sub(/\357\273\277/, ''), s
+    assert_equal orig_str.sub(/\357\273\277/, ""), s
 
     exp_err = ""
 
@@ -163,9 +160,9 @@ class TestStackState < MiniTest::Unit::TestCase
   end
 end
 
-class TestEnvironment < MiniTest::Unit::TestCase
+class TestEnvironment < Minitest::Test
   def deny t
-    assert ! t
+    assert !t
   end
 
   def setup
@@ -174,36 +171,6 @@ class TestEnvironment < MiniTest::Unit::TestCase
     assert_equal 42, @env[:blah]
   end
 
-  def test_use
-    @env.use :blah
-    expected = [{ :blah => true }]
-    assert_equal expected, @env.instance_variable_get(:"@use")
-  end
-
-  def test_use_scoped
-    @env.use :blah
-    @env.extend
-    expected = [{}, { :blah => true }]
-    assert_equal expected, @env.instance_variable_get(:"@use")
-  end
-
-  def test_used_eh
-    @env.extend :dynamic
-    @env[:x] = :dvar
-    @env.use :x
-    assert_equal true, @env.used?(:x)
-  end
-
-  def test_used_eh_none
-    assert_equal nil, @env.used?(:x)
-  end
-
-  def test_used_eh_scoped
-    self.test_used_eh
-    @env.extend :dynamic
-    assert_equal true, @env.used?(:x)
-  end
-
   def test_var_scope_dynamic
     @env.extend :dynamic
     assert_equal 42, @env[:blah]
@@ -213,35 +180,11 @@ class TestEnvironment < MiniTest::Unit::TestCase
 
   def test_var_scope_static
     @env.extend
-    assert_equal nil, @env[:blah]
+    assert_nil @env[:blah]
     @env.unextend
     assert_equal 42, @env[:blah]
   end
 
-  def test_dynamic
-    expected1 = {}
-    expected2 = { :x => 42 }
-
-    assert_equal expected1, @env.dynamic
-    begin
-      @env.extend :dynamic
-      assert_equal expected1, @env.dynamic
-
-      @env[:x] = 42
-      assert_equal expected2, @env.dynamic
-
-      begin
-        @env.extend :dynamic
-        assert_equal expected2, @env.dynamic
-        @env.unextend
-      end
-
-      assert_equal expected2, @env.dynamic
-      @env.unextend
-    end
-    assert_equal expected1, @env.dynamic
-  end
-
   def test_all_dynamic
     expected = { :blah => 42 }
 
@@ -261,14 +204,6 @@ class TestEnvironment < MiniTest::Unit::TestCase
     assert_equal expected, @env.all
   end
 
-  def test_dynamic_eh
-    assert_equal false, @env.dynamic?
-    @env.extend :dynamic
-    assert_equal true, @env.dynamic?
-    @env.extend
-    assert_equal false, @env.dynamic?
-  end
-
   def test_all_static_deeper
     expected0 = { :blah => 42 }
     expected1 = { :blah => 42, :blah2 => 24 }
@@ -278,7 +213,7 @@ class TestEnvironment < MiniTest::Unit::TestCase
     @env[:blah2] = 24
     assert_equal expected1, @env.all
 
-    @env.extend 
+    @env.extend
     @env[:blah] = 27
     assert_equal expected2, @env.all
 
@@ -289,3 +224,38 @@ class TestEnvironment < MiniTest::Unit::TestCase
     assert_equal expected0, @env.all
   end
 end
+
+class Fake20
+  include RubyParserStuff
+
+  def initialize
+  end
+
+  def s(*a) # bypass lexer/lineno stuff that RP overrides in
+    Kernel.send :s, *a
+  end
+end
+
+class TestValueExpr < Minitest::Test
+  def assert_value_expr exp, input
+    assert_equal exp, Fake20.new.value_expr(input.line(1))
+  end
+
+  def assert_remove_begin exp, input
+    assert_equal exp, Fake20.new.remove_begin(input.line(1))
+  end
+
+  def test_value_expr
+    assert_value_expr s(:nil),                     s(:begin)
+    assert_value_expr s(:nil),                     s(:begin, s(:nil))
+    assert_value_expr s(:nil),                     s(:begin, s(:begin, s(:nil)))
+    assert_value_expr s(:begin, s(:nil), s(:nil)), s(:begin, s(:nil), s(:nil))
+  end
+
+  def test_remove_begin
+    assert_remove_begin s(:nil),                     s(:begin)
+    assert_remove_begin s(:nil),                     s(:begin, s(:nil))
+    assert_remove_begin s(:nil),                     s(:begin, s(:begin, s(:nil)))
+    assert_remove_begin s(:begin, s(:nil), s(:nil)), s(:begin, s(:nil), s(:nil))
+  end
+end

data/tools/munge.rb

@@ -0,0 +1,250 @@
+#!/usr/bin/env ruby -ws
+
+$v ||= false
+
+stack = []
+last_token = nil
+reduce_line = nil
+
+def munge s
+  renames = [
+             "'='",             "tEQL",
+             "'!'",             "tBANG",
+             "'%'",             "tPERCENT",
+             "'&'",             "tAMPER2",
+             "'('",             "tLPAREN2",
+             "')'",             "tRPAREN",
+             "'*'",             "tSTAR2",
+             "'+'",             "tPLUS",
+             "','",             "tCOMMA",
+             "'-'",             "tMINUS",
+             "'.'",             "tDOT",
+             "'/'",             "tDIVIDE",
+             "';'",             "tSEMI",
+             "':'",             "tCOLON",
+             "'<'",             "tLT",
+             "'>'",             "tGT",
+             "'?'",             "tEH",
+             "'['",             "tLBRACK",
+             "'\\n'",           "tNL",
+             "']'",             "tRBRACK",
+             "'^'",             "tCARET",
+             "'`'",             "tBACK_REF2",
+             "'{'",             "tLCURLY",
+             "'|'",             "tPIPE",
+             "'}'",             "tRCURLY",
+             "'~'",             "tTILDE",
+             '"["',             "tLBRACK",
+
+             # 2.0 changes?
+             '"<=>"',            "tCMP",
+             '"=="',             "tEQ",
+             '"==="',            "tEQQ",
+             '"!~"',             "tNMATCH",
+             '"=~"',             "tMATCH",
+             '">="',             "tGEQ",
+             '"<="',             "tLEQ",
+             '"!="',             "tNEQ",
+             '"<<"',             "tLSHFT",
+             '">>"',             "tRSHFT",
+             '"*"',              "tSTAR",
+
+             '".."',             "tDOT2",
+
+             '"&"',              "tAMPER",
+             '"&&"',             "tANDOP",
+             '"&."',             "tLONELY",
+             '"||"',             "tOROP",
+
+             '"..."',            "tDOT3",
+             '"**"',             "tPOW",
+             '"unary+"',         "tUPLUS",
+             '"unary-"',         "tUMINUS",
+             '"[]"',             "tAREF",
+             '"[]="',            "tASET",
+             '"::"',             "tCOLON2",
+             '"{ arg"',          "tLBRACE_ARG",
+             '"( arg"',          "tLPAREN_ARG",
+             '"("',              "tLPAREN",
+             'rparen',           "tRPAREN",
+             '"{"',              "tLBRACE",
+             '"=>"',             "tASSOC",
+             '"->"',             "tLAMBDA",
+             '":: at EXPR_BEG"', "tCOLON3",
+             '"**arg"',          "tDSTAR",
+             '","',              "tCOMMA",
+
+             # other
+
+             'kTERMINATOR',     "tSTRING_END",
+             '"kTERMINATOR"',   "tSTRING_END",
+             'kTRCURLY',        "tSTRING_DEND",
+
+             '"symbol literal"',       "tSYMBEG",
+             '"string literal"',       "tSTRING_BEG",
+             '"backtick literal"',     "tXSTRING_BEG",
+             '"regexp literal"',       "tREGEXP_BEG",
+             '"word list"',            "tWORDS_BEG",
+             '"verbatim word list"',   "tQWORDS_BEG",
+             '"symbol list"',          "tSYMBOLS_BEG",
+             '"verbatim symbol list"', "tQSYMBOLS_BEG",
+             '"terminator"',           "tSTRING_END",
+             '"\'}\'"',                  "tSTRING_DEND",
+
+             '"string literal"',"tSTRING_BEG",
+             '"literal content"', "tSTRING_CONTENT",
+             /\$/,              "", # try to remove these lumps?
+
+             'tLBRACK2',        "tLBRACK", # HACK
+
+             "' '",             "tSPACE", # needs to be later to avoid bad hits
+
+             "/* empty */",     "none",
+             /^\s*$/,           "",
+
+             "keyword_BEGIN",   "klBEGIN",
+             "keyword_END",     "klEND",
+             /keyword_(\w+)/,   proc { "k#{$1.upcase}" },
+             /\bk_([a-z_]+)/,   proc { "k#{$1.upcase}" },
+             /modifier_(\w+)/,  proc { "k#{$1.upcase}_MOD" },
+             "kVARIABLE",       "keyword_variable", # ugh
+             "tCONST",          "kCONST",
+
+             # 2.6 collapses klBEGIN to kBEGIN
+             "klBEGIN",   "kBEGIN",
+             "klEND",     "kEND",
+
+             /keyword_(\w+)/,          proc { "k#{$1.upcase}" },
+             /\bk_([^_][a-z_]+)/,      proc { "k#{$1.upcase}" },
+             /modifier_(\w+)/,         proc { "k#{$1.upcase}_MOD" },
+
+             "kVARIABLE",       "keyword_variable", # ugh: this is a rule name
+
+             # UGH
+             "k_LINE__",       "k__LINE__",
+             "k_FILE__",       "k__FILE__",
+             "k_ENCODING__",   "k__ENCODING__",
+
+             '"defined?"',     "kDEFINED",
+
+             "<none>",         "none",
+
+             '"do (for condition)"', "kDO_COND",
+             '"do (for lambda)"',    "kDO_LAMBDA",
+             '"do (for block)"',     "kDO_BLOCK",
+             '"local variable or method"', "tIDENTIFIER",
+
+             /\"(\w+) \(modifier\)\"/, proc { |x| "k#{$1.upcase}_MOD" },
+             /\"(\w+)\"/,              proc { |x| "k#{$1.upcase}" },
+             /\"`(\w+)'\"/,            proc { |x| "k#{$1.upcase}" },
+
+             /@(\d+)(\s+|$)/,       "",
+             /\$?@(\d+) */,         "", # TODO: remove?
+
+             /_EXPR/,               "",
+            ]
+
+  renames.each_slice(2) do |(a, b)|
+    if Proc === b then
+      s.gsub!(a, &b)
+    else
+      s.gsub!(a, b)
+    end
+  end
+
+  if s.empty? then
+    nil
+  else
+    s.strip.squeeze " "
+  end
+end
+
+ARGF.each_line do |line|
+  case line
+  when /^(Stack now|Entering state|Shifting|Cleanup|Starting)/ then
+    # do nothing
+  when /^vtable_/ then
+    # do nothing
+  when /Gem::MissingSpecError/ then
+    # do nothing -- ruby 2.5 is being bitchy?
+  when /^Reading a token: Next token is token (.*?) \(\)/ then
+    token = munge $1
+    next if last_token == token
+    puts "next token is %p" % [token]
+    last_token = token
+  when /^Reading a token: / then
+    next # skip
+  when /^read\s+:(\w+)/ then # read    :tNL(tNL) nil
+    token = munge $1
+    next if last_token == token
+    puts "next token is %p" % [token]
+    last_token = token
+  when /^Next token is token ("[^"]+"|\S+)/ then
+    token = munge $1
+    next if last_token == token
+    puts "next token is %p" % [token]
+    last_token = token
+  when /^read\s+false/ then # read    false($end) "$end"
+    puts "next token is EOF"
+  when /^Now at end of input./ then
+    # do nothing
+  when /^.:scan=>\["([^"]+)"/ then
+    puts "scan = %p" % [$1]
+  when /^.:getch=>\["([^"]+)/ then
+    puts "SCAN = %p" % [$1]
+  when /^Reducing stack by rule (\d+) \(line (\d+)\):/ then
+    reduce_line = $2.to_i
+  when /^   \$\d+ = (?:token|nterm) (.+) \(.*\)/ then
+    item = $1
+    stack << munge(item)
+  when /^-> \$\$ = (?:token|nterm) (.+) \(.*\)/ then
+    stack << "none" if stack.empty?
+    item = munge $1
+    x = stack.compact.map { |s| munge s.strip }.compact.join " "
+    if x != item then # prevent kdef -> kdef
+      if $v && reduce_line then
+        puts "reduce #{x} --> #{item} at #{reduce_line}".squeeze " "
+      else
+        puts "reduce #{x} --> #{item}".squeeze " "
+      end
+      puts
+    end
+    reduce_line = nil
+    stack.clear
+  when /^reduce/ then # ruby_parser side
+    puts munge line.chomp
+    puts
+  when /^(\w+_stack)\.(\w+)/ then
+    # TODO: make pretty, but still informative w/ line numbers etc
+    puts line.gsub("true", "1").gsub("false", "0")
+    # puts "#{$1}(#{$2})"
+  when /^(\w+_stack(\(\w+\))?: \S+)/ then
+    # _data = $v ? line.chomp : $1
+    # puts line
+    # TODO: make pretty, but still informative w/ line numbers etc
+    puts line.gsub("true", "1").gsub("false", "0")
+  when /^lex_state: :?([\w|]+) -> :?([\w|]+)(?: (?:at|from) (.*))?/ then
+    a, b, c = $1.upcase, $2.upcase, $3
+    a.gsub!(/EXPR_/, "")
+    b.gsub!(/EXPR_/, "")
+    if c && $v then
+      puts "lex_state: #{a} -> #{b} at #{c}"
+    else
+      puts "lex_state: #{a} -> #{b}"
+    end
+  when /debug|FUCK/ then
+    puts line.chomp
+  when /^(#.*parse error|on )/ then
+    puts line.chomp
+  when /^(goto|shift| +\[|$)/ then # racc
+    # do nothing
+  # when /^Reading a token: Now at end of input./ then
+  #   # puts "EOF"
+  # when /^Reading a token: Next token is token (.+)/ then
+  #   puts "READ: #{$1.inspect}"
+  when /^accept/ then
+    puts "DONE"
+  else
+    puts "unparsed: #{line.chomp}"
+  end
+end

data/tools/ripper.rb

@@ -0,0 +1,44 @@
+#!/usr/bin/env ruby -ws
+
+$b ||= false # bug mode -- ripper is buggy, use Ripper.sexp
+$d ||= false # debug -- turn on yydebug
+$p ||= false # Use pp
+
+require "ripper/sexp"
+require "pp" if $p
+
+if ARGV.empty? then
+  warn "reading from stdin"
+  ARGV << "-"
+end
+
+class MySexpBuilder < Ripper::SexpBuilderPP
+  def on_parse_error msg
+    Kernel.warn msg
+  end
+end
+
+ARGV.each do |path|
+  src = path == "-" ? $stdin.read : File.read(path)
+
+  sexp = if $b then
+           Ripper.sexp src
+         else
+           rip = MySexpBuilder.new src
+           rip.yydebug = $d
+           rip.parse
+
+           if rip.error? then
+             warn "skipping"
+             next
+           end
+         end
+
+  puts "accept"
+
+  if $p then
+    pp sexp
+  else
+    p sexp
+  end
+end

data.tar.gz.sig

@@ -1 +1 @@
-FȔCd�qyC0m�cScI�=��
+��C-]̓�˺��)5J��������I�[���Y5��S�t?���e��z�1�Q����e��i��ł��^���BU4�'b�����o!_�ٝ:�Q�"l�l�$c��5n�V6L5{І���	0�}���By�p3�^�H�{���5�����j�y���
I�N^�6�����IZ�DQ݆M�	2w|(|x�uaP�F��L�i���6��M�>5s��@_�%�|�P����[��GP��i���