ruby_parser 3.8.4 → 3.9.0

data/lib/ruby_parser_extras.rb

@@ -1,97 +1,13 @@
 # encoding: ASCII-8BIT
 
-require 'stringio'
-require 'racc/parser'
-require 'sexp'
-require 'strscan'
-require 'ruby_lexer'
+require "sexp"
+require "ruby_lexer"
 require "timeout"
-
-# :stopdoc:
-# WHY do I have to do this?!?
-class Regexp
-  ONCE = 0 unless defined? ONCE # FIX: remove this - it makes no sense
-
-  unless defined? ENC_NONE then
-    ENC_NONE = /x/n.options
-    ENC_EUC  = /x/e.options
-    ENC_SJIS = /x/s.options
-    ENC_UTF8 = /x/u.options
-  end
-end
-
-# I hate ruby 1.9 string changes
-class Fixnum
-  def ord
-    self
-  end
-end unless "a"[0] == "a"
-# :startdoc:
-
-class RPStringScanner < StringScanner
-#   if ENV['TALLY'] then
-#     alias :old_getch :getch
-#     def getch
-#       warn({:getch => caller[0]}.inspect)
-#       old_getch
-#     end
-#   end
-
-  if "".respond_to? :encoding then
-    if "".respond_to? :byteslice then
-      def string_to_pos
-        string.byteslice(0, pos)
-      end
-    else
-      def string_to_pos
-        string.bytes.first(pos).pack("c*").force_encoding(string.encoding)
-      end
-    end
-
-    def charpos
-      string_to_pos.length
-    end
-  else
-    alias :charpos :pos
-
-    def string_to_pos
-      string[0..pos]
-    end
-  end
-
-  def unread_many str # TODO: remove this entirely - we should not need it
-    warn({:unread_many => caller[0]}.inspect) if ENV['TALLY']
-    begin
-      string[charpos, 0] = str
-    rescue IndexError
-      # HACK -- this is a bandaid on a dirty rag on an open festering wound
-    end
-  end
-
-  if ENV['DEBUG'] then
-    alias :old_getch :getch
-    def getch
-      c = self.old_getch
-      p :getch => [c, caller.first]
-      c
-    end
-
-    alias :old_scan :scan
-    def scan re
-      s = old_scan re
-      where = caller[1].split(/:/).first(2).join(":")
-      d :scan => [s, where] if s
-      s
-    end
-  end
-
-  def d o
-    $stderr.puts o.inspect
-  end
-end
+require "rp_extensions"
+require "rp_stringscanner"
 
 module RubyParserStuff
-  VERSION = "3.8.4" unless constants.include? "VERSION" # SIGH
+  VERSION = "3.9.0"
 
   attr_accessor :lexer, :in_def, :in_single, :file
   attr_reader :env, :comments
@@ -107,7 +23,14 @@ module RubyParserStuff
     raise "not yet #{n} #{v.inspect} => #{r.inspect}" unless $good20[n]
   end
 
-  ruby19 = "".respond_to? :encoding
+  def self.deprecate old, new
+    define_method old do |*args|
+      warn "DEPRECATED: #{old} -> #{new} from #{caller.first}"
+      send new, *args
+    end
+  end
+
+  has_enc = "".respond_to? :encoding
 
   # This is in sorted order of occurrence according to
   # charlock_holmes against 500k files, with UTF_8 forced
@@ -122,7 +45,7 @@ module RubyParserStuff
     Encoding::SHIFT_JIS,
     Encoding::WINDOWS_1252,
     Encoding::EUC_JP
-  ] if ruby19
+  ] if has_enc
 
   def syntax_error msg
     raise RubyParser::SyntaxError, msg
@@ -341,7 +264,7 @@ module RubyParserStuff
     _racc_do_parse_rb(_racc_setup, false)
   end if ENV['PURE_RUBY']
 
-  def get_match_node lhs, rhs # TODO: rename to new_match
+  def new_match lhs, rhs
     if lhs then
       case lhs[0]
       when :dregx, :dregx_once then
@@ -363,6 +286,9 @@ module RubyParserStuff
     return new_call(lhs, :"=~", argl(rhs)).line(lhs.line)
   end
 
+  # TODO: remove in 4.0 or 2018-01, whichever is first
+  deprecate :get_match_node, :new_match
+
   def gettable(id)
     lineno = id.lineno if id.respond_to? :lineno
     id = id.to_sym if String === id
@@ -477,7 +403,7 @@ module RubyParserStuff
     return head
   end
 
-  def logop(type, left, right) # TODO: rename logical_op
+  def logical_op type, left, right
     left = value_expr left
 
     if left and left[0] == type and not left.paren then
@@ -495,6 +421,9 @@ module RubyParserStuff
     return s(type, left, right)
   end
 
+  # TODO: remove in 4.0 or 2018-01, whichever is first
+  deprecate :logop, :logical_op
+
   def new_aref val
     val[2] ||= s(:arglist)
     val[2][0] = :arglist if val[2][0] == :array # REFACTOR
@@ -691,6 +620,10 @@ module RubyParserStuff
     result
   end
 
+  def new_hash val
+    s(:hash, *val[2].values).line(val[1])
+  end
+
   def new_if c, t, f
     l = [c.line, t && t.line, f && f.line].compact.min
     c = cond c
@@ -1028,7 +961,7 @@ module RubyParserStuff
     end
   end
 
-  def node_assign(lhs, rhs) # TODO: rename new_assign
+  def new_assign lhs, rhs
     return nil unless lhs
 
     rhs = value_expr rhs
@@ -1046,6 +979,9 @@ module RubyParserStuff
     lhs
   end
 
+  # TODO: remove in 4.0 or 2018-01, whichever is first
+  deprecate :node_assign, :new_assign
+
   ##
   # Returns a UTF-8 encoded string after processing BOMs and magic
   # encoding comments.
@@ -1066,11 +1002,11 @@ module RubyParserStuff
 
   def handle_encoding str
     str = str.dup
-    ruby19 = str.respond_to? :encoding
+    has_enc = str.respond_to? :encoding
     encoding = nil
 
     header = str.lines.first(2)
-    header.map! { |s| s.force_encoding "ASCII-8BIT" } if ruby19
+    header.map! { |s| s.force_encoding "ASCII-8BIT" } if has_enc
 
     first = header.first || ""
     encoding, str = "utf-8", str[3..-1] if first =~ /\A\xEF\xBB\xBF/
@@ -1081,7 +1017,7 @@ module RubyParserStuff
     }
 
     if encoding then
-      if ruby19 then
+      if has_enc then
         encoding.sub!(/utf-8-.+$/, 'utf-8') # HACK for stupid emacs formats
         hack_encoding str, encoding
       else
@@ -1089,7 +1025,7 @@ module RubyParserStuff
       end
     else
       # nothing specified... ugh. try to encode as utf-8
-      hack_encoding str if ruby19
+      hack_encoding str if has_enc
     end
 
     str
@@ -1421,142 +1357,3 @@ module RubyParserStuff
     end
   end
 end
-
-class Ruby23Parser < Racc::Parser
-  include RubyParserStuff
-end
-
-class Ruby22Parser < Racc::Parser
-  include RubyParserStuff
-end
-
-class Ruby21Parser < Racc::Parser
-  include RubyParserStuff
-end
-
-class Ruby20Parser < Racc::Parser
-  include RubyParserStuff
-end
-
-class Ruby19Parser < Racc::Parser
-  include RubyParserStuff
-end
-
-class Ruby18Parser < Racc::Parser
-  include RubyParserStuff
-end
-
-##
-# RubyParser is a compound parser that first attempts to parse using
-# the 1.9 syntax parser and falls back to the 1.8 syntax parser on a
-# parse error.
-
-class RubyParser
-  class SyntaxError < RuntimeError; end
-
-  def initialize
-    @p18 = Ruby18Parser.new
-    @p19 = Ruby19Parser.new
-    @p20 = Ruby20Parser.new
-    @p21 = Ruby21Parser.new
-    @p22 = Ruby22Parser.new
-    @p23 = Ruby23Parser.new
-  end
-
-  def process s, f = "(string)", t = 10
-    e = nil
-    [@p23, @p22, @p21, @p20, @p19, @p18].each do |parser|
-      begin
-        return parser.process s, f, t
-      rescue Racc::ParseError, RubyParser::SyntaxError => exc
-        e = exc
-      end
-    end
-    raise e
-  end
-
-  alias :parse :process
-
-  def reset
-    @p18.reset
-    @p19.reset
-    @p20.reset
-    @p21.reset
-    @p22.reset
-    @p23.reset
-  end
-
-  def self.for_current_ruby
-    case RUBY_VERSION
-    when /^1\.8/ then
-      Ruby18Parser.new
-    when /^1\.9/ then
-      Ruby19Parser.new
-    when /^2.0/ then
-      Ruby20Parser.new
-    when /^2.1/ then
-      Ruby21Parser.new
-    when /^2.2/ then
-      Ruby22Parser.new
-    when /^2.3/ then
-      Ruby23Parser.new
-    else
-      raise "unrecognized RUBY_VERSION #{RUBY_VERSION}"
-    end
-  end
-end
-
-############################################################
-# HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK
-
-unless "".respond_to?(:grep) then
-  class String
-    def grep re
-      lines.grep re
-    end
-  end
-end
-
-class String
-  ##
-  # This is a hack used by the lexer to sneak in line numbers at the
-  # identifier level. This should be MUCH smaller than making
-  # process_token return [value, lineno] and modifying EVERYTHING that
-  # reduces tIDENTIFIER.
-
-  attr_accessor :lineno
-end
-
-class Sexp
-  attr_writer :paren
-
-  def paren
-    @paren ||= false
-  end
-
-  def value
-    raise "multi item sexp" if size > 2
-    last
-  end
-
-  def to_sym
-    raise "no: #{self.inspect}.to_sym is a bug"
-    self.value.to_sym
-  end
-
-  alias :add :<<
-
-  def add_all x
-    self.concat x.sexp_body
-  end
-
-  def block_pass?
-    any? { |s| Sexp === s && s[0] == :block_pass }
-  end
-
-  alias :node_type :sexp_type
-  alias :values :sexp_body # TODO: retire
-end
-
-# END HACK
-############################################################

data/test/test_ruby_lexer.rb

@@ -1,10 +1,8 @@
 # encoding: US-ASCII
 
-require 'rubygems'
-require 'minitest/autorun'
-require 'ruby_lexer'
-require 'ruby18_parser'
-require 'ruby20_parser'
+require "minitest/autorun"
+require "ruby_lexer"
+require "ruby_parser"
 
 class TestRubyLexer < Minitest::Test
   attr_accessor :processor, :lex, :parser_class, :lex_state
@@ -14,7 +12,7 @@ class TestRubyLexer < Minitest::Test
 
   def setup
     self.lex_state = :expr_beg
-    setup_lexer_class Ruby20Parser
+    setup_lexer_class RubyParser.latest.class
   end
 
   def setup_lexer input, exp_sexp = nil
@@ -53,6 +51,10 @@ class TestRubyLexer < Minitest::Test
     assert_lex(input, exp_sexp, *args, &block)
   end
 
+  def ruby18
+    RubyParser::V18 === lexer.parser
+  end
+
   def refute_lex input, *args # TODO: re-sort
     args = args.each_slice(2).map { |a, b| [a, b, nil, nil, nil] }.flatten
 
@@ -248,13 +250,13 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_yylex_and_dot
-    setup_lexer_class Ruby23Parser
+    setup_lexer_class RubyParser::V23
 
     assert_lex3("&.", nil, :tLONELY, "&.", :expr_dot)
   end
 
   def test_yylex_and_dot_call
-    setup_lexer_class Ruby23Parser
+    setup_lexer_class RubyParser::V23
 
     assert_lex3("x&.y", nil,
                 :tIDENTIFIER, "x", :expr_cmdarg,
@@ -262,6 +264,15 @@ class TestRubyLexer < Minitest::Test
                 :tIDENTIFIER, "y")
   end
 
+  def test_yylex_and_dot_call_newline
+    setup_lexer_class Ruby23Parser
+
+    assert_lex3("x\n&.y", nil,
+                :tIDENTIFIER, "x", :expr_cmdarg,
+                :tLONELY, "&.", :expr_dot,
+                :tIDENTIFIER, "y")
+  end
+
   def test_yylex_and_arg
     self.lex_state = :expr_arg
 
@@ -294,7 +305,7 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_yylex_label__18
-    setup_lexer_class Ruby18Parser
+    setup_lexer_class RubyParser::V18
 
     assert_lex3("{a:",
                 nil,
@@ -304,7 +315,7 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_yylex_label_in_params__18
-    setup_lexer_class Ruby18Parser
+    setup_lexer_class RubyParser::V18
 
     assert_lex3("foo(a:",
                 nil,
@@ -315,7 +326,7 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_yylex_label__19
-    setup_lexer_class Ruby19Parser
+    setup_lexer_class RubyParser::V19
 
     assert_lex3("{a:",
                 nil,
@@ -324,7 +335,7 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_yylex_label_in_params__19
-    setup_lexer_class Ruby19Parser
+    setup_lexer_class RubyParser::V19
 
     assert_lex3("foo(a:",
                 nil,
@@ -383,7 +394,9 @@ class TestRubyLexer < Minitest::Test
     refute_lexeme
   end
 
-  def test_yylex_not_at_defn
+  def test_yylex_not_at_defn__20
+    setup_lexer_class RubyParser::V20
+
     assert_lex("def +@; end",
                s(:defn, :+@, s(:args), s(:nil)),
 
@@ -467,7 +480,7 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_yylex_lambda_args__20
-    setup_lexer_class Ruby20Parser
+    setup_lexer_class RubyParser::V20
 
     assert_lex("-> (a) { }",
                s(:iter, s(:call, nil, :lambda),
@@ -482,7 +495,7 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_yylex_lambda_args_opt__20
-    setup_lexer_class Ruby20Parser
+    setup_lexer_class RubyParser::V20
 
     assert_lex("-> (a=nil) { }",
                s(:iter, s(:call, nil, :lambda),
@@ -499,7 +512,7 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_yylex_lambda_hash__20
-    setup_lexer_class Ruby20Parser
+    setup_lexer_class RubyParser::V20
 
     assert_lex("-> (a={}) { }",
                s(:iter, s(:call, nil, :lambda),
@@ -1277,13 +1290,13 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_yylex_identifier_def__18
-    setup_lexer_class Ruby18Parser
+    setup_lexer_class RubyParser::V18
 
     assert_lex_fname "identifier", :tIDENTIFIER, :expr_end
   end
 
   def test_yylex_identifier_def__1920
-    setup_lexer_class Ruby19Parser
+    setup_lexer_class RubyParser::V19
 
     assert_lex_fname "identifier", :tIDENTIFIER, :expr_endfn
   end
@@ -1320,13 +1333,13 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_yylex_identifier_equals_def__18
-    setup_lexer_class Ruby18Parser
+    setup_lexer_class RubyParser::V18
 
     assert_lex_fname "identifier=", :tIDENTIFIER, :expr_end
   end
 
   def test_yylex_identifier_equals_def__1920
-    setup_lexer_class Ruby19Parser
+    setup_lexer_class RubyParser::V19
 
     assert_lex_fname "identifier=", :tIDENTIFIER, :expr_endfn
   end
@@ -1429,25 +1442,25 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_yylex_question_eh_a__18
-    setup_lexer_class Ruby18Parser
+    setup_lexer_class RubyParser::V18
 
     assert_lex3("?a", nil, :tINTEGER, 97, :expr_end)
   end
 
   def test_yylex_question_eh_a__19
-    setup_lexer_class Ruby19Parser
+    setup_lexer_class RubyParser::V19
 
     assert_lex3("?a", nil, :tSTRING, "a", :expr_end)
   end
 
   def test_yylex_question_eh_escape_M_escape_C__18
-    setup_lexer_class Ruby18Parser
+    setup_lexer_class RubyParser::V18
 
     assert_lex3("?\\M-\\C-a", nil, :tINTEGER, 129, :expr_end)
   end
 
   def test_yylex_question_eh_escape_M_escape_C__19
-    setup_lexer_class Ruby19Parser
+    setup_lexer_class RubyParser::V19
 
     assert_lex3("?\\M-\\C-a", nil, :tSTRING, "\M-\C-a", :expr_end)
   end
@@ -1632,14 +1645,14 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_yylex_open_bracket_exprarg__18
-    setup_lexer_class Ruby18Parser
+    setup_lexer_class RubyParser::V18
     self.lex_state = :expr_arg
 
     assert_lex3(" (", nil, :tLPAREN2, "(", :expr_beg)
   end
 
   def test_yylex_open_bracket_exprarg__19
-    setup_lexer_class Ruby19Parser
+    setup_lexer_class RubyParser::V19
     self.lex_state = :expr_arg
 
     assert_lex3(" (", nil, :tLPAREN_ARG, "(", :expr_beg)
@@ -1804,13 +1817,13 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_yylex_question__18
-    setup_lexer_class Ruby18Parser
+    setup_lexer_class RubyParser::V18
 
     assert_lex3("?*", nil, :tINTEGER, 42, :expr_end)
   end
 
   def test_yylex_question__19
-    setup_lexer_class Ruby19Parser
+    setup_lexer_class RubyParser::V19
 
     assert_lex3("?*", nil, :tSTRING, "*", :expr_end)
   end
@@ -1829,7 +1842,7 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_yylex_question_ws_backslashed__18
-    setup_lexer_class Ruby18Parser
+    setup_lexer_class RubyParser::V18
 
     assert_lex3("?\\ ", nil, :tINTEGER, 32, :expr_end)
     assert_lex3("?\\n", nil, :tINTEGER, 10, :expr_end)
@@ -1840,7 +1853,7 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_yylex_question_ws_backslashed__19
-    setup_lexer_class Ruby19Parser
+    setup_lexer_class RubyParser::V19
 
     assert_lex3("?\\ ", nil, :tSTRING, " ",  :expr_end)
     assert_lex3("?\\n", nil, :tSTRING, "\n", :expr_end)
@@ -2243,7 +2256,7 @@ class TestRubyLexer < Minitest::Test
 
   def test_yylex_string_double_escape_M
     chr = "\341"
-    chr.force_encoding("UTF-8") if RubyLexer::RUBY19
+    chr.force_encoding("UTF-8") if RubyLexer::HAS_ENC
 
     assert_lex3("\"\\M-a\"", nil, :tSTRING, chr, :expr_end)
   end
@@ -2333,7 +2346,7 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_yylex_string_double_pound_dollar_bad
-    skip if Ruby18Parser === lexer.parser
+    skip if ruby18
 
     assert_lex3('"#$%"', nil,
 
@@ -2515,7 +2528,7 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_yylex_symbol_zero_byte__18
-    setup_lexer_class Ruby18Parser
+    setup_lexer_class RubyParser::V18
 
     refute_lex(":\"symbol\0\"", :tSYMBEG, ":")
   end
@@ -2650,7 +2663,7 @@ class TestRubyLexer < Minitest::Test
 
   def test_pct_w_backslashes
     ["\t", "\n", "\r", "\v", "\f"].each do |char|
-      next if !RubyLexer::RUBY19 and char == "\v"
+      next if !RubyLexer::HAS_ENC and char == "\v"
 
       assert_lex("%w[foo#{char}bar]",
                  s(:array, s(:str, "foo"), s(:str, "bar")),
@@ -2682,7 +2695,7 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_yylex_hash_colon_quoted_22
-    setup_lexer_class Ruby22Parser
+    setup_lexer_class RubyParser::V22
 
     assert_lex("{'a':1}",
                s(:hash, s(:lit, :a), s(:lit, 1)),
@@ -2694,7 +2707,7 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_yylex_hash_colon_quoted_symbol_22
-    setup_lexer_class Ruby22Parser
+    setup_lexer_class RubyParser::V22
 
     assert_lex("{'abc': :b}",
                s(:hash, s(:lit, :abc), s(:lit, :b)),
@@ -2706,7 +2719,7 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_yylex_hash_colon_double_quoted_symbol_22
-    setup_lexer_class Ruby22Parser
+    setup_lexer_class RubyParser::V22
 
     assert_lex('{"abc": :b}',
                s(:hash, s(:lit, :abc), s(:lit, :b)),
@@ -2718,7 +2731,7 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_ruby21_rational_literal
-    setup_lexer_class Ruby21Parser
+    setup_lexer_class RubyParser::V21
 
     assert_lex3("10r",      nil, :tRATIONAL, Rational(10), :expr_end)
     assert_lex3("0x10r",      nil, :tRATIONAL, Rational(16), :expr_end)
@@ -2737,7 +2750,7 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_ruby21_imaginary_literal
-    setup_lexer_class Ruby21Parser
+    setup_lexer_class RubyParser::V21
 
     assert_lex3("1i",       nil, :tIMAGINARY, Complex(0, 1), :expr_end)
     assert_lex3("0x10i",    nil, :tIMAGINARY, Complex(0, 16), :expr_end)
@@ -2756,7 +2769,7 @@ class TestRubyLexer < Minitest::Test
   end
 
   def test_ruby21_rational_imaginary_literal
-    setup_lexer_class Ruby21Parser
+    setup_lexer_class RubyParser::V21
 
     assert_lex3("1ri",       nil, :tIMAGINARY, Complex(0, Rational(1)), :expr_end)
     assert_lex3("0x10ri",    nil, :tIMAGINARY, Complex(0, Rational(16)), :expr_end)
@@ -2777,7 +2790,7 @@ class TestRubyLexer < Minitest::Test
   def test_ruby21_imaginary_literal_with_succeeding_keyword
     skip "Currently does not tokenize correctly"
 
-    setup_lexer_class Ruby21Parser
+    setup_lexer_class RubyParser::V21
 
     assert_lex3("1if", nil,
                 :tINTEGER, 1, :expr_end,