ruby_parser 3.0.0.a8 → 3.0.0.a9

data/lib/ruby_parser_extras.rb

@@ -1,3 +1,5 @@
+# encoding: ASCII-8BIT
+
 require 'stringio'
 require 'racc/parser'
 require 'sexp'
@@ -54,7 +56,11 @@ class RPStringScanner < StringScanner
   def unread_many str # TODO: remove this entirely - we should not need it
     warn({:unread_many => caller[0]}.inspect) if ENV['TALLY']
     self.extra_lines_added += str.count("\n")
-    string[pos, 0] = str
+    begin
+      string[pos, 0] = str
+    rescue IndexError
+      # HACK -- this is a bandaid on a dirty rag on an open festering wound
+    end
   end
 
   if ENV['DEBUG'] then
@@ -79,11 +85,15 @@ class RPStringScanner < StringScanner
 end
 
 module RubyParserStuff
-  VERSION = '3.0.0.a8' unless constants.include? "VERSION" # SIGH
+  VERSION = '3.0.0.a9' unless constants.include? "VERSION" # SIGH
 
   attr_accessor :lexer, :in_def, :in_single, :file
   attr_reader :env, :comments
 
+  def syntax_error msg
+    raise RubyParser::SyntaxError, msg
+  end
+
   def arg_add(node1, node2) # TODO: nuke
     return s(:arglist, node2) unless node1
 
@@ -192,7 +202,7 @@ module RubyParserStuff
   def block_args19 val, id
     # HACK OMG THIS CODE IS SOOO UGLY! CLEAN ME
     untested = %w[1 2 3 4 7 9 10 12 14]
-    raise "no block_args19 #{id} #{val.inspect}" if untested.include? id
+    raise "no block_args19 #{id}\non: #{val.inspect}" if untested.include? id
 
     r = s(:array)
 
@@ -247,7 +257,7 @@ module RubyParserStuff
         raise "oh noes!: #{r.inspect}"
       end
     else
-      raise "fuck no #{r.inspect}"
+      raise "totally borked: #{r.inspect}"
     end
 
     r
@@ -261,6 +271,8 @@ module RubyParserStuff
     id = lhs.to_sym
     id = id.to_sym if Sexp === id
 
+    raise "write a test 1" if id.to_s =~ /^(?:self|nil|true|false|__LINE__|__FILE__)$/
+
     raise SyntaxError, "Can't change the value of #{id}" if
       id.to_s =~ /^(?:self|nil|true|false|__LINE__|__FILE__)$/
 
@@ -547,8 +559,10 @@ module RubyParserStuff
     # TODO: need a test for this... obviously
     case ref.first
     when :nth_ref then
+      raise "write a test 2"
       raise SyntaxError, "Can't set variable %p" % ref.last
     when :back_ref then
+      raise "write a test 3"
       raise SyntaxError, "Can't set back reference %p" % ref.last
     else
       raise "Unknown backref type: #{ref.inspect}"
@@ -861,6 +875,7 @@ module RubyParserStuff
 
   def new_yield args = nil
     # TODO: raise args.inspect unless [:arglist].include? args.first # HACK
+    raise "write a test 4" if args && args.node_type == :block_pass
     raise SyntaxError, "Block argument should not be given." if
       args && args.node_type == :block_pass
 
@@ -902,6 +917,99 @@ module RubyParserStuff
     lhs
   end
 
+  ##
+  # Returns a UTF-8 encoded string after processing BOMs and magic
+  # encoding comments.
+  #
+  # Holy crap... ok. Here goes:
+  #
+  # Ruby's file handling and encoding support is insane. We need to be
+  # able to lex a file. The lexer file is explicitly UTF-8 to make
+  # things cleaner. This allows us to deal with extended chars in
+  # class and method names. In order to do this, we need to encode all
+  # input source files as UTF-8. First, we look for a UTF-8 BOM by
+  # looking at the first line while forcing its encoding to
+  # ASCII-8BIT. If we find a BOM, we strip it and set the expected
+  # encoding to UTF-8. Then, we search for a magic encoding comment.
+  # If found, it overrides the BOM. Finally, we force the encoding of
+  # the input string to whatever was found, and then encode that to
+  # UTF-8 for compatibility with the lexer.
+
+  def handle_encoding str
+    str = str.dup
+    ruby19 = str.respond_to? :encoding
+    encoding = nil
+
+    header = str.lines.first(2)
+    header.map! { |s| s.force_encoding "ASCII-8BIT" } if ruby19
+
+    first = header.first || ""
+    encoding, str = "utf-8", str[3..-1] if first =~ /\A\xEF\xBB\xBF/
+
+    encoding = $1.strip if header.find { |s|
+      s[/^#.*?-\*-.*?coding:\s*([^ ;]+).*?-\*-/, 1] ||
+      s[/^#.*(?:en)?coding(?:\s*[:=])\s*([\w-]+)/, 1]
+    }
+
+    if encoding then
+      if ruby19 then
+        encoding.sub!(/utf-8-.+$/, 'utf-8') # HACK for stupid emacs formats
+        hack_encoding str, encoding
+      else
+        warn "Skipping magic encoding comment"
+      end
+    else
+      # nothing specified... ugh. try to encode as utf-8
+      if ruby19 then
+        begin
+          str.encode! "utf-8"
+        rescue Encoding::InvalidByteSequenceError, Encoding::UndefinedConversionError
+          # OK... You really suck. You have extended chars but didn't
+          # specify what they were. Now we try to force it and double
+          # check that it is valid.
+
+          hack_encoding str
+        end
+      end
+    end
+
+    str
+  end
+
+  def hack_encoding str, extra = nil
+    # this is in sorted order of occurrence according to
+    # charlock_holmes against 500k files
+    encodings = [
+                 extra,
+                 Encoding::ISO_8859_1,
+                 Encoding::UTF_8,
+                 Encoding::ISO_8859_2,
+                 Encoding::ISO_8859_9,
+                 Encoding::SHIFT_JIS,
+                 Encoding::WINDOWS_1252,
+                 Encoding::EUC_JP,
+                ].compact
+
+    # terrible, horrible, no good, very bad, last ditch effort.
+    encodings.each do |enc|
+      begin
+        str.force_encoding enc
+        if str.valid_encoding? then
+          str.encode! Encoding::UTF_8
+          break
+        end
+      rescue Encoding::InvalidByteSequenceError
+        # do nothing
+      rescue Encoding::UndefinedConversionError
+        # do nothing
+      end
+    end
+
+    # no amount of pain is enough for you.
+    raise "Bad encoding. Need a magic encoding comment." unless
+      str.encoding.name == "UTF-8"
+  end
+
   ##
   # Parse +str+ at path +file+ and return a sexp. Raises
   # Timeout::Error if it runs for more than +time+ seconds.
@@ -910,20 +1018,9 @@ module RubyParserStuff
     Timeout.timeout time do
       raise "bad val: #{str.inspect}" unless String === str
 
-      str.lines.first(2).find { |s| s[/^# encoding: (.+)/, 1] }
-      encoding = $1
-
-      str = str.dup
-
-      if encoding then
-        if defined?(Encoding) then
-          str.force_encoding(encoding).encode! "utf-8"
-        else
-          warn "Skipping magic encoding comment"
-        end
-      end
+      str = handle_encoding str
 
-      self.file = file
+      self.file = file.dup
       self.lexer.src = str
 
       @yydebug = ENV.has_key? 'DEBUG'
@@ -951,8 +1048,15 @@ module RubyParserStuff
     self.comments.clear
   end
 
+  def block_dup_check call_or_args, block
+    syntax_error "Both block arg and actual block given." if
+      block and call_or_args.block_pass?
+  end
+
   def ret_args node
     if node then
+      raise "write a test 5" if node[0] == :block_pass
+
       raise SyntaxError, "block argument should not be given" if
         node[0] == :block_pass
 
@@ -1169,11 +1273,14 @@ module RubyParserStuff
   end
 
   class StackState
+    attr_reader :name
     attr_reader :stack
+    attr_accessor :debug
 
     def initialize(name)
       @name = name
       @stack = [false]
+      @debug = false
     end
 
     def inspect
@@ -1181,10 +1288,12 @@ module RubyParserStuff
     end
 
     def is_in_state
+      p :stack_is_in_state => [name, @stack.last, caller.first] if debug
       @stack.last
     end
 
     def lexpop
+      p :stack_lexpop => caller.first if debug
       raise if @stack.size == 0
       a = @stack.pop
       b = @stack.pop
@@ -1193,12 +1302,15 @@ module RubyParserStuff
 
     def pop
       r = @stack.pop
+      p :stack_pop => [name, r, @stack, caller.first] if debug
       @stack.push false if @stack.size == 0
       r
     end
 
     def push val
       @stack.push val
+      p :stack_push => [name, @stack, caller.first] if debug
+      nil
     end
   end
 end
@@ -1217,6 +1329,8 @@ end
 # parse error.
 
 class RubyParser
+  class SyntaxError < RuntimeError; end
+
   def initialize
     @p18 = Ruby18Parser.new
     @p19 = Ruby19Parser.new
@@ -1239,9 +1353,11 @@ end
 ############################################################
 # HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK
 
-class Symbol
-  def is_argument # TODO: phase this out
-    return self == :expr_arg || self == :expr_cmdarg
+unless "".respond_to?(:grep) then
+  class String
+    def grep re
+      lines.grep re
+    end
   end
 end
 
@@ -1270,6 +1386,10 @@ class Sexp
     raise "no: #{self.inspect}.add_all #{x.inspect}" # TODO: need a test to trigger this
   end
 
+  def block_pass?
+    any? { |s| Sexp === s && s[0] == :block_pass }
+  end
+
   alias :node_type :sexp_type
   alias :values :sexp_body # TODO: retire
 end

data/test/test_ruby_lexer.rb

@@ -318,7 +318,7 @@ class TestRubyLexer < MiniTest::Unit::TestCase
   end
 
   def test_yylex_cvar_bad
-    assert_raises SyntaxError do
+    assert_raises RubyParser::SyntaxError do
       util_lex_token "@@1"
     end
   end
@@ -744,6 +744,22 @@ class TestRubyLexer < MiniTest::Unit::TestCase
                    :tASSOC, "=>")
   end
 
+  def test_yylex_identifier_equals3
+    # @lex.lex_state = :expr_fname
+    util_lex_token(":a===b",
+                   :tSYMBOL, "a",
+                   :tEQQ, "===",
+                   :tIDENTIFIER, "b")
+  end
+
+  def test_yylex_identifier_equals_equals_arrow
+    # @lex.lex_state = :expr_fname
+    util_lex_token(":a==>b",
+                   :tSYMBOL, "a=",
+                   :tASSOC, "=>",
+                   :tIDENTIFIER, "b")
+  end
+
   def test_yylex_identifier_equals_caret
     util_lex_fname "^", :tCARET
   end
@@ -1136,6 +1152,45 @@ class TestRubyLexer < MiniTest::Unit::TestCase
     util_lex_token "+@", :tUPLUS, "+@"
   end
 
+  def test_yylex_numbers
+    util_lex_token "0b10", :tINTEGER, 2
+    util_lex_token "0B10", :tINTEGER, 2
+
+    util_lex_token "0d10", :tINTEGER, 10
+    util_lex_token "0D10", :tINTEGER, 10
+
+    util_lex_token "0x10", :tINTEGER, 16
+    util_lex_token "0X10", :tINTEGER, 16
+
+    util_lex_token "0o10", :tINTEGER, 8
+    util_lex_token "0O10", :tINTEGER, 8
+    util_lex_token "0o",   :tINTEGER, 0
+    util_lex_token "0O",   :tINTEGER, 0
+
+    util_lex_token "0o",   :tINTEGER, 0
+    util_lex_token "0O",   :tINTEGER, 0
+
+    util_lex_token "0",    :tINTEGER, 0
+
+    util_bad_token "0x"
+    util_bad_token "0X"
+    util_bad_token "0b"
+    util_bad_token "0B"
+    util_bad_token "0d"
+    util_bad_token "0D"
+
+    util_bad_token "08"
+    util_bad_token "09"
+    util_bad_token "0o8"
+    util_bad_token "0o9"
+    util_bad_token "0O8"
+    util_bad_token "0O9"
+
+    util_bad_token "1_e1"
+    util_bad_token "1_.1"
+    util_bad_token "1__1"
+  end
+
   def test_yylex_plus_unary_number
     util_lex_token("+42",
                    :tINTEGER, 42)
@@ -1867,7 +1922,7 @@ class TestRubyLexer < MiniTest::Unit::TestCase
   ############################################################
 
   def util_bad_token s, *args
-    assert_raises SyntaxError do
+    assert_raises RubyParser::SyntaxError do
       util_lex_token s, *args
     end
   end
@@ -1879,7 +1934,7 @@ class TestRubyLexer < MiniTest::Unit::TestCase
 
   def util_escape_bad input
     @lex.src = input
-    assert_raises SyntaxError do
+    assert_raises RubyParser::SyntaxError do
       @lex.read_escape
     end
   end

data/test/test_ruby_parser.rb

@@ -33,6 +33,17 @@ class RubyParserTestCase < ParseTreeTestCase
     assert_equal pt, result
   end
 
+  def assert_syntax_error rb, emsg
+    e = nil
+    assert_silent do
+      e = assert_raises RubyParser::SyntaxError do
+        processor.parse rb
+      end
+    end
+
+    assert_equal emsg, e.message.strip # TODO: why strip?
+  end
+
   def assert_parse_error rb, emsg
     e = nil
     out, err = capture_io do
@@ -59,6 +70,58 @@ module TestRubyParserShared
     # p :test => [self.class, __name__]
   end
 
+  BLOCK_DUP_MSG = "Both block arg and actual block given."
+
+  def test_double_block_error_01
+    assert_syntax_error "a(1, &b) { }", BLOCK_DUP_MSG
+  end
+
+  def test_double_block_error_02
+    assert_syntax_error "a(1, &b) do end", BLOCK_DUP_MSG
+  end
+
+  def test_double_block_error_03
+    assert_syntax_error "a 1, &b do end", BLOCK_DUP_MSG
+  end
+
+  def test_double_block_error_04
+    assert_syntax_error "m.a(1, &b) { }", BLOCK_DUP_MSG
+  end
+
+  def test_double_block_error_05
+    assert_syntax_error "m.a(1, &b) do end", BLOCK_DUP_MSG
+  end
+
+  def test_double_block_error_06
+    assert_syntax_error "m.a 1, &b do end", BLOCK_DUP_MSG
+  end
+
+  def test_double_block_error_07
+    assert_syntax_error "m::a(1, &b) { }", BLOCK_DUP_MSG
+  end
+
+  def test_double_block_error_08
+    assert_syntax_error "m::a(1, &b) do end", BLOCK_DUP_MSG
+  end
+
+  def test_double_block_error_09
+    assert_syntax_error "m::a 1, &b do end", BLOCK_DUP_MSG
+  end
+
+  def test_wtf_7
+    assert_parse "a.b (1) {c}", s(:iter,
+                                  s(:call, s(:call, nil, :a), :b, s(:lit, 1)),
+                                  nil,
+                                  s(:call, nil, :c))
+  end
+
+  def test_wtf_8
+    assert_parse "a::b (1) {c}", s(:iter,
+                                  s(:call, s(:call, nil, :a), :b, s(:lit, 1)),
+                                  nil,
+                                  s(:call, nil, :c))
+  end
+
   def test_attrasgn_array_lhs
     rb = '[1, 2, 3, 4][from .. to] = ["a", "b", "c"]'
     pt = s(:attrasgn,
@@ -808,6 +871,8 @@ module TestRubyParserShared
       end
     EOM
 
+    rb.force_encoding "ASCII-8BIT" if rb.respond_to? :force_encoding
+
     # TODO: class vars
     # TODO: odd-ternary: a ?bb : c
     # TODO: globals
@@ -968,6 +1033,34 @@ class TestRuby18Parser < RubyParserTestCase
 
     assert_parse rb, pt
   end
+
+  def test_double_block_error_10
+    assert_syntax_error "a.b (&b) {}", BLOCK_DUP_MSG
+  end
+
+  def test_double_block_error_11
+    assert_syntax_error "a (1, &b) { }", BLOCK_DUP_MSG
+  end
+
+  def test_double_block_error_12
+    assert_syntax_error "a (1, &b) do end", BLOCK_DUP_MSG
+  end
+
+  def test_double_block_error_13
+    assert_syntax_error "m.a (1, &b) { }", BLOCK_DUP_MSG
+  end
+
+  def test_double_block_error_14
+    assert_syntax_error "m.a (1, &b) do end", BLOCK_DUP_MSG
+  end
+
+  def test_double_block_error_15
+    assert_syntax_error "m::a (1, &b) { }", BLOCK_DUP_MSG
+  end
+
+  def test_double_block_error_16
+    assert_syntax_error "m::a (1, &b) do end", BLOCK_DUP_MSG
+  end
 end
 
 class TestRuby19Parser < RubyParserTestCase
@@ -1193,7 +1286,7 @@ class TestRuby19Parser < RubyParserTestCase
     rb = "a { |b = 1| }"
     pt = s(:iter,
            s(:call, nil, :a),
-           s(:masgn, s(:array, s(:lasgn, :b, s(:lit, 1)))))
+           s(:args, :b, s(:block, s(:lasgn, :b, s(:lit, 1)))))
 
     assert_parse rb, pt
   end
@@ -1238,7 +1331,7 @@ class TestRuby19Parser < RubyParserTestCase
   #   pt = s(:iter,
   #          s(:call, nil, :lambda),
   #          s(:args, :a, :b,
-  #            s(:block, s(:lasgn, :b, s(nil)))),
+  #            s(:block, s(:lasgn, :b, s(:nil)))),
   #          s(:call, nil, :p, s(:array, s(:lvar, :a), s(:lvar, :b))))
   #
   #   assert_parse rb, pt
@@ -1247,4 +1340,102 @@ class TestRuby19Parser < RubyParserTestCase
   #
   #   assert_parse rb, pt
   # end
+
+  def test_block_args_opt1
+    rb = "f { |a, b = 42| [a, b] }"
+    pt = s(:iter,
+           s(:call, nil, :f),
+           s(:args, :a, :b,
+             s(:block, s(:lasgn, :b, s(:lit, 42)))),
+           s(:array, s(:lvar, :a), s(:lvar, :b)))
+
+    assert_parse rb, pt
+  end
+
+  def test_block_args_opt2
+    rb = "f { |a, b = 42, c = 24| [a, b, c] }"
+    pt = s(:iter,
+           s(:call, nil, :f),
+           s(:args, :a, :b, :c,
+             s(:block, s(:lasgn, :b, s(:lit, 42)), s(:lasgn, :c, s(:lit, 24)))),
+           s(:array, s(:lvar, :a), s(:lvar, :b), s(:lvar, :c)))
+
+    assert_parse rb, pt
+  end
+
+  def test_block_args_opt3
+    rb = "f { |a, b = 42, c = 24, &d| [a, b, c, d] }"
+    pt = s(:iter,
+           s(:call, nil, :f),
+           s(:args, :a, :b, :c, :"&d",
+             s(:block, s(:lasgn, :b, s(:lit, 42)), s(:lasgn, :c, s(:lit, 24)))),
+           s(:array, s(:lvar, :a), s(:lvar, :b), s(:lvar, :c), s(:lvar, :d)))
+
+    assert_parse rb, pt
+  end
+
+  def test_i_have_no_freakin_clue
+    rb = "1 ? b('') : 2\na d: 3"
+    pt = s(:block,
+           s(:if, s(:lit, 1), s(:call, nil, :b, s(:str, "")), s(:lit, 2)),
+           s(:call, nil, :a, s(:hash, s(:lit, :d), s(:lit, 3))))
+
+    assert_parse rb, pt
+  end
+
+  def test_motherfuckin_leading_dots
+    rb = "a\n.b"
+    pt = s(:call, s(:call, nil, :a), :b)
+
+    assert_parse rb, pt
+  end
+
+  def test_motherfuckin_leading_dots2
+    rb = "a\n..b"
+
+    assert_parse_error rb, 'parse error on value ".." (tDOT2)'
+  end
+
+  def test_kill_me
+    rb = "f { |a, (b, *c)| }"
+    pt = s(:iter,
+           s(:call, nil, :f),
+           s(:masgn,
+             s(:array,
+               s(:lasgn, :a),
+               s(:masgn,
+                 s(:array,
+                   s(:lasgn, :b),
+                   s(:splat, :c)))))) # TODO: omg this is so horrible
+
+    assert_parse rb, pt
+  end
+
+  def test_kill_me2
+    rb = "f { |*a, b| }"
+    pt = s(:iter, s(:call, nil, :f), s(:args, :"*a", :b))
+
+    assert_parse rb, pt
+  end
+
+  def test_kill_me3
+    rb = "f { |*a, b, &c| }"
+    pt = s(:iter, s(:call, nil, :f), s(:args, :"*a", :b, :"&c"))
+
+    assert_parse rb, pt
+  end
+
+  def test_kill_me4
+    rb = "a=b ? true: false"
+    pt = s(:lasgn, :a, s(:if, s(:call, nil, :b), s(:true), s(:false)))
+
+    assert_parse rb, pt
+  end
+
+  # def test_kill_me5
+  #   rb = "f ->() { g do end }"
+  #   pt = 42
+  #
+  #   assert_parse rb, pt
+  # end
 end