ruby_parser 3.0.0.a10 → 3.0.0

data.tar.gz.sig

@@ -1 +1 @@
-�%���Ed�=�b�|��7�R46�ʼn�3w9��Cw�=�O��
+FȔCd�qyC0m�cScI�=��

data/History.txt

@@ -1,3 +1,23 @@
+=== 3.0.0 / 2012-11-02
+
+I've hit 99.967% success rate against 558k files! 3.6σ!! 182 files
+failed to parse and spot checking them shows that they're pretty much
+lost causes. I'm sure I missed some stuff, but it is more important at
+this point to release a version to get more people using it in more
+diverse ways than I can come up with.
+
+* 3 minor enhancements:
+
+  * Added RBStringScanner#charpos using #string_to_pos.
+  * Added RBStringScanner#string_to_pos using String#byteslice.
+  * Optimized regexp used for escape chars in strings. (presidentbeef)
+
+* 3 bug fixes:
+
+  * Fixed current_line and unread_many depending on StringScanner#pos
+  * Fixed parsing of 'a[] = b'
+  * Fixed the worst abuse of heredocs EVER. Just because it is valid doesn't mean you should.
+
 === 3.0.0.a10 / 2012-10-26
 
 * 1 major enhancement:

data/Rakefile

@@ -14,7 +14,7 @@ Hoe.spec 'ruby_parser' do
 
   self.rubyforge_name = 'parsetree'
 
-  dependency 'sexp_processor', '~> 4.0'
+  dependency 'sexp_processor', '~> 4.1'
 
   if plugin? :perforce then
     self.perforce_ignore << "lib/ruby18_parser.rb"

data/lib/ruby_lexer.rb

@@ -21,7 +21,7 @@ class RubyLexer
   attr_accessor :tern # TODO: rename ternary damnit... wtf
   attr_accessor :nest
 
-  ESC_RE = /\\([0-7]{1,3}|x[0-9a-fA-F]{1,2}|M-[^\\]|(C-|c)[^\\]|[^0-7xMCc])/
+  ESC_RE = /\\((?>[0-7]{1,3}|x[0-9a-fA-F]{1,2}|M-[^\\]|(C-|c)[^\\]|[^0-7xMCc]))/
 
   ##
   # What version of ruby to parse. 18 and 19 are the only valid values
@@ -156,7 +156,7 @@ class RubyLexer
         string_buffer << '#'
       end
 
-      until src.check(eos_re) do
+      begin
         c = tokadd_string func, "\n", nil
 
         rb_compile_error err_msg if
@@ -171,7 +171,7 @@ class RubyLexer
 
         rb_compile_error err_msg if
           src.eos?
-      end
+      end until src.check(eos_re)
     else
       until src.check(eos_re) do
         string_buffer << src.scan(/.*(\n|\z)/)
@@ -193,9 +193,7 @@ class RubyLexer
     case
     when src.scan(/(-?)(['"`])(.*?)\2/) then
       term = src[2]
-      unless src[1].empty? then
-        func |= STR_FUNC_INDENT
-      end
+      func |= STR_FUNC_INDENT unless src[1].empty?
       func |= case term
               when "\'" then
                 STR_SQUOTE
@@ -989,9 +987,7 @@ class RubyLexer
                                 :expr_endarg, :expr_class) &&
                 (!is_arg? || space_seen)) then
               tok = self.heredoc_identifier
-              if tok then
-                return tok
-              end
+              return tok if tok
             end
 
             self.fix_arg_lex_state

data/lib/ruby_parser_extras.rb

@@ -34,8 +34,25 @@ class RPStringScanner < StringScanner
 #       old_getch
 #     end
 #   end
+
+  if "".respond_to? :encoding then
+    def string_to_pos
+      string.byteslice(0, pos)
+    end
+
+    def charpos
+      string_to_pos.length
+    end
+  else
+    alias :charpos :pos
+
+    def string_to_pos
+      string[0..pos]
+    end
+  end
+
   def current_line # HAHA fuck you (HACK)
-    string[0..pos][/\A.*__LINE__/m].split(/\n/).size
+    string_to_pos[/\A.*__LINE__/m].split(/\n/).size
   end
 
   def extra_lines_added
@@ -57,7 +74,7 @@ class RPStringScanner < StringScanner
     warn({:unread_many => caller[0]}.inspect) if ENV['TALLY']
     self.extra_lines_added += str.count("\n")
     begin
-      string[pos, 0] = str
+      string[charpos, 0] = str
     rescue IndexError
       # HACK -- this is a bandaid on a dirty rag on an open festering wound
     end
@@ -85,7 +102,7 @@ class RPStringScanner < StringScanner
 end
 
 module RubyParserStuff
-  VERSION = '3.0.0.a10' unless constants.include? "VERSION" # SIGH
+  VERSION = '3.0.0' unless constants.include? "VERSION" # SIGH
 
   attr_accessor :lexer, :in_def, :in_single, :file
   attr_reader :env, :comments
@@ -186,6 +203,7 @@ module RubyParserStuff
   end
 
   def aryset receiver, index
+    index ||= []
     s(:attrasgn, receiver, :"[]=", *index[1..-1])
   end
 
@@ -890,17 +908,7 @@ module RubyParserStuff
       end
     else
       # nothing specified... ugh. try to encode as utf-8
-      if ruby19 then
-        begin
-          str.encode! "utf-8"
-        rescue Encoding::InvalidByteSequenceError, Encoding::UndefinedConversionError
-          # OK... You really suck. You have extended chars but didn't
-          # specify what they were. Now we try to force it and double
-          # check that it is valid.
-
-          hack_encoding str
-        end
-      end
+      hack_encoding str if ruby19
     end
 
     str

data/test/test_ruby_lexer.rb

@@ -642,6 +642,15 @@ class TestRubyLexer < MiniTest::Unit::TestCase
                    :tNL,             nil)
   end
 
+  def test_yylex_heredoc_empty
+    util_lex_token("<<\"\"\n\#{x}\nblah2\n\n",
+                   :tSTRING_BEG,     "\"",
+                   :tSTRING_DBEG,    "\#{",
+                   :tSTRING_CONTENT, "x}\nblah2\n",
+                   :tSTRING_END,     "",
+                   :tNL,             nil)
+  end
+
   def test_yylex_heredoc_none
     util_lex_token("a = <<EOF\nblah\nblah\nEOF",
                    :tIDENTIFIER,     "a",

data/test/test_ruby_parser.rb

@@ -872,6 +872,13 @@ module TestRubyParserShared
 
     assert_parse rb, pt
   end
+
+  def test_str_heredoc_interp
+    rb = "<<\"\"\n\#{x}\nblah2\n\n"
+    pt = s(:dstr, "", s(:evstr, s(:call, nil, :x)), s(:str, "\nblah2\n"))
+
+    assert_parse rb, pt
+  end
 end
 
 class TestRubyParser < MiniTest::Unit::TestCase
@@ -1642,4 +1649,11 @@ class TestRuby19Parser < RubyParserTestCase
 
     assert_parse rb, pt
   end
+
+  def test_index_0
+    rb = "a[] = b"
+    pt = s(:attrasgn, s(:call, nil, :a), :[]=, s(:call, nil, :b))
+
+    assert_parse rb, pt
+  end
 end

metadata

@@ -1,15 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: ruby_parser
 version: !ruby/object:Gem::Version 
-  hash: 614431433
-  prerelease: 6
+  hash: 7
+  prerelease: 
   segments: 
   - 3
   - 0
   - 0
-  - a
-  - 10
-  version: 3.0.0.a10
+  version: 3.0.0
 platform: ruby
 authors: 
 - Ryan Davis
@@ -38,7 +36,7 @@ cert_chain:
   FBHgymkyj/AOSqKRIpXPhjC6
   -----END CERTIFICATE-----
 
-date: 2012-10-26 00:00:00 Z
+date: 2012-11-02 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: sexp_processor
@@ -48,11 +46,11 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 27
+        hash: 25
         segments: 
         - 4
-        - 0
-        version: "4.0"
+        - 1
+        version: "4.1"
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
@@ -189,14 +187,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version 
-      hash: 25
+      hash: 3
       segments: 
-      - 1
-      - 3
-      - 1
-      version: 1.3.1
+      - 0
+      version: "0"
 requirements: []
 
 rubyforge_project: parsetree