ruby_parser 3.3.0 → 3.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b4e320a2bd80671b5ce1434876d1f85f76e63eff
-  data.tar.gz: 9354638a4b05c8bacc5a5a50cbaa20387dc7d1d4
+  metadata.gz: c54a714696b5a3055f3f423bb81b3a6f4ecc2584
+  data.tar.gz: 5215e8bb9dccbd5f4dd6b515844fc1525ad56334
 SHA512:
-  metadata.gz: 0332c2ba2307398ab331029a4c04e7bc4b0246a7a71735b56770ea5ab2edd94ae44da50be63c4288bf372616114bd3a038ae2ab9457a3a32a766a5034fa34b1d
-  data.tar.gz: fcdde142a09539086dfd8b8c2371b0fddd08df31d3c1f63c3c2d6bf3518732ec2f1b1a3425a1a14d1e341420f350e9c8c9bd035cd83af9418c9c8ca7044901db
+  metadata.gz: ed81b97bf7ec8fc1800ab0c247cbb22850cb5e0559f9372eb19061f2d361237bc2c88cd2512ccfe7896c1515da4ded3bcadd8357b22028c9fd186ef4cbb55ed4
+  data.tar.gz: adc086eee5342a953f578aae3daf22685b1462f7e5444476f83babb09ff2b1940b40de5dc7a83cf249b56915057f594612849c9b2155074156cb80568e6c5d19

data.tar.gz.sig

@@ -1,2 +1,2 @@
-���{P���S����M#�)���I��0�m�s��CyP�(�!x?�`�I�ܤ<�\�H+W��a��LqIy�[16�vo��POZQ�
-��=	Ɇn�a�~2;}����K��
���1C��y%��$)�M�B�0GX���QR�7����?���?~Y�ۺ�-
+���ְ�A%��s�g���<���1­��6ǭ9��X��樀tlѽJ������<������_h�<2)`k�|.�-�J���e�g����l^ ��»I������t+xHS>�e|~�@؍�
+���a��5���SN	 $-b*I�J�65�

data/.autotest

@@ -7,20 +7,17 @@ require 'autotest/rcov' if ENV['RCOV']
 Autotest.add_hook :initialize do |at|
   at.extra_files << "../../sexp_processor/dev/lib/pt_testcase.rb"
   at.libs << ":../../sexp_processor/dev/lib"
-  at.add_exception 'unit'
-  at.add_exception 'coverage'
-  at.add_exception 'coverage.info'
-  at.add_exception '.diff'
-  at.add_exception '.output'
-  at.add_exception 'rubycorpus'
+  at.add_exception "unit"
+  at.add_exception "coverage"
+  at.add_exception "coverage.info"
+  at.add_exception ".diff"
+  at.add_exception ".output"
   at.add_exception "lib/ruby18_parser.rb"
   at.add_exception "lib/ruby19_parser.rb"
   at.add_exception "lib/ruby20_parser.rb"
-  at.add_exception "lib/gauntlet_rubyparser.rb"
+  at.add_exception "lib/ruby_lexer.rex.rb"
 
-  dirs = Dir["corpus*"] + Dir["gauntlet*"]
-
-  dirs.each do |f|
+  Dir["gauntlet*"].each do |f|
     at.add_exception f
   end
 
@@ -28,7 +25,17 @@ Autotest.add_hook :initialize do |at|
   at.testlib = "minitest/autorun"
 
   at.add_mapping(/^lib\/.*\.y$/) do |f, _|
-    at.files_matching %r%^test/.*#{File.basename(f, '.y').gsub '_', '_?'}.rb$%
+    g = File.basename(f, ".y").gsub("_", "_?").gsub(/1[89]|20/, '')
+    at.files_matching %r%^test/.*#{g}.rb$%
+  end
+
+  at.add_mapping(/^lib\/ruby_lexer\.rex\.rb$/) do |f, _|
+    at.files_matching %r%^test/.*ruby_lexer\.rb$%
+  end
+
+  at.add_mapping(/^lib\/.*\.rex$/) do |f, _|
+    g = File.basename(f, ".rex").gsub("_", "_?")
+    at.files_matching %r%^test/.*#{g}.rb$%
   end
 
   at.add_mapping(/pt_testcase.rb/) do |f, _|
@@ -45,17 +52,17 @@ Autotest.add_hook :initialize do |at|
 end
 
 Autotest.add_hook :run_command do |at, _|
-  system "rake parser DEBUG=1"
+  system "rake parser lexer DEBUG=1"
 end
 
 class Autotest
   def ruby
     File.expand_path "~/.multiruby/install/1.9.3-p194/bin/ruby"
   end
-end if ENV['R19']
+end if ENV["R19"]
 
 class Autotest
   def ruby
     File.expand_path "~/.multiruby/install/2.0.0-p195/bin/ruby"
   end
-end if ENV['R20']
+end if ENV["R20"]

data/History.txt

@@ -1,3 +1,53 @@
+=== 3.4.0 / 2014-02-04
+
+* 1 major enhancement:
+
+  * Replaced hand-written/optimized f'd-up lexer with an oedipus_lex
+    generated lexer. This makes it roughly 40-50% faster.
+
+* 30 minor enhancements:
+
+  * 2.0: Added support for a.b c() do d end.e do |f| g end
+  * 2.0: Added support for a.b c() do d end.e f do |g| h end
+  * Added -s flag to ruby_parse_extract_error to output timings.
+  * Added RubyLexer #command_state and #last_state to deal with oedipus_lex differences.
+  * Added String#lineno and #lineno= because I'm a bad bad person.
+  * Added a bunch of RubyLexer scanning methods: beginning_of_line?, check, scan, etc.
+  * Added a bunch of process_* methods extracted from old yylex. process_amper, etc.
+  * Added lib/.document to save my laptop's battery from pain and suffering
+  * Adjust lineno when we lex a bunch of blank lines.
+  * Attach lineno to tIDENTIFIER values (strings, ugh)
+  * Cleaned up and re-ordered node_assign to be faster (ordered by actual occurrance).
+  * Extend RubyParserStuff#gettable to set the lineno if it comes in with the id.
+  * Extended RubyParserStuff#new_case to take line number.
+  * Finally dropped RPStringScanner's BS #current_line.
+  * Finally dropped RPStringScanner's BS line number calculation (lineno). 
+  * Implemented Sexp#add_all since we now have a test case for it.
+  * Removed :call case of node_assign. I don't think it is possible.
+  * Removed RubyLexer #extra_lines_added. No longer used. Complex heredoc lineno's possible screwed up.
+  * Removed RubyLexer#parse_number. Handled by oedipus_lex.
+  * Removed RubyLexer#yacc_value now that next_token returns pairs.
+  * Removed RubyLexer's @src. Now taken care of by oedipus_lex.
+  * Removed RubyParser#advance. RubyParser#next_token takes care of everything now.
+  * Removed RubyParserExtras#arg_add. (presidentbeef! YAY!)
+  * Removed lib/gauntlet_rubyparser.rb. I just don't use it anymore. Too slow.
+  * RubyLexer#is_label_possible? doesn't need an arg
+  * RubyLexer#process_token is now a normal oedipal lexer method.
+  * RubyParser#next_token now expects RubyLexer#next_token to return a pair (type, val).
+  * TRYING a new scheme to figure out encodings... but I'm about to throw in the towel. I hate this stuff so much.
+  * Turned off oedipus_lex's automatic line counting. (pushing to oedipus_lex soon).
+  * Updated to oedipus_lex 2.1+.
+
+* 7 bug fixes:
+
+  * 1.8: Properly parse `a (:b, :c, :d => :e)`. (presidentbeef)
+  * Fixed lexing symbol!= vs symbol!. Please use your spacebar. Think of the children.
+  * Fixed line for dstr spanning multiple lines via backslash. (presidentbeef)
+  * Fixed line numbers for odd cases with trailing whitespace. (presidentbeef)
+  * Fixed line numbers on ambiguous calls w/ gvar/ivar args. (presidentbeef)
+  * Max out unicode hex values to 2-4 or 2-6 chars or pack will overflow and puke.
+  * Removed ESC_RE from RubyLexer. Must have slipped through.
+
 === 3.3.0 / 2014-01-14
 
 * Notes:

data/Manifest.txt

@@ -5,7 +5,7 @@ README.txt
 Rakefile
 bin/ruby_parse
 bin/ruby_parse_extract_error
-lib/gauntlet_rubyparser.rb
+lib/.document
 lib/ruby18_parser.rb
 lib/ruby18_parser.y
 lib/ruby19_parser.rb
@@ -13,6 +13,8 @@ lib/ruby19_parser.y
 lib/ruby20_parser.rb
 lib/ruby20_parser.y
 lib/ruby_lexer.rb
+lib/ruby_lexer.rex
+lib/ruby_lexer.rex.rb
 lib/ruby_parser.rb
 lib/ruby_parser_extras.rb
 test/test_ruby_lexer.rb

data/Rakefile

@@ -1,7 +1,9 @@
 # -*- ruby -*-
 
-require 'rubygems'
-require 'hoe'
+$:.unshift "../../hoe/dev/lib"
+
+require "rubygems"
+require "hoe"
 
 Hoe.plugin :seattlerb
 Hoe.plugin :racc
@@ -10,18 +12,20 @@ Hoe.plugin :isolate
 Hoe.add_include_dirs "../../sexp_processor/dev/lib"
 Hoe.add_include_dirs "../../minitest/dev/lib"
 
-Hoe.spec 'ruby_parser' do
-  developer 'Ryan Davis', 'ryand-ruby@zenspider.com'
+Hoe.spec "ruby_parser" do
+  developer "Ryan Davis", "ryand-ruby@zenspider.com"
 
   license "MIT"
 
-  dependency 'sexp_processor', '~> 4.1'
-  dependency 'rake', '< 11', :developer
+  dependency "sexp_processor", "~> 4.1"
+  dependency "rake", "< 11", :developer
+  dependency "oedipus_lex", "~> 2.1", :developer
 
   if plugin? :perforce then
     self.perforce_ignore << "lib/ruby18_parser.rb"
     self.perforce_ignore << "lib/ruby19_parser.rb"
     self.perforce_ignore << "lib/ruby20_parser.rb"
+    self.perforce_ignore << "lib/ruby_lexer.rex.rb"
   end
 
   self.racc_flags << " -t" if plugin?(:racc) && ENV["DEBUG"]
@@ -30,10 +34,11 @@ end
 file "lib/ruby18_parser.rb" => "lib/ruby18_parser.y"
 file "lib/ruby19_parser.rb" => "lib/ruby19_parser.y"
 file "lib/ruby20_parser.rb" => "lib/ruby20_parser.y"
+file "lib/ruby_lexer.rex.rb" => "lib/ruby_lexer.rex"
 
 task :clean do
   rm_rf(Dir["**/*~"] +
-        Dir["**/*.diff"] +
+        Dir["diff.diff"] + # not all diffs. bit me too many times
         Dir["coverage.info"] +
         Dir["coverage"] +
         Dir["lib/*.output"])
@@ -51,12 +56,12 @@ task :compare do
     puts file
     system "./cmp.rb -q #{file} && rm #{file}"
   end
-  system 'find -d unit -type d -empty -exec rmdir {} \;'
+  system "find -d unit -type d -empty -exec rmdir {} \;"
 end
 
 task :sort do
-  sh 'grepsort "^ +def" lib/ruby_lexer.rb'
-  sh 'grepsort "^ +def (test|util)" test/test_ruby_lexer.rb'
+  sh "grepsort '^ +def' lib/ruby_lexer.rb"
+  sh "grepsort '^ +def (test|util)' test/test_ruby_lexer.rb"
 end
 
 task :loc do
@@ -78,7 +83,7 @@ task :validate do
 end
 
 def run_and_log cmd, prefix
-  files = ENV['FILES'] || 'unit/*.rb'
+  files = ENV["FILES"] || "unit/*.rb"
   p, x = prefix, "txt"
   n = Dir["#{p}.*.#{x}"].map { |s| s[/\d+/].to_i }.max + 1 rescue 1
   f = "#{p}.#{n}.#{x}"
@@ -140,10 +145,11 @@ end
 task :debug => :isolate do
   ENV["V"] ||= "20"
   Rake.application[:parser].invoke # this way we can have DEBUG set
+  Rake.application[:lexer].invoke # this way we can have DEBUG set
 
   $: << "lib"
-  require 'ruby_parser'
-  require 'pp'
+  require "ruby_parser"
+  require "pp"
 
   parser = case ENV["V"]
            when "18" then
@@ -169,7 +175,7 @@ task :debug => :isolate do
     pp parser.process(ruby, file, time)
   rescue Racc::ParseError => e
     p e
-    ss = parser.lexer.src
+    ss = parser.lexer.ss
     src = ss.string
     lines = src[0..ss.pos].split(/\n/)
     abort "on #{file}:#{lines.size}"

data/bin/ruby_parse_extract_error

@@ -9,6 +9,7 @@ $m ||= ENV["MOVE_TIMEOUT"]
 $q ||= false
 $q ||= ENV["QUIET"]
 $v ||= ENV["V"] || "20"
+$s ||= ENV["SPEED"] || false
 
 require 'rubygems'
 require 'ruby_parser'
@@ -37,7 +38,7 @@ ARGV.push "-" if ARGV.empty?
 
 class Racc::Parser
   def extract_defs
-    ss = lexer.src
+    ss = lexer.ss
 
     raise "can't access source. possible encoding issue" unless ss
 
@@ -113,8 +114,13 @@ def process file
 
   $stderr.print "# Validating #{file}: "
   parser = $parser_class.new
+  t0 = Time.now if $s
   parser.process(ruby, file, time)
-  warn "good"
+  if $s then
+    warn "good: #{Time.now - t0}"
+  else
+    warn "good"
+  end
   File.unlink file if $d
 rescue Timeout::Error
   $exit = 1

data/lib/.document

@@ -0,0 +1 @@
+*.rb

data/lib/ruby18_parser.rb

@@ -4253,14 +4253,16 @@ def _reduce_246(val, _values, result)
 end
 
 def _reduce_247(val, _values, result)
-                      result = s(:array, val[0]).add_all(val[2]).add(s(:hash, *val[4].values))
-                      result = arg_blk_pass result, val[5]
+                      arg_value, _, args, _, assocs, opt_block = val
+                      result = s(:array, arg_value).add_all(args)
+                      result.add(s(:hash, *assocs.sexp_body))
+                      result = arg_blk_pass result, opt_block
                     
     result
 end
 
 def _reduce_248(val, _values, result)
-                      result = arg_concat s(:array, val[0]).add(s(:hash, *val[2].values)), val[5]
+                      result = arg_concat s(:array, val[0]).add(s(:hash, *val[2].sexp_body)), val[5]
                       result = arg_blk_pass result, val[6]
                     
     result
@@ -4558,19 +4560,22 @@ def _reduce_301(val, _values, result)
 end
 
 def _reduce_302(val, _values, result)
-                      result = new_case val[1], val[3]
+                      (_, line), expr, _, body, _ = val
+                      result = new_case expr, body, line
                     
     result
 end
 
 def _reduce_303(val, _values, result)
-                      result = new_case nil, val[2]
+                      (_, line), _, body, _ = val
+                      result = new_case nil, body, line
                     
     result
 end
 
 def _reduce_304(val, _values, result)
-                      result = new_case nil, val[3]
+                      (_, line), _, _, elsebody, _ = val
+                      result = new_case nil, elsebody, line
                     
     result
 end
@@ -5413,7 +5418,7 @@ def _reduce_447(val, _values, result)
 end
 
 def _reduce_448(val, _values, result)
- result = s(:lit, lexer.src.current_line) 
+ result = s(:lit, lexer.lineno) 
     result
 end
 

data/lib/ruby18_parser.y

@@ -829,12 +829,14 @@ rule
                     }
                 | arg_value tCOMMA args tCOMMA assocs opt_block_arg
                     {
-                      result = s(:array, val[0]).add_all(val[2]).add(s(:hash, *val[4].values))
-                      result = arg_blk_pass result, val[5]
+                      arg_value, _, args, _, assocs, opt_block = val
+                      result = s(:array, arg_value).add_all(args)
+                      result.add(s(:hash, *assocs.sexp_body))
+                      result = arg_blk_pass result, opt_block
                     }
                 | arg_value tCOMMA assocs tCOMMA tSTAR arg_value opt_block_arg
                     {
-                      result = arg_concat s(:array, val[0]).add(s(:hash, *val[2].values)), val[5]
+                      result = arg_concat s(:array, val[0]).add(s(:hash, *val[2].sexp_body)), val[5]
                       result = arg_blk_pass result, val[6]
                     }
                 | arg_value tCOMMA args tCOMMA assocs tCOMMA tSTAR arg_value opt_block_arg
@@ -1042,15 +1044,18 @@ rule
                     }
                 | kCASE expr_value opt_terms case_body kEND
                     {
-                      result = new_case val[1], val[3]
+                      (_, line), expr, _, body, _ = val
+                      result = new_case expr, body, line
                     }
                 | kCASE            opt_terms case_body kEND
                     {
-                      result = new_case nil, val[2]
+                      (_, line), _, body, _ = val
+                      result = new_case nil, body, line
                     }
                 | kCASE opt_terms kELSE compstmt kEND # TODO: need a test
                     {
-                      result = new_case nil, val[3]
+                      (_, line), _, _, elsebody, _ = val
+                      result = new_case nil, elsebody, line
                     }
                 | kFOR for_var kIN
                     {
@@ -1653,7 +1658,7 @@ xstring_contents: none
                 | kTRUE     { result = s(:true)  }
                 | kFALSE    { result = s(:false) }
                 | k__FILE__ { result = s(:str, self.file) }
-                | k__LINE__ { result = s(:lit, lexer.src.current_line) }
+                | k__LINE__ { result = s(:lit, lexer.lineno) }
 
          var_ref: variable
                     {

data/lib/ruby19_parser.rb

@@ -4650,13 +4650,15 @@ def _reduce_306(val, _values, result)
 end
 
 def _reduce_307(val, _values, result)
-                      result = new_case val[1], val[3]
+                      (_, line), expr, _, body, _ = val
+                      result = new_case expr, body, line
                     
     result
 end
 
 def _reduce_308(val, _values, result)
-                      result = new_case nil, val[2]
+                      (_, line), _, body, _ = val
+                      result = new_case nil, body, line
                     
     result
 end
@@ -5708,7 +5710,7 @@ def _reduce_489(val, _values, result)
 end
 
 def _reduce_490(val, _values, result)
- result = s(:lit, lexer.src.current_line) 
+ result = s(:lit, lexer.lineno) 
     result
 end
 

data/lib/ruby19_parser.y

@@ -1025,11 +1025,13 @@ rule
                     }
                 | kCASE expr_value opt_terms case_body kEND
                     {
-                      result = new_case val[1], val[3]
+                      (_, line), expr, _, body, _ = val
+                      result = new_case expr, body, line
                     }
                 | kCASE            opt_terms case_body kEND
                     {
-                      result = new_case nil, val[2]
+                      (_, line), _, body, _ = val
+                      result = new_case nil, body, line
                     }
                 | kFOR for_var kIN
                     {
@@ -1787,7 +1789,7 @@ keyword_variable: kNIL      { result = s(:nil)   }
                 | kTRUE     { result = s(:true)  }
                 | kFALSE    { result = s(:false) }
                 | k__FILE__ { result = s(:str, self.file) }
-                | k__LINE__ { result = s(:lit, lexer.src.current_line) }
+                | k__LINE__ { result = s(:lit, lexer.lineno) }
                 | k__ENCODING__ 
                     { 
                       result =