ruby_parser 2.0.1 → 2.0.2

data/History.txt

@@ -1,3 +1,17 @@
+=== 2.0.2 / 2009-01-20
+
+* 2 minor enhancements:
+
+  * Added gauntlet_rubyparser plugin. YAY for easy massive bug-hunting.
+  * Promoted Sexp's file/line/comments to sexp_processor.
+
+* 4 bug fixes:
+
+  * Fixed and improved the readme
+  * Fixed lexing heredoc newlines.
+  * Fixed line numbers on defns.
+  * Fixed rdoc generation bug pointed out by hugh sasse (who rocks)
+
 === 2.0.1 / 2008-11-04
 
 * 2 minor enhancements:

data/Manifest.txt

@@ -4,6 +4,7 @@ Manifest.txt
 README.txt
 Rakefile
 bin/ruby_parse
+lib/gauntlet_rubyparser.rb
 lib/ruby_lexer.rb
 lib/ruby_parser.y
 lib/ruby_parser_extras.rb

data/README.txt

@@ -1,6 +1,6 @@
-ruby_parser
-    by Ryan Davis
-    http://parsetree.rubyforge.org/
+= ruby_parser
+
+* http://parsetree.rubyforge.org/
 
 == DESCRIPTION:
 
@@ -9,6 +9,27 @@ racc--which does by default use a C extension). RP's output is
 the same as ParseTree's output: s-expressions using ruby's arrays and
 base types.
 
+As an example:
+
+  def conditional1(arg1)
+    if arg1 == 0 then
+      return 1
+    end
+    return 0
+  end
+
+becomes:
+
+  s(:defn, :conditional1,
+   s(:args, :arg1),
+   s(:scope,
+    s(:block,
+     s(:if,
+      s(:call, s(:lvar, :arg1), :==, s(:arglist, s(:lit, 0))),
+      s(:return, s(:lit, 1)),
+      nil),
+     s(:return, s(:lit, 0)))))
+
 == FEATURES/PROBLEMS:
 
 * Pure ruby, no compiles.
@@ -16,11 +37,12 @@ base types.
 * Incredibly simple interface.
 * Output is 100% equivalent to ParseTree.
   * Can utilize PT's SexpProcessor and UnifiedRuby for language processing.
-* Known Issue: Speed sucks currently. 5500 tests currently run in 21 min.
-* Known Issue: Code is waaay ugly. Port of a port. Not my fault. Will fix RSN.
-* Known Issue: I don't currently support newline nodes.
+* Known Issue: Speed is now pretty good, but can always improve:
+  * RP parses a corpus of 3702 files in 125s (avg 108 Kb/s)
+  * MRI+PT parsed the same in 67.38s (avg 200.89 Kb/s)
+* Known Issue: Code is much better, but still has a long way to go.
 * Known Issue: Totally awesome.
-* Known Issue: dasgn_curr decls can be out of order from ParseTree's.
+* Known Issue: line number values can be slightly off. Parsing LR sucks.
 
 == SYNOPSIS:
 
@@ -30,9 +52,9 @@ base types.
 == REQUIREMENTS:
 
 * ruby. woot.
-* ParseTree is needed for Sexp class... crap. I might break that out.
+* sexp_processor for Sexp and SexpProcessor classes.
 * ParseTree for testing.
-* racc full package for parser development.
+* racc full package for parser development (compiling .y to .rb).
 
 == INSTALL:
 
@@ -42,7 +64,7 @@ base types.
 
 (The MIT License)
 
-Copyright (c) 2007 Ryan Davis
+Copyright (c) 2007-2008 Ryan Davis
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/Rakefile

@@ -15,7 +15,7 @@ hoe = Hoe.new('ruby_parser', RubyParser::VERSION) do |parser|
   parser.developer('Ryan Davis', 'ryand-ruby@zenspider.com')
 
   parser.extra_dev_deps << 'ParseTree'
-  parser.extra_deps << ['sexp_processor', '>= 3.0.0']
+  parser.extra_deps << ['sexp_processor', '>= 3.0.1']
 end
 
 hoe.spec.files += ['lib/ruby_parser.rb'] # jim.... cmon man

data/lib/gauntlet_rubyparser.rb

@@ -0,0 +1,120 @@
+#!/usr/bin/ruby -ws
+
+$f ||= false
+
+$:.unshift "../../ruby_parser/dev/lib"
+$:.unshift "../../ruby2ruby/dev/lib"
+
+require 'rubygems'
+require 'ruby2ruby'
+require 'ruby_parser'
+
+require 'gauntlet'
+
+class RubyParserGauntlet < Gauntlet
+  def initialize
+    super
+
+    self.data = Hash.new { |h,k| h[k] = {} }
+    old_data = load_yaml data_file
+    self.data.merge! old_data
+  end
+
+  def should_skip? name
+    if $f then
+      if Hash === data[name] then
+        ! data[name].empty?
+      else
+        data[name]
+      end
+    else
+      data[name] == true # yes, == true on purpose
+    end
+  end
+
+  def diff_pp o1, o2
+    require 'pp'
+
+    File.open("/tmp/a.#{$$}", "w") do |f|
+      PP.pp o1, f
+    end
+
+    File.open("/tmp/b.#{$$}", "w") do |f|
+      PP.pp o2, f
+    end
+
+    `diff -u /tmp/a.#{$$} /tmp/b.#{$$}`
+  ensure
+    File.unlink "/tmp/a.#{$$}" rescue nil
+    File.unlink "/tmp/b.#{$$}" rescue nil
+  end
+
+  def broke name, file, msg
+    warn "bad"
+    self.data[name][file] = msg
+    self.dirty = true
+  end
+
+  def process path, name
+    begin
+      $stderr.print "  #{path}: "
+      rp = RubyParser.new
+      r2r = Ruby2Ruby.new
+
+      old_ruby = File.read(path)
+
+      begin
+        old_sexp = rp.process old_ruby
+      rescue Racc::ParseError => e
+        self.data[name][path] = :unparsable
+        self.dirty = true
+        return
+      end
+
+      new_ruby = r2r.process old_sexp.deep_clone
+
+      begin
+        new_sexp = rp.process new_ruby
+      rescue Racc::ParseError => e
+        broke name, path, "couldn't parse new_ruby: #{e.message.strip}"
+        return
+      end
+
+      if old_sexp != new_sexp then
+        broke name, path, diff_pp(old_sexp, new_sexp)
+        return
+      end
+
+      self.data[name][path] = true
+      self.dirty = true
+
+      warn "good"
+    rescue Interrupt
+      puts "User cancelled"
+      exit 1
+    rescue Exception => e
+      broke name, path, "    UNKNOWN ERROR: #{e}: #{e.message.strip}"
+    end
+  end
+
+  def run name
+    warn name
+    Dir["**/*.rb"].sort.each do |path|
+      next if path =~ /gemspec.rb/ # HACK
+      next if data[name][path] == true
+      process path, name
+    end
+
+    if self.data[name].values.all? { |v| v == true } then
+      warn "  ALL GOOD!"
+      self.data[name] = true
+      self.dirty = true
+    end
+  end
+end
+
+filter = ARGV.shift
+filter = Regexp.new filter if filter
+
+gauntlet = RubyParserGauntlet.new
+gauntlet.run_the_gauntlet filter

data/lib/ruby_lexer.rb

@@ -188,7 +188,8 @@ class RubyLexer
     if src.check(/.*\n/) then
       # TODO: think about storing off the char range instead
       line = src.string[src.pos, src.matched_size]
-      src.string[src.pos, src.matched_size] = ''
+      src.string[src.pos, src.matched_size] = "\n"
+      src.pos += 1
     else
       line = nil
     end
@@ -651,7 +652,6 @@ class RubyLexer
             if src.eos? then
               return RubyLexer::EOF
             end
-          else
           end
 
           # Replace a string of newlines with a single one
@@ -662,7 +662,6 @@ class RubyLexer
             next
           end
 
-
           self.command_start = true
           self.lex_state = :expr_beg
           return :tNL

data/lib/ruby_parser.rb

@@ -1,6 +1,6 @@
 #
 # DO NOT MODIFY!!!!
-# This file is automatically generated by Racc 1.4.5
+# This file is automatically generated by Racc 1.4.6
 # from Racc grammer file "".
 #
 
@@ -4564,7 +4564,7 @@ def _reduce_317(val, _values, result)
                       self.comments.push self.lexer.comments
                       self.in_def = true
                       self.env.extend
-                      result = self.lexer.lineno
+                      result = lexer.lineno, lexer.src.beginning_of_line?
                     
     result
 end

data/lib/ruby_parser.y

@@ -1110,7 +1110,7 @@ rule
                       self.comments.push self.lexer.comments
                       self.in_def = true
                       self.env.extend
-                      result = self.lexer.lineno
+                      result = lexer.lineno, lexer.src.beginning_of_line?
                     }
                     f_arglist bodystmt kEND
                     {

data/lib/ruby_parser_extras.rb

@@ -113,7 +113,7 @@ class RPStringScanner < StringScanner
 end
 
 class RubyParser < Racc::Parser
-  VERSION = '2.0.1'
+  VERSION = '2.0.2'
 
   attr_accessor :lexer, :in_def, :in_single, :file
   attr_reader :env, :comments
@@ -502,7 +502,7 @@ class RubyParser < Racc::Parser
   end
 
   def new_defn val
-    line, name, args, body = val[2], val[1], val[3], val[4]
+    (line, bol), name, args, body = val[2], val[1], val[3], val[4]
     body ||= s(:nil)
 
     body ||= s(:block)
@@ -510,6 +510,7 @@ class RubyParser < Racc::Parser
 
     result = s(:defn, name.to_sym, args, s(:scope, body))
     result.line = line
+    result.line -= 1 if bol
     result.comments = self.comments.pop
     result
   end
@@ -827,6 +828,8 @@ class Keyword
   end
 
   ##
+  # :stopdoc:
+  #
   # :expr_beg    = ignore newline, +/- is a sign.
   # :expr_end    = newline significant, +/- is a operator.
   # :expr_arg    = newline significant, +/- is a operator.
@@ -880,6 +883,8 @@ class Keyword
               ["alias",    [:kALIAS,    :kALIAS      ], :expr_fname ],
              ].map { |args| KWtable.new(*args) }
 
+  # :startdoc:
+
   WORDLIST = Hash[*wordlist.map { |o| [o.name, o] }.flatten]
 
   def self.keyword str
@@ -1004,25 +1009,6 @@ end
 
 class Sexp
   attr_writer :paren
-  attr_accessor :comments
-  attr_accessor :file
-
-  def line(n=nil)
-    if n then
-      @line = n
-      self
-    else
-      @line ||= nil
-    end
-  end
-
-  def line= n
-    @line = n
-  end
-
-  def node_type
-    first
-  end
 
   def paren
     @paren ||= false
@@ -1037,19 +1023,8 @@ class Sexp
     self.value.to_sym
   end
 
-  def values
-    self[1..-1]
-  end
-
-  alias :real_inspect :inspect
-  def inspect # :nodoc:
-    sexp_str = self.map {|x|x.inspect}.join(', ')
-    if line && ENV['VERBOSE'] then
-      "s(#{sexp_str}).line(#{line})"
-    else
-      "s(#{sexp_str})"
-    end
-  end
+  alias :node_type :sexp_type
+  alias :values :sexp_body # TODO: retire
 end
 
 # END HACK

data/test/test_ruby_parser.rb

@@ -374,6 +374,13 @@ class TestRubyParser < RubyParserTestCase
     assert_equal pt, @processor.parse(rb)
   end
 
+#   def test_str_pct_nested_nested
+#     rb = "%{ { #\{ \"#\{1}\" } } }"
+#     pt = s(:dstr, " { ", s(:evstr, s(:lit, 1)), s(:str, " } "))
+
+#     assert_equal pt, @processor.parse(rb)
+#   end
+
   def test_str_str
     rb = "\"a #\{'b'}\""
     pt = s(:str, "a b")
@@ -389,30 +396,21 @@ class TestRubyParser < RubyParserTestCase
   end
 
   STARTING_LINE = {
-    "begin_def"                           => 2,
-    "block_stmt_after"                    => 2,
-    "block_stmt_after_mri_verbose_flag"   => 2,
-    "block_stmt_before"                   => 2,
-    "block_stmt_before_mri_verbose_flag"  => 2,
-    "block_stmt_both"                     => 2,
-    "block_stmt_both_mri_verbose_flag"    => 2,
-    "case_nested_inner_no_expr"           => 2,
-    "case_no_expr"                        => 2,
-    "case_splat"                          => 2,
-    "cvasgn"                              => 2,
-    "defn_args_none"                      => 2,
-    "defn_zarray"                         => 2,
-    "structure_unused_literal_wwtt"       => 3, # yes, 3... odd test
-    "super_0"                             => 2,
-    "super_1"                             => 2,
-    "super_1_array"                       => 2,
-    "super_n"                             => 2,
-    "super_multi"                         => 2,
-    "undef_block_1"                       => 2,
-    "undef_block_2"                       => 2,
-    "undef_block_3"                       => 2,
-    "undef_block_wtf"                     => 2,
-    "zsuper"                              => 2,
+    "case_nested_inner_no_expr"          => 2,
+    "case_no_expr"                       => 2,
+    "case_splat"                         => 2,
+    "dstr_heredoc_expand"                => 2,
+    "dstr_heredoc_windoze_sucks"         => 2,
+    "dstr_heredoc_yet_again"             => 2,
+    "str_heredoc"                        => 2,
+    "str_heredoc_call"                   => 2,
+    "str_heredoc_empty"                  => 2,
+    "str_heredoc_indent"                 => 2,
+    "structure_unused_literal_wwtt"      => 3, # yes, 3... odd test
+    "undef_block_1"                      => 2,
+    "undef_block_2"                      => 2,
+    "undef_block_3"                      => 2,
+    "undef_block_wtf"                    => 2,
   }
 
   def after_process_hook klass, node, data, input_name, output_name

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: ruby_parser
 version: !ruby/object:Gem::Version 
-  version: 2.0.1
+  version: 2.0.2
 platform: ruby
 authors: 
 - Ryan Davis
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-11-04 00:00:00 -08:00
+date: 2009-01-20 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -20,7 +20,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 3.0.0
+        version: 3.0.1
     version: 
 - !ruby/object:Gem::Dependency 
   name: ParseTree
@@ -40,9 +40,9 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 1.8.0
+        version: 1.8.2
     version: 
-description: "ruby_parser (RP) is a ruby parser written in pure ruby (utilizing racc--which does by default use a C extension). RP's output is the same as ParseTree's output: s-expressions using ruby's arrays and base types."
+description: "ruby_parser (RP) is a ruby parser written in pure ruby (utilizing racc--which does by default use a C extension). RP's output is the same as ParseTree's output: s-expressions using ruby's arrays and base types.  As an example:  def conditional1(arg1) if arg1 == 0 then return 1 end return 0 end  becomes:  s(:defn, :conditional1, s(:args, :arg1), s(:scope, s(:block, s(:if, s(:call, s(:lvar, :arg1), :==, s(:arglist, s(:lit, 0))), s(:return, s(:lit, 1)), nil), s(:return, s(:lit, 0)))))"
 email: 
 - ryand-ruby@zenspider.com
 executables: 
@@ -60,6 +60,7 @@ files:
 - README.txt
 - Rakefile
 - bin/ruby_parse
+- lib/gauntlet_rubyparser.rb
 - lib/ruby_lexer.rb
 - lib/ruby_parser.y
 - lib/ruby_parser_extras.rb
@@ -68,7 +69,7 @@ files:
 - test/test_ruby_parser_extras.rb
 - lib/ruby_parser.rb
 has_rdoc: true
-homepage: ruby_parser (RP) is a ruby parser written in pure ruby (utilizing
+homepage: http://parsetree.rubyforge.org/
 post_install_message: 
 rdoc_options: 
 - --main