redparse 0.8.3 → 0.8.4

data/lib/redparse/parse_tree_server.rb

@@ -0,0 +1,129 @@
+require 'tempfile'
+#require 'yaml'
+#require 'marshal'
+
+module ParseTreeComm
+  SERIALIZE=Marshal
+  def put o
+    o=SERIALIZE.dump o
+    msg= o.size.to_s+"\n"+o+"\n"
+    begin
+      @out.write msg
+      @out.flush
+    rescue Exception
+      @out=@in=nil
+      raise
+    end
+  end
+  def get
+    begin
+      len=@in.gets.to_i
+      msg=@in.read(len)
+      @in.getc  #read trailing \n
+    rescue Exception
+      @in=@out=nil
+      raise
+    end
+    result=SERIALIZE.load msg
+    return result
+  end
+end
+
+class ParseTreeServer
+  include ParseTreeComm
+
+  def self.path_to_server_command
+    File.expand_path __FILE__
+  end
+
+  def ensure_parse_tree_and_1_8
+    if ::RUBY_VERSION[/^\d+\.\d+/].to_f>1.8
+      ruby18=ENV['RUBY1_8']||fail("you must use ruby <= 1.8 (with parse_tree) or set RUBY1_8 env to a 1.8 interpreter")
+      exec ruby18, $0
+    else
+      begin require 'rubygems'; rescue LoadError; end
+
+      if File.exist? find_home+"/.redparse/parse_tree_server.rc"
+        $:.concat File.readlines(find_home+"/.redparse/parse_tree_server.rc").map{|l| l.chop }
+      end
+
+      require 'parse_tree'
+    end
+  rescue Exception=>e
+    put e
+    put nil
+    put nil
+    raise
+  end
+
+  def main
+    si=STDIN
+    so=STDOUT
+    @out=so; @in=si
+    ensure_parse_tree_and_1_8
+      begin
+      warnstash=Tempfile.new "warnstash"
+      STDERR.reopen warnstash
+      instance=ParseTree.new
+      while true
+        str=get
+        exit! if str==:exit!
+        if str==:version
+          put ::RUBY_VERSION
+          next
+        end
+
+        pos=STDERR.pos
+
+        tree=
+        begin
+          instance.parse_tree_for_string(str) #tree
+        rescue Exception=>e; 
+          tree=e
+        end
+        put tree
+
+        open(STDERR.path){|f| 
+          f.pos=pos
+          put warnings=f.read.split #warnings
+        }
+      end
+      rescue Exception=>e; put e; raise
+      ensure exit!
+      end
+  end
+
+    ##
+    # Finds the user's home directory.
+    #--
+    # Some comments from the ruby-talk list regarding finding the home
+    # directory:
+    #
+    #   I have HOME, USERPROFILE and HOMEDRIVE + HOMEPATH. Ruby seems
+    #   to be depending on HOME in those code samples. I propose that
+    #   it should fallback to USERPROFILE and HOMEDRIVE + HOMEPATH (at
+    #   least on Win32).
+    #(originally stolen from rubygems)
+    def find_home
+      ['HOME', 'USERPROFILE'].each do |homekey|
+        return ENV[homekey] if ENV[homekey]
+      end
+
+      if ENV['HOMEDRIVE'] && ENV['HOMEPATH'] then
+        return "#{ENV['HOMEDRIVE']}#{ENV['HOMEPATH']}"
+      end
+
+      begin
+        File.expand_path("~")
+      rescue
+        if File::ALT_SEPARATOR then
+            "C:/"
+        else
+            "/"
+        end
+      end
+    end
+
+end
+
+ParseTreeServer.new.main if $0==__FILE__

data/lib/redparse/pthelper.rb

@@ -0,0 +1,43 @@
+class RedParse
+  def self.remove_silly_begins(pt)
+    pt.each_with_index{|x,i|
+      if Array===x
+        remove_silly_begins(x)
+        if x.size==2 and x.first==:begin
+          pt[i]=x=x.last
+        end
+      end
+    }
+  end
+end
+
+__END__
+begin require 'rubygems'
+rescue LoadError; #do nothing
+end
+
+have_graphwalk=true
+begin require 'ron/graphedge'
+rescue LoadError; 
+  warn 'Ron::GraphWalk not found; some tests will be too strict'
+  have_graphwalk=false
+end
+
+unless have_graphwalk
+  class RedParse
+    def self.remove_silly_begins(pt) pt end
+  end
+else
+  class RedParse
+    def self.remove_silly_begins(pt)
+      munger=proc{|cntr,o,i,ty,useit|
+        if Array===o and o.size==2 and o.first==:begin
+          useit[0]=true
+          Ron::GraphWalk.graphcopy(o.last,&munger)
+        end
+      }
+      Ron::GraphWalk.graphcopy(pt,&munger)
+    end
+  end
+end
+

data/lib/redparse/reg_more_sugar.rb

@@ -20,7 +20,7 @@
   unless Array===(Class*5).subregs
     module ::Reg
       class Repeat
-        undef subregs if instance_methods.include? "subregs"
+        undef subregs if allocate.respond_to? :subregs
         def subregs
           [@reg]
         end
@@ -29,7 +29,7 @@
   end
 
 
-  unless defined? ::Reg::Transform and ::Reg::Transform.ancestors.include? ::Reg::HasCmatch
+  unless defined? ::Reg::Transform #and ::Reg::Transform.ancestors.include? ::Reg::HasBmatch
   #hack, until support for this syntax makes it into the release of reg
     module ::Reg
       class Transform;
@@ -46,7 +46,7 @@
     end    
   end
 
-  unless ::Reg::Reg.instance_methods.include? "lb"
+  unless Object.allocate.extend(::Reg::Reg).respond_to? :lb
     module ::Reg
       module Reg
         def lb
@@ -73,7 +73,7 @@
     end 
   end
  
-  unless ::Reg::Reg.instance_methods.include? "la"
+  unless Object.allocate.extend(::Reg::Reg).respond_to? :la
     module ::Reg
       module Reg
         def la
@@ -100,7 +100,7 @@
     end 
   end
 
-  unless ::Reg::Reg.instance_methods.include? "watch"
+  unless Object.allocate.extend(::Reg::Reg).respond_to? :watch
     module ::Reg
       module Reg
         def watch

data/lib/redparse/version.rb

@@ -1,3 +1,3 @@
 class RedParse
-  VERSION='0.8.3'
+  VERSION='0.8.4'
 end

data/redparse.gemspec

@@ -0,0 +1,43 @@
+# -*- encoding: utf-8 -*-
+
+require "#{File.dirname(__FILE__)}/lib/redparse/version"
+RedParse::Description=open("README.txt"){|f| f.read[/^==+ ?description[^\n]*?\n *\n?(.*?\n *\n.*?)\n *\n/im,1] }
+RedParse::Latest_changes="###"+open("History.txt"){|f| f.read[/\A===(.*?)(?====)/m,1] }
+
+Gem::Specification.new do |s|
+  s.name = "redparse"
+  s.version = RedParse::VERSION
+  s.date = Time.now.strftime("%Y-%m-%d")
+  s.authors = ["Caleb Clausen"]
+  s.email = %q{caleb (at) inforadical (dot) net}
+  s.summary = "RedParse is a ruby parser written in pure ruby."
+  s.description = RedParse::Description
+  s.homepage = %{http://github.com/coatl/redparse}
+  s.rubyforge_project = %q{redparse}
+
+  s.files = `git ls-files`.split
+  s.test_files = %w[test/test_all.rb]
+  s.require_paths = ["lib"]
+  s.extra_rdoc_files = ["README.txt", "COPYING.LGPL"]
+  s.has_rdoc = true
+  s.rdoc_options = %w[--main README.txt]
+
+  s.rubygems_version = %q{1.3.0}
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 2
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency("rubylexer", '0.7.7')
+      s.add_runtime_dependency("reg", ['>= 0.4.8'])
+    else
+      s.add_dependency("rubylexer", '0.7.7')
+      s.add_dependency("reg", ['>= 0.4.8'])
+    end
+  else
+    s.add_dependency("rubylexer", '0.7.7')
+    s.add_dependency("reg", ['>= 0.4.8'])
+  end
+end

data/test/data/skkdictools.rb

@@ -0,0 +1,3 @@
+#!/usr/bin/ruby -Ke
+  okuri = "p" if can_postfix =~ /^��[��-��]/
+  okuri = "k" if can_postfix =~ /^��[��-��]/

data/test/generate_parse_tree_server_rc.rb

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'parse_tree'
+
+    ##
+    # Finds the user's home directory. 
+    #--
+    # Some comments from the ruby-talk list regarding finding the home
+    # directory:
+    #
+    #   I have HOME, USERPROFILE and HOMEDRIVE + HOMEPATH. Ruby seems
+    #   to be depending on HOME in those code samples. I propose that
+    #   it should fallback to USERPROFILE and HOMEDRIVE + HOMEPATH (at
+    #   least on Win32).
+    #(originally stolen from rubygems)
+    def find_home
+      ['HOME', 'USERPROFILE'].each do |homekey|
+        return ENV[homekey] if ENV[homekey]
+      end
+
+      if ENV['HOMEDRIVE'] && ENV['HOMEPATH'] then
+        return "#{ENV['HOMEDRIVE']}#{ENV['HOMEPATH']}"
+      end
+
+      begin
+        File.expand_path("~")
+      rescue
+        if File::ALT_SEPARATOR then
+            "C:/"
+        else
+            "/"
+        end
+      end
+    end
+
+PARSETREE_AND_DEPENDANCIES=
+  %r{/(?:ZenTest|RubyInline|sexp_processor|ParseTree)-}
+
+pt_needs_dirs=$:.grep PARSETREE_AND_DEPENDANCIES
+
+open find_home+"/.redparse/parse_tree_server.rc","w" do |rcfile|
+  rcfile.write pt_needs_dirs.join("\n")+"\n"  
+end

data/test/rp-locatetest.rb

@@ -17,6 +17,43 @@
     Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 =end
 
+
+=begin 
+rp-locatetest passes all the ruby files it can find on the system (using 
+the unix command locate) to bin/redparse --vsparsetree. In other words, 
+it's comparing the output of RedParse (when cvt'd to a ParseTree-style 
+tree) to the actual output of ParseTree.
+
+I usually run it like this:
+
+  nice ruby test/rp-locatetest.rb >test.log 2>&1 &
+
+That runs rp-locatetest at low priority, in the background, saving 
+stdout and stderr to test.log.
+
+You can then trawl through test.log, looking for lines that don't start 
+with 'no differences in '. (Other than the page of warnings in the 
+beginning, which is repeated every so often.) Or, you can look for 
+non-commented lines in the file 'problemfiles'. (Which is just a list of 
+files that caused failures in rp-locatetest.)
+
+Ok, this is more complicated than it ought to be. Sorry.
+
+rp-locatetest takes a LONG time to run, and hogs the cpu at 100%. That's 
+why I was using nice. So, make sure it's ok to do that on whatever 
+server you run it on. I always get impatient and kill it after some 
+hours or days. On a subsequent run it'll pick up again where it left 
+off.
+
+Right now, about 0.5% of my ruby files cause problems for rp-locatetest.  
+Most of those failures correspond to known problems (the file is listed 
+at the end of the README and/or there's a case in test_redparse.rb for 
+that failure).
+=end
+
+$VERBOSE=1
+$Debug=1
+
 #require 'test/code/rubylexervsruby'
 #require 'test/code/strgen'
 require "redparse/problemfiles"
@@ -152,7 +189,8 @@ until testdata.empty?
 
 #  puts "testing: "+chunk.join(" ")
   chunk.empty? and next
-  system $RUBY, "-Ilib", "bin/redparse", "--update-problemfiles", "-q", "--vsparsetree", *chunk
+  system $RUBY, "-Ilib", "bin/redparse", "--update-problemfiles", "--ignore-silly-begins", "--unparse", 
+    "-q", "--vsparsetree", *chunk
   exit 2 if $?>>8 == 2  #exit if child got ^c
 end
 end
@@ -343,3 +381,5 @@ __END__
 /usr/lib/ruby/1.8/hpricot/traverse.rb
 /home/caleb/sandbox/rubylexer/jewels/net-netrc-0.2.1/lib/net/netrc.rb
 /home/caleb/sandbox/rubylexer/jewels/etc-0.2.0/lib/etc.rb
+
+

data/test/test_1.9.rb

@@ -0,0 +1,114 @@
+require 'test/unit'
+require 'rubylexer/test/oneliners_1.9'
+require "redparse"
+
+class TestsFor1_9 < Test::Unit::TestCase
+  RUBY_1_9_TO_1_8_EQUIVALENCES=[
+    '{a:  b}'...'{:a=>b}',
+    '{a:  b, c:  d}'...'{:a=>b, :c=>d}',
+    "a ? b\n :  c"..."a ? b : \n c",
+
+    'not(true)'...'not true',
+    'not(+1)'...'not +1',
+    'not (true).to_s'...'not (true).to_s', #equivalent, but parser gets there different ways
+  ]
+
+  RUBY_1_9_TO_1_8_EQUIVALENCES_BUT_FOR_STRESC=[
+    '"foo"'...'"foo"',
+    '"foo#{bar}"'...'"foo#{bar}"',
+
+    '%W[foo]'...'%W[foo]',
+    '%W[foo#{bar}]'...'%W[foo#{bar}]',
+
+    '`foo`'...'`foo`',
+    '`foo#{bar}`'...'`foo#{bar}`',
+
+    '//'...'//',
+    '/aa/i'...'/aa/i',
+    '/a#{a}/u'...'/a#{a}/u',
+    '/a#{"a"}/e'...'/a#{"a"}/e',
+    '/b#{__FILE__}/s'...'/b#{__FILE__}/s',
+    '/bb/m'...'/bb/m',
+    '/b b/x'...'/b b/x',
+    '/b#{b}/o'...'/b#{b}/o',
+  ]
+
+
+  RUBY_1_9_VALID=[
+    'not (true).to_s',
+  ]
+
+  include RedParse::Nodes
+  RUBY_1_9_PATTERNS={
+    'not(true).to_s'=>+CallNode[+UnOpNode["not", +VarLikeNode["true"]], "to_s"],
+  }
+
+  def test_ruby19_equivs
+    RUBY_1_9_TO_1_8_EQUIVALENCES.each{|pair|
+      new,old=pair.first,pair.last
+      pt19=RedParse.new(new,'(eval)',1,[],:rubyversion=>1.9,:cache_mode=>:none).parse
+      pt18=RedParse.new(old,'(eval)',1,[],:cache_mode=>:none).parse
+      assert_equal pt18,pt19
+    }
+  end
+ 
+  def test_ruby19_equivs_but_for_stresc
+    RUBY_1_9_TO_1_8_EQUIVALENCES_BUT_FOR_STRESC.each{|pair|
+      new,old=pair.first,pair.last
+      pt19=RedParse.new(new,'(eval)',1,[],:rubyversion=>1.9,:cache_mode=>:none).parse
+      pt18=RedParse.new(old,'(eval)',1,[],:cache_mode=>:none).parse
+      if pt18.instance_variable_get(:@bs_handler)==:dquote_esc_seq
+        pt18.instance_variable_set :@bs_handler,:dquote19_esc_seq
+      else
+        pt18.instance_variable_set :@bs_handler,:Wquote19_esc_seq
+        pt18.instance_variable_get(:@parses_like).each{|x|
+          x.instance_variable_set :@bs_handler,:Wquote19_esc_seq if x.instance_variable_get :@bs_handler
+        }
+      end
+      assert_equal pt18,pt19
+    }
+  end
+ 
+  def test_ruby19_valid
+    RUBY_1_9_VALID.each{|xmpl|
+      pt19=RedParse.new(xmpl,'(eval)',1,[],:rubyversion=>1.9,:cache_mode=>:none).parse
+      assert_nil pt19.errors
+    }
+  end
+
+  def test_ruby19_patterns
+    RUBY_1_9_PATTERNS.each_pair{|code,pattern|
+      pt=RedParse.new(code,'(eval)',1,[],:rubyversion=>1.9,:cache_mode=>:none).parse
+      assert_match pattern, pt
+    }
+  end
+
+
+
+
+  include Ruby1_9OneLiners
+
+  def count_methods(tree)
+    count=0
+    tree.walk{|node|
+      case node
+      when CallSiteNode, MethodNode; count+=1
+      end
+    }
+    return count
+  end
+
+  def test_1_9
+    EXPECT_NO_METHODS.each{|snippet|
+      tree=RedParse.new(snippet,"-e").parse
+      count=count_methods(tree)
+      assert_equal count,0
+    }
+
+    EXPECT_1_METHOD.each{|snippet|
+      tree=RedParse.new(snippet,"-e").parse
+      count=count_methods(tree)
+      assert_equal count,1
+    }
+  end
+end