redparse 0.8.4 → 1.0.0

data/lib/redparse/babynodes.rb

@@ -1,3 +1,20 @@
+=begin
+    redparse - a ruby parser written in ruby
+    Copyright (C) 2008,2009, 2012, 2016  Caleb Clausen
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+=end
 require 'pp'
 require 'rubygems'
 require 'rubylexer'

data/lib/redparse/babyparser.rb

@@ -1,3 +1,20 @@
+=begin
+    redparse - a ruby parser written in ruby
+    Copyright (C) 2012, 2016  Caleb Clausen
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+=end
 require 'pp'
 require 'rubygems'
 require 'rubylexer'

data/lib/redparse/cache.rb

@@ -1,7 +1,38 @@
+=begin
+    redparse - a ruby parser written in ruby
+    Copyright (C) 2012, 2016  Caleb Clausen
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Lesser General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+=end
 require 'digest/sha2'
+require 'fileutils'
+begin
+  require 'etc'
+rescue LoadError #ignore it
+end
 class RedParse
   class Cache
-    def initialize *params
+    def initialize(is_file,name,*params)
+      divider=params.index(:/)
+      @input_id=params[0...divider]
+      @output_id=params[divider+1..-1]
+      @rubyversion=@input_id.pop
+      @name=name
+      @is_file=is_file
+    end
+
+    def old_initialize *params
       @callersfile=Digest::SHA2.hexdigest params.join(',')
       @homedir=find_home+"/.redparse/"
       Dir.mkdir @homedir unless File.exist? @homedir
@@ -13,6 +44,8 @@ class RedParse
         File.open(@homedir+"/parserdigest","wb"){|fd| fd.puts actual_digest } #update saved digest
       end
       retire_old_entries
+    rescue Errno::EACCES
+      #do nothing
     end
 
     def cachedir
@@ -76,7 +109,16 @@ class RedParse
     #   it should fallback to USERPROFILE and HOMEDRIVE + HOMEPATH (at
     #   least on Win32).
     #(originally stolen from rubygems)
-    def find_home
+    #before trying env variables, try getpwuid, since the environment
+    #might have been cleansed (eg by a cgi server) or altered (eg by 
+    #rubygems tests).
+    def Cache.find_home
+      begin
+        return Etc.getpwuid.dir
+      rescue Exception
+        #do nothing
+      end
+
       ['HOME', 'USERPROFILE'].each do |homekey|
         return ENV[homekey] if ENV[homekey]
       end
@@ -95,6 +137,7 @@ class RedParse
         end
       end
     end
+    def find_home; Cache.find_home end
     private :find_home, :entry_files, :redparse_rb_hexdigest, :retire_old_entries, :max_size, :hexdigest_of_file
 
     def hash_of_input input
@@ -105,7 +148,7 @@ class RedParse
       end
     end
 
-    def get input
+    def old_get input
       hash=hash_of_input input
       cachefile=cachedir+hash
       if File.exist? cachefile
@@ -138,17 +181,17 @@ class RedParse
         end
         return result
       end
-    rescue EOFError
+    rescue EOFError,Errno::EACCES
       return nil
     end
 
-    def put input,result
+    def old_put input,result
       hash=hash_of_input input
       File.open(cachedir+hash, "wb"){|fd|
         begin
           Thread.current["Marshal.ignore_sclass"]=true
           Marshal.dump(result,fd)
-        rescue TypeError=>e #dump failed
+        rescue TypeError #dump failed
           File.unlink cachedir+hash
           begin
             require 'ron'
@@ -168,5 +211,246 @@ class RedParse
       warn "cache write failed for:\n#{result.inspect}"
       File.unlink cachedir+hash
     end
+
+    def get input,output_type="parsed"
+      output_type=@rubyversion.to_s+output_type
+      if @is_file
+        Cache.read_for_file(input,@name,@input_id,output_type,@output_id)
+      else
+        Cache.read_for_str(input,@input_id,output_type,@output_id)
+      end
+    end
+
+    def put input,result,output_type="parsed"
+      output_type=@rubyversion.to_s+output_type
+      if @is_file
+        Cache.write_for_file(input,@name,@input_id,output_type,@output_id,result)
+      else
+        Cache.write_for_str(input,@input_id,output_type,@output_id,result)
+      end
+    end
+  end
+
+  CACHEDIRNAME="obj"
+  GLOBALCACHEDIRNAME=".ruby-"+CACHEDIRNAME
+
+  def Cache.get input,name,input_id,output_type,outputter_id
+    if String===input
+      read_for_str(input,input_id,output_type,outputter_id)
+    else
+      read_for_file(input,name,input_id,output_type,outputter_id)
+    end
+  end
+
+  def Cache.put input,name,input_id,output_type,outputter_id,result
+    if String===input
+      write_for_str(input,input_id,output_type,outputter_id,result)
+    else
+      write_for_file(input,name,input_id,output_type,outputter_id,result)
+    end
+  end
+
+  def Cache.read input,name,input_id,output_type,outputter_id
+    result=get input,name,input_id,output_type,outputter_id
+    return result if result
+
+    result=yield
+    put input,name,input_id,output_type,outputter_id,result
+  end
+
+  def Cache.cachename_for_file(name,output_type)
+    cache_fn=File.join(File.dirname(name),CACHEDIRNAME,File.basename(name))+"."+output_type
+    dir=File.dirname(cache_fn)
+    begin
+      Dir.mkdir dir unless File.exists?(dir)
+      raise unless File.writable?(dir)
+    rescue Exception #chosen dir is not writeable
+      cache_fn=File.join(find_home,GLOBALCACHEDIRNAME,name)+"."+output_type
+      FileUtils.mkdir_p( File.dirname( cache_fn ) )
+    end
+    return cache_fn
+  end
+
+  def Cache.cachename_for_str(str,options,output_type)
+    options=options.to_a.sort_by{|k,v| k}.map{|k,v| k+'='+v} if Hash===options
+    options=options.join(',') unless String===options
+    #options.gsub!('%n','%%n'); options.gsub!("\n",'%n')
+    #fail if /\n/===options
+    #str=options+"\n"+str
+    options=Digest::SHA2.hexdigest(options) if options.size>100
+    digest=Digest::SHA2.hexdigest(str)
+    cache_fn=File.join(find_home, GLOBALCACHEDIRNAME,"-e",options,digest)+"."+output_type
+    FileUtils.mkdir_p( File.dirname( cache_fn ) )
+    return cache_fn
+  end
+
+  HEADER_REX=/\A\n\#encoded\ with\ (ascii|Ron|Marshal)\n
+                  \#(.*)\n
+                  \#([0-9A-Fa-f]+)\n
+                  \#([0-9A-Fa-f]+)\n
+                  \#\s{0,9}([0-9]{1,9})\s{0,9}\n\z
+  /x
+
+  #encoder, options, inputhash, outputterhash, size
+  def Cache.read_trailer(f)
+    f.seek(-(9*3+2+1),IO::SEEK_END)
+    size=f.read[/\n\# *([0-9]+) *\n\z/,1].to_i
+    return if size.zero?
+    f.seek(-size,IO::SEEK_END)
+    buf=f.read
+    return unless result=buf.match(HEADER_REX)
+    result=result.to_a
+    result.shift
+    return result
+  end
+
+  def Cache.attempt_read(cache_fn,inputdigest,options,output_identity,want_file=false)
+    return if !File.exist? cache_fn
+    
+    #warn "...reading from cache file #{cache_fn}"
+    #warn "...options=#{options.inspect}"
+
+      outid=Digest::SHA2.hexdigest output_identity.join(',')
+
+      cache_f=File.open(cache_fn,"rb")
+      pos=cache_f.pos
+      options=options.to_s
+
+      encoder,saved_options,saved_inputdigest,saved_outid=read_trailer cache_f
+      error=case
+      when !encoder; "trailer not found"
+      when saved_inputdigest!=inputdigest; "input changed"
+      when saved_outid!=outid; "outputter changed"
+      when saved_options!=options; "options changed from #{saved_options.inspect} to #{options.inspect}"
+      end
+      if error
+        #warn "...cache read failed because #{error}"
+        cache_f.close
+        cache_f=nil
+        return
+      end
+     
+      case encoder
+      when 'ascii'
+        return cache_f if want_file
+        return cache_f.read
+      when 'Ron'
+        begin
+          require 'ron'
+          return Ron.load( cache_f.read )
+        rescue Exception=>e
+          warn "#{e.class}: #{e}"
+          warn "cache ron read failed for:#{cache_fn}"
+          return nil
+        end
+      when 'Marshal'
+        cache_f.pos=pos
+        begin
+          return Marshal.load( cache_f )
+        rescue Exception=>e
+          warn "#{e.class}: #{e}"
+          warn "cache read failed for:#{cache_fn}"
+          return nil
+        end
+      else 
+        warn "unrecognized RedParse::Cache encoder type: #{encoder}"
+        return nil
+      end
+  end
+
+  def Cache.attempt_write(cache_fn,inputdigest,options,output_identity,result)
+    #STDERR.write "...writing to cache file #{cache_fn}... "
+
+    if result.respond_to? :sysread
+      path=result.path
+      begin
+        FileUtils.move(path,cache_fn)
+      rescue Exception
+        FileUtils.copy(path,cache_fn)
+      end
+      encoder= "ascii"
+      #STDERR.puts 'file'
+      cache_f=File.open(cache_fn,"a")
+    else
+      cache_f=File.open(cache_fn,"wb")
+
+      if String===result
+        cache_f.write result
+        encoder= "ascii"
+        #STDERR.puts 'ascii'
+      else begin
+        Thread.current["Marshal.ignore_sclass"]=true
+        Marshal.dump(result,cache_f)
+        encoder= "Marshal"
+        #STDERR.puts 'Marshal'
+      rescue TypeError #dump failed
+        #STDERR.write "Marshal failed => "
+        cache_f.close
+        cache_f=File.open(cache_fn,"wb")
+        begin
+          require 'ron'
+          cache_f.write Ron.dump(result)
+          encoder='Ron'
+          #STDERR.puts "Ron"
+        rescue Exception
+          #STDERR.puts "Ron failed"
+          File.unlink(cache_fn)
+          return
+        end
+      ensure
+        Thread.current["Marshal.ignore_sclass"]=nil
+      end end
+    end
+
+    outid=Digest::SHA2.hexdigest output_identity.join(',')
+    trailer= "\n#encoded with #{encoder}\n##{options}\n##{inputdigest}\n##{outid}\n"
+    sz=trailer.size+2
+    szsz=sz.to_s.size
+    sz+=szsz
+    sz+=1 unless szsz==sz.to_s.size
+    trailer<< "##{sz}\n"
+    fail 'bad trailer size' unless trailer[/^\#([0-9]+)\n\Z/,1].to_i==trailer.size
+    cache_f.write trailer
+
+    return result
+  ensure 
+    cache_f.close if cache_f
+    #STDERR.puts("...options=#{options.inspect}")
+  end
+
+  def Cache.read_for_file(input,name,options,output_type,output_identity,&if_missing)
+    name=File.expand_path(name)
+    inputdigest=Digest::SHA2.file(name).hexdigest
+    cache_fn=cachename_for_file name,output_type
+    result=attempt_read(cache_fn,inputdigest,options,output_identity,:want_file)
+    return result if result
+
+    if if_missing and result=if_missing.call
+      write_for_file(input,name,options,output_type,output_identity,result)
+    end
+  end
+
+  def Cache.write_for_file(input,name,options,output_type,output_identity,result)
+    name=File.expand_path(name)
+    inputdigest=Digest::SHA2.file(name).hexdigest
+    cache_fn=cachename_for_file name,output_type
+    attempt_write(cache_fn,inputdigest,options,output_identity,result)
+  end
+
+  def Cache.read_for_str(input,options,output_type,output_identity,&if_missing)
+    inputdigest=Digest::SHA2.hexdigest(input)
+    cache_fn= cachename_for_str input,options,output_type
+    result=attempt_read(cache_fn,inputdigest,options,output_identity)
+    return result if result
+
+    if if_missing and result=if_missing.call
+      write_for_str(input,options,output_type,output_identity,result)
+    end
+  end
+
+  def Cache.write_for_str(input,options,output_type,output_identity,result)
+    inputdigest=Digest::SHA2.hexdigest(input)
+    cache_fn= cachename_for_str input,options,output_type
+    attempt_write(cache_fn,inputdigest,options,output_identity,result)
   end
 end

data/lib/redparse/compile.rb

@@ -1,6 +1,6 @@
 =begin
     redparse - a ruby parser written in ruby
-    Copyright (C) 2008,2009  Caleb Clausen
+    Copyright (C) 2008,2009, 2012, 2016  Caleb Clausen
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU Lesser General Public License as published by
@@ -35,6 +35,7 @@ require "redparse/node"
 require "redparse/reg_more_sugar"
 require "redparse/generate"
 require "redparse/cache"
+require "redparse/stackableclasses"
 
 class RedParse
   
@@ -143,101 +144,8 @@ if defined? END_ATTACK
 
     end
 
-    
-
-
-    #just the left side (the stack/lookahead matchers)
-    def LEFT
-      @rules.map{|r| r.left.subregs }.flatten
-    end
-
-    #remove lookahead and lookback decoration
-    def LEFT_NO_LOOKING
-    l=LEFT()
-    l.map!{|m| 
-      case m #
-      when Reg::LookAhead,Reg::LookBack; m.subregs[0]
-      when Proc; []
-      else m #
-      end #
-    } 
-    l
-    end
-
-    #all classes mentioned in rules, on left and right sides
-    def STACKABLE_CLASSES #
-      return @sc_result unless 	@sc_result.nil? 
-      @sc_result=false
-      l=LEFT_NO_LOOKING()
-      l=l.map{|lm| sc_juice lm}.flatten.compact
-      r=  @rules.map{|rr| rr.right }.grep(Class) #classes in productions
-      result=l+r 
-      @sc_result=result.grep(Class).uniq
-      fail if @sc_result.empty?
-      return @sc_result
-    end
+    include RedParse::StackableClasses
 
-    def juice(m)
-      case m  #
-      when Class; 
-        return [m] unless @subclasses_of
-        result=[m]  # and subclasses too
-        i=0
-        while item=result[i]
-          #p item
-          result.concat @subclasses_of[item]
-          i += 1
-        end
-        result
-      when String,Regexp; juice(RedParse.KW(m))
-      when Reg::And; m.subregs.map{|x| juice(x).flatten.compact}.inject{|sum,rr| sum&rr} 
-      when Reg::Or; m.subregs.map( &method(:juice) )
-      when Reg::Not;  
-        m=m.subregs[0]
-        if Class===m or (Reg::Or===m  and 
-             m.subregs.inject{|sum,x| sum && (Class===x) })
-          j=juice(m)
-          STACKABLE_CLASSES()-j.flatten.compact rescue j
-        else 
-          STACKABLE_CLASSES()
-        end
-      else STACKABLE_CLASSES()
-      end
-    end
-
-    def sc_juice(m) 
-      case m #
-      when Class; [m]
-      when String,Regexp; juice(RedParse.KW(m))
-#      when String,Regexp; [KeywordToken]
-      when Reg::And; m.subregs.map{|x| sc_juice(x)}.compact.map{|x| x.flatten.compact}.inject{|sum,rr| sum&rr }
-      when Reg::Or; m.subregs.map( &method(:sc_juice) )
-      when Reg::Not; sc_juice(m.subregs[0])
-      when Reg::LookAhead, Reg::LookBack; sc_juice(m.subregs[0])
-      else []
-      end
-    end
-
-    def LOOKAHEAD_CLASSES rule
-      last=rule.left.subregs.last 
-      return STACKABLE_CLASSES() unless Reg::LookAhead===last
-      la= last.subregs[0]
-      return juice(la).flatten.compact
-    end
-  #
-    def TOS_CLASSES rule
-      i=-1
-      mats=rule.left.subregs
-      m=mats[i]
-      m=mats[i-=1] if Reg::LookAhead===m || Proc===m
-      result=[]
-      while Reg::Repeat===m and m.times.min.zero?
-        result<<juice(m.subregs[0])
-        m=mats[i-=1]
-      end
-      return (result+juice(m)).flatten.compact
-    end
-   
     def [](i)
       @rules[i]
     end
@@ -1386,9 +1294,10 @@ end
 
       states=all_states
 #      @rules=expanded_RULES 
-      @inputs=nil #Marshal no like it
 
+      false && \
       begin
+        @inputs=nil #Marshal no like it
         p :dumping
         Marshal.dump(self,f=open("cached_parse_tables.drb","wb"))
         p :dump_done!
@@ -1399,7 +1308,7 @@ end
         @inputs=enumerate_exemplars
       end
     end
-    f.close
+    f.close if f
    
     #look for unused dotted rules and actions 
     #also states with drs past the end