crystalizer 0.2.2

data/lib/ruby2cext/compiler.rb

@@ -0,0 +1,311 @@
+
+require "rubynode"
+require "rbconfig"
+require 'sane'
+require 'logger'
+require_rel '.' 
+require "rubynode"
+require "rbconfig"
+require "ruby2cext/parser"
+require "ruby2cext/error"
+require "ruby2cext/tools"
+require "ruby2cext/c_function"
+require "ruby2cext/version"
+require_rel 'plugins' 
+require 'sane'
+
+module Ruby2CExtension
+
+  class Compiler
+
+    attr_reader :name, :logger, :plugins
+
+    def initialize(name, logger = nil)
+      @name = name
+      @logger = logger
+      @funs = []
+      @funs_reuseable = {}
+      @toplevel_funs = []
+      @sym_man = Tools::SymbolManager.new
+      @global_man = Tools::GlobalManager.new
+      @uniq_names = Tools::UniqueNames.new
+      @helpers = {}
+      @plugins = []
+      @preprocessors = {}
+    end
+
+    # plugins is {:optimizations => :all || [:name1, :name2]}, also {:warnings => true} or something 
+    # logger if require 'logger'; Logger.new
+    # include_paths = [] # dirs
+    # only_c -- pass true if you just want the source, not compiled
+    def Compiler.compile_file(file_name, plugins, include_paths, only_c, logger)
+      bn = File.basename(file_name)
+      unless bn =~ /\A(.*)\.rb\w?\z/
+        raise "#{file_name} is no ruby file"
+      end
+      name = $1;
+      unless name =~ /\A\w+\z/
+        raise "'#{name}' is not a valid extension name"
+      end
+      file_name = File.join(File.dirname(file_name), bn)
+
+      logger.info("reading #{file_name}")
+      source_str = IO.read(file_name)
+
+      logger.info("translating #{file_name} to C")
+      c = Compiler.new(name, logger)
+      unless include_paths.empty?
+        plugins = plugins.merge({:require_include => [include_paths, [file_name]]})
+      end
+      logger.debug("plugins = #{plugins.inspect}")
+      c.add_plugins(plugins)
+      logger.debug("plugins used: #{c.plugins.map { |pi| pi.class }.inspect}")
+      c.add_rb_file(source_str, file_name)
+      c_code = c.to_c_code
+
+      c_file_name = File.join(File.dirname(file_name), "#{name}.c")
+      logger.info("writing #{c_file_name}")
+      File.open(c_file_name, "w") { |f| f.puts(c_code) }
+
+      if only_c
+         c_code
+      else
+        logger.info("compiling #{c_file_name}")
+        Compiler.compile_c_file_to_dllib(c_file_name, logger)
+      end
+    end
+
+
+
+
+    def to_c_code(time_stamp = Time.now)
+      plugins_global = @plugins.map { |plugin| plugin.global_c_code }
+      plugins_init = @plugins.map { |plugin| plugin.init_c_code }
+      res = [
+        "/* generated by #{FULL_VERSION_STRING} on #{time_stamp} */",
+        "#include <ruby.h>",
+        "#include <node.h>",
+        "#include <env.h>",
+        "#include <st.h>",
+        "extern VALUE ruby_top_self;",
+        "static VALUE org_ruby_top_self;",
+        @sym_man.to_c_code,
+        @global_man.to_c_code,
+      ]
+      res.concat(@helpers.keys.sort)
+      res.concat(plugins_global)
+      res.concat(@funs)
+      res << "void Init_#{@name}() {"
+      res << "org_ruby_top_self = ruby_top_self;"
+      # just to be sure
+      res << "rb_global_variable(&org_ruby_top_self);"
+      res << "init_syms();"
+      res << "init_globals();"
+      res << "NODE *cref = rb_node_newnode(NODE_CREF, rb_cObject, 0, 0);"
+      res.concat(plugins_init)
+      @toplevel_funs.each { |f| res << "#{f}(ruby_top_self, cref);" }
+      res << "}"
+      res.join("\n").split("\n").map { |l| l.strip }.reject { |l| l.empty? }.join("\n")
+    end
+
+    def add_toplevel(function_name)
+      @toplevel_funs << function_name
+    end
+
+    # non destructive: node_tree will not be changed
+    def compile_toplevel_function(node_tree, private_vmode = true)
+      CFunction::ToplevelScope.compile(self, node_tree, private_vmode)
+    end
+
+    NODE_TRANSFORM_OPTIONS = {:include_node => true, :keep_newline_nodes => true}
+
+    def rb_file_to_toplevel_functions(source_str, file_name)
+      res = []
+      hash = Parser.parse_string(source_str, file_name)
+      # add all BEGIN blocks, if available
+      if (beg_tree = hash[:begin])
+        beg_tree = beg_tree.transform(NODE_TRANSFORM_OPTIONS)
+        if beg_tree.first == :block
+          beg_tree.last.each { |s| res << compile_toplevel_function(s, false) }
+        else
+          res << compile_toplevel_function(beg_tree, false)
+        end
+      end
+      # add toplevel scope
+      if (tree = hash[:tree])
+        res << compile_toplevel_function(tree.transform(NODE_TRANSFORM_OPTIONS))
+      end
+      res
+    end
+
+    def add_rb_file(source_str, file_name)
+      rb_file_to_toplevel_functions(source_str, file_name).each { |fn|
+        add_toplevel(fn)
+      }
+    end
+
+    # uniq name
+    def un(str)
+      @uniq_names.get(str)
+    end
+    def sym(sym)
+      @sym_man.get(sym)
+    end
+    def global_const(str, register_gc = true)
+      @global_man.get(str, true, register_gc)
+    end
+    def global_var(str)
+      @global_man.get(str, false, true)
+    end
+
+    def log(str, warning = false)
+      if logger
+        if warning
+          logger.warn(str)
+        else
+          logger.info(str)
+        end
+      end
+    end
+
+    def add_helper(str)
+      @helpers[str] ||= true
+    end
+
+    def add_fun(code, base_name)
+      unless (name = @funs_reuseable[code])
+        name = un(base_name)
+        lines = code.split("\n")
+        unless lines.shift =~ /^\s*static / # first line needs static
+          raise Ruby2CExtError::Bug, "trying to add a non static function"
+        end
+        if lines.grep(/^\s*static /).empty? # only reuseably without static variables
+          @funs_reuseable[code] = name
+        end
+        unless code.sub!("FUNNAME", name)
+          raise Ruby2CExtError::Bug, "trying to add a function without FUNNAME"
+        end
+        @funs << code
+      end
+      name
+    end
+
+    def add_plugin(plugin_class, *args)
+      @plugins << plugin_class.new(self, *args)
+    end
+
+    def add_plugins(options)
+      if options[:warnings]
+        add_plugin(Plugins::Warnings)
+      end
+      if (opt = options[:optimizations])
+        if opt == :all
+          opt = {
+            :const_cache=>true,
+            :case_optimize=>true,
+           # :direct_self_call=>true, # also buggy...
+           # :inline_builtin=>true, # causes a bug [just itself, too]
+            :cache_call=>true,
+            :builtin_methods=>true,
+            :inline_methods=>true,
+            :ivar_cache=>true
+          }
+        end
+        if opt[:const_cache]
+          add_plugin(Plugins::ConstCache)
+        end
+        if opt[:case_optimize]
+          add_plugin(Plugins::CaseOptimize)
+        end
+        if opt[:direct_self_call]
+          add_plugin(Plugins::DirectSelfCall)
+        end
+        if opt[:inline_builtin]
+          add_plugin(Plugins::InlineBuiltin)
+        end
+        if opt[:inline_methods]
+          add_plugin(Plugins::InlineMethods)
+        end
+        if opt[:cache_call]
+          add_plugin(Plugins::CacheCall)
+        end
+        if (builtins = opt[:builtin_methods])
+          if Array === builtins
+            builtins = builtins.map { |b| b.to_s.to_sym } # allow symbols, strings and the actual classes to work
+          else
+            builtins = Plugins::BuiltinMethods::SUPPORTED_BUILTINS
+          end
+          add_plugin(Plugins::BuiltinMethods, builtins)
+        end
+        if opt[:ivar_cache]
+          add_plugin(Plugins::IVarCache)
+        end
+      end
+      if (ri_args = options[:require_include])
+        unless Array === ri_args.first
+          ri_args = [ri_args] # to allow just an array of include paths to also work
+        end
+        add_plugin(Plugins::RequireInclude, *ri_args)
+      end
+    end
+
+    # preprocessors can be added by plugins. preprocessors are procs that
+    # take two arguments: the current cfun and the node (tree) to
+    # preprocess (which will have type node_type)
+    #
+    # The proc can either return a (modified) node (tree) or string. If a
+    # node (tree) is returned then that will be translated as usual, if a
+    # string is returned, that string will be the result
+    #
+    # Example, a preprocessor that replaces 23 with 42:
+    # add_preprocessor(:lit) { |cfun, node|
+    #   node.last[:lit] == 23 ? [:lit, {:lit=>42}] : node
+    # }
+    #
+    # Another way to do the same:
+    # add_preprocessor(:lit) { |cfun, node|
+    #   node.last[:lit] == 23 ? cfun.comp_lit(:lit=>42) : node
+    # }
+    #
+    # If multiple preprocessors are added for the same node type then they
+    # will be called after each other with the result of the previous one
+    # unless it is a string, then the following preprocessors are ignored
+    def add_preprocessor(node_type, &pp_proc)
+      (@preprocessors[node_type] ||= []) << pp_proc
+    end
+
+    def preprocessors_for(node_type)
+      @preprocessors[node_type]
+    end
+
+    conf = ::Config::CONFIG
+    cflags = [conf["CCDLFLAGS"], conf["CFLAGS"], conf["ARCH_FLAG"]].join(" ")
+    COMPILE_COMMAND = "#{conf["LDSHARED"]} #{cflags} -I . -I #{conf["archdir"]} " # added -c
+    DLEXT = conf["DLEXT"]
+
+    # compiles a C file using the compiler from rbconfig
+    def self.compile_c_file_to_dllib(c_file_name, logger = nil)
+      conf = ::Config::CONFIG
+      unless c_file_name =~ /\.c\z/
+        raise Ruby2CExtError, "#{c_file_name} is no C file"
+      end
+      dl_name = c_file_name.sub(/c\z/, DLEXT)
+      cmd = "#{COMPILE_COMMAND} -o #{dl_name} #{c_file_name}"
+      if RUBY_PLATFORM =~ /mswin32/
+        cmd << " -link /INCREMENTAL:no /EXPORT:Init_#{File.basename(c_file_name, ".c")}"
+      end
+      if RUBY_PLATFORM =~ /mingw/
+        cmd << " #{ conf['DLDFLAGS'] } #{ conf['SOLIBS'] }  "
+        cmd << " -L#{ conf['libdir'] } #{ conf["LIBRUBYARG_SHARED"] } "
+      end
+      logger.info(cmd) if logger
+
+      unless system(cmd) # run it
+        raise Ruby2CExtError, "error while executing '#{cmd}' #{`cmd`}"
+      end
+      dl_name
+    end
+
+  end
+
+end

data/lib/ruby2cext/concretize.rb

@@ -0,0 +1,269 @@
+require 'rubygems'
+require 'ruby2ruby'
+require 'parse_tree'
+require 'thread' # mutex
+require 'sane'
+# LTODO singleton methods, class methods, [procs?]
+
+module Ruby2CExtension
+
+  class Concretize
+
+    def Concretize.source_for klass, method_name
+      raw = @@pt.parse_tree_for_method(klass, method_name)
+      begin
+        processed = @@pt.process(raw)
+      rescue
+        return nil
+      end
+      code = @@r2r.process( processed )
+      if code.include?('&block') || code.include?('binding')
+        # eval?
+        # for now, try to avoid the tripsy case of ruby2cext not yielding arrays right yet...
+        # TODO add some test cases for it in 'broken.rb' or something
+        # LTODO fix it :)
+
+        # TODO test it with rdoc...yeah. A real test sniff
+        # compare with 1.9...
+        nil
+      else
+        code
+      end
+    end
+
+
+    begin
+      @@good_codes ||= YAML.load File.read('known_good_codes')
+    rescue Exception
+      @@good_codes = {}
+    end
+
+    at_exit {
+      puts 'saving'
+      File.write 'known_good_codes', YAML.dump(@@good_codes)
+    }
+    @@count = 0
+    @@mutex = Mutex.new
+    @@r2r = Ruby2Ruby.new
+    @@pt = ParseTree.new
+
+    # returns rb code, or c code, or true on successfull load
+    # returns nil if want_rb_afterward && had to compile
+    # which is ok
+    def Concretize.c_ify! klass, method_name, want_rb_afterward = false, want_just_c = false
+      count = @@mutex.synchronize { @@count += 1 }
+      rb = "temp_#{count}.rb"
+      ruby_code = source_for klass, method_name
+
+      # TODO it probably catches railsy class style poorly [re-use my other desc_method?]
+      return nil unless ruby_code
+      
+
+      assert klass.class.in?( [Class, Module]) # sanity check
+      comment = "# concretize temp file: autogenerated: #{ Time.now }\n"
+      if klass.class == Module
+        klass_string = "module #{klass}" # modules don't descend
+      else
+        nearest_ancestor = klass.ancestors[1..-1].find{|ancestor| ancestor.class == Class}
+        if nearest_ancestor
+          klass_string = "class #{klass} < #{nearest_ancestor}"
+        else
+          # Object, I suppose
+          klass_string = "class #{klass}"
+        end
+      end
+      
+      ruby_code_wrapped = comment + klass_string + "\npublic\n" + ruby_code +  "\nend\n"
+      @@log.debug ruby_code_wrapped
+      if @@good_codes[ruby_code] && want_rb_afterward
+        @@log.debug 'cache hit'
+        @@cache_hits+= 1
+        return ruby_code_wrapped
+      else
+        @@log.debug 'cache miss' + @@good_codes.length.to_s
+      end
+
+      # sometimes ruby2ruby unpacks them wrong, so test for that
+      File.open(rb, 'w') do |file|
+        file.write ruby_code_wrapped
+      end
+      output = `ruby -c #{rb} 2>&1`
+      if($?.exitstatus != 0)
+        # unparsable ruby was generated...hmm...
+        @@log.warn "got bad code generation", klass, method_name, ruby_code
+        File.delete rb
+        return nil # LTODO re-parse it [other parser?] make sure reparsing matches...
+      else
+        # see if it passes, below, too
+      end
+
+
+      if want_just_c
+        return Concretize.compile_string(ruby_code_wrapped, want_just_c)
+      else
+        success = Concretize.compile_string(ruby_code_wrapped) rescue nil # can fail, like ...ensure; return nil
+        if success
+          @@good_codes[ruby_code] = true          
+          
+          if want_rb_afterward
+            return ruby_code_wrapped # no new [helpful] ruby code here, but pass it out anyway..
+          end
+        end
+        success
+      end
+    end
+
+    @@cache_hits = 0
+    def self.cache_hits
+      @@cache_hits #ltodo attr_reader?
+    end
+    def self.good_codes
+      @@good_codes
+    end
+    
+    @@log = Logger.new (STDOUT)
+    private
+    # returns the c code, or the .so filename if it was compiled and run
+
+
+    def Concretize.compile_string(ruby_string, want_just_c = false)
+      file_name = "temp_#{@@count += 1}" # has to be .rb for some reason
+      File.write file_name + '.rb', ruby_string
+
+      Compiler.compile_file(file_name + '.rb', {:optimizations => :all}, [], false, @@log)
+      #LTODO delete all temp files -- except we can't delete .so files once loaded...
+      c_file = file_name + '.c'
+      so_file = file_name + '.so'
+
+      if(want_just_c)
+        return File.read(c_file)
+      end
+      # LTODO make it multi process friendly, too :)
+      require so_file # return just the name? huh?
+    end
+
+    public
+    
+    @@already_cified = {}
+    # pass in a class name instnace
+    # like c_ify_class! ClassName
+    # currently only cifys the singleton methods...
+    # add_to_string is either nil or a string.
+    # returns whether that class and all ancestors had something ruby-y that it successfully converted to C
+    def Concretize.c_ify_class! klass, add_to_string = nil, skip_ancestors = false
+      
+      local_add_to_string = ''
+      
+      Ruby2CExtension::Plugins::DirectSelfCall.allow_public_methods # we're concretizing, so public methods are ok
+      # TODO test that this actually does something to the C code :)
+      success = false
+      # LTODO class methods, singleton methods...sure! :)
+      ancestors = [klass]
+      
+      if(!skip_ancestors)
+        ancestors = klass.ancestors
+      end
+      
+      for klass in ancestors # ltodo reverse...
+        if @@already_cified[klass]
+          # for now we keep all classes as real classes [not really concretize anything...we just c-ify everything aggressively]
+          # when that is fixed be careful to get the inheritance wrong if two ancestors define the same method
+          next
+        end
+        # TODO optionally take out all private checks :)
+        for method_name in klass.instance_methods(false)
+          string = Concretize.c_ify!(klass, method_name, true)
+          if(string)              
+              local_add_to_string << " " << string
+              success = true
+          end
+          
+        end
+        
+        if add_to_string
+          add_to_string << local_add_to_string
+        else
+          Concretize.compile_string(local_add_to_string)
+        end
+        
+        @@already_cified[klass] = true
+
+        print klass.to_s + " has "
+        if(!success)
+          print "no "
+        end
+        puts "ruby methods"
+      end
+
+      return success
+    end
+
+    # turn all classes' ruby methods into their C equivalents
+    # deemed unstable as of yet :(
+    def Concretize.concretize_all! classes_to_do = nil, all_together = true
+      @@already_cified = {} # in case things have changed...maybe?
+      if !classes_to_do
+        classes_to_do = []
+        ObjectSpace.each_object(Class){|c| classes_to_do << c}
+      end
+
+      all_successful = []
+      if all_together
+        all_ruby_codes = ''
+      else
+        all_ruby_codes = nil
+      end
+
+      classes_to_do.each{|klass|
+        all_successful << klass if Concretize.c_ify_class!(klass, all_ruby_codes)
+      }
+      # puts 'none found' if all_ruby_codes == '' this is expected currently anytime after the first run...
+      Concretize.compile_string(all_ruby_codes)
+      all_successful
+    end
+  end
+
+  require 'event_hook' # 1.8 only for now [sigh]
+  class Tracer < EventHook
+    class << self
+      def all_classes
+        @all_classes.keys
+      end
+      def start
+        @all_classes = {}
+        start_hook
+      end
+      def process(*args)
+        # like [16, Tracer, :get_line, Tracer]
+        # not sure what [1] is versus [3]
+        @all_classes[args[3]]= true
+      end
+    end
+  end
+
+  def Concretize.crystalize_after_first_time_through
+    raise unless block_given?
+    Tracer.start
+    yield
+    Tracer.stop_hook
+    raise unless Tracer.all_classes.length > 0
+    puts 'crystalizing', Tracer.all_classes.inspect
+    Concretize.concretize_all! Tracer.all_classes
+  end
+end
+
+class Class
+  # somewhat of a misnomer TODO rdoc
+  def concretize!
+    Ruby2CExtension::Concretize.c_ify_class! self
+  end
+end
+
+class Object
+  def concretize_ruby!
+    Ruby2CExtension::Concretize.concretize_all!
+  end
+end
+
+
+