ruby2cext 0.2.0

data/lib/ruby2cext/compiler.rb

@@ -0,0 +1,242 @@
+
+require "rubynode"
+require "rbconfig"
+require "ruby2cext/parser"
+require "ruby2cext/error"
+require "ruby2cext/tools"
+require "ruby2cext/c_function"
+require "ruby2cext/version"
+
+module Ruby2CExtension
+
+	class Compiler
+
+		attr_reader :name, :logger, :plugins
+
+		def initialize(name, logger = nil)
+			@name = name
+			@logger = logger
+			@funs = []
+			@funs_reuseable = {}
+			@toplevel_funs = []
+			@sym_man = Tools::SymbolManager.new
+			@global_man = Tools::GlobalManager.new
+			@uniq_names = Tools::UniqueNames.new
+			@helpers = {}
+			@plugins = []
+			@preprocessors = {}
+		end
+
+		def to_c_code(time_stamp = Time.now)
+			plugins_global = @plugins.map { |plugin| plugin.global_c_code }
+			plugins_init = @plugins.map { |plugin| plugin.init_c_code }
+			res = [
+				"/* generated by #{FULL_VERSION_STRING} on #{time_stamp} */",
+				"#include <ruby.h>",
+				"#include <node.h>",
+				"#include <env.h>",
+				"#include <st.h>",
+				"extern VALUE ruby_top_self;",
+				"static VALUE org_ruby_top_self;",
+				@sym_man.to_c_code,
+				@global_man.to_c_code,
+			]
+			res.concat(@helpers.keys.sort)
+			res.concat(plugins_global)
+			res.concat(@funs)
+			res << "void Init_#{@name}() {"
+			res << "org_ruby_top_self = ruby_top_self;"
+			# just to be sure
+			res << "rb_global_variable(&org_ruby_top_self);"
+			res << "init_syms();"
+			res << "init_globals();"
+			res << "NODE *cref = rb_node_newnode(NODE_CREF, rb_cObject, 0, 0);"
+			res.concat(plugins_init)
+			@toplevel_funs.each { |f| res << "#{f}(ruby_top_self, cref);" }
+			res << "}"
+			res.join("\n").split("\n").map { |l| l.strip }.reject { |l| l.empty? }.join("\n")
+		end
+
+		def add_toplevel(function_name)
+			@toplevel_funs << function_name
+		end
+
+		# non destructive: node_tree will not be changed
+		def compile_toplevel_function(node_tree, private_vmode = true)
+			CFunction::ToplevelScope.compile(self, node_tree, private_vmode)
+		end
+
+		NODE_TRANSFORM_OPTIONS = {:include_node => true, :keep_newline_nodes => true}
+
+		def rb_file_to_toplevel_functions(source_str, file_name)
+			res = []
+			hash = Parser.parse_string(source_str, file_name)
+			# add all BEGIN blocks, if available
+			if (beg_tree = hash[:begin])
+				beg_tree = beg_tree.transform(NODE_TRANSFORM_OPTIONS)
+				if beg_tree.first == :block
+					beg_tree.last.each { |s| res << compile_toplevel_function(s, false) }
+				else
+					res << compile_toplevel_function(beg_tree, false)
+				end
+			end
+			# add toplevel scope
+			if (tree = hash[:tree])
+				res << compile_toplevel_function(tree.transform(NODE_TRANSFORM_OPTIONS))
+			end
+			res
+		end
+
+		def add_rb_file(source_str, file_name)
+			rb_file_to_toplevel_functions(source_str, file_name).each { |fn|
+				add_toplevel(fn)
+			}
+		end
+
+		# uniq name
+		def un(str)
+			@uniq_names.get(str)
+		end
+		def sym(sym)
+			@sym_man.get(sym)
+		end
+		def global_const(str, register_gc = true)
+			@global_man.get(str, true, register_gc)
+		end
+		def global_var(str)
+			@global_man.get(str, false, true)
+		end
+
+		def log(str, warning = false)
+			if logger
+				if warning
+					logger.warn(str)
+				else
+					logger.info(str)
+				end
+			end
+		end
+
+		def add_helper(str)
+			@helpers[str] ||= true
+		end
+
+		def add_fun(code, base_name)
+			unless (name = @funs_reuseable[code])
+				name = un(base_name)
+				lines = code.split("\n")
+				unless lines.shift =~ /^\s*static / # first line needs static
+					raise Ruby2CExtError::Bug, "trying to add a non static function"
+				end
+				if lines.grep(/^\s*static /).empty? # only reuseably without static variables
+					@funs_reuseable[code] = name
+				end
+				unless code.sub!("FUNNAME", name)
+					raise Ruby2CExtError::Bug, "trying to add a function without FUNNAME"
+				end
+				@funs << code
+			end
+			name
+		end
+
+		def add_plugin(plugin_class, *args)
+			@plugins << plugin_class.new(self, *args)
+		end
+
+		def add_plugins(options)
+			if options[:warnings]
+				require "ruby2cext/plugins/warnings"
+				add_plugin(Plugins::Warnings)
+			end
+			if (opt = options[:optimizations])
+				if opt == :all
+					opt = {
+						:const_cache=>true, :case_optimize=>true,
+						:builtin_methods=>true, :inline_methods=>true
+					}
+				end
+				if opt[:const_cache]
+					require "ruby2cext/plugins/const_cache"
+					add_plugin(Plugins::ConstCache)
+				end
+				if opt[:case_optimize]
+					require "ruby2cext/plugins/case_optimize"
+					add_plugin(Plugins::CaseOptimize)
+				end
+				if opt[:inline_methods]
+					require "ruby2cext/plugins/inline_methods"
+					add_plugin(Plugins::InlineMethods)
+				end
+				if (builtins = opt[:builtin_methods])
+					require "ruby2cext/plugins/builtin_methods"
+					if Array === builtins
+						builtins = builtins.map { |b| b.to_s.to_sym } # allow symbols, strings and the actual classes to work
+					else
+						builtins = Plugins::BuiltinMethods::SUPPORTED_BUILTINS
+					end
+					add_plugin(Plugins::BuiltinMethods, builtins)
+				end
+			end
+			if (ri_args = options[:require_include])
+				require "ruby2cext/plugins/require_include"
+				unless Array === ri_args.first
+					ri_args = [ri_args] # to allow just an array of include paths to also work
+				end
+				add_plugin(Plugins::RequireInclude, *ri_args)
+			end
+		end
+
+		# preprocessors can be added by plugins. preprocessors are procs that
+		# take two arguments: the current cfun and the node (tree) to
+		# preprocess (which will have type node_type)
+		#
+		# The proc can either return a (modified) node (tree) or string. If a
+		# node (tree) is returned then that will be translated as usual, if a
+		# string is returned, that string will be the result
+		#
+		# Example, a preprocessor that replaces 23 with 42:
+		# add_preprocessor(:lit) { |cfun, node|
+		#   node.last[:lit] == 23 ? [:lit, {:lit=>42}] : node
+		# }
+		#
+		# Another way to do the same:
+		# add_preprocessor(:lit) { |cfun, node|
+		#   node.last[:lit] == 23 ? cfun.comp_lit(:lit=>42) : node
+		# }
+		#
+		# If multiple preprocessors are added for the same node type then they
+		# will be called after each other with the result of the previous one
+		# unless it is a string, then the following preprocessors are ignored
+		def add_preprocessor(node_type, &pp_proc)
+			(@preprocessors[node_type] ||= []) << pp_proc
+		end
+
+		def preprocessors_for(node_type)
+			@preprocessors[node_type]
+		end
+
+		conf = ::Config::CONFIG
+		cflags = [conf["CCDLFLAGS"], conf["CFLAGS"], conf["ARCH_FLAG"]].join(" ")
+		COMPILE_COMMAND = "#{conf["LDSHARED"]} #{cflags} -I . -I #{conf["archdir"]}"
+		DLEXT = conf["DLEXT"]
+
+		# compiles a C file using the compiler from rbconfig
+		def self.compile_c_file_to_dllib(c_file_name, logger = nil)
+			unless c_file_name =~ /\.c\z/
+				raise Ruby2CExtError, "#{c_file_name} is no C file"
+			end
+			dl_name = c_file_name.sub(/c\z/, DLEXT)
+			cmd = "#{COMPILE_COMMAND} -o #{dl_name} #{c_file_name}"
+			if RUBY_PLATFORM =~ /mswin32/
+				cmd << " -link /INCREMENTAL:no /EXPORT:Init_#{File.basename(c_file_name, ".c")}"
+			end
+			logger.info(cmd) if logger
+			unless system(cmd) # run it
+				raise Ruby2CExtError, "error while executing '#{cmd}'"
+			end
+			dl_name
+		end
+
+	end
+
+end

data/lib/ruby2cext/error.rb

@@ -0,0 +1,15 @@
+
+module Ruby2CExtension
+
+	class Ruby2CExtError < StandardError
+		class NotSupported < self
+		end
+
+		class Bug < self
+			def initialize(msg)
+				super("BUG! #{msg}")
+			end
+		end
+	end
+
+end

data/lib/ruby2cext/eval2c.rb

@@ -0,0 +1,129 @@
+
+require "rubynode"
+require "digest/sha1"
+require "ruby2cext/compiler"
+require "ruby2cext/error"
+require "ruby2cext/version"
+
+module Ruby2CExtension
+
+	class Eval2C
+
+		attr_reader :path, :prefix, :plugins, :logger, :force_recompile
+
+		DEFAULT_PATH = File.join(File.expand_path(ENV["HOME"] || ENV["USERPROFILE"] || ENV["HOMEPATH"] || "."), ".ruby2cext")
+
+		def initialize(options = {})
+			unless (@path = options[:path])
+				@path = DEFAULT_PATH
+				Dir.mkdir(@path, 0700) unless File.exists?(@path)
+			end
+			@path = File.expand_path(@path)
+			unless File.directory?(@path)
+				raise Ruby2CExtError, "#{@path} is no directory"
+			end
+			unless File.stat(@path).mode & 022 == 0 # no writing for group and others
+				warn "Ruby2CExtension::Eval2C warning: #{@path} is insecure"
+			end
+			@prefix = options[:prefix] || "eval2c"
+			@plugins = options[:plugins] || {:optimizations => :all}
+			@logger = options[:logger]
+			@force_recompile = options[:force_recompile]
+			@done = {}
+			@digest_extra = Ruby2CExtension::FULL_VERSION_STRING + @plugins.inspect.split(//).sort.join("")
+		end
+
+		private
+
+		def compile_helper(digest_str)
+			name = "#{prefix}_#{Digest::SHA1.hexdigest(digest_str + @digest_extra)}"
+			gvar_name = "$__#{name}__"
+			dl_filename = File.join(path, "#{name}.#{Compiler::DLEXT}")
+			if !File.exists?(dl_filename) || (force_recompile && !@done[name])
+				c = Compiler.new(name, logger)
+				c.add_plugins(plugins)
+				yield c, name, gvar_name
+				c_filename = File.join(path, "#{name}.c")
+				File.open(c_filename, "w") { |f| f.puts(c.to_c_code) }
+				unless Compiler.compile_c_file_to_dllib(c_filename, logger) == dl_filename
+					raise Ruby2CExtError::Bug, "unexpected return value from compile_c_file_to_dllib"
+				end
+				@done[name] = true
+			end
+			require dl_filename
+			eval(gvar_name) # return the proc
+		end
+
+		public
+
+		def compile_to_proc(code_str)
+			compile_helper(code_str) { |compiler, name, gvar_name|
+				compiler.add_rb_file("#{gvar_name} = proc { #{code_str} }", name)
+			}
+		end
+
+		def module_eval(mod, code_str)
+			mod.module_eval(&compile_to_proc(code_str))
+		end
+		alias :class_eval :module_eval
+
+		def instance_eval(object, code_str)
+			object.instance_eval(&compile_to_proc(code_str))
+		end
+
+		def toplevel_eval(code_str)
+			compile_to_proc(code_str).call
+		end
+
+		def compile_methods(mod, *methods)
+			methods = methods.map { |m| m.to_sym }.uniq
+			defs = methods.map { |m|
+				bnode = mod.instance_method(m).body_node
+				unless bnode.type == :scope
+					raise Ruby2CExtError, "the method #{m} cannot be compiled, only methods defined using def can be compiled"
+				end
+				[:defn, {:mid => m, :defn => bnode.transform(Compiler::NODE_TRANSFORM_OPTIONS)}]
+			}
+			node_tree = [:scope, {:next => [:gasgn, {:vid => :$test, :value =>
+				[:iter, {
+					:var => false,
+					:iter => [:fcall, {:args => false, :mid => :proc}],
+					:body => [:block, defs]
+				}]
+			}]}]
+			# save current visibility of the methods
+			publ_methods = mod.public_instance_methods.map { |m| m.to_sym } & methods
+			prot_methods = mod.protected_instance_methods.map { |m| m.to_sym } & methods
+			priv_methods = mod.private_instance_methods.map { |m| m.to_sym } & methods
+			# compile to test if the methods don't need a cref and to get a string for the hash
+			c = Compiler.new("test")
+			c.add_toplevel(c.compile_toplevel_function(node_tree))
+			test_code = c.to_c_code(nil) # no time_stamp
+			# don't allow methods that need a cref, because the compiled version would get a different cref
+			if test_code =~ /^static void def_only_once/ # a bit hackish ...
+				raise Ruby2CExtError, "the method(s) cannot be compiled, because at least one needs a cref"
+			end
+			# compile the proc
+			def_proc = compile_helper(test_code) { |compiler, name, gvar_name|
+				node_tree.last[:next].last[:vid] = gvar_name.to_sym
+				compiler.add_toplevel(compiler.compile_toplevel_function(node_tree))
+			}
+			# try to remove all the methods
+			mod.module_eval {
+				methods.each { |m|
+					remove_method(m) rescue nil
+				}
+			}
+			# add the compiled methods
+			mod.module_eval &def_proc
+			# restore original visibility
+			mod.module_eval {
+				public *publ_methods unless publ_methods.empty?
+				protected *prot_methods unless prot_methods.empty?
+				private *priv_methods unless priv_methods.empty?
+			}
+		end
+
+	end
+
+end

data/lib/ruby2cext/parser.rb

@@ -0,0 +1,36 @@
+
+require "rubynode"
+
+module Ruby2CExtension
+
+	# not really a parser, uses rubynode
+	module Parser
+		def self.parse_string(str, file_name = "(parse)")
+			res = {}
+			# for the first parsing use original str, because it doesn't matter
+			# for BEGIN stuff and we get better exceptions this way.
+			if (tmp = str.parse_begin_to_nodes(file_name, 1))
+				res[:begin] = tmp
+			end
+			# now wrap str in a class scope and strip the class node of
+			# afterwards, to get a clean scope in the result. src should
+			# not have syntax errors if str didn't.
+			src = "class Object\n#{str}\nend"
+			begin
+				old_verb = $VERBOSE
+				# turn warnings of here to avoid the repetition of parse warnings
+				$VERBOSE = nil
+				if (tmp = src.parse_to_nodes(file_name, 0))
+					tmp = tmp.nd_next.nd_body
+					if tmp.type == :scope && tmp.nd_next
+						res[:tree] = tmp
+					end
+				end
+			ensure
+				$VERBOSE = old_verb
+			end
+			res
+		end
+	end
+
+end

data/lib/ruby2cext/plugin.rb

@@ -0,0 +1,24 @@
+
+module Ruby2CExtension
+
+	class Plugin
+		attr_reader :compiler
+
+		def initialize(compiler)
+			@compiler = compiler
+		end
+
+		# C code returned by this method will be inserted into the final C file
+		# between the helpers and the C functions
+		def global_c_code
+			nil
+		end
+
+		# C code returned by this method will be inserted into the Init_*()
+		# function before the calling of the toplevel scopes
+		def init_c_code
+			nil
+		end
+	end
+
+end