crystalruby 0.1.12 → 0.2.0

data/lib/crystalruby/function.rb

@@ -0,0 +1,228 @@
+module CrystalRuby
+  # This class represents a single Crystalized function.
+  # Each such function belongs a shared lib (See: CrystalRuby::Library)
+  # and is attached to a single owner (a class or a module).
+  class Function
+    include Typemaps
+    include Config
+
+    attr_accessor :owner, :method_name, :args, :returns, :function_body, :lib, :async, :block
+
+    def initialize(method:, args:, returns:, function_body:, lib:, async: false, &block)
+      self.owner = method.owner
+      self.method_name = method.name
+      self.args = args
+      self.returns = returns
+      self.function_body = function_body
+      self.lib = lib
+      self.async = async
+      self.block = block
+    end
+
+    # This is where we write/overwrite the class and instance methods
+    # with their crystalized equivalents.
+    # We also perform JIT compilation and JIT attachment of the FFI functions.
+    # Crystalized methods work in a live-reloading manner environment.
+    # If they are redefined with a different function body, the new function body
+    # will result in a new digest and the FFI function will be recompiled and reattached.
+    def define_crystalized_methods!(lib)
+      func = self
+      [owner, owner.singleton_class].each do |receiver|
+        receiver.undef_method(method_name) if receiver.method_defined?(method_name)
+        receiver.define_method(method_name) do |*args|
+          unless lib.compiled?
+            lib.build!
+            return send(func.method_name, *args)
+          end
+          unless lib.attached?(func.owner)
+            should_reenter = func.attach_ffi_lib_functions!
+            return send(func.method_name, *args) if should_reenter
+          end
+          # All crystalruby functions are executed on the reactor to ensure
+          # all Crystal interop is executed from the same thread. (Needed to make GC and Fiber scheduler happy)
+          # Type mapping (if required) is applied on arguments and on return values.
+          func.map_retval(
+            Reactor.schedule_work!(
+              func.owner,
+              func.ffi_name,
+              *func.map_args(args),
+              func.ffi_ret_type,
+              async: func.async,
+              lib: lib
+            )
+          )
+        end
+      end
+    end
+
+    # This is where we attach the top-level FFI functions of the shared object
+    # to our library (yield and init) needed for successful operation of the reactor.
+    # We also initialize the shared object (needed to start the GC) and
+    # start the reactor unless we are in single-thread mode.
+    def attach_ffi_lib_functions!
+      should_reenter = unwrapped?
+      lib_file = lib.lib_file
+      lib.attachments[owner] = true
+      lib.methods.each_value do |method|
+        method.attach_ffi_func!
+      end
+      lib.singleton_class.class_eval do
+        extend FFI::Library
+        ffi_lib lib_file
+        %i[yield init].each do |method_name|
+          singleton_class.undef_method(method_name) if singleton_class.method_defined?(method_name)
+          undef_method(method_name) if method_defined?(method_name)
+        end
+        attach_function :init, %i[string pointer], :void
+        attach_function :yield, %i[], :int
+      end
+
+      if CrystalRuby.config.single_thread_mode
+        Reactor.init_single_thread_mode!
+      else
+        Reactor.start!
+      end
+      Reactor.schedule_work!(lib, :init, lib.name, Reactor::ERROR_CALLBACK, :void, blocking: true, async: false)
+      should_reenter
+    end
+
+    # This is where we attach the crystalized FFI functions to their related
+    # Ruby modules and classes. If a wrapper block has been passed to the crystalize function,
+    # then the we also wrap the crystalized function using a prepended Module.
+    def attach_ffi_func!
+      argtypes = ffi_types
+      rettype = ffi_ret_type
+      if async && !config.single_thread_mode
+        argtypes += %i[int pointer]
+        rettype = :void
+      end
+
+      owner.extend FFI::Library unless owner.is_a?(FFI::Library)
+
+      unless (owner.instance_variable_get(:@ffi_libs) || [])
+             .map(&:name)
+             .map(&File.method(:basename))
+             .include?(File.basename(lib.lib_file))
+        owner.ffi_lib lib.lib_file
+      end
+
+      if owner.method_defined?(ffi_name)
+        owner.undef_method(ffi_name)
+        owner.singleton_class.undef_method(ffi_name)
+      end
+
+      owner.attach_function ffi_name, argtypes, rettype, blocking: true
+      around_wrapper_block = block
+      method_name = self.method_name
+
+      return unless around_wrapper_block
+
+      @around_wrapper ||= begin
+        wrapper_module = Module.new {}
+        [owner, owner.singleton_class].each do |receiver|
+          receiver.prepend(wrapper_module)
+        end
+        wrapper_module
+      end
+      @around_wrapper.undef_method(method_name) if @around_wrapper.method_defined?(method_name)
+      @around_wrapper.define_method(method_name, &around_wrapper_block)
+    rescue StandardError => e
+      CrystalRuby.log_error("Error attaching #{method_name} as #{ffi_name} to #{owner.name}")
+      CrystalRuby.log_error(e.message)
+      CrystalRuby.log_error(e.backtrace.join("\n"))
+    end
+
+    def unwrapped?
+      block && !@around_wrapper
+    end
+
+    def ffi_name
+      lib_fn_name + (async && !config.single_thread_mode ? "_async" : "")
+    end
+
+    def lib_fn_name
+      @lib_fn_name ||= "#{owner.name.downcase.gsub("::", "_")}_#{method_name}_#{Digest::MD5.hexdigest(function_body)}"
+    end
+
+    def arg_type_map
+      @arg_type_map ||= args.transform_values(&method(:build_type_map))
+    end
+
+    def lib_fn_args
+      @lib_fn_args ||= arg_type_map.map { |k, arg_type|
+        "_#{k} : #{arg_type[:lib_type]}"
+      }.join(",") + (arg_type_map.empty? ? "" : ", ")
+    end
+
+    def lib_fn_arg_names
+      @lib_fn_arg_names ||= arg_type_map.map { |k, _arg_type|
+        "_#{k}"
+      }.join(",") + (arg_type_map.empty? ? "" : ", ")
+    end
+
+    def return_type_map
+      @return_type_map ||= build_type_map(returns)
+    end
+
+    def ffi_types
+      @ffi_types ||= arg_type_map.map { |_k, arg_type| arg_type[:ffi_type] }
+    end
+
+    def arg_maps
+      @arg_maps ||= arg_type_map.map { |_k, arg_type| arg_type[:arg_mapper] }
+    end
+
+    def ffi_ret_type
+      @ffi_ret_type ||= return_type_map[:ffi_ret_type]
+    end
+
+    def register_custom_types!(lib)
+      [*arg_type_map.values, return_type_map].map { |t| t[:crystal_ruby_type] }.each do |crystalruby_type|
+        if crystalruby_type.is_a?(Types::TypeSerializer) && !crystalruby_type.anonymous?
+          lib.register_type!(crystalruby_type)
+        end
+      end
+    end
+
+    def map_args(args)
+      return args unless arg_maps.any?
+
+      arg_maps.each_with_index do |argmap, index|
+        next unless argmap
+
+        args[index] = argmap[args[index]]
+      end
+      args
+    end
+
+    def map_retval(retval)
+      return retval unless return_type_map[:retval_mapper]
+
+      return_type_map[:retval_mapper][retval]
+    end
+
+    def chunk
+      @chunk ||= Template::Function.render(
+        {
+          module_name: owner.name,
+          lib_fn_name: lib_fn_name,
+          fn_name: method_name,
+          fn_body: function_body,
+          callback_call: returns == :void ? "callback.call(thread_id)" : "callback.call(thread_id, converted)",
+          callback_type: return_type_map[:ffi_type] == :void ? "UInt32 -> Void" : " UInt32, #{return_type_map[:lib_type]} -> Void",
+          fn_args: arg_type_map.map { |k, arg_type| "#{k} : #{arg_type[:crystal_type]}" }.join(","),
+          fn_ret_type: return_type_map[:crystal_type],
+          lib_fn_args: lib_fn_args,
+          lib_fn_arg_names: lib_fn_arg_names,
+          lib_fn_ret_type: return_type_map[:lib_type],
+          convert_lib_args: arg_type_map.map do |k, arg_type|
+            "#{k} = #{arg_type[:convert_lib_to_crystal_type]["_#{k}"]}"
+          end.join("\n    "),
+          arg_names: args.keys.join(","),
+          convert_return_type: return_type_map[:convert_crystal_to_lib_type]["return_value"],
+          error_value: return_type_map[:error_value]
+        }
+      )
+    end
+  end
+end

data/lib/crystalruby/library.rb

@@ -0,0 +1,198 @@
+module CrystalRuby
+  class Library
+    include Typemaps
+    include Config
+
+    # *CR_ATTACH_MUX* and *CR_COMPILE_MUX* are used to only allow a single FFI compile or attach operation at once
+    # to avoid a rare scenario where the same function is attached simultaneously across two or more threads.
+    CR_COMPILE_MUX = Mutex.new
+    CR_ATTACH_MUX = Mutex.new
+
+    attr_accessor :name, :methods, :chunks, :root_dir, :lib_dir, :src_dir, :codegen_dir, :attachments, :reactor
+
+    @libs_by_name = {}
+
+    def self.all
+      @libs_by_name.values
+    end
+
+    def self.[](name)
+      @libs_by_name[name] ||= begin
+        CrystalRuby.initialize_crystal_ruby! unless CrystalRuby.initialized?
+        Library.new(name)
+      end
+    end
+
+    # A Library represents a single Crystal shared object.
+    # It holds code as either methods (invokable from Ruby and attached) or
+    # anonymous chunks, which are just raw Crystal code.
+    def initialize(name)
+      self.name = name
+      self.methods = {}
+      self.chunks = []
+      self.attachments = Hash.new(false)
+      initialize_library!
+    end
+
+    # This method is used to
+    # bootstrap a library filesystem,
+    # and generate a top level index.cr and shard file if
+    # these do not already exist.
+    def initialize_library!
+      @root_dir, @lib_dir, @src_dir, @codegen_dir = [
+        config.crystal_src_dir_abs / name,
+        config.crystal_src_dir_abs / name / "lib",
+        config.crystal_src_dir_abs / name / "src",
+        config.crystal_src_dir_abs / name / "src" / config.crystal_codegen_dir
+      ].each do |dir|
+        FileUtils.mkdir_p(dir)
+      end
+      IO.write  main_file, "require \"./#{config.crystal_codegen_dir}/index\"\n" unless File.exist?(main_file)
+
+      return if File.exist?(shard_file)
+
+      IO.write(shard_file, <<~YAML)
+        name: src
+        version: 0.1.0
+      YAML
+    end
+
+    # This is where we instantiate the crystalized method as a CrystalRuby::Function
+    # and trigger the generation of the crystal code.
+    def crystalize_method(method, args, returns, function_body, async, &block)
+      CR_ATTACH_MUX.synchronize do
+        attachments.delete(method.owner)
+        method_key = "#{method.owner.name}/#{method.name}"
+        methods[method_key] = Function.new(
+          method: method,
+          args: args,
+          returns: returns,
+          function_body: function_body,
+          async: async,
+          lib: self,
+          &block
+        ).tap do |func|
+          func.define_crystalized_methods!(self)
+          func.register_custom_types!(self)
+          write_chunk(method.owner.name, method.name, func.chunk)
+        end
+      end
+    end
+
+    def main_file
+      src_dir / "#{name}.cr"
+    end
+
+    def lib_file
+      lib_dir / "#{name}_#{digest}"
+    end
+
+    def shard_file
+      src_dir / "shard.yml"
+    end
+
+    def crystalize_chunk(mod, chunk_name, body)
+      write_chunk(mod.name, chunk_name, body)
+    end
+
+    def instantiated?
+      @instantiated
+    end
+
+    def compiled?
+      index_contents = self.index_contents
+      File.exist?(lib_file) && chunks.all? do |chunk|
+        chunk_data = chunk[:body]
+        file_digest = Digest::MD5.hexdigest chunk_data
+        fname = chunk[:chunk_name]
+        index_contents.include?("#{chunk[:module_name]}/#{fname}_#{file_digest}.cr")
+      end
+    end
+
+    def index_contents
+      IO.read(codegen_dir / "index.cr")
+    rescue StandardError
+      ""
+    end
+
+    def attached?(owner)
+      attachments[owner]
+    end
+
+    def register_type!(type)
+      @types_cache ||= {}
+      @types_cache[type.name] = type.type_defn
+    end
+
+    def type_modules
+      (@types_cache || {}).map do |type_name, expr|
+        parts = type_name.split("::")
+        typedef = parts[0...-1].each_with_index.reduce("") do |acc, (part, index)|
+          acc + "#{"  " * index}module #{part}\n"
+        end
+        typedef += "#{"  " * (parts.size - 1)}alias #{parts.last} = #{expr}\n"
+        typedef + parts[0...-1].reverse.each_with_index.reduce("") do |acc, (_part, index)|
+          acc + "#{"  " * (parts.size - 2 - index)}end\n"
+        end
+      end.join("\n")
+    end
+
+    def requires
+      chunks.map do |chunk|
+        chunk_data = chunk[:body]
+        file_digest = Digest::MD5.hexdigest chunk_data
+        fname = chunk[:chunk_name]
+        "require \"./#{chunk[:module_name]}/#{fname}_#{file_digest}.cr\"\n"
+      end.join("\n")
+    end
+
+    def build!
+      CR_COMPILE_MUX.synchronize do
+        File.write codegen_dir / "index.cr", Template::Index.render(
+          type_modules: type_modules,
+          requires: requires
+        )
+
+        unless compiled?
+          CrystalRuby::Compilation.compile!(
+            verbose: config.verbose,
+            debug: config.debug,
+            src: main_file,
+            lib: lib_file
+          )
+        end
+      end
+    end
+
+    def digest
+      Digest::MD5.hexdigest(File.read(codegen_dir / "index.cr")) if File.exist?(codegen_dir / "index.cr")
+    end
+
+    def self.chunk_store
+      @chunk_store ||= []
+    end
+
+    def write_chunk(module_name, chunk_name, body)
+      chunks.delete_if { |chnk| chnk[:module_name] == module_name && chnk[:chunk_name] == chunk_name }
+      chunks << { module_name: module_name, chunk_name: chunk_name, body: body }
+      existing = Dir.glob(codegen_dir / "**/*.cr")
+      chunks.each do |chunk|
+        module_name, chunk_name, body = chunk.values_at(:module_name, :chunk_name, :body)
+
+        file_digest = Digest::MD5.hexdigest body
+        filename = (codegen_dir / module_name / "#{chunk_name}_#{file_digest}.cr").to_s
+
+        unless existing.delete(filename)
+          @attached = false
+          FileUtils.mkdir_p(codegen_dir / module_name)
+          File.write(filename, body)
+        end
+        existing.select do |f|
+          f =~ /#{config.crystal_codegen_dir / module_name / "#{chunk_name}_[a-f0-9]{32}\.cr"}/
+        end.each do |fl|
+          File.delete(fl) unless fl.eql?(filename)
+        end
+      end
+    end
+  end
+end

data/lib/crystalruby/reactor.rb

@@ -0,0 +1,155 @@
+module CrystalRuby
+  # The Reactor represents a singleton Thread
+  # responsible for running all Ruby/crystal interop code.
+  # Crystal's Fiber scheduler and GC assumes all code is run on a single thread.
+  # This class is responsible for multiplexing Ruby and Crystal code on a single thread,
+  # to allow safe invocation of Crystal code from across any number of Ruby threads.
+  # Functions annotated with async: true, are executed using callbacks to allow these to be multi-plexed in a non-blocking manner.
+
+  module Reactor
+    module_function
+
+    class SingleThreadViolation < StandardError; end
+
+    REACTOR_QUEUE = Queue.new
+
+    # We maintain a map of threads, each with a mutex, condition variable, and result
+    THREAD_MAP = Hash.new do |h, tid_or_thread, tid = tid_or_thread|
+      if tid_or_thread.is_a?(Thread)
+        ObjectSpace.define_finalizer(tid_or_thread) do
+          THREAD_MAP.delete(tid_or_thread)
+          THREAD_MAP.delete(tid_or_thread.object_id)
+        end
+        tid = tid_or_thread.object_id
+      end
+
+      h[tid] = {
+        mux: Mutex.new,
+        cond: ConditionVariable.new,
+        result: nil,
+        thread_id: tid
+      }
+      h[tid_or_thread] = h[tid] if tid_or_thread.is_a?(Thread)
+    end
+
+    # We memoize callbacks, once per return type
+    CALLBACKS_MAP = Hash.new do |h, rt|
+      h[rt] = FFI::Function.new(:void, [:int, *(rt == :void ? [] : [rt])]) do |tid, ret|
+        THREAD_MAP[tid][:error] = nil
+        THREAD_MAP[tid][:result] = ret
+        THREAD_MAP[tid][:cond].signal
+      end
+    end
+
+    ERROR_CALLBACK = FFI::Function.new(:void, %i[string string int]) do |error_type, message, tid|
+      error_type = error_type.to_sym
+      is_exception_type = Object.const_defined?(error_type) && Object.const_get(error_type).ancestors.include?(Exception)
+      error_type = is_exception_type ? Object.const_get(error_type) : RuntimeError
+      tid = tid.zero? ? Reactor.current_thread_id : tid
+      raise error_type.new(message) unless THREAD_MAP.key?(tid)
+
+      THREAD_MAP[tid][:error] = error_type.new(message)
+      THREAD_MAP[tid][:result] = nil
+      THREAD_MAP[tid][:cond].signal
+    end
+
+    def thread_conditions
+      THREAD_MAP[Thread.current]
+    end
+
+    def await_result!
+      mux, cond = thread_conditions.values_at(:mux, :cond)
+      cond.wait(mux)
+      raise THREAD_MAP[thread_id][:error] if THREAD_MAP[thread_id][:error]
+
+      THREAD_MAP[thread_id][:result]
+    end
+
+    def start!
+      @main_loop ||= Thread.new do
+        CrystalRuby.log_debug("Starting reactor")
+        CrystalRuby.log_debug("CrystalRuby initialized")
+        loop do
+          REACTOR_QUEUE.pop[]
+        end
+      rescue StandardError => e
+        CrystalRuby.log_error "Error: #{e}"
+        CrystalRuby.log_error e.backtrace
+      end
+    end
+
+    def thread_id
+      Thread.current.object_id
+    end
+
+    def yield!(lib: nil, time: 0.0)
+      schedule_work!(lib, :yield, nil, async: false, blocking: false, lib: lib) if running? && lib
+      nil
+    end
+
+    def current_thread_id=(val)
+      @current_thread_id = val
+    end
+
+    def current_thread_id
+      @current_thread_id
+    end
+
+    def schedule_work!(receiver, op_name, *args, return_type, blocking: true, async: true, lib: nil)
+      if @single_thread_mode
+        unless Thread.current.object_id == @main_thread_id
+          raise SingleThreadViolation,
+                "Single thread mode is enabled, cannot run in multi-threaded mode. " \
+                "Reactor was started from: #{@main_thread_id}, then called from #{Thread.current.object_id}"
+        end
+
+        return receiver.send(op_name, *args)
+      end
+
+      tvars = thread_conditions
+      tvars[:mux].synchronize do
+        REACTOR_QUEUE.push(
+          case true
+          when async
+            lambda {
+              receiver.send(
+                op_name, *args, tvars[:thread_id],
+                CALLBACKS_MAP[return_type]
+              )
+              yield!(lib: lib, time: 0)
+            }
+          when blocking
+            lambda {
+              tvars[:error] = nil
+              Reactor.current_thread_id = tvars[:thread_id]
+              begin
+                result = receiver.send(op_name, *args)
+              rescue StandardError => e
+                tvars[:error] = e
+              end
+              tvars[:result] = result unless tvars[:error]
+              tvars[:cond].signal
+            }
+          else
+            lambda {
+              outstanding_jobs = receiver.send(op_name, *args)
+              yield!(lib: lib, time: 0) unless outstanding_jobs == 0
+            }
+          end
+        )
+        return await_result! if blocking
+      end
+    end
+
+    def running?
+      @main_loop&.alive?
+    end
+
+    def init_single_thread_mode!
+      @single_thread_mode ||= begin
+        @main_thread_id = Thread.current.object_id
+        true
+      end
+    end
+  end
+end

data/lib/crystalruby/template.rb

@@ -2,11 +2,11 @@ module CrystalRuby
   module Template
     Dir[File.join(File.dirname(__FILE__), "templates", "*.cr")].each do |file|
       template_name = File.basename(file, File.extname(file)).split("_").map(&:capitalize).join
-      const_set(template_name, File.read(file))
-    end
-
-    def self.render(template, context)
-      template % context
+      template_value = File.read(file)
+      template_value.define_singleton_method(:render) do |context|
+        self % context
+      end
+      const_set(template_name, template_value)
     end
   end
 end

data/lib/crystalruby/templates/function.cr

@@ -12,7 +12,6 @@ end
 # This function is the entry point for the CrystalRuby code, exposed through FFI.
 # We apply some basic error handling here, and convert the arguments and return values
 # to ensure that we are using Crystal native types.
-
 fun %{lib_fn_name}(%{lib_fn_args}): %{lib_fn_ret_type}
   begin
     %{convert_lib_args}
@@ -20,10 +19,44 @@ fun %{lib_fn_name}(%{lib_fn_args}): %{lib_fn_ret_type}
       return_value = %{module_name}.%{fn_name}(%{arg_names})
       return %{convert_return_type}
     rescue ex
-      CrystalRuby.report_error("RuntimeError", ex.message.to_s)
+      CrystalRuby.report_error("RuntimeError", ex.message.to_s, 0)
     end
   rescue ex
-    CrystalRuby.report_error("ArgumentError", ex.message.to_s)
+    CrystalRuby.report_error("ArgumentError", ex.message.to_s, 0)
   end
   return %{error_value}
 end
+
+
+# This function is the async entry point for the CrystalRuby code, exposed through FFI.
+# We apply some basic error handling here, and convert the arguments and return values
+# to ensure that we are using Crystal native types.
+fun %{lib_fn_name}_async(%{lib_fn_args} thread_id: UInt32,  callback : %{callback_type}): Void
+  begin
+    %{convert_lib_args}
+    CrystalRuby.increment_task_counter
+    spawn do
+      begin
+        return_value = %{module_name}.%{fn_name}(%{arg_names})
+        converted = %{convert_return_type}
+        CrystalRuby.queue_callback(->{
+          %{callback_call}
+          CrystalRuby.decrement_task_counter
+        })
+      rescue ex
+        exception = ex.message.to_s
+        CrystalRuby.queue_callback(->{
+          CrystalRuby.error_callback.call("RuntimeError".to_unsafe, exception.to_unsafe, thread_id)
+          CrystalRuby.decrement_task_counter
+        })
+      end
+    end
+  rescue ex
+
+    exception = ex.message.to_s
+    CrystalRuby.queue_callback(->{
+      CrystalRuby.error_callback.call("RuntimeError".to_unsafe, ex.message.to_s.to_unsafe, thread_id)
+      CrystalRuby.decrement_task_counter
+    })
+  end
+end