crystalruby 0.1.13 → 0.2.1

data/lib/crystalruby/function.rb

@@ -0,0 +1,234 @@
+module CrystalRuby
+  # This class represents a single Crystalized function.
+  # Each such function belongs a shared lib (See: CrystalRuby::Library)
+  # and is attached to a single owner (a class or a module).
+  class Function
+    include Typemaps
+    include Config
+
+    attr_accessor :owner, :method_name, :args, :returns, :function_body, :lib, :async, :block, :attached
+
+    def initialize(method:, args:, returns:, function_body:, lib:, async: false, &block)
+      self.owner = method.owner
+      self.method_name = method.name
+      self.args = args
+      self.returns = returns
+      self.function_body = function_body
+      self.lib = lib
+      self.async = async
+      self.block = block
+      self.attached = false
+    end
+
+    # This is where we write/overwrite the class and instance methods
+    # with their crystalized equivalents.
+    # We also perform JIT compilation and JIT attachment of the FFI functions.
+    # Crystalized methods can be redefined without restarting, if running in a live-reloading environment.
+    # If they are redefined with a different function body, the new function body
+    # will result in a new digest and the FFI function will be recompiled and reattached.
+    def define_crystalized_methods!(lib)
+      func = self
+      [owner, owner.singleton_class].each do |receiver|
+        receiver.undef_method(method_name) if receiver.method_defined?(method_name)
+        receiver.define_method(method_name) do |*args|
+          unless lib.compiled?
+            lib.build!
+            return send(func.method_name, *args)
+          end
+          unless func.attached?
+            should_reenter = func.attach_ffi_lib_functions!
+            return send(func.method_name, *args) if should_reenter
+          end
+          # All crystalruby functions are executed on the reactor to ensure Crystal/Ruby interop code is executed
+          # from a single same thread. (Needed to make GC and Fiber scheduler happy)
+          # Type mapping (if required) is applied on arguments and on return values.
+          func.map_retval(
+            Reactor.schedule_work!(
+              func.owner,
+              func.ffi_name,
+              *func.map_args(args),
+              func.ffi_ret_type,
+              async: func.async,
+              lib: lib
+            )
+          )
+        end
+      end
+    end
+
+    # This is where we attach the top-level FFI functions of the shared object
+    # to our library (yield and init) needed for successful operation of the reactor.
+    # We also initialize the shared object (needed to start the GC) and
+    # start the reactor, unless we are in single-thread mode.
+    def attach_ffi_lib_functions!
+      should_reenter = unwrapped?
+      lib_file = lib.lib_file
+      lib.methods.each_value(&:attach_ffi_func!)
+      lib.singleton_class.class_eval do
+        extend FFI::Library
+        ffi_lib lib_file
+        %i[yield init].each do |method_name|
+          singleton_class.undef_method(method_name) if singleton_class.method_defined?(method_name)
+          undef_method(method_name) if method_defined?(method_name)
+        end
+        attach_function :init, %i[string pointer], :void
+        attach_function :yield, %i[], :int
+      end
+
+      if CrystalRuby.config.single_thread_mode
+        Reactor.init_single_thread_mode!
+      else
+        Reactor.start!
+      end
+      Reactor.schedule_work!(lib, :init, lib.name, Reactor::ERROR_CALLBACK, :void, blocking: true, async: false)
+      should_reenter
+    end
+
+    # Attaches the crystalized FFI functions to their related Ruby modules and classes.
+    # If a wrapper block has been passed to the crystalize function,
+    # then the we also wrap the crystalized function using a prepended Module.
+    def attach_ffi_func!
+      argtypes = ffi_types
+      rettype = ffi_ret_type
+      if async && !config.single_thread_mode
+        argtypes += %i[int pointer]
+        rettype = :void
+      end
+
+      owner.extend FFI::Library unless owner.is_a?(FFI::Library)
+
+      unless (owner.instance_variable_get(:@ffi_libs) || [])
+             .map(&:name)
+             .map(&File.method(:basename))
+             .include?(File.basename(lib.lib_file))
+        owner.ffi_lib lib.lib_file
+      end
+
+      if owner.method_defined?(ffi_name)
+        owner.undef_method(ffi_name)
+        owner.singleton_class.undef_method(ffi_name)
+      end
+
+      owner.attach_function ffi_name, argtypes, rettype, blocking: true
+      around_wrapper_block = block
+      method_name = self.method_name
+      @attached = true
+      return unless around_wrapper_block
+
+      @around_wrapper ||= begin
+        wrapper_module = Module.new {}
+        [owner, owner.singleton_class].each do |receiver|
+          receiver.prepend(wrapper_module)
+        end
+        wrapper_module
+      end
+      @around_wrapper.undef_method(method_name) if @around_wrapper.method_defined?(method_name)
+      @around_wrapper.define_method(method_name, &around_wrapper_block)
+    rescue StandardError => e
+      CrystalRuby.log_error("Error attaching #{method_name} as #{ffi_name} to #{owner.name}")
+      CrystalRuby.log_error(e.message)
+      CrystalRuby.log_error(e.backtrace.join("\n"))
+    end
+
+    def unwrapped?
+      block && !@around_wrapper
+    end
+
+    def attached?
+      @attached
+    end
+
+    def unattach!
+      @attached = false
+    end
+
+    def ffi_name
+      lib_fn_name + (async && !config.single_thread_mode ? "_async" : "")
+    end
+
+    def lib_fn_name
+      @lib_fn_name ||= "#{owner.name.downcase.gsub("::", "_")}_#{method_name}_#{Digest::MD5.hexdigest(function_body)}"
+    end
+
+    def arg_type_map
+      @arg_type_map ||= args.transform_values(&method(:build_type_map))
+    end
+
+    def lib_fn_args
+      @lib_fn_args ||= arg_type_map.map { |k, arg_type|
+        "_#{k} : #{arg_type[:lib_type]}"
+      }.join(",") + (arg_type_map.empty? ? "" : ", ")
+    end
+
+    def lib_fn_arg_names
+      @lib_fn_arg_names ||= arg_type_map.map { |k, _arg_type|
+        "_#{k}"
+      }.join(",") + (arg_type_map.empty? ? "" : ", ")
+    end
+
+    def return_type_map
+      @return_type_map ||= build_type_map(returns)
+    end
+
+    def ffi_types
+      @ffi_types ||= arg_type_map.map { |_k, arg_type| arg_type[:ffi_type] }
+    end
+
+    def arg_maps
+      @arg_maps ||= arg_type_map.map { |_k, arg_type| arg_type[:arg_mapper] }
+    end
+
+    def ffi_ret_type
+      @ffi_ret_type ||= return_type_map[:ffi_ret_type]
+    end
+
+    def register_custom_types!(lib)
+      [*arg_type_map.values, return_type_map].map { |t| t[:crystal_ruby_type] }.each do |crystalruby_type|
+        if crystalruby_type.is_a?(Types::TypeSerializer) && !crystalruby_type.anonymous?
+          lib.register_type!(crystalruby_type)
+        end
+      end
+    end
+
+    def map_args(args)
+      return args unless arg_maps.any?
+
+      arg_maps.each_with_index do |argmap, index|
+        next unless argmap
+
+        args[index] = argmap[args[index]]
+      end
+      args
+    end
+
+    def map_retval(retval)
+      return retval unless return_type_map[:retval_mapper]
+
+      return_type_map[:retval_mapper][retval]
+    end
+
+    def chunk
+      @chunk ||= Template::Function.render(
+        {
+          module_name: owner.name,
+          lib_fn_name: lib_fn_name,
+          fn_name: method_name,
+          fn_body: function_body,
+          callback_call: returns == :void ? "callback.call(thread_id)" : "callback.call(thread_id, converted)",
+          callback_type: return_type_map[:ffi_type] == :void ? "UInt32 -> Void" : " UInt32, #{return_type_map[:lib_type]} -> Void",
+          fn_args: arg_type_map.map { |k, arg_type| "#{k} : #{arg_type[:crystal_type]}" }.join(","),
+          fn_ret_type: return_type_map[:crystal_type],
+          lib_fn_args: lib_fn_args,
+          lib_fn_arg_names: lib_fn_arg_names,
+          lib_fn_ret_type: return_type_map[:lib_type],
+          convert_lib_args: arg_type_map.map do |k, arg_type|
+            "#{k} = #{arg_type[:convert_lib_to_crystal_type]["_#{k}"]}"
+          end.join("\n    "),
+          arg_names: args.keys.join(","),
+          convert_return_type: return_type_map[:convert_crystal_to_lib_type]["return_value"],
+          error_value: return_type_map[:error_value]
+        }
+      )
+    end
+  end
+end

data/lib/crystalruby/library.rb

@@ -0,0 +1,191 @@
+module CrystalRuby
+  class Library
+    include Typemaps
+    include Config
+
+    # *CR_ATTACH_MUX* and *CR_COMPILE_MUX* are used to only allow a single FFI compile or attach operation at once
+    # to avoid a rare scenario where the same function is attached simultaneously across two or more threads.
+    CR_COMPILE_MUX = Mutex.new
+    CR_ATTACH_MUX = Mutex.new
+
+    attr_accessor :name, :methods, :chunks, :root_dir, :lib_dir, :src_dir, :codegen_dir, :reactor
+
+    @libs_by_name = {}
+
+    def self.all
+      @libs_by_name.values
+    end
+
+    def self.[](name)
+      @libs_by_name[name] ||= begin
+        CrystalRuby.initialize_crystal_ruby! unless CrystalRuby.initialized?
+        Library.new(name)
+      end
+    end
+
+    # A Library represents a single Crystal shared object.
+    # It holds code as either methods (invokable from Ruby and attached) or
+    # anonymous chunks, which are just raw Crystal code.
+    def initialize(name)
+      self.name = name
+      self.methods = {}
+      self.chunks = []
+      initialize_library!
+    end
+
+    # Bootstraps the library filesystem and generates top level index.cr and shard files if
+    # these do not already exist.
+    def initialize_library!
+      @root_dir, @lib_dir, @src_dir, @codegen_dir = [
+        config.crystal_src_dir_abs / name,
+        config.crystal_src_dir_abs / name / "lib",
+        config.crystal_src_dir_abs / name / "src",
+        config.crystal_src_dir_abs / name / "src" / config.crystal_codegen_dir
+      ].each do |dir|
+        FileUtils.mkdir_p(dir)
+      end
+      IO.write  main_file, "require \"./#{config.crystal_codegen_dir}/index\"\n" unless File.exist?(main_file)
+
+      return if File.exist?(shard_file)
+
+      IO.write(shard_file, <<~YAML)
+        name: src
+        version: 0.1.0
+      YAML
+    end
+
+    # Generates and stores a reference to a new CrystalRuby::Function
+    # and triggers the generation of the crystal code. (See write_chunk)
+    def crystalize_method(method, args, returns, function_body, async, &block)
+      CR_ATTACH_MUX.synchronize do
+        methods.each_value(&:unattach!)
+        method_key = "#{method.owner.name}/#{method.name}"
+        methods[method_key] = Function.new(
+          method: method,
+          args: args,
+          returns: returns,
+          function_body: function_body,
+          async: async,
+          lib: self,
+          &block
+        ).tap do |func|
+          func.define_crystalized_methods!(self)
+          func.register_custom_types!(self)
+          write_chunk(method.owner.name, method.name, func.chunk)
+        end
+      end
+    end
+
+    def main_file
+      src_dir / "#{name}.cr"
+    end
+
+    def lib_file
+      lib_dir / "#{name}_#{digest}"
+    end
+
+    def shard_file
+      src_dir / "shard.yml"
+    end
+
+    def crystalize_chunk(mod, chunk_name, body)
+      write_chunk(mod.name, chunk_name, body)
+    end
+
+    def instantiated?
+      @instantiated
+    end
+
+    def compiled?
+      @compiled ||= File.exist?(lib_file) && chunks.all? do |chunk|
+        chunk_data = chunk[:body]
+        file_digest = Digest::MD5.hexdigest chunk_data
+        fname = chunk[:chunk_name]
+        index_contents.include?("#{chunk[:module_name]}/#{fname}_#{file_digest}.cr")
+      end
+    end
+
+    def index_contents
+      IO.read(codegen_dir / "index.cr")
+    rescue StandardError
+      ""
+    end
+
+    def register_type!(type)
+      @types_cache ||= {}
+      @types_cache[type.name] = type.type_defn
+    end
+
+    def type_modules
+      (@types_cache || {}).map do |type_name, expr|
+        parts = type_name.split("::")
+        typedef = parts[0...-1].each_with_index.reduce("") do |acc, (part, index)|
+          acc + "#{"  " * index}module #{part}\n"
+        end
+        typedef += "#{"  " * (parts.size - 1)}alias #{parts.last} = #{expr}\n"
+        typedef + parts[0...-1].reverse.each_with_index.reduce("") do |acc, (_part, index)|
+          acc + "#{"  " * (parts.size - 2 - index)}end\n"
+        end
+      end.join("\n")
+    end
+
+    def requires
+      chunks.map do |chunk|
+        chunk_data = chunk[:body]
+        file_digest = Digest::MD5.hexdigest chunk_data
+        fname = chunk[:chunk_name]
+        "require \"./#{chunk[:module_name]}/#{fname}_#{file_digest}.cr\"\n"
+      end.join("\n")
+    end
+
+    def build!
+      CR_COMPILE_MUX.synchronize do
+        File.write codegen_dir / "index.cr", Template::Index.render(
+          type_modules: type_modules,
+          requires: requires
+        )
+
+        unless compiled?
+          CrystalRuby::Compilation.compile!(
+            verbose: config.verbose,
+            debug: config.debug,
+            src: main_file,
+            lib: lib_file
+          )
+        end
+      end
+    end
+
+    def digest
+      Digest::MD5.hexdigest(File.read(codegen_dir / "index.cr")) if File.exist?(codegen_dir / "index.cr")
+    end
+
+    def self.chunk_store
+      @chunk_store ||= []
+    end
+
+    def write_chunk(module_name, chunk_name, body)
+      chunks.delete_if { |chnk| chnk[:module_name] == module_name && chnk[:chunk_name] == chunk_name }
+      chunks << { module_name: module_name, chunk_name: chunk_name, body: body }
+      existing = Dir.glob(codegen_dir / "**/*.cr")
+      chunks.each do |chunk|
+        module_name, chunk_name, body = chunk.values_at(:module_name, :chunk_name, :body)
+
+        file_digest = Digest::MD5.hexdigest body
+        filename = (codegen_dir / module_name / "#{chunk_name}_#{file_digest}.cr").to_s
+
+        unless existing.delete(filename)
+          methods.each_value(&:unattach!)
+          @compiled = false
+          FileUtils.mkdir_p(codegen_dir / module_name)
+          File.write(filename, body)
+        end
+        existing.select do |f|
+          f =~ /#{config.crystal_codegen_dir / module_name / "#{chunk_name}_[a-f0-9]{32}\.cr"}/
+        end.each do |fl|
+          File.delete(fl) unless fl.eql?(filename)
+        end
+      end
+    end
+  end
+end

data/lib/crystalruby/reactor.rb

@@ -1,8 +1,12 @@
 module CrystalRuby
+  # The Reactor represents a singleton Thread responsible for running all Ruby/crystal interop code.
+  # Crystal's Fiber scheduler and GC assume all code is run on a single thread.
+  # This class is responsible for multiplexing Ruby and Crystal code on a single thread.
+  # Functions annotated with async: true, are executed using callbacks to allow these to be interleaved without blocking.
+
   module Reactor
     module_function
 
-    class ReactorStoppedException < StandardError; end
     class SingleThreadViolation < StandardError; end
 
     REACTOR_QUEUE = Queue.new
@@ -39,7 +43,8 @@ module CrystalRuby
       error_type = error_type.to_sym
       is_exception_type = Object.const_defined?(error_type) && Object.const_get(error_type).ancestors.include?(Exception)
       error_type = is_exception_type ? Object.const_get(error_type) : RuntimeError
-      tid = tid.zero? ? Reactor.current_thread_id : tid
+      raise error_type.new(message) unless THREAD_MAP.key?(tid)
+
       THREAD_MAP[tid][:error] = error_type.new(message)
       THREAD_MAP[tid][:result] = nil
       THREAD_MAP[tid][:cond].signal
@@ -57,15 +62,26 @@ module CrystalRuby
       THREAD_MAP[thread_id][:result]
     end
 
+    def start!
+      @main_loop ||= Thread.new do
+        CrystalRuby.log_debug("Starting reactor")
+        CrystalRuby.log_debug("CrystalRuby initialized")
+        loop do
+          REACTOR_QUEUE.pop[]
+        end
+      rescue StandardError => e
+        CrystalRuby.log_error "Error: #{e}"
+        CrystalRuby.log_error e.backtrace
+      end
+    end
+
     def thread_id
       Thread.current.object_id
     end
 
-    def yield!(time: 0)
-      Thread.new do
-        sleep time
-        schedule_work!(Reactor, :yield, nil, async: false, blocking: false)
-      end
+    def yield!(lib: nil, time: 0.0)
+      schedule_work!(lib, :yield, nil, async: false, blocking: false, lib: lib) if running? && lib
+      nil
     end
 
     def current_thread_id=(val)
@@ -76,9 +92,7 @@ module CrystalRuby
       @current_thread_id
     end
 
-    def schedule_work!(receiver, op_name, *args, return_type, blocking: true, async: true)
-      raise ReactorStoppedException, "Reactor has been terminated, no new work can be scheduled" if @stopped
-
+    def schedule_work!(receiver, op_name, *args, return_type, blocking: true, async: true, lib: nil)
       if @single_thread_mode
         unless Thread.current.object_id == @main_thread_id
           raise SingleThreadViolation,
@@ -99,7 +113,7 @@ module CrystalRuby
                 op_name, *args, tvars[:thread_id],
                 CALLBACKS_MAP[return_type]
               )
-              yield!(time: 0)
+              yield!(lib: lib, time: 0)
             }
           when blocking
             lambda {
@@ -116,7 +130,7 @@ module CrystalRuby
           else
             lambda {
               outstanding_jobs = receiver.send(op_name, *args)
-              yield!(time: 0.01) unless outstanding_jobs.zero?
+              yield!(lib: lib, time: 0) unless outstanding_jobs == 0
             }
           end
         )
@@ -124,61 +138,15 @@ module CrystalRuby
       end
     end
 
-    def init_single_thread_mode!
-      @single_thread_mode = true
-      @main_thread_id = Thread.current.object_id
-      init_crystal_ruby!
-    end
-
-    def init_crystal_ruby!
-      attach_lib!
-      init(ERROR_CALLBACK)
-    end
-
-    def attach_lib!
-      CrystalRuby.log_debug("Attaching lib")
-      extend FFI::Library
-      ffi_lib CrystalRuby.config.crystal_lib_dir / CrystalRuby.config.crystal_lib_name
-      attach_function :init, [:pointer], :void
-      attach_function :stop, [], :void
-      attach_function :yield, %i[], :int
-    end
-
-    def stop!
-      CrystalRuby.log_debug("Stopping reactor")
-      @stopped = true
-      sleep 1
-      @main_loop&.kill
-      @main_loop = nil
-      CrystalRuby.log_debug("Reactor stopped")
-    end
-
     def running?
       @main_loop&.alive?
     end
 
-    def start!
-      @main_loop ||= begin
-        attach_lib!
-        Thread.new do
-          CrystalRuby.log_debug("Starting reactor")
-          init(ERROR_CALLBACK)
-          CrystalRuby.log_debug("CrystalRuby initialized")
-          loop do
-            REACTOR_QUEUE.pop[]
-            break if @stopped
-          end
-          stop
-          CrystalRuby.log_debug("Stopping reactor")
-        rescue StandardError => e
-          puts "Error: #{e}"
-          puts e.backtrace
-        end
+    def init_single_thread_mode!
+      @single_thread_mode ||= begin
+        @main_thread_id = Thread.current.object_id
+        true
       end
     end
-
-    at_exit do
-      @stopped = true
-    end
   end
 end

data/lib/crystalruby/template.rb

@@ -4,7 +4,6 @@ module CrystalRuby
       template_name = File.basename(file, File.extname(file)).split("_").map(&:capitalize).join
       template_value = File.read(file)
       template_value.define_singleton_method(:render) do |context|
-        CrystalRuby.log_debug("Template.render: #{template_name}")
         self % context
       end
       const_set(template_name, template_value)

data/lib/crystalruby/templates/index.cr

@@ -1,13 +1,15 @@
-alias ErrorCallback = (Pointer(UInt8), Pointer(UInt8), UInt32 -> Void)
+module CrystalRuby
 
-ARGV1 = "crystalruby"
-CALLBACK_MUX = Mutex.new
+  ARGV1 = "crystalruby"
+  CALLBACK_MUX = Mutex.new
+
+  alias ErrorCallback = (Pointer(UInt8), Pointer(UInt8), UInt32 -> Void)
 
-module CrystalRuby
   # Initializing Crystal Ruby invokes init on the Crystal garbage collector.
   # We need to be sure to only do this once.
   @@initialized = false
 
+  @@libname = "crystalruby"
   # Our Ruby <-> Crystal Reactor uses Fibers, with callbacks to allow
   # multiple concurrent Crystal operations to be queued
   @@callbacks = [] of Proc(Nil)
@@ -25,13 +27,13 @@ module CrystalRuby
   # 1. Initialize the Crystal garbage collector
   # 2. Set the error callback
   # 3. Call the Crystal main function
-  def self.init(error_callback : ErrorCallback)
+  def self.init(libname : Pointer(UInt8), error_callback : ErrorCallback)
     return if @@initialized
     @@initialized = true
-    GC.init
     argv_ptr = ARGV1.to_unsafe
-    Crystal.main(0, pointerof(argv_ptr))
+    Crystal.main_user_code(0, pointerof(argv_ptr))
     @@error_callback = error_callback
+    @@libname = String.new(libname)
   end
 
   # Explicit error handling (triggers exception within Ruby on the same thread)
@@ -70,6 +72,10 @@ module CrystalRuby
     @@callbacks.size
   end
 
+  def self.libname : String
+    @@libname
+  end
+
   # Flush all callbacks
   def self.flush_callbacks : Int32
     CALLBACK_MUX.synchronize do
@@ -84,12 +90,28 @@ module CrystalRuby
 end
 
 # Initialize CrystalRuby
-fun init(cb : ErrorCallback): Void
-  CrystalRuby.init(cb)
+fun init(libname : Pointer(UInt8), cb : CrystalRuby::ErrorCallback): Void
+  CrystalRuby.init(libname, cb)
+end
+
+fun stop : Void
+  LibGC.deinit()
 end
 
-fun stop(): Void
-  GC.disable
+@[Link("gc")]
+lib LibGC
+  $stackbottom = GC_stackbottom : Void*
+  fun deinit = GC_deinit
+end
+
+module GC
+  def self.current_thread_stack_bottom
+    {Pointer(Void).null, LibGC.stackbottom}
+  end
+
+  def self.set_stackbottom(stack_bottom : Void*)
+    LibGC.stackbottom = stack_bottom
+  end
 end
 
 # Yield to the Crystal scheduler from Ruby
@@ -99,7 +121,6 @@ end
 # once this figure reaches 0).
 fun yield() : Int32
   if CrystalRuby.count_callbacks == 0
-
     Fiber.yield
 
     # TODO: We should apply backpressure here to prevent busy waiting if the number of outstanding tasks is not decreasing.

data/lib/crystalruby/typebuilder.rb

@@ -19,6 +19,7 @@ module CrystalRuby
           nil
         end
         restores << [context, method_name, old_method]
+        context.singleton_class.undef_method(method_name) if old_method
         context.define_singleton_method(method_name) do |*args|
           Types.send(method_name, *args)
         end
@@ -26,6 +27,7 @@ module CrystalRuby
       yield
     ensure
       restores.each do |context, method_name, old_method|
+        context.singleton_class.undef_method(method_name)
         context.define_singleton_method(method_name, old_method) if old_method
       end
     end