crystalruby 0.2.3 → 0.3.1

data/lib/crystalruby/templates/index.cr

@@ -1,48 +1,40 @@
 module CrystalRuby
-
   ARGV1 = "crystalruby"
-  CALLBACK_MUX = Mutex.new
 
-  alias ErrorCallback = (Pointer(UInt8), Pointer(UInt8), UInt32 -> Void)
+  alias ErrorCallback = (Pointer(::UInt8), Pointer(::UInt8), Pointer(::UInt8), ::UInt32 -> Void)
+
+  class_property libname : String = "crystalruby"
+  class_property callbacks : Channel(Proc(Nil)) = Channel(Proc(Nil)).new
+  class_property rc_mux : Pointer(Void) = Pointer(Void).null
+  class_property task_counter : Atomic(Int32) = Atomic(Int32).new(0)
 
   # Initializing Crystal Ruby invokes init on the Crystal garbage collector.
   # We need to be sure to only do this once.
-  @@initialized = false
-
-  @@libname = "crystalruby"
-  # Our Ruby <-> Crystal Reactor uses Fibers, with callbacks to allow
-  # multiple concurrent Crystal operations to be queued
-  @@callbacks = [] of Proc(Nil)
-
-  # We only continue to yield to the Crystal scheduler from Ruby
-  # while there are outstanding tasks.
-  @@task_counter : Atomic(Int32) = Atomic.new(0)
+  class_property initialized : Bool = false
 
   # We can override the error callback to catch errors in Crystal,
   # and explicitly expose them to Ruby.
-  @@error_callback : ErrorCallback = ->(t : UInt8* , s : UInt8*, tid : UInt32){ puts "Error: #{t}:#{s}" }
+  @@error_callback : ErrorCallback?
 
   # This is the entry point for instantiating CrystalRuby
   # We:
   # 1. Initialize the Crystal garbage collector
   # 2. Set the error callback
   # 3. Call the Crystal main function
-  def self.init(libname : Pointer(UInt8), error_callback : ErrorCallback)
-    return if @@initialized
-    @@initialized = true
+  def self.init(libname : Pointer(::UInt8), @@error_callback : ErrorCallback, @@rc_mux : Pointer(Void))
+    return if self.initialized
+    self.initialized = true
     argv_ptr = ARGV1.to_unsafe
     Crystal.main_user_code(0, pointerof(argv_ptr))
-    @@error_callback = error_callback
-    @@libname = String.new(libname)
+    self.libname = String.new(libname)
+    LibGC.set_finalize_on_demand(1)
   end
 
   # Explicit error handling (triggers exception within Ruby on the same thread)
-  def self.report_error(error_type : String, str : String, thread_id : UInt32, )
-    @@error_callback.call(error_type.to_unsafe, str.to_unsafe, thread_id)
-  end
-
-  def self.error_callback : ErrorCallback
-    @@error_callback
+  def self.report_error(error_type : String, message : String, backtrace : String, thread_id : UInt32)
+    if error_reporter = @@error_callback
+      error_reporter.call(error_type.to_unsafe, message.to_unsafe, backtrace.to_unsafe, thread_id)
+    end
   end
 
   # New async task started
@@ -57,51 +49,40 @@ module CrystalRuby
 
   # Get number of outstanding tasks
   def self.get_task_counter : Int32
-    @@task_counter.get()
+    @@task_counter.get
   end
 
   # Queue a callback for an async task
   def self.queue_callback(callback : Proc(Nil))
-    CALLBACK_MUX.synchronize do
-      @@callbacks << callback
-    end
-  end
-
-  # Get number of queued callbacks
-  def self.count_callbacks : Int32
-    @@callbacks.size
+    self.callbacks.send(callback)
   end
 
-  def self.libname : String
-    @@libname
-  end
-
-  # Flush all callbacks
-  def self.flush_callbacks : Int32
-    CALLBACK_MUX.synchronize do
-      count = @@callbacks.size
-      @@callbacks.each do |callback|
-        result = callback.call()
-      end
-      @@callbacks.clear
-    end
-    get_task_counter
+  def self.synchronize
+    LibC.pthread_mutex_lock(self.rc_mux)
+    yield
+    LibC.pthread_mutex_unlock(self.rc_mux)
   end
 end
 
 # Initialize CrystalRuby
-fun init(libname : Pointer(UInt8), cb : CrystalRuby::ErrorCallback): Void
-  CrystalRuby.init(libname, cb)
+fun init(libname : Pointer(::UInt8), cb : CrystalRuby::ErrorCallback, rc_mux : Pointer(Void)) : Void
+  CrystalRuby.init(libname, cb, rc_mux)
 end
 
 fun stop : Void
-  LibGC.deinit()
+  LibGC.deinit
 end
 
 @[Link("gc")]
 lib LibGC
   $stackbottom = GC_stackbottom : Void*
   fun deinit = GC_deinit
+  fun set_finalize_on_demand = GC_set_finalize_on_demand(Int32)
+  fun invoke_finalizers = GC_invoke_finalizers : Int
+end
+
+lib LibC
+  fun calloc = calloc(Int32, Int32) : Void*
 end
 
 module GC
@@ -112,6 +93,11 @@ module GC
   def self.set_stackbottom(stack_bottom : Void*)
     LibGC.stackbottom = stack_bottom
   end
+
+  def self.collect
+    LibGC.collect
+    LibGC.invoke_finalizers
+  end
 end
 
 # Yield to the Crystal scheduler from Ruby
@@ -119,24 +105,25 @@ end
 # Otherwise, we yield to the Crystal scheduler and let Ruby know
 # how many outstanding tasks still remain (it will stop yielding to Crystal
 # once this figure reaches 0).
-fun yield() : Int32
-  if CrystalRuby.count_callbacks == 0
-    Fiber.yield
-
-    # TODO: We should apply backpressure here to prevent busy waiting if the number of outstanding tasks is not decreasing.
-    # Use a simple exponential backoff strategy, to increase the time between each yield up to a maximum of 1 second.
-
-    CrystalRuby.get_task_counter
-  else
-    CrystalRuby.flush_callbacks()
+fun yield : Int32
+  Fiber.yield
+  loop do
+    select
+    when callback = CrystalRuby.callbacks.receive
+      callback.call
+    else
+      break
+    end
   end
+  CrystalRuby.get_task_counter
 end
 
+class Array(T)
+  def initialize(size : Int32, @buffer : Pointer(T))
+    @size = size.to_i32
+    @capacity = @size
+  end
+end
 
-# This is where we define all our Crystal modules and types
-# derived from their Ruby counterparts.
-%{type_modules}
-
-# Require all generated crystal files
 require "json"
 %{requires}

data/lib/crystalruby/templates/inline_chunk.cr

@@ -1,6 +1,6 @@
 # This is the template used for writing inline chunks of Crystal code without direct
 # Ruby integration
 
-module %{module_name}
+%{mod_or_class} %{module_name} %{superclass}
   %{body}
 end

data/lib/crystalruby/templates/ruby_interface.cr

@@ -0,0 +1,34 @@
+# This is the template used for all CrystalRuby functions
+# Calls to this method *from ruby* are first transformed through the lib function.
+# Crystal code can simply call this method directly, enabling generated crystal code
+# to call other generated crystal code without overhead.
+
+%{module_or_class} %{module_name} %{superclass}
+  def %{fn_scope}%{fn_name}(%{fn_args}) : %{fn_ret_type}
+    %{convert_lib_args}
+    cb = %{module_name}.%{callback_name}
+    unless cb.nil?
+      callback_done_channel = Channel(Nil).new
+      return_value = nil
+      if Fiber.current == Thread.current.main_fiber
+        return_value = cb.call(%{lib_fn_arg_names})
+        return %{convert_return_type}
+      else
+        CrystalRuby.queue_callback(->{
+          return_value = cb.call(%{lib_fn_arg_names})
+          callback_done_channel.send(nil)
+        })
+      end
+      callback_done_channel.receive
+      return %{convert_return_type}
+    end
+    raise "No callback registered for %{fn_name}"
+
+  end
+
+  class_property %{callback_name} : Proc(%{lib_fn_types} %{lib_fn_ret_type})?
+end
+
+fun register_%{callback_name}(callback : Proc(%{lib_fn_types} %{lib_fn_ret_type})) : Void
+  %{module_name}.%{callback_name} = callback
+end

data/lib/crystalruby/templates/top_level_function.cr

@@ -0,0 +1,62 @@
+# This is the template used for all CrystalRuby functions
+# Calls to this method *from ruby* are first transformed through the lib function.
+# Crystal code can simply call this method directly, enabling generated crystal code
+# to call other generated crystal code without overhead.
+
+def %{fn_name}(%{fn_args}) : %{fn_ret_type}
+  %{fn_body}
+end
+
+# This function is the entry point for the CrystalRuby code, exposed through FFI.
+# We apply some basic error handling here, and convert the arguments and return values
+# to ensure that we are using Crystal native types.
+fun %{lib_fn_name}(%{lib_fn_args}): %{lib_fn_ret_type}
+  begin
+    %{convert_lib_args}
+    begin
+      return_value = %{fn_name}(%{arg_names})%{block_converter}
+      return %{convert_return_type}
+    rescue ex
+      CrystalRuby.report_error("RuntimeError", ex.message.to_s, ex.backtrace.to_json, 0)
+    end
+  rescue ex
+    CrystalRuby.report_error("ArgumentError", ex.message.to_s, ex.backtrace.to_json, 0)
+  end
+  return %{error_value}
+end
+
+
+# This function is the async entry point for the CrystalRuby code, exposed through FFI.
+# We apply some basic error handling here, and convert the arguments and return values
+# to ensure that we are using Crystal native types.
+fun %{lib_fn_name}_async(%{lib_fn_args} thread_id: UInt32,  callback : %{callback_type}): Void
+  begin
+    %{convert_lib_args}
+    CrystalRuby.increment_task_counter
+    spawn do
+      begin
+        return_value = %{fn_name}(%{arg_names})%{block_converter}
+        CrystalRuby.queue_callback(->{
+          converted = %{convert_return_type}
+          %{callback_call}
+          CrystalRuby.decrement_task_counter
+        })
+      rescue ex
+        exception = ex.message.to_s
+        backtrace = ex.backtrace.to_json
+        CrystalRuby.queue_callback(->{
+          CrystalRuby.report_error("RuntimeError", exception, backtrace, thread_id)
+          CrystalRuby.decrement_task_counter
+        })
+      end
+    end
+  rescue ex
+
+    exception = ex.message.to_s
+    backtrace = ex.backtrace.to_json
+    CrystalRuby.queue_callback(->{
+      CrystalRuby.report_error("RuntimeError", ex.message.to_s, backtrace, thread_id)
+      CrystalRuby.decrement_task_counter
+    })
+  end
+end

data/lib/crystalruby/templates/top_level_ruby_interface.cr

@@ -0,0 +1,33 @@
+# This is the template used for all CrystalRuby functions
+# Calls to this method *from ruby* are first transformed through the lib function.
+# Crystal code can simply call this method directly, enabling generated crystal code
+# to call other generated crystal code without overhead.
+
+module TopLevelCallbacks
+  class_property %{fn_name}_callback : Proc(%{lib_fn_types} %{lib_fn_ret_type})?
+end
+
+def %{fn_scope}%{fn_name}(%{fn_args}) : %{fn_ret_type}
+  %{convert_lib_args}
+  cb = TopLevelCallbacks.%{fn_name}_callback
+  unless cb.nil?
+    callback_done_channel = Channel(Nil).new
+    return_value = nil
+    if Fiber.current == Thread.current.main_fiber
+      return_value = cb.call(%{lib_fn_arg_names})
+      return %{convert_return_type}
+    else
+      CrystalRuby.queue_callback(->{
+        return_value = cb.call(%{lib_fn_arg_names})
+        callback_done_channel.send(nil)
+      })
+    end
+    callback_done_channel.receive
+    return %{convert_return_type}
+  end
+  raise "No callback registered for %{fn_name}"
+end
+
+fun register_%{fn_name}_callback(callback : Proc(%{lib_fn_types} %{lib_fn_ret_type})) : Void
+  TopLevelCallbacks.%{fn_name}_callback = callback
+end

data/lib/crystalruby/typebuilder.rb

@@ -4,66 +4,22 @@ module CrystalRuby
   module TypeBuilder
     module_function
 
-    def with_injected_type_dsl(context, &block)
-      with_constants(context) do
-        with_methods(context, &block)
+    def build_from_source(src, context: )
+      source_type = Types.with_binding_fallback(context) do |binding|
+        eval(src.is_a?(String) ? src : SourceReader.extract_source_from_proc(src), binding)
       end
-    end
 
-    def with_methods(context)
-      restores = []
-      %i[Array Hash NamedTuple Tuple].each do |method_name|
-        old_method = begin
-          context.instance_method(method_name)
-        rescue StandardError
-          nil
-        end
-        restores << [context, method_name, old_method]
-        context.singleton_class.undef_method(method_name) if old_method
-        context.define_singleton_method(method_name) do |*args|
-          Types.send(method_name, *args)
-        end
-      end
-      yield
-    ensure
-      restores.each do |context, method_name, old_method|
-        context.singleton_class.undef_method(method_name)
-        context.define_singleton_method(method_name, old_method) if old_method
-      end
-    end
+      return source_type if source_type.is_a?(Types::Root::Symbol)
 
-    def with_constants(context)
-      previous_const_pairs = CrystalRuby::Types.constants.map do |type|
-        [type, begin
-          context.const_get(type)
-        rescue StandardError
-          nil
-        end]
-      end
-      CrystalRuby::Types.constants.each do |type|
-        begin
-          context.send(:remove_const, type)
-        rescue StandardError
-          nil
-        end
-        context.const_set(type, CrystalRuby::Types.const_get(type))
+      unless source_type.kind_of?(Class) && source_type < Types::Type
+        raise "Invalid type #{source_type.inspect}"
       end
-      yield
-    ensure
-      previous_const_pairs.each do |const_name, const_value|
-        begin
-          context.send(:remove_const, const_name)
-        rescue StandardError
-          nil
-        end
-        context.const_set(const_name, const_value)
-      end
-    end
 
-    def build
-      result = yield
-      Types::Type.validate!(result)
-      Types::Typedef(result)
+      return source_type unless source_type.anonymous?
+
+      source_type.tap do |new_type|
+        Types::Type.validate!(new_type)
+      end
     end
   end
 end

data/lib/crystalruby/typemaps.rb

@@ -3,55 +3,60 @@
 module CrystalRuby
   module Typemaps
     CRYSTAL_TYPE_MAP = {
-      char: "Int8",        # In Crystal, :char is typically represented as Int8
-      uchar: "UInt8",      # Unsigned char
-      int8: "Int8",        # Same as :char
-      uint8: "UInt8",      # Same as :uchar
-      short: "Int16",      # Short integer
-      ushort: "UInt16",    # Unsigned short integer
-      int16: "Int16",      # Same as :short
-      uint16: "UInt16",    # Same as :ushort
-      int: "Int32",        # Integer, Crystal defaults to 32 bits
-      uint: "UInt32",      # Unsigned integer
-      int32: "Int32",      # 32-bit integer
-      uint32: "UInt32",    # 32-bit unsigned integer
-      long: "Int32 | Int64", # Long integer, size depends on the platform (32 or 64 bits)
+      char: "Int8",             # In Crystal, :char is typically represented as Int8
+      uchar: "UInt8",           # Unsigned char
+      int8: "Int8",             # Same as :char
+      uint8: "UInt8",           # Same as :uchar
+      short: "Int16",           # Short integer
+      ushort: "UInt16",         # Unsigned short integer
+      int16: "Int16",           # Same as :short
+      uint16: "UInt16",         # Same as :ushort
+      int: "Int32",             # Integer, Crystal defaults to 32 bits
+      uint: "UInt32",           # Unsigned integer
+      int32: "Int32",           # 32-bit integer
+      uint32: "UInt32",         # 32-bit unsigned integer
+      long: "Int32 | Int64",    # Long integer, size depends on the platform (32 or 64 bits)
       ulong: "UInt32 | UInt64", # Unsigned long integer, size depends on the platform
-      int64: "Int64",      # 64-bit integer
-      uint64: "UInt64",    # 64-bit unsigned integer
-      long_long: "Int64",  # Same as :int64
-      ulong_long: "UInt64", # Same as :uint64
-      float: "Float32",    # Floating point number (single precision)
-      double: "Float64",   # Double precision floating point number
-      bool: "Bool",        # Boolean type
-      void: "Void",        # Void type
-      string: "String"     # String type
+      int64: "Int64",           # 64-bit integer
+      uint64: "UInt64",         # 64-bit unsigned integer
+      long_long: "Int64",       # Same as :int64
+      ulong_long: "UInt64",     # Same as :uint64
+      float: "Float32",         # Floating point number (single precision)
+      double: "Float64",        # Double precision floating point number
+      bool: "Bool",             # Boolean type
+      void: "Void",             # Void type
+      string: "String",         # String type
+      pointer: "Pointer(Void)"  # Pointer type
+
     }
 
+    FFI_TYPE_MAP = CRYSTAL_TYPE_MAP.invert
+
     ERROR_VALUE = {
-      char: "0", # In Crystal, :char is typically represented as Int8
-      uchar: "0", # Unsigned char
-      int8: "0", # Same as :char
-      uint8: "0",      # Same as :uchar
-      short: "0",      # Short integer
-      ushort: "0", # Unsigned short integer
-      int16: "0", # Same as :short
-      uint16: "0", # Same as :ushort
-      int: "0", # Integer, Crystal defaults to 32 bits
-      uint: "0", # Unsigned integer
-      int32: "0", # 32-bit integer
-      uint32: "0", # 32-bit unsigned integer
-      long: "0", # Long integer, size depends on the platform (32 or 64 bits)
-      ulong: "0", # Unsigned long integer, size depends on the platform
-      int64: "0", # 64-bit integer
-      uint64: "0", # 64-bit unsigned integer
-      long_long: "0",  # Same as :int64
-      ulong_long: "0", # Same as :uint64
-      float: "0.0",    # Floating point number (single precision)
-      double: "0.0",   # Double precision floating point number
+      char: "0i8", # In Crystal, :char is typically represented as Int8
+      uchar: "0u8", # Unsigned char
+      int8: "0i8", # Same as :char
+      uint8: "0u8", # Same as :uchar
+      short: "0i16", # Short integer
+      ushort: "0u16", # Unsigned short integer
+      int16: "0i16", # Same as :short
+      uint16: "0u16", # Same as :ushort
+      int: "0i32", # Integer, Crystal defaults to 32 bits
+      uint: "0u32", # Unsigned integer
+      int32: "0i32", # 32-bit integer
+      uint32: "0u32", # 32-bit unsigned integer
+      long: "0i64", # Long integer, size depends on the platform (32 or 64 bits)
+      ulong: "0u64", # Unsigned long integer, size depends on the platform
+      int64: "0_i64", # 64-bit integer
+      uint64: "0_u64", # 64-bit unsigned integer
+      long_long: "0_i64",  # Same as :int64
+      ulong_long: "0_u64", # Same as :uint64
+      float: "0.0f32",    # Floating point number (single precision)
+      double: "0.0f64",   # Double precision floating point number
       bool: "false", # Boolean type
       void: "Void", # Void type
-      string: '"".to_unsafe' # String type
+      string: '"".to_unsafe', # String type
+      pointer: "Pointer(Void).null" # Pointer type
     }
 
     C_TYPE_MAP = CRYSTAL_TYPE_MAP.merge(
@@ -68,32 +73,40 @@ module CrystalRuby
       void: {
         to: "nil"
       }
-      }.tap do |hash|
-        hash.define_singleton_method(:convert) do |type, dir, expr|
-          if hash.key?(type)
-            conversion_string = hash[type][dir]
-            conversion_string =~ /%/ ? conversion_string % expr : conversion_string
-          else
-            expr
-          end
+    }.tap do |hash|
+      hash.define_singleton_method(:convert) do |type, dir, expr|
+        if hash.key?(type)
+          conversion_string = hash[type][dir]
+          conversion_string =~ /%/ ? conversion_string % expr : conversion_string
+        else
+          expr
         end
       end
+    end
 
     def build_type_map(crystalruby_type)
+      crystalruby_type = CRType(&crystalruby_type) if crystalruby_type.is_a?(Proc)
       {
         ffi_type: ffi_type(crystalruby_type),
         ffi_ret_type: ffi_type(crystalruby_type),
         crystal_type: crystal_type(crystalruby_type),
+        crystalruby_type: crystalruby_type,
         lib_type: lib_type(crystalruby_type),
         error_value: error_value(crystalruby_type),
-        arg_mapper: if crystalruby_type.is_a?(Types::TypeSerializer)
+        arg_mapper: if crystalruby_type.is_a?(Class) && crystalruby_type < Types::Type
                       lambda { |arg|
-                        crystalruby_type.prepare_argument(arg)
+                        arg = crystalruby_type.new(arg.memory) if arg.is_a?(Types::Type) && !arg.is_a?(crystalruby_type)
+                        arg = crystalruby_type.new(arg) unless arg.is_a?(Types::Type)
+                        arg
                       }
                     end,
-        retval_mapper: if crystalruby_type.is_a?(Types::TypeSerializer)
+        retval_mapper: if crystalruby_type.is_a?(Class) && crystalruby_type < Types::Type
                          lambda { |arg|
-                           crystalruby_type.prepare_retval(arg)
+                           if arg.is_a?(Types::Type) && !arg.is_a?(crystalruby_type)
+                             arg = crystalruby_type.new(arg.memory)
+                           end
+                           arg = crystalruby_type.new(arg) unless arg.is_a?(Types::Type)
+                           crystalruby_type.anonymous? ? arg.native : arg
                          }
                        end,
         convert_crystal_to_lib_type: ->(expr) { convert_crystal_to_lib_type(expr, crystalruby_type) },
@@ -104,13 +117,20 @@ module CrystalRuby
     def ffi_type(type)
       case type
       when Symbol then type
-      when Types::TypeSerializer then type.ffi_type
+      when Class
+        if type < Types::FixedWidth
+          :pointer
+        elsif type < Types::Primitive
+          type.ffi_type
+        end
       end
     end
 
     def lib_type(type)
-      if type.is_a?(Types::TypeSerializer)
-        type.lib_type
+      if type.is_a?(Class) && type < Types::FixedWidth
+        "Pointer(::UInt8)"
+      elsif type.is_a?(Class) && type < Types::Type
+        C_TYPE_MAP.fetch(type.ffi_type)
       else
         C_TYPE_MAP.fetch(type)
       end
@@ -119,8 +139,10 @@ module CrystalRuby
     end
 
     def error_value(type)
-      if type.is_a?(Types::TypeSerializer)
-        type.error_value
+      if type.is_a?(Class) && type < Types::FixedWidth
+        "Pointer(::UInt8).null"
+      elsif type.is_a?(Class) && type < Types::Type
+        ERROR_VALUE.fetch(type.ffi_type)
       else
         ERROR_VALUE.fetch(type)
       end
@@ -129,8 +151,8 @@ module CrystalRuby
     end
 
     def crystal_type(type)
-      if type.is_a?(Types::TypeSerializer)
-        type.crystal_type
+      if type.is_a?(Class) && type < Types::Type
+        type.anonymous? ? type.native_type_expr : type.inspect
       else
         CRYSTAL_TYPE_MAP.fetch(type)
       end
@@ -139,16 +161,19 @@ module CrystalRuby
     end
 
     def convert_lib_to_crystal_type(expr, type)
-      if type.is_a?(Types::TypeSerializer)
-        type.lib_to_crystal_type_expr(expr)
+      if type.is_a?(Class) && type < Types::Type
+        expr = "#{expr}.not_nil!" unless type.nil?
+        type.pointer_to_crystal_type_conversion(expr)
+      elsif type == :void
+        "nil"
       else
-        C_TYPE_CONVERSIONS.convert(type, :from, expr)
+        "#{C_TYPE_CONVERSIONS.convert(type, :from, expr)}.not_nil!"
       end
     end
 
     def convert_crystal_to_lib_type(expr, type)
-      if type.is_a?(Types::TypeSerializer)
-        type.crystal_to_lib_type_expr(expr)
+      if type.is_a?(Class) && type < Types::Type
+        type.crystal_type_to_pointer_type_conversion(expr)
       else
         C_TYPE_CONVERSIONS.convert(type, :to, expr)
       end