crystalruby 0.2.3 → 0.3.0

data/lib/crystalruby/reactor.rb

@@ -1,13 +1,17 @@
 module CrystalRuby
+  require 'json'
   # The Reactor represents a singleton Thread responsible for running all Ruby/crystal interop code.
   # Crystal's Fiber scheduler and GC assume all code is run on a single thread.
-  # This class is responsible for multiplexing Ruby and Crystal code on a single thread.
-  # Functions annotated with async: true, are executed using callbacks to allow these to be interleaved without blocking.
-
+  # This class is responsible for multiplexing Ruby and Crystal code onto a single thread.
+  # Functions annotated with async: true, are executed using callbacks to allow these to be interleaved
+  # without blocking multiple Ruby threads.
   module Reactor
     module_function
 
     class SingleThreadViolation < StandardError; end
+    class StopReactor < StandardError; end
+
+    @single_thread_mode = false
 
     REACTOR_QUEUE = Queue.new
 
@@ -39,13 +43,15 @@ module CrystalRuby
       end
     end
 
-    ERROR_CALLBACK = FFI::Function.new(:void, %i[string string int]) do |error_type, message, tid|
+    ERROR_CALLBACK = FFI::Function.new(:void, %i[string string string int]) do |error_type, message, backtrace, tid|
       error_type = error_type.to_sym
       is_exception_type = Object.const_defined?(error_type) && Object.const_get(error_type).ancestors.include?(Exception)
       error_type = is_exception_type ? Object.const_get(error_type) : RuntimeError
-      raise error_type.new(message) unless THREAD_MAP.key?(tid)
+      error = error_type.new(message)
+      error.set_backtrace(JSON.parse(backtrace))
+      raise error  unless THREAD_MAP.key?(tid)
 
-      THREAD_MAP[tid][:error] = error_type.new(message)
+      THREAD_MAP[tid][:error] = error
       THREAD_MAP[tid][:result] = nil
       THREAD_MAP[tid][:cond].signal
     end
@@ -57,18 +63,35 @@ module CrystalRuby
     def await_result!
       mux, cond = thread_conditions.values_at(:mux, :cond)
       cond.wait(mux)
-      raise THREAD_MAP[thread_id][:error] if THREAD_MAP[thread_id][:error]
+      if error = thread_conditions[:error]
+        combined_backtrace = error.backtrace[0..(error.backtrace.index{|m| m.include?('call_blocking_function')} || 2) - 3] + caller[5..-1]
+        error.set_backtrace(combined_backtrace)
+        raise error
+      end
+
+      thread_conditions[:result]
+    end
+
+    def halt_loop!
+      raise StopReactor
+    end
 
-      THREAD_MAP[thread_id][:result]
+    def stop!
+      if @main_loop
+        schedule_work!(self, :halt_loop!, :void, blocking: true, async: false)
+        @main_loop.join
+        @main_loop = nil
+        CrystalRuby.log_info "Reactor loop stopped"
+      end
     end
 
     def start!
       @main_loop ||= Thread.new do
+        @main_thread_id = Thread.current.object_id
         CrystalRuby.log_debug("Starting reactor")
         CrystalRuby.log_debug("CrystalRuby initialized")
-        loop do
-          REACTOR_QUEUE.pop[]
-        end
+        REACTOR_QUEUE.pop[] while true
+      rescue StopReactor => e
       rescue StandardError => e
         CrystalRuby.log_error "Error: #{e}"
         CrystalRuby.log_error e.backtrace
@@ -80,26 +103,17 @@ module CrystalRuby
     end
 
     def yield!(lib: nil, time: 0.0)
-      schedule_work!(lib, :yield, nil, async: false, blocking: false, lib: lib) if running? && lib
+      schedule_work!(lib, :yield, :int, async: false, blocking: false, lib: lib) if running? && lib
       nil
     end
 
-    def current_thread_id=(val)
-      @current_thread_id = val
-    end
-
-    def current_thread_id
-      @current_thread_id
-    end
-
     def schedule_work!(receiver, op_name, *args, return_type, blocking: true, async: true, lib: nil)
-      if @single_thread_mode
+      if @single_thread_mode || (Thread.current.object_id == @main_thread_id && op_name != :yield)
         unless Thread.current.object_id == @main_thread_id
           raise SingleThreadViolation,
                 "Single thread mode is enabled, cannot run in multi-threaded mode. " \
                 "Reactor was started from: #{@main_thread_id}, then called from #{Thread.current.object_id}"
         end
-
         return receiver.send(op_name, *args)
       end
 
@@ -118,14 +132,17 @@ module CrystalRuby
           when blocking
             lambda {
               tvars[:error] = nil
-              Reactor.current_thread_id = tvars[:thread_id]
+              should_halt = false
               begin
                 result = receiver.send(op_name, *args)
+              rescue StopReactor => e
+                should_halt = true
               rescue StandardError => e
                 tvars[:error] = e
               end
               tvars[:result] = result unless tvars[:error]
               tvars[:cond].signal
+              raise StopReactor if should_halt
             }
           else
             lambda {

data/lib/crystalruby/source_reader.rb

@@ -0,0 +1,86 @@
+module CrystalRuby
+  module SourceReader
+    module_function
+
+    # Reads code line by line from a given source location and returns the first valid Ruby expression found
+    def extract_expr_from_source_location(source_location)
+      lines = source_location.then{|f,l| IO.readlines(f)[l-1..]}
+      lines[0] = lines[0][/CRType.*/] if lines[0] =~ /<\s+CRType/ || lines[0] =~ /= CRType/
+      lines.each.with_object("") do |line, expr_source|
+        break expr_source if (SyntaxTree.parse(expr_source << line) rescue nil)
+      end
+    rescue
+      raise "Failed to extract expression from source location: #{source_location}. Ensure the file exists and the line number is correct. Extraction from a REPL is not supported"
+    end
+
+    # Given a proc, extracts the source code of the block passed to it
+    # If raw is true, the source is expected to be Raw Crystal code captured
+    # in a string or Heredoc literal. Otherwise the Ruby code (assumed to be valid Crystal)
+    # is extracted.
+    def extract_source_from_proc(block, raw: false)
+      block_source = extract_expr_from_source_location(block.source_location)
+      body_node =  SyntaxTree.search(block_source, :Statements).to_a[1]
+      return raw ?
+        SyntaxTree.search(node_to_s(body_node), :TStringContent).map(&method(:node_to_s)).join :
+        node_to_s(body_node)
+    end
+
+    @visitor = SyntaxTree::MutationVisitor.new
+
+    # Specify that it should mutate If nodes with assignments in their predicates
+    @visitor.mutate("ReturnNode") do |node|
+      node
+    end
+
+    # Simple helper function to turn a SyntaxTree node back into a Ruby string
+    # The default formatter will turn a break/return of [1,2,3] into a brackless 1,2,3
+    # Can't have that in Crystal as it turns it into a Tuple
+    def node_to_s(node)
+      @_syxt_transform ||= SyntaxTree::FlowControlFormatter.prepend(Module.new do
+        def format(quer)
+          first_arg = self.node.arguments.child_nodes
+          if first_arg[0].kind_of?(SyntaxTree::ArrayLiteral)
+            return format_arguments(quer, " [", "]")
+          end
+          super(quer)
+        end
+      end)
+      SyntaxTree::Formatter.format("", node.accept(@visitor))
+    end
+
+    # Given a method, extracts the source code of the block passed to it
+    # and also converts any keyword arguments given in the method definition as a
+    # named map of keyword names to Crystal types.
+    # Also supports basic ffi symbol types.
+    #
+    # E.g.
+    #
+    # def add a: Int32 | Int64, b: :int
+    #
+    # The above will be converted to:
+    # {
+    #   a: Int32 | Int64, # Int32 | Int64 is a Crystal type
+    #   b: :int           # :int is an FFI type shorthand
+    # }
+    # If raw is true, the source is expected to be Raw Crystal code captured
+    # in a string or Heredoc literal. Otherwise the Ruby code (assumed to be valid Crystal)
+    # is extracted.
+    def extract_args_and_source_from_method(method, raw: false)
+      method_source = extract_expr_from_source_location(method.source_location)
+      params = SyntaxTree.search(method_source, :Params).first
+      args = params ? params.keywords.map{|k,v| [k.value[0...-1].to_sym, node_to_s(v)] }.to_h : {}
+      body_node = SyntaxTree.search(method_source, :BodyStmt).first || SyntaxTree.search(method_source, :Statements).to_a[1]
+      body = node_to_s(body_node)
+      body = SyntaxTree.search(body, :TStringContent).map(&method(:node_to_s)).join if raw
+      args.transform_values! do |type_exp|
+        if CrystalRuby::Typemaps::CRYSTAL_TYPE_MAP.key?(type_exp[1..-1].to_sym)
+          type_exp[1..-1].to_sym
+        else
+          TypeBuilder.build_from_source(type_exp, context: method.owner)
+        end
+      end.to_h
+      return args, body
+    end
+
+  end
+end

data/lib/crystalruby/template.rb

@@ -1,12 +1,23 @@
 module CrystalRuby
   module Template
-    Dir[File.join(File.dirname(__FILE__), "templates", "*.cr")].each do |file|
+    class Renderer < Struct.new(:raw_value)
+      require 'erb'
+      def render(context)
+        if context.kind_of?(::Hash)
+          raw_value % context
+        else
+          ERB.new(raw_value, trim_mode: "%").result(context)
+        end
+      end
+    end
+
+    (
+      Dir[File.join(File.dirname(__FILE__), "templates", "**", "*.cr")] +
+      Dir[File.join(File.dirname(__FILE__), "types", "**", "*.cr")]
+    ).each do |file|
       template_name = File.basename(file, File.extname(file)).split("_").map(&:capitalize).join
       template_value = File.read(file)
-      template_value.define_singleton_method(:render) do |context|
-        self % context
-      end
-      const_set(template_name, template_value)
+      const_set(template_name, Renderer.new(template_value))
     end
   end
 end

data/lib/crystalruby/templates/function.cr

@@ -3,8 +3,8 @@
 # Crystal code can simply call this method directly, enabling generated crystal code
 # to call other generated crystal code without overhead.
 
-module %{module_name}
-  def self.%{fn_name}(%{fn_args}) : %{fn_ret_type}
+%{module_or_class} %{module_name} %{superclass}
+  def %{fn_scope}%{fn_name}(%{fn_args}) : %{fn_ret_type}
     %{fn_body}
   end
 end
@@ -16,13 +16,13 @@ fun %{lib_fn_name}(%{lib_fn_args}): %{lib_fn_ret_type}
   begin
     %{convert_lib_args}
     begin
-      return_value = %{module_name}.%{fn_name}(%{arg_names})
+      return_value = %{receiver}.%{fn_name}(%{arg_names})%{block_converter}
       return %{convert_return_type}
     rescue ex
-      CrystalRuby.report_error("RuntimeError", ex.message.to_s, 0)
+      CrystalRuby.report_error("RuntimeError", ex.message.to_s, ex.backtrace.to_json, 0)
     end
   rescue ex
-    CrystalRuby.report_error("ArgumentError", ex.message.to_s, 0)
+    CrystalRuby.report_error("ArgumentError", ex.message.to_s, ex.backtrace.to_json, 0)
   end
   return %{error_value}
 end
@@ -37,25 +37,26 @@ fun %{lib_fn_name}_async(%{lib_fn_args} thread_id: UInt32,  callback : %{callbac
     CrystalRuby.increment_task_counter
     spawn do
       begin
-        return_value = %{module_name}.%{fn_name}(%{arg_names})
-        converted = %{convert_return_type}
+        return_value = %{receiver}.%{fn_name}(%{arg_names})%{block_converter}
         CrystalRuby.queue_callback(->{
+          converted = %{convert_return_type}
           %{callback_call}
           CrystalRuby.decrement_task_counter
         })
       rescue ex
         exception = ex.message.to_s
+        backtrace = ex.backtrace.to_json
         CrystalRuby.queue_callback(->{
-          CrystalRuby.error_callback.call("RuntimeError".to_unsafe, exception.to_unsafe, thread_id)
+          CrystalRuby.report_error("RuntimeError", exception, backtrace, thread_id)
           CrystalRuby.decrement_task_counter
         })
       end
     end
   rescue ex
-
     exception = ex.message.to_s
+    backtrace = ex.backtrace.to_json
     CrystalRuby.queue_callback(->{
-      CrystalRuby.error_callback.call("RuntimeError".to_unsafe, ex.message.to_s.to_unsafe, thread_id)
+      CrystalRuby.report_error("RuntimeError", ex.message.to_s, backtrace, thread_id)
       CrystalRuby.decrement_task_counter
     })
   end

data/lib/crystalruby/templates/index.cr

@@ -1,48 +1,40 @@
 module CrystalRuby
-
   ARGV1 = "crystalruby"
-  CALLBACK_MUX = Mutex.new
 
-  alias ErrorCallback = (Pointer(UInt8), Pointer(UInt8), UInt32 -> Void)
+  alias ErrorCallback = (Pointer(::UInt8), Pointer(::UInt8), Pointer(::UInt8), ::UInt32 -> Void)
+
+  class_property libname : String = "crystalruby"
+  class_property callbacks : Channel(Proc(Nil)) = Channel(Proc(Nil)).new
+  class_property rc_mux : Pointer(Void) = Pointer(Void).null
+  class_property task_counter : Atomic(Int32) = Atomic(Int32).new(0)
 
   # Initializing Crystal Ruby invokes init on the Crystal garbage collector.
   # We need to be sure to only do this once.
-  @@initialized = false
-
-  @@libname = "crystalruby"
-  # Our Ruby <-> Crystal Reactor uses Fibers, with callbacks to allow
-  # multiple concurrent Crystal operations to be queued
-  @@callbacks = [] of Proc(Nil)
-
-  # We only continue to yield to the Crystal scheduler from Ruby
-  # while there are outstanding tasks.
-  @@task_counter : Atomic(Int32) = Atomic.new(0)
+  class_property initialized : Bool = false
 
   # We can override the error callback to catch errors in Crystal,
   # and explicitly expose them to Ruby.
-  @@error_callback : ErrorCallback = ->(t : UInt8* , s : UInt8*, tid : UInt32){ puts "Error: #{t}:#{s}" }
+  @@error_callback : ErrorCallback?
 
   # This is the entry point for instantiating CrystalRuby
   # We:
   # 1. Initialize the Crystal garbage collector
   # 2. Set the error callback
   # 3. Call the Crystal main function
-  def self.init(libname : Pointer(UInt8), error_callback : ErrorCallback)
-    return if @@initialized
-    @@initialized = true
+  def self.init(libname : Pointer(::UInt8), @@error_callback : ErrorCallback, @@rc_mux : Pointer(Void))
+    return if self.initialized
+    self.initialized = true
     argv_ptr = ARGV1.to_unsafe
     Crystal.main_user_code(0, pointerof(argv_ptr))
-    @@error_callback = error_callback
-    @@libname = String.new(libname)
+    self.libname = String.new(libname)
+    LibGC.set_finalize_on_demand(1)
   end
 
   # Explicit error handling (triggers exception within Ruby on the same thread)
-  def self.report_error(error_type : String, str : String, thread_id : UInt32, )
-    @@error_callback.call(error_type.to_unsafe, str.to_unsafe, thread_id)
-  end
-
-  def self.error_callback : ErrorCallback
-    @@error_callback
+  def self.report_error(error_type : String, message : String, backtrace : String, thread_id : UInt32)
+    if error_reporter = @@error_callback
+      error_reporter.call(error_type.to_unsafe, message.to_unsafe, backtrace.to_unsafe, thread_id)
+    end
   end
 
   # New async task started
@@ -57,51 +49,40 @@ module CrystalRuby
 
   # Get number of outstanding tasks
   def self.get_task_counter : Int32
-    @@task_counter.get()
+    @@task_counter.get
   end
 
   # Queue a callback for an async task
   def self.queue_callback(callback : Proc(Nil))
-    CALLBACK_MUX.synchronize do
-      @@callbacks << callback
-    end
-  end
-
-  # Get number of queued callbacks
-  def self.count_callbacks : Int32
-    @@callbacks.size
+    self.callbacks.send(callback)
   end
 
-  def self.libname : String
-    @@libname
-  end
-
-  # Flush all callbacks
-  def self.flush_callbacks : Int32
-    CALLBACK_MUX.synchronize do
-      count = @@callbacks.size
-      @@callbacks.each do |callback|
-        result = callback.call()
-      end
-      @@callbacks.clear
-    end
-    get_task_counter
+  def self.synchronize
+    LibC.pthread_mutex_lock(self.rc_mux)
+    yield
+    LibC.pthread_mutex_unlock(self.rc_mux)
   end
 end
 
 # Initialize CrystalRuby
-fun init(libname : Pointer(UInt8), cb : CrystalRuby::ErrorCallback): Void
-  CrystalRuby.init(libname, cb)
+fun init(libname : Pointer(::UInt8), cb : CrystalRuby::ErrorCallback, rc_mux : Pointer(Void)) : Void
+  CrystalRuby.init(libname, cb, rc_mux)
 end
 
 fun stop : Void
-  LibGC.deinit()
+  LibGC.deinit
 end
 
 @[Link("gc")]
 lib LibGC
   $stackbottom = GC_stackbottom : Void*
   fun deinit = GC_deinit
+  fun set_finalize_on_demand = GC_set_finalize_on_demand(Int32)
+  fun invoke_finalizers = GC_invoke_finalizers : Int
+end
+
+lib LibC
+  fun calloc = calloc(Int32, Int32) : Void*
 end
 
 module GC
@@ -112,6 +93,11 @@ module GC
   def self.set_stackbottom(stack_bottom : Void*)
     LibGC.stackbottom = stack_bottom
   end
+
+  def self.collect
+    LibGC.collect
+    LibGC.invoke_finalizers
+  end
 end
 
 # Yield to the Crystal scheduler from Ruby
@@ -119,24 +105,25 @@ end
 # Otherwise, we yield to the Crystal scheduler and let Ruby know
 # how many outstanding tasks still remain (it will stop yielding to Crystal
 # once this figure reaches 0).
-fun yield() : Int32
-  if CrystalRuby.count_callbacks == 0
-    Fiber.yield
-
-    # TODO: We should apply backpressure here to prevent busy waiting if the number of outstanding tasks is not decreasing.
-    # Use a simple exponential backoff strategy, to increase the time between each yield up to a maximum of 1 second.
-
-    CrystalRuby.get_task_counter
-  else
-    CrystalRuby.flush_callbacks()
+fun yield : Int32
+  Fiber.yield
+  loop do
+    select
+    when callback = CrystalRuby.callbacks.receive
+      callback.call
+    else
+      break
+    end
   end
+  CrystalRuby.get_task_counter
 end
 
+class Array(T)
+  def initialize(size : Int32, @buffer : Pointer(T))
+    @size = size.to_i32
+    @capacity = @size
+  end
+end
 
-# This is where we define all our Crystal modules and types
-# derived from their Ruby counterparts.
-%{type_modules}
-
-# Require all generated crystal files
 require "json"
 %{requires}

data/lib/crystalruby/templates/inline_chunk.cr

@@ -1,6 +1,6 @@
 # This is the template used for writing inline chunks of Crystal code without direct
 # Ruby integration
 
-module %{module_name}
+%{mod_or_class} %{module_name} %{superclass}
   %{body}
 end

data/lib/crystalruby/templates/ruby_interface.cr

@@ -0,0 +1,34 @@
+# This is the template used for all CrystalRuby functions
+# Calls to this method *from ruby* are first transformed through the lib function.
+# Crystal code can simply call this method directly, enabling generated crystal code
+# to call other generated crystal code without overhead.
+
+%{module_or_class} %{module_name} %{superclass}
+  def %{fn_scope}%{fn_name}(%{fn_args}) : %{fn_ret_type}
+    %{convert_lib_args}
+    cb = %{module_name}.%{callback_name}
+    unless cb.nil?
+      callback_done_channel = Channel(Nil).new
+      return_value = nil
+      if Fiber.current == Thread.current.main_fiber
+        return_value = cb.call(%{lib_fn_arg_names})
+        return %{convert_return_type}
+      else
+        CrystalRuby.queue_callback(->{
+          return_value = cb.call(%{lib_fn_arg_names})
+          callback_done_channel.send(nil)
+        })
+      end
+      callback_done_channel.receive
+      return %{convert_return_type}
+    end
+    raise "No callback registered for %{fn_name}"
+
+  end
+
+  class_property %{callback_name} : Proc(%{lib_fn_types} %{lib_fn_ret_type})?
+end
+
+fun register_%{callback_name}(callback : Proc(%{lib_fn_types} %{lib_fn_ret_type})) : Void
+  %{module_name}.%{callback_name} = callback
+end

data/lib/crystalruby/templates/top_level_function.cr

@@ -0,0 +1,62 @@
+# This is the template used for all CrystalRuby functions
+# Calls to this method *from ruby* are first transformed through the lib function.
+# Crystal code can simply call this method directly, enabling generated crystal code
+# to call other generated crystal code without overhead.
+
+def %{fn_name}(%{fn_args}) : %{fn_ret_type}
+  %{fn_body}
+end
+
+# This function is the entry point for the CrystalRuby code, exposed through FFI.
+# We apply some basic error handling here, and convert the arguments and return values
+# to ensure that we are using Crystal native types.
+fun %{lib_fn_name}(%{lib_fn_args}): %{lib_fn_ret_type}
+  begin
+    %{convert_lib_args}
+    begin
+      return_value = %{fn_name}(%{arg_names})%{block_converter}
+      return %{convert_return_type}
+    rescue ex
+      CrystalRuby.report_error("RuntimeError", ex.message.to_s, ex.backtrace.to_json, 0)
+    end
+  rescue ex
+    CrystalRuby.report_error("ArgumentError", ex.message.to_s, ex.backtrace.to_json, 0)
+  end
+  return %{error_value}
+end
+
+
+# This function is the async entry point for the CrystalRuby code, exposed through FFI.
+# We apply some basic error handling here, and convert the arguments and return values
+# to ensure that we are using Crystal native types.
+fun %{lib_fn_name}_async(%{lib_fn_args} thread_id: UInt32,  callback : %{callback_type}): Void
+  begin
+    %{convert_lib_args}
+    CrystalRuby.increment_task_counter
+    spawn do
+      begin
+        return_value = %{fn_name}(%{arg_names})%{block_converter}
+        CrystalRuby.queue_callback(->{
+          converted = %{convert_return_type}
+          %{callback_call}
+          CrystalRuby.decrement_task_counter
+        })
+      rescue ex
+        exception = ex.message.to_s
+        backtrace = ex.backtrace.to_json
+        CrystalRuby.queue_callback(->{
+          CrystalRuby.report_error("RuntimeError", exception, backtrace, thread_id)
+          CrystalRuby.decrement_task_counter
+        })
+      end
+    end
+  rescue ex
+
+    exception = ex.message.to_s
+    backtrace = ex.backtrace.to_json
+    CrystalRuby.queue_callback(->{
+      CrystalRuby.report_error("RuntimeError", ex.message.to_s, backtrace, thread_id)
+      CrystalRuby.decrement_task_counter
+    })
+  end
+end

data/lib/crystalruby/templates/top_level_ruby_interface.cr

@@ -0,0 +1,33 @@
+# This is the template used for all CrystalRuby functions
+# Calls to this method *from ruby* are first transformed through the lib function.
+# Crystal code can simply call this method directly, enabling generated crystal code
+# to call other generated crystal code without overhead.
+
+module TopLevelCallbacks
+  class_property %{fn_name}_callback : Proc(%{lib_fn_types} %{lib_fn_ret_type})?
+end
+
+def %{fn_scope}%{fn_name}(%{fn_args}) : %{fn_ret_type}
+  %{convert_lib_args}
+  cb = TopLevelCallbacks.%{fn_name}_callback
+  unless cb.nil?
+    callback_done_channel = Channel(Nil).new
+    return_value = nil
+    if Fiber.current == Thread.current.main_fiber
+      return_value = cb.call(%{lib_fn_arg_names})
+      return %{convert_return_type}
+    else
+      CrystalRuby.queue_callback(->{
+        return_value = cb.call(%{lib_fn_arg_names})
+        callback_done_channel.send(nil)
+      })
+    end
+    callback_done_channel.receive
+    return %{convert_return_type}
+  end
+  raise "No callback registered for %{fn_name}"
+end
+
+fun register_%{fn_name}_callback(callback : Proc(%{lib_fn_types} %{lib_fn_ret_type})) : Void
+  TopLevelCallbacks.%{fn_name}_callback = callback
+end