crystalruby 0.1.12 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 41e8370627d23725b55bb1916254b88690e93b384c94692d59d73b0cf5ceda03
-  data.tar.gz: 7d7192ea627331a30f6cd9ff9dcc7482ed2b1e373f115cc0e3cfa19edd5db506
+  metadata.gz: 117525a6b5905a6c4aa86b0b2fed0c25e46d0e97b6de8ce187b90f1438394f19
+  data.tar.gz: 55b35d8731c2d7e68f6cb0a1a4ea3eb068805bc4297f1b85fceac08cb288682a
 SHA512:
-  metadata.gz: 56d644c482b04bb8f5b39b6626dda93079578f3f95809294a441e0fd1707aeb71eb253da7a549c351fe448750f6dd2acfdbda3782b37e5aaebb4019332c64d0c
-  data.tar.gz: c529c61b9564f6d23dc551209ac7ce426d56a02424a970b57825f5e993953c376fa57afb3880db0647d581391e7d88881775cd0e2284408997fbaf4e8b37a690
+  metadata.gz: 585ad3697b06ba3660989ea538ddae90816d0e93a947e95ab5e6bb83ace64d35988c436cf8e07cad077529691baebbdbdeac6bcc0e7a991297d67f2efada5438
+  data.tar.gz: bd06431cc0c059481db8215676e493f16d485cb14a8b15dedc3de6e79df79716c4e6499bb4f9678257659601224b249798dc8c4c492c1659c46157714e45499e

data/.dockerignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+crystalruby-*.gem
+/crystalruby/

data/README.md

@@ -43,33 +43,45 @@ E.g.
 require 'crystalruby'
 require 'benchmark'
 
-module Fibonnaci
+module PrimeCounter
   crystalize [n: :int32] => :int32
-  def fib_cr(n)
-    a = 0
-    b = 1
-    n.times { a, b = b, a + b }
-    a
+  def count_primes_upto_cr(n)
+    (2..n).each.count do |i|
+      is_prime = true
+      (2..Math.sqrt(i).to_i).each do |j|
+        if i % j == 0
+          is_prime = false
+          break
+        end
+      end
+      is_prime
+    end
   end
 
   module_function
 
-  def fib_rb(n)
-    a = 0
-    b = 1
-    n.times { a, b = b, a + b }
-    a
+  def count_primes_upto_rb(n)
+    (2..n).each.count do |i|
+      is_prime = true
+      (2..Math.sqrt(i).to_i).each do |j|
+        if i % j == 0
+          is_prime = false
+          break
+        end
+      end
+      is_prime
+    end
   end
 end
 
-puts(Benchmark.realtime { 1_000_000.times { Fibonnaci.fib_rb(30) } })
-puts(Benchmark.realtime { 1_000_000.times { Fibonnaci.fib_cr(30) } })
-
+include PrimeCounter
+puts(Benchmark.realtime { count_primes_upto_rb(1000_000) })
+puts(Benchmark.realtime { count_primes_upto_cr(1000_000) })
 ```
 
 ```bash
-3.193121999996947 # Ruby
-0.29086600001028273 # Crystal
+2.8195170001126826 # Ruby
+0.3402599999681115 # Crystal
 ```
 
 _Note_: The first run of the Crystal code will be slower, as it needs to compile the code first. The subsequent runs will be much faster.
@@ -121,10 +133,10 @@ end
 ### Crystal Compatible
 
 Some Crystal syntax is not valid Ruby, for methods of this form, we need to
-define our functions using a :raw parameter.
+define our functions using a raw: true option
 
 ```ruby
-crystalize :raw, [a: :int, b: :int] => :int
+crystalize [a: :int, b: :int] => :int, raw: true
 def add(a, b)
   <<~CRYSTAL
     c = 0_u64
@@ -453,6 +465,78 @@ CrystalRuby.compile!
 
 Then you can run this file as part of your build step, to ensure all Crystal code is compiled ahead of time.
 
+## Concurrency
+
+While Ruby programs allow multi-threading, Crystal by default uses only a single thread, while utilising Fiber based cooperative-multitasking to allow for concurrent execution. This means that by default, Crystal libraries can not safely be invoked in parallel across multiple Ruby threads.
+
+To safely expose this behaviour, `crystalruby` implements a Reactor, which multiplexes all Ruby calls to Crystal across a single thread. This way you can safely use `crystalruby` in a multi-threaded Ruby environment.
+
+By default `crystalruby` methods are blocking/synchronous, this means that for blocking operations, a single crystalruby call can block the entire reactor.
+
+To allow you to benefit from Crystal's fiber based concurrency, you can use the `async` option on crystalized ruby methods. This allows several Ruby threads to invoke Crystal code simultaneously.
+
+E.g.
+
+```ruby
+module Sleeper
+  crystalize [] => :void
+  def sleep_sync
+    sleep 2
+  end
+
+  crystalize [] => :void, async: true
+  def sleep_async
+    sleep 2
+  end
+end
+```
+
+```ruby
+5.times.map{ Thread.new{ Sleeper.sleep_sync } }.each(&:join) # Will take 10 seconds
+5.times.map{ Thread.new{ Sleeper.sleep_async } }.each(&:join) # Will take 2 seconds (the sleeps are processed concurrently)
+```
+
+### Reactor performance
+
+There is a small amount of synchronization overhead to multiplexing calls across a single thread. Ad-hoc testing amounts this to be around 10 nanoseconds per call.
+For most use-cases this overhead is negligible, especially if the bulk of your CPU heavy task occurs exclusively in Crystal code. However, if you are invoking very fast Crystal code from Ruby in a tight loop (e.g. a simple 1 + 2)
+then the overhead of the reactor can become significant.
+
+In this case you can use the `crystalruby` in a single-threaded mode to avoid the reactor overhead and greatly increase performance, with the caveat that _all_ calls to Crystal must occur from a single thread. If your Ruby program is already single-threaded this is not a problem.
+
+```ruby
+CrystalRuby.configure do |config|
+  config.single_thread_mode = true
+end
+```
+
+## Live Reloading
+
+`crystalruby` supports live reloading of Crystal code. It will intelligently
+recompile Crystal code only when it detects changes to the embedded function or block bodies. This allows you to iterate quickly on your Crystal code without having to restart your Ruby process in live-reloading environments like Rails.
+
+## Multi-library support
+
+Large Crystal projects are known to have long compile times. To mitigate this, `crystalruby` supports splitting your Crystal code into multiple libraries. This allows you to only recompile the library that has changed, rather than the entire project.
+To indicate which library a piece of embedded Crystal code belongs to, you can use the `library` option in the `crystalize` and `crystal` methods.
+If the "lib" option is not provided, the code will be compiled into the default library (simply named `crystalruby`).
+
+```ruby
+module Foo
+  crystalize lib: "foo"
+  def bar
+    puts "Hello from Foo"
+  end
+
+  crystal lib: "foo" do
+    REDIS = Redis.new
+  end
+end
+```
+
+Naturally Crystal code must be in the same library to interact directly.
+Interaction across multiple libraries can be coordinated via Ruby code.
+
 ## Troubleshooting
 
 The logic to detect when to JIT recompile is not robust and can end up in an inconsistent state. To remedy this it is useful to clear out all generated assets and build from scratch.
@@ -509,24 +593,24 @@ crystalruby init
 ```
 
 ```yaml
-crystal_src_dir: "./crystalruby/src"
-crystal_lib_dir: "./crystalruby/lib"
+crystal_src_dir: "./crystalruby"
+crystal_codegen_dir: "generated"
 crystal_main_file: "main.cr"
 crystal_lib_name: "crlib"
 crystal_codegen_dir: "generated"
 debug: true
 ```
 
-Alternatively, these can be set programmatically:
+Alternatively, these can be set programmatically, e.g:
 
 ```ruby
 CrystalRuby.configure do |config|
-  config.crystal_src_dir = "./crystalruby/src"
-  config.crystal_lib_dir = "./crystalruby/lib"
-  config.crystal_main_file = "main.cr"
-  config.crystal_lib_name = "crlib"
+  config.crystal_src_dir = "./crystalruby"
   config.crystal_codegen_dir = "generated"
   config.debug = true
+  config.verbose = false
+  config.colorize_log_output = false
+  config.log_level = :info
 end
 ```
 

data/exe/crystalruby

@@ -1,6 +1,5 @@
 #!/usr/bin/env ruby
 
-require "bundler/setup"
 require "crystalruby"
 require "fileutils"
 
@@ -9,11 +8,10 @@ def init
   # Define some dummy content for the YAML file
   yaml_content = <<~YAML
     # crystalruby configuration file
-    crystal_src_dir: "./crystalruby/src"
-    crystal_lib_dir: "./crystalruby/lib"
-    crystal_main_file: "main.cr"
-    crystal_lib_name: "crlib"
+    crystal_src_dir: "./crystalruby"
     crystal_codegen_dir: "generated"
+    log_level: "info"
+    single_thread_mode: false
     debug: true
   YAML
 
@@ -23,23 +21,25 @@ def init
 end
 
 def install
-  Dir.chdir("#{CrystalRuby.config.crystal_src_dir}") do
-    if system("shards update")
-      puts "Shards installed successfully."
-    else
-      puts "Error installing shards."
+  Dir["#{CrystalRuby.config.crystal_src_dir}/**/src"].each do |src_dir|
+    Dir.chdir(src_dir) do
+      if system("shards check") || system("shards update")
+        puts "Shards installed successfully."
+      else
+        puts "Error installing shards."
+      end
     end
   end
-  clean
 end
 
 def clean
-  # This is a stub for the clear command
-  FileUtils.rm_rf("#{CrystalRuby.config.crystal_src_dir}/generated")
+  Dir["#{CrystalRuby.config.crystal_src_dir}/**/src/generated"].each do |codegen_dir|
+    FileUtils.rm_rf(codegen_dir)
+  end
 end
 
 def build
-  # This is a stub for the build command
+  # TODO: Iterate through all generated libs and build
   puts "Build command is not implemented yet."
 end
 

data/lib/crystalruby/adapter.rb

@@ -0,0 +1,133 @@
+module CrystalRuby
+  module Adapter
+    # Use this method to annotate a Ruby method that should be crystalized.
+    # Compilation and attachment of the method is done lazily.
+    # You can force compilation by calling `CrystalRuby.compile!`
+    # It's important that all code using crystalized methods is
+    # loaded before any manual calls to compile.
+    #
+    # E.g.
+    #
+    # crystalize [a: :int32, b: :int32] => :int32
+    # def add(a, b)
+    #  a + b
+    # end
+    #
+    # Pass `raw: true` to pass Raw crystal code to the compiler as a string instead.
+    # (Useful for cases where the Crystal method body is not valid Ruby)
+    # E.g.
+    # crystalize raw: true [a: :int32, b: :int32] => :int32
+    # def add(a, b)
+    #   <<~CRYSTAL
+    #   a + b
+    #   CRYSTAL
+    # end
+    #
+    # Pass `async: true` to make the method async.
+    # Crystal methods will always block the currently executing Ruby thread.
+    # With async: false, all other Crystal code will be blocked while this Crystal method is executing (similar to Ruby code with the GVL)
+    # With async: true, several Crystal methods can be executing concurrently.
+    #
+    # Pass lib: "name_of_lib" to compile Crystal code into several distinct libraries.
+    # This can help keep compilation times low, by packaging your Crystal code into separate shared objects.
+    def crystalize(raw: false, async: false, lib: "crystalruby", **options, &block)
+      (args,), returns = options.first || [[], :void]
+      args ||= {}
+      raise "Arguments should be of the form name: :type. Got #{args}" unless args.is_a?(Hash)
+
+      @crystalize_next = {
+        raw: raw,
+        async: async,
+        args: args,
+        returns: returns,
+        block: block,
+        lib: lib
+      }
+    end
+
+    # This method provides a useful DSL for defining Crystal types in pure Ruby.
+    # These types can not be passed directly to Ruby, and must be serialized as either:
+    #   JSON or
+    #   C-Structures (WIP)
+    #
+    # See #json for an example of how to define arguments or return types for complex objects.
+    # E.g.
+    #
+    # MyType = crtype{ Int32 | Hash(String, Array(Bool) | Float65 | Nil) }
+    def crtype(&block)
+      TypeBuilder.with_injected_type_dsl(self) do
+        TypeBuilder.build(&block)
+      end
+    end
+
+    # Use the json{} helper for defining complex method arguments or return types
+    # that should be serialized to and from Crystal using JSON. (This conversion is applied automatically)
+    #
+    # E.g.
+    # crystalize [a: json{ Int32 | Float64 | Nil } ] => NamedStruct(result: Int32 | Float64 | Nil)
+    def json(&block)
+      crtype(&block).serialize_as(:json)
+    end
+
+    # We trigger attaching of crystalized instance methods here.
+    # If a method is added after a crystalize annotation we assume it's the target of the crystalize annotation.
+    def method_added(method_name)
+      define_crystalized_method(method_name, instance_method(method_name)) if @crystalize_next
+      super
+    end
+
+    # We trigger attaching of crystalized class methods here.
+    # If a method is added after a crystalize annotation we assume it's the target of the crystalize annotation.
+    def singleton_method_added(method_name)
+      define_crystalized_method(method_name, singleton_method(method_name)) if @crystalize_next
+      super
+    end
+
+    # Use this method to define inline Crystal code that does not need to be bound to a Ruby method.
+    # This is useful for defining classes, modules, performing set-up tasks etc.
+    # See: docs for .crystalize to understand the `raw` and `lib` parameters.
+    def crystal(raw: false, lib: "crystalruby", &block)
+      inline_crystal_body = Template::InlineChunk.render(
+        {
+          module_name: name, body: extract_source(block, raw: raw)
+        }
+      )
+      CrystalRuby::Library[lib].crystalize_chunk(
+        self,
+        Digest::MD5.hexdigest(inline_crystal_body),
+        inline_crystal_body
+      )
+    end
+
+    # We attach crystalized class methods here.
+    # This function is responsible for
+    # - Generating the Crystal source code
+    # - Overwriting the method and class methods by the same name in the caller.
+    # - Lazily triggering compilation and attachment of the Ruby method to the Crystal code.
+    # - We also optionally prepend a block (if given) to the owner, to allow Ruby code to wrap around Crystal code.
+    def define_crystalized_method(method_name, method)
+      CrystalRuby.log_debug("Defining crystalized method #{name}.#{method_name}")
+
+      args, returns, block, async, lib, raw = @crystalize_next.values_at(:args, :returns, :block, :async, :lib, :raw)
+      @crystalize_next = nil
+
+      CrystalRuby::Library[lib].crystalize_method(
+        method,
+        args,
+        returns,
+        extract_source(method, raw: raw),
+        async,
+        &block
+      )
+    end
+
+    # Extract Ruby source to serve as Crystal code directly.
+    # If it's a raw method, we'll strip the string delimiters at either end of the definition.
+    # We need to clear the MethodSource cache here to allow for code reloading.
+    def extract_source(method_or_block, raw: false)
+      method_or_block.source.lines[raw ? 2...-2 : 1...-1].join("\n").tap do
+        MethodSource.instance_variable_get(:@lines_for_file).delete(method_or_block.source_location[0])
+      end
+    end
+  end
+end

data/lib/crystalruby/compilation.rb

@@ -1,75 +1,28 @@
 require "open3"
 require "tmpdir"
+require "shellwords"
 
 module CrystalRuby
   module Compilation
+    class CompilationFailedError < StandardError; end
+
     def self.compile!(
-      src: config.crystal_src_dir_abs / config.crystal_main_file,
-      lib: config.crystal_lib_dir_abs / config.crystal_lib_name,
-      verbose: config.verbose,
-      debug: config.debug
+      src:,
+      lib:,
+      verbose: CrystalRuby.config.verbose,
+      debug: CrystalRuby.config.debug
     )
-      Dir.chdir(config.crystal_src_dir_abs) do
-        compile_command = compile_command!(verbose: verbose, debug: debug, lib: lib, src: src)
-        link_command = link_cmd!(verbose: verbose, lib: lib, src: src)
-
-        puts "[crystalruby] Compiling Crystal code: #{compile_command}" if verbose
-        unless system(compile_command)
-          puts "Failed to build Crystal object file."
-          return false
-        end
-
-        puts "[crystalruby] Linking Crystal code: #{link_command}" if verbose
-        unless system(link_command)
-          puts "Failed to link Crystal library."
-          return false
-        end
-      end
-
-      true
+      compile_command = build_compile_command(verbose: verbose, debug: debug, lib: lib, src: src)
+      CrystalRuby.log_debug "Compiling Crystal code #{verbose ? ": #{compile_command}" : ""}"
+      raise CompilationFailedError, "Compilation failed" unless system(compile_command)
     end
 
-    def self.compile_command!(verbose:, debug:, lib:, src:)
-      @compile_command ||= begin
-        verbose_flag = verbose ? "--verbose" : ""
-        debug_flag = debug ? "" : "--release --no-debug"
-        redirect_output = " > /dev/null " unless verbose
-
-        %(crystal build #{verbose_flag} #{debug_flag} --cross-compile -o #{lib} #{src}#{redirect_output})
-      end
-    end
-
-    # Here we misuse the crystal compiler to build a valid linking command
-    # with all of the platform specific flags that we need.
-    # We then use this command to link the object file that we compiled in the previous step.
-    # This is not robust and is likely to need revision in the future.
-    def self.link_cmd!(verbose:, lib:, src:)
-      @link_cmd ||= begin
-        result = nil
-
-        Dir.mktmpdir do |tmp|
-          output, status = Open3.capture2("crystal build --verbose #{src} -o #{Pathname.new(tmp) / "main"}")
-          unless status.success?
-            puts "Failed to compile the Crystal code."
-            exit 1
-          end
-
-          # Parse the output to find the last invocation of the C compiler, which is likely the linking stage
-          # and strip off the targets that the crystal compiler added.
-          link_command_suffix = output.lines.select { |line| line.strip.start_with?("cc") }.last.strip[/.*(-o.*)/, 1]
-
-          # Replace the output file with the path to the object file we compiled
-          link_command_suffix.gsub!(
-            /-o.*main/,
-            "-o #{lib}"
-          )
-          result = %(cc #{lib}.o -shared #{link_command_suffix})
-          result << " > /dev/null 2>&1" unless verbose
-          result
-        end
-
-        result
-      end
+    def self.build_compile_command(verbose:, debug:, lib:, src:)
+      verbose_flag = verbose ? "--verbose --progress" : ""
+      debug_flag = debug ? "" : "--release --no-debug"
+      redirect_output = " &> /dev/null " unless verbose
+      lib, src, lib_dir = [lib, src, File.dirname(src)].map(&Shellwords.method(:escape))
+      %(cd #{lib_dir} && crystal build #{verbose_flag} #{debug_flag} --single-module --link-flags "-shared" -o #{lib} #{src}#{redirect_output})
     end
   end
 end

data/lib/crystalruby/config.rb

@@ -1,15 +1,23 @@
 require "singleton"
 require "yaml"
+require "logger"
 
 module CrystalRuby
-  def self.config
-    Config.instance
+  # Config mixin to easily access the configuration
+  # from anywhere in the code
+  module Config
+    def config
+      Configuration.instance
+    end
   end
 
-  # Define a nested Config class
-  class Config
+  # Defines our configuration singleton
+  # Config can be specified through either:
+  # - crystalruby.yaml OR
+  # - CrystalRuby.configure block
+  class Configuration
     include Singleton
-    attr_accessor :debug, :verbose
+    attr_accessor :debug, :verbose, :logger, :colorize_log_output, :single_thread_mode
 
     def initialize
       @debug = true
@@ -19,17 +27,19 @@ module CrystalRuby
       rescue StandardError
         nil
       end || {}
-      @crystal_src_dir      = config.fetch("crystal_src_dir", "./crystalruby/src")
-      @crystal_lib_dir      = config.fetch("crystal_lib_dir", "./crystalruby/lib")
-      @crystal_main_file    = config.fetch("crystal_main_file", "main.cr")
-      @crystal_lib_name     = config.fetch("crystal_lib_name", "crlib")
+      @crystal_src_dir      = config.fetch("crystal_src_dir", "./crystalruby")
       @crystal_codegen_dir  = config.fetch("crystal_codegen_dir", "generated")
       @crystal_project_root = config.fetch("crystal_project_root", Pathname.pwd)
       @debug                = config.fetch("debug", true)
       @verbose              = config.fetch("verbose", false)
+      @single_thread_mode   = config.fetch("single_thread_mode", false)
+      @colorize_log_output  = config.fetch("colorize_log_output", false)
+      @log_level            = config.fetch("log_level", ENV.fetch("CRYSTALRUBY_LOG_LEVEL", "info"))
+      @logger               = Logger.new(STDOUT)
+      @logger.level         = Logger.const_get(@log_level.to_s.upcase)
     end
 
-    %w[crystal_main_file crystal_lib_name crystal_project_root].each do |method_name|
+    %w[crystal_project_root].each do |method_name|
       define_method(method_name) do
         @paths_cache[method_name] ||= Pathname.new(instance_variable_get(:"@#{method_name}"))
       end
@@ -46,7 +56,7 @@ module CrystalRuby
       end
     end
 
-    %w[crystal_src_dir crystal_lib_dir].each do |method_name|
+    %w[crystal_src_dir].each do |method_name|
       abs_method_name = "#{method_name}_abs"
       define_method(abs_method_name) do
         @paths_cache[abs_method_name] ||= crystal_project_root / instance_variable_get(:"@#{method_name}")
@@ -56,8 +66,14 @@ module CrystalRuby
         @paths_cache[method_name] ||= Pathname.new instance_variable_get(:"@#{method_name}")
       end
     end
+
+    def log_level=(level)
+      @log_level = level
+      @logger.level = Logger.const_get(level.to_s.upcase)
+    end
   end
 
+  extend Config
   def self.configure
     yield(config)
     @paths_cache = {}