crystalruby 0.1.13 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 49ff0856e24d9a62e6dda014115652431074d4a45e1cb4270c92beeae2baa7ea
-  data.tar.gz: adaa86c18d254bf07704997050133db592dd60dda5ca9aed9f4c2bc34edfd7a9
+  metadata.gz: 117525a6b5905a6c4aa86b0b2fed0c25e46d0e97b6de8ce187b90f1438394f19
+  data.tar.gz: 55b35d8731c2d7e68f6cb0a1a4ea3eb068805bc4297f1b85fceac08cb288682a
 SHA512:
-  metadata.gz: 599c397e9710bb7dfe528e3ec49b64cfbf61f0d73089dc3476d85587317c894367b09f8b47e062d5cc98646575512421f28860aadeceac62f05588a80830726b
-  data.tar.gz: 60b9896f92cf41389933c80bb21e5c4e899e54648575ca778dcea2ba881876ef3f2539bdc46ab1c22b3f234f1237019454f8726957f5858c83ba59a2065e406f
+  metadata.gz: 585ad3697b06ba3660989ea538ddae90816d0e93a947e95ab5e6bb83ace64d35988c436cf8e07cad077529691baebbdbdeac6bcc0e7a991297d67f2efada5438
+  data.tar.gz: bd06431cc0c059481db8215676e493f16d485cb14a8b15dedc3de6e79df79716c4e6499bb4f9678257659601224b249798dc8c4c492c1659c46157714e45499e

data/.dockerignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+crystalruby-*.gem
+/crystalruby/

data/README.md

@@ -46,8 +46,7 @@ require 'benchmark'
 module PrimeCounter
   crystalize [n: :int32] => :int32
   def count_primes_upto_cr(n)
-    primes = 0
-    (2..n).each do |i|
+    (2..n).each.count do |i|
       is_prime = true
       (2..Math.sqrt(i).to_i).each do |j|
         if i % j == 0
@@ -55,16 +54,14 @@ module PrimeCounter
           break
         end
       end
-      primes += 1
+      is_prime
     end
-    primes
   end
 
   module_function
 
   def count_primes_upto_rb(n)
-    primes = 0
-    (2..n).each do |i|
+    (2..n).each.count do |i|
       is_prime = true
       (2..Math.sqrt(i).to_i).each do |j|
         if i % j == 0
@@ -72,9 +69,8 @@ module PrimeCounter
           break
         end
       end
-      primes += 1
+      is_prime
     end
-    primes
   end
 end
 
@@ -477,7 +473,7 @@ To safely expose this behaviour, `crystalruby` implements a Reactor, which multi
 
 By default `crystalruby` methods are blocking/synchronous, this means that for blocking operations, a single crystalruby call can block the entire reactor.
 
-To allow you to benefit from Crystal's fiber based concurrency, you can use use the `async` option on crystalized ruby methods. This allows several Ruby threads to invoke Crystal code simultaneously.
+To allow you to benefit from Crystal's fiber based concurrency, you can use the `async` option on crystalized ruby methods. This allows several Ruby threads to invoke Crystal code simultaneously.
 
 E.g.
 
@@ -502,11 +498,11 @@ end
 
 ### Reactor performance
 
-There is a small amount of overhead to multiplexing calls across a single thread. Ad-hoc testing amounts this to be around 10 nanoseconds per call.
+There is a small amount of synchronization overhead to multiplexing calls across a single thread. Ad-hoc testing amounts this to be around 10 nanoseconds per call.
 For most use-cases this overhead is negligible, especially if the bulk of your CPU heavy task occurs exclusively in Crystal code. However, if you are invoking very fast Crystal code from Ruby in a tight loop (e.g. a simple 1 + 2)
 then the overhead of the reactor can become significant.
 
-In this case you can use the `crystalruby` in a single-threaded mode to avoid the reactor overhead and greatly increase performance, with the caveat that _all_ calls to Crystal must occur from a single thread. If you Ruby program is already single-threaded this is not a problem.
+In this case you can use the `crystalruby` in a single-threaded mode to avoid the reactor overhead and greatly increase performance, with the caveat that _all_ calls to Crystal must occur from a single thread. If your Ruby program is already single-threaded this is not a problem.
 
 ```ruby
 CrystalRuby.configure do |config|
@@ -514,6 +510,33 @@ CrystalRuby.configure do |config|
 end
 ```
 
+## Live Reloading
+
+`crystalruby` supports live reloading of Crystal code. It will intelligently
+recompile Crystal code only when it detects changes to the embedded function or block bodies. This allows you to iterate quickly on your Crystal code without having to restart your Ruby process in live-reloading environments like Rails.
+
+## Multi-library support
+
+Large Crystal projects are known to have long compile times. To mitigate this, `crystalruby` supports splitting your Crystal code into multiple libraries. This allows you to only recompile the library that has changed, rather than the entire project.
+To indicate which library a piece of embedded Crystal code belongs to, you can use the `library` option in the `crystalize` and `crystal` methods.
+If the "lib" option is not provided, the code will be compiled into the default library (simply named `crystalruby`).
+
+```ruby
+module Foo
+  crystalize lib: "foo"
+  def bar
+    puts "Hello from Foo"
+  end
+
+  crystal lib: "foo" do
+    REDIS = Redis.new
+  end
+end
+```
+
+Naturally Crystal code must be in the same library to interact directly.
+Interaction across multiple libraries can be coordinated via Ruby code.
+
 ## Troubleshooting
 
 The logic to detect when to JIT recompile is not robust and can end up in an inconsistent state. To remedy this it is useful to clear out all generated assets and build from scratch.
@@ -570,8 +593,8 @@ crystalruby init
 ```
 
 ```yaml
-crystal_src_dir: "./crystalruby/src"
-crystal_lib_dir: "./crystalruby/lib"
+crystal_src_dir: "./crystalruby"
+crystal_codegen_dir: "generated"
 crystal_main_file: "main.cr"
 crystal_lib_name: "crlib"
 crystal_codegen_dir: "generated"
@@ -582,10 +605,7 @@ Alternatively, these can be set programmatically, e.g:
 
 ```ruby
 CrystalRuby.configure do |config|
-  config.crystal_src_dir = "./crystalruby/src"
-  config.crystal_lib_dir = "./crystalruby/lib"
-  config.crystal_main_file = "main.cr"
-  config.crystal_lib_name = "crlib"
+  config.crystal_src_dir = "./crystalruby"
   config.crystal_codegen_dir = "generated"
   config.debug = true
   config.verbose = false

data/exe/crystalruby

@@ -1,6 +1,5 @@
 #!/usr/bin/env ruby
 
-require "bundler/setup"
 require "crystalruby"
 require "fileutils"
 
@@ -9,10 +8,7 @@ def init
   # Define some dummy content for the YAML file
   yaml_content = <<~YAML
     # crystalruby configuration file
-    crystal_src_dir: "./crystalruby/src"
-    crystal_lib_dir: "./crystalruby/lib"
-    crystal_main_file: "main.cr"
-    crystal_lib_name: "crlib"
+    crystal_src_dir: "./crystalruby"
     crystal_codegen_dir: "generated"
     log_level: "info"
     single_thread_mode: false
@@ -25,23 +21,25 @@ def init
 end
 
 def install
-  Dir.chdir("#{CrystalRuby.config.crystal_src_dir}") do
-    if system("shards update")
-      puts "Shards installed successfully."
-    else
-      puts "Error installing shards."
+  Dir["#{CrystalRuby.config.crystal_src_dir}/**/src"].each do |src_dir|
+    Dir.chdir(src_dir) do
+      if system("shards check") || system("shards update")
+        puts "Shards installed successfully."
+      else
+        puts "Error installing shards."
+      end
     end
   end
-  clean
 end
 
 def clean
-  # This is a stub for the clear command
-  FileUtils.rm_rf("#{CrystalRuby.config.crystal_src_dir}/generated")
+  Dir["#{CrystalRuby.config.crystal_src_dir}/**/src/generated"].each do |codegen_dir|
+    FileUtils.rm_rf(codegen_dir)
+  end
 end
 
 def build
-  # This is a stub for the build command
+  # TODO: Iterate through all generated libs and build
   puts "Build command is not implemented yet."
 end
 

data/lib/crystalruby/adapter.rb

@@ -1,104 +1,132 @@
 module CrystalRuby
   module Adapter
-    # Define a method to set the @crystalize proc if it doesn't already exist
-    def crystalize(raw: false, async: false, **options, &block)
-      (args,), returns = options.first
+    # Use this method to annotate a Ruby method that should be crystalized.
+    # Compilation and attachment of the method is done lazily.
+    # You can force compilation by calling `CrystalRuby.compile!`
+    # It's important that all code using crystalized methods is
+    # loaded before any manual calls to compile.
+    #
+    # E.g.
+    #
+    # crystalize [a: :int32, b: :int32] => :int32
+    # def add(a, b)
+    #  a + b
+    # end
+    #
+    # Pass `raw: true` to pass Raw crystal code to the compiler as a string instead.
+    # (Useful for cases where the Crystal method body is not valid Ruby)
+    # E.g.
+    # crystalize raw: true [a: :int32, b: :int32] => :int32
+    # def add(a, b)
+    #   <<~CRYSTAL
+    #   a + b
+    #   CRYSTAL
+    # end
+    #
+    # Pass `async: true` to make the method async.
+    # Crystal methods will always block the currently executing Ruby thread.
+    # With async: false, all other Crystal code will be blocked while this Crystal method is executing (similar to Ruby code with the GVL)
+    # With async: true, several Crystal methods can be executing concurrently.
+    #
+    # Pass lib: "name_of_lib" to compile Crystal code into several distinct libraries.
+    # This can help keep compilation times low, by packaging your Crystal code into separate shared objects.
+    def crystalize(raw: false, async: false, lib: "crystalruby", **options, &block)
+      (args,), returns = options.first || [[], :void]
       args ||= {}
       raise "Arguments should be of the form name: :type. Got #{args}" unless args.is_a?(Hash)
 
-      @crystalize_next = { raw: raw, async: async, args: args, returns: returns, block: block }
-    end
-
-    def crystal(raw: false, &block)
-      inline_crystal_body = Template::InlineChunk.render(
-        {
-          module_name: name,
-          body: block.source.lines[
-            raw ? 2...-2 : 1...-1
-          ].join("\n")
-        }
-      )
-      CrystalRuby.write_chunk(self, body: inline_crystal_body)
+      @crystalize_next = {
+        raw: raw,
+        async: async,
+        args: args,
+        returns: returns,
+        block: block,
+        lib: lib
+      }
     end
 
+    # This method provides a useful DSL for defining Crystal types in pure Ruby.
+    # These types can not be passed directly to Ruby, and must be serialized as either:
+    #   JSON or
+    #   C-Structures (WIP)
+    #
+    # See #json for an example of how to define arguments or return types for complex objects.
+    # E.g.
+    #
+    # MyType = crtype{ Int32 | Hash(String, Array(Bool) | Float65 | Nil) }
     def crtype(&block)
       TypeBuilder.with_injected_type_dsl(self) do
         TypeBuilder.build(&block)
       end
     end
 
+    # Use the json{} helper for defining complex method arguments or return types
+    # that should be serialized to and from Crystal using JSON. (This conversion is applied automatically)
+    #
+    # E.g.
+    # crystalize [a: json{ Int32 | Float64 | Nil } ] => NamedStruct(result: Int32 | Float64 | Nil)
     def json(&block)
       crtype(&block).serialize_as(:json)
     end
 
+    # We trigger attaching of crystalized instance methods here.
+    # If a method is added after a crystalize annotation we assume it's the target of the crystalize annotation.
     def method_added(method_name)
-      if @crystalize_next
-        define_crystalized_method(method_name, instance_method(method_name))
-        @crystalize_next = nil
-      end
+      define_crystalized_method(method_name, instance_method(method_name)) if @crystalize_next
       super
     end
 
+    # We trigger attaching of crystalized class methods here.
+    # If a method is added after a crystalize annotation we assume it's the target of the crystalize annotation.
     def singleton_method_added(method_name)
-      if @crystalize_next
-        define_crystalized_method(method_name, singleton_method(method_name))
-        @crystalize_next = nil
-      end
+      define_crystalized_method(method_name, singleton_method(method_name)) if @crystalize_next
       super
     end
 
+    # Use this method to define inline Crystal code that does not need to be bound to a Ruby method.
+    # This is useful for defining classes, modules, performing set-up tasks etc.
+    # See: docs for .crystalize to understand the `raw` and `lib` parameters.
+    def crystal(raw: false, lib: "crystalruby", &block)
+      inline_crystal_body = Template::InlineChunk.render(
+        {
+          module_name: name, body: extract_source(block, raw: raw)
+        }
+      )
+      CrystalRuby::Library[lib].crystalize_chunk(
+        self,
+        Digest::MD5.hexdigest(inline_crystal_body),
+        inline_crystal_body
+      )
+    end
+
+    # We attach crystalized class methods here.
+    # This function is responsible for
+    # - Generating the Crystal source code
+    # - Overwriting the method and class methods by the same name in the caller.
+    # - Lazily triggering compilation and attachment of the Ruby method to the Crystal code.
+    # - We also optionally prepend a block (if given) to the owner, to allow Ruby code to wrap around Crystal code.
     def define_crystalized_method(method_name, method)
       CrystalRuby.log_debug("Defining crystalized method #{name}.#{method_name}")
-      CrystalRuby.instantiate_crystal_ruby! unless CrystalRuby.instantiated?
-
-      function_body = method.source.lines[
-        @crystalize_next[:raw] ? 2...-2 : 1...-1
-      ].join("\n")
 
-      MethodSource.instance_variable_get(:@lines_for_file).delete(method.source_location[0])
-      lib_fname = "#{name.downcase}_#{method_name}_#{Digest::MD5.hexdigest(function_body)}"
-      args, returns, block, async = @crystalize_next.values_at(:args, :returns, :block, :async)
-      args ||= {}
+      args, returns, block, async, lib, raw = @crystalize_next.values_at(:args, :returns, :block, :async, :lib, :raw)
       @crystalize_next = nil
-      function = CrystalRuby.build_function(self, lib_fname, method_name, args, returns, function_body)
-      CrystalRuby.write_chunk(self, name: function[:name], body: function[:body]) do
-        CrystalRuby.log_debug("attaching #{lib_fname} to #{name}")
-        extend FFI::Library
-        ffi_lib CrystalRuby.config.crystal_lib_dir / CrystalRuby.config.crystal_lib_name
-        if async
-          attach_function lib_fname, "#{lib_fname}_async", function[:ffi_types] + %i[int pointer], :void,
-                          blocking: true
-        else
-          attach_function lib_fname, function[:ffi_types], function[:ffi_ret_type], blocking: true
-        end
-        if block
-          [self, singleton_class].each do |receiver|
-            receiver.prepend(Module.new do
-              define_method(method_name, &block)
-            end)
-          end
-        end
-      end
-
-      [self, singleton_class].each do |receiver|
-        receiver.define_method(method_name) do |*args|
-          CrystalRuby.build! unless CrystalRuby.compiled?
-          unless CrystalRuby.attached?
-            CrystalRuby.attach!
-            return send(method_name, *args) if block
-          end
-          args.each_with_index do |arg, i|
-            args[i] = function[:arg_maps][i][arg] if function[:arg_maps][i]
-          end
 
-          result = Reactor.schedule_work!(self, lib_fname, *args, function[:ffi_ret_type], async: async)
+      CrystalRuby::Library[lib].crystalize_method(
+        method,
+        args,
+        returns,
+        extract_source(method, raw: raw),
+        async,
+        &block
+      )
+    end
 
-          if function[:retval_map]
-            function[:retval_map][result]
-          else
-            result
-          end
-        end
+    # Extract Ruby source to serve as Crystal code directly.
+    # If it's a raw method, we'll strip the string delimiters at either end of the definition.
+    # We need to clear the MethodSource cache here to allow for code reloading.
+    def extract_source(method_or_block, raw: false)
+      method_or_block.source.lines[raw ? 2...-2 : 1...-1].join("\n").tap do
+        MethodSource.instance_variable_get(:@lines_for_file).delete(method_or_block.source_location[0])
       end
     end
   end

data/lib/crystalruby/compilation.rb

@@ -4,94 +4,25 @@ require "shellwords"
 
 module CrystalRuby
   module Compilation
+    class CompilationFailedError < StandardError; end
+
     def self.compile!(
-      src: CrystalRuby.config.crystal_src_dir_abs / CrystalRuby.config.crystal_main_file,
-      lib: CrystalRuby.config.crystal_lib_dir_abs / CrystalRuby.config.crystal_lib_name,
+      src:,
+      lib:,
       verbose: CrystalRuby.config.verbose,
       debug: CrystalRuby.config.debug
     )
-      Dir.chdir(CrystalRuby.config.crystal_src_dir_abs) do
-        compile_command = compile_command!(verbose: verbose, debug: debug, lib: lib, src: src)
-        link_command = link_cmd!(verbose: verbose, lib: lib, src: src)
-
-        CrystalRuby.log_debug "Compiling Crystal code: #{compile_command}"
-        unless system(compile_command)
-          CrystalRuby.log_error "Failed to build Crystal object file."
-          return false
-        end
-
-        CrystalRuby.log_debug "Linking Crystal code: #{link_command}"
-        unless system(link_command)
-          CrystalRuby.log_error "Failed to link Crystal library."
-          return false
-        end
-        CrystalRuby.log_info "Compilation successful"
-      end
-
-      true
+      compile_command = build_compile_command(verbose: verbose, debug: debug, lib: lib, src: src)
+      CrystalRuby.log_debug "Compiling Crystal code #{verbose ? ": #{compile_command}" : ""}"
+      raise CompilationFailedError, "Compilation failed" unless system(compile_command)
     end
 
-    def self.compile_command!(verbose:, debug:, lib:, src:)
-      @compile_command ||= begin
-        verbose_flag = verbose ? "--verbose" : ""
-        debug_flag = debug ? "" : "--release --no-debug"
-        redirect_output = " > /dev/null " unless verbose
-
-        src = Shellwords.escape(src)
-        lib = Shellwords.escape(lib)
-
-        %(crystal build #{verbose_flag} #{debug_flag} --cross-compile -o #{lib} #{src}#{redirect_output})
-      end
-    end
-
-    # Here we misuse the crystal compiler to build a valid linking command
-    # with all of the platform specific flags that we need.
-    # We then use this command to link the object file that we compiled in the previous step.
-    # This is not robust and is likely to need revision in the future.
-    def self.link_cmd!(verbose:, lib:, src:)
-      @link_cmd ||= begin
-        result = nil
-
-        Dir.mktmpdir do |tmp|
-          CrystalRuby.log_debug "Building link command"
-          src = Shellwords.escape(src)
-          lib_dir = Shellwords.escape(CrystalRuby.config.crystal_src_dir_abs / "lib")
-          escaped_output_path = Shellwords.escape(Pathname.new(tmp) / "main")
-
-          command = "timeout -k 2s 2s bash -c \"export CRYSTAL_PATH=$(crystal env CRYSTAL_PATH):#{lib_dir} && crystal build --verbose #{src} -o #{escaped_output_path} \""
-
-          output = ""
-          pid = nil
-
-          CrystalRuby.log_debug "Running command: #{command}"
-
-          Open3.popen2e(command) do |_stdin, stdout_and_stderr, _wait_thr|
-            while line = stdout_and_stderr.gets
-              puts line if verbose
-              output += line # Capture the output
-            end
-          end
-
-          CrystalRuby.log_debug "Parsing link command"
-
-          # Parse the output to find the last invocation of the C compiler, which is likely the linking stage
-          # and strip off the targets that the crystal compiler added.
-          link_command_suffix = output.lines.select { |line| line.strip.start_with?("cc") }.last.strip[/.*(-o.*)/, 1]
-
-          # Replace the output file with the path to the object file we compiled
-          link_command_suffix.gsub!(
-            /-o.*main/,
-            "-o #{lib}"
-          )
-          result = %(cc #{lib}.o -shared #{link_command_suffix})
-          result << " > /dev/null 2>&1" unless verbose
-          result
-        end
-
-        result
-      end
-    rescue StandardError
-      ""
+    def self.build_compile_command(verbose:, debug:, lib:, src:)
+      verbose_flag = verbose ? "--verbose --progress" : ""
+      debug_flag = debug ? "" : "--release --no-debug"
+      redirect_output = " &> /dev/null " unless verbose
+      lib, src, lib_dir = [lib, src, File.dirname(src)].map(&Shellwords.method(:escape))
+      %(cd #{lib_dir} && crystal build #{verbose_flag} #{debug_flag} --single-module --link-flags "-shared" -o #{lib} #{src}#{redirect_output})
     end
   end
 end

data/lib/crystalruby/config.rb

@@ -3,20 +3,19 @@ require "yaml"
 require "logger"
 
 module CrystalRuby
-  def self.config
-    Config.instance
-  end
-
-  %w[debug info warn error].each do |level|
-    define_singleton_method("log_#{level}") do |*msg|
-      prefix = config.colorize_log_output ? "\e[33mcrystalruby\e[0m\e[90m [#{Thread.current.object_id}]\e[0m" : "[crystalruby] #{Thread.current.object_id}"
-
-      config.logger.send(level, "#{prefix} #{msg.join(", ")}")
+  # Config mixin to easily access the configuration
+  # from anywhere in the code
+  module Config
+    def config
+      Configuration.instance
     end
   end
 
-  # Define a nested Config class
-  class Config
+  # Defines our configuration singleton
+  # Config can be specified through either:
+  # - crystalruby.yaml OR
+  # - CrystalRuby.configure block
+  class Configuration
     include Singleton
     attr_accessor :debug, :verbose, :logger, :colorize_log_output, :single_thread_mode
 
@@ -28,10 +27,7 @@ module CrystalRuby
       rescue StandardError
         nil
       end || {}
-      @crystal_src_dir      = config.fetch("crystal_src_dir", "./crystalruby/src")
-      @crystal_lib_dir      = config.fetch("crystal_lib_dir", "./crystalruby/lib")
-      @crystal_main_file    = config.fetch("crystal_main_file", "main.cr")
-      @crystal_lib_name     = config.fetch("crystal_lib_name", "crlib")
+      @crystal_src_dir      = config.fetch("crystal_src_dir", "./crystalruby")
       @crystal_codegen_dir  = config.fetch("crystal_codegen_dir", "generated")
       @crystal_project_root = config.fetch("crystal_project_root", Pathname.pwd)
       @debug                = config.fetch("debug", true)
@@ -43,7 +39,7 @@ module CrystalRuby
       @logger.level         = Logger.const_get(@log_level.to_s.upcase)
     end
 
-    %w[crystal_main_file crystal_lib_name crystal_project_root].each do |method_name|
+    %w[crystal_project_root].each do |method_name|
       define_method(method_name) do
         @paths_cache[method_name] ||= Pathname.new(instance_variable_get(:"@#{method_name}"))
       end
@@ -60,7 +56,7 @@ module CrystalRuby
       end
     end
 
-    %w[crystal_src_dir crystal_lib_dir].each do |method_name|
+    %w[crystal_src_dir].each do |method_name|
       abs_method_name = "#{method_name}_abs"
       define_method(abs_method_name) do
         @paths_cache[abs_method_name] ||= crystal_project_root / instance_variable_get(:"@#{method_name}")
@@ -77,6 +73,7 @@ module CrystalRuby
     end
   end
 
+  extend Config
   def self.configure
     yield(config)
     @paths_cache = {}