crystalruby 0.2.2 → 0.3.0

data/Rakefile

@@ -1,12 +1,13 @@
 # frozen_string_literal: true
 
 require "bundler/gem_tasks"
-require "minitest/test_task"
-
-Minitest::TestTask.create
 
 require "rubocop/rake_task"
 
 RuboCop::RakeTask.new
 
 task default: %i[test rubocop]
+
+task :test do
+  require_relative "test/test_all"
+end

data/crystalruby.gemspec

@@ -4,7 +4,7 @@ require_relative "lib/crystalruby/version"
 
 Gem::Specification.new do |spec|
   spec.name = "crystalruby"
-  spec.version = Crystalruby::VERSION
+  spec.version = CrystalRuby::VERSION
   spec.authors = ["Wouter Coppieters"]
   spec.email = ["wc@pico.net.nz"]
 
@@ -35,7 +35,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "digest"
   spec.add_dependency "ffi"
   spec.add_dependency "fileutils"
-  spec.add_dependency "method_source"
+  spec.add_dependency "syntax_tree"
   # For more information and examples about making a new gem, check out our
   # guide at: https://bundler.io/guides/creating_gem.html
 end

data/exe/crystalruby

@@ -10,6 +10,7 @@ def init
     # crystalruby configuration file
     crystal_src_dir: "./crystalruby"
     crystal_codegen_dir: "generated"
+    crystal_missing_ignore: false
     log_level: "info"
     single_thread_mode: false
     debug: true

data/lib/crystalruby/adapter.rb

@@ -8,16 +8,16 @@ module CrystalRuby
     #
     # E.g.
     #
-    # crystalize [a: :int32, b: :int32] => :int32
-    # def add(a, b)
+    # crystalize :int32
+    # def add(a: :int32, b: :int32)
     #  a + b
     # end
     #
     # Pass `raw: true` to pass Raw crystal code to the compiler as a string instead.
     # (Useful for cases where the Crystal method body is not valid Ruby)
     # E.g.
-    # crystalize raw: true [a: :int32, b: :int32] => :int32
-    # def add(a, b)
+    # crystalize :int32, raw: true
+    # def add(a: :int32, b: :int32)
     #   <<~CRYSTAL
     #   a + b
     #   CRYSTAL
@@ -30,73 +30,135 @@ module CrystalRuby
     #
     # Pass lib: "name_of_lib" to compile Crystal code into several distinct libraries.
     # This can help keep compilation times low, by packaging your Crystal code into separate shared objects.
-    def crystalize(raw: false, async: false, lib: "crystalruby", **options, &block)
-      (args,), returns = options.first || [[], :void]
-      args ||= {}
-      raise "Arguments should be of the form name: :type. Got #{args}" unless args.is_a?(Hash)
-
-      @crystalize_next = {
-        raw: raw,
-        async: async,
-        args: args,
-        returns: returns,
-        block: block,
-        lib: lib
-      }
+    # @param returns The return type of the method. Optional (defaults to :void).
+    # @param [Hash] options The options hash.
+    # @option options [Boolean] :raw (false) Pass raw Crystal code to the compiler as a string.
+    # @option options [Boolean] :async (false) Mark the method as async (allows multiplexing).
+    # @option options [String] :lib ("crystalruby") The name of the library to compile the Crystal code into.
+    # @option options [Proc] :block An optional wrapper Ruby block that wraps around any invocations of the crystal code
+    def crystalize( returns=:void, raw: false, async: false, lib: "crystalruby", &block)
+      (self == TOPLEVEL_BINDING.receiver ? Object : self).instance_eval do
+        @crystalize_next = {
+          raw: raw,
+          async: async,
+          returns: returns,
+          block: block,
+          lib: lib
+        }
+      end
     end
 
-    # This method provides a useful DSL for defining Crystal types in pure Ruby.
-    # These types can not be passed directly to Ruby, and must be serialized as either:
-    #   JSON or
-    #   C-Structures (WIP)
-    #
-    # See #json for an example of how to define arguments or return types for complex objects.
-    # E.g.
-    #
-    # MyType = crtype{ Int32 | Hash(String, Array(Bool) | Float65 | Nil) }
-    def crtype(&block)
-      TypeBuilder.with_injected_type_dsl(self) do
-        TypeBuilder.build(&block)
+    # Exposes a Ruby method to one or more Crystal libraries.
+    # Type annotations follow the same rules as the `crystalize` method, but are
+    # applied in reverse.
+    # @param returns The return type of the method. Optional (defaults to :void).
+    # @param [Hash] options The options hash.
+    # @option options [Boolean] :raw (false) Pass raw Crystal code to the compiler as a string.
+    # @option options [String] :libs (["crystalruby"]) The name of the Crystal librarie(s) to expose the Ruby code to.
+    def expose_to_crystal( returns=:void, libs: ["crystalruby"])
+      (self == TOPLEVEL_BINDING.receiver ? Object : self).instance_eval do
+        @expose_next_to_crystal = {
+          returns: returns,
+          libs: libs
+        }
       end
     end
 
-    # Use the json{} helper for defining complex method arguments or return types
-    # that should be serialized to and from Crystal using JSON. (This conversion is applied automatically)
-    #
-    # E.g.
-    # crystalize [a: json{ Int32 | Float64 | Nil } ] => NamedStruct(result: Int32 | Float64 | Nil)
-    def json(&block)
-      crtype(&block).serialize_as(:json)
+    # Define a shard dependency
+    # This dependency will be automatically injected into the shard.yml file for
+    # the given library and installed upon compile if it is not already installed.
+    def shard(shard_name, lib: 'crystalruby', **opts)
+      CrystalRuby::Library[lib].require_shard(shard_name, **opts)
+    end
+
+    # Use this method to define inline Crystal code that does not need to be bound to a Ruby method.
+    # This is useful for defining classes, modules, performing set-up tasks etc.
+    # See: docs for .crystalize to understand the `raw` and `lib` parameters.
+    def crystal(raw: false, lib: "crystalruby", &block)
+      inline_crystal_body = respond_to?(:name) ? Template::InlineChunk.render(
+        {
+          module_name: name,
+          body: SourceReader.extract_source_from_proc(block, raw: raw),
+          mod_or_class: self.kind_of?(Class) && self < Types::Type ? "class" : "module",
+          superclass: self.kind_of?(Class) && self < Types::Type ? "< #{self.crystal_supertype}" : ""
+        }) :
+        SourceReader.extract_source_from_proc(block, raw: raw)
+
+      CrystalRuby::Library[lib].crystalize_chunk(
+        self,
+        Digest::MD5.hexdigest(inline_crystal_body),
+        inline_crystal_body
+      )
+    end
+
+
+    # This method provides a useful DSL for defining Crystal types in pure Ruby
+    # MyType = CRType{ Int32 | Hash(String, Array(Bool) | Float65 | Nil) }
+    # @param [Proc] block The block within which we build the type definition.
+    def CRType(&block)
+      TypeBuilder.build_from_source(block, context: self)
     end
 
+    private
+
     # We trigger attaching of crystalized instance methods here.
     # If a method is added after a crystalize annotation we assume it's the target of the crystalize annotation.
+    # @param [Symbol] method_name The name of the method being added.
     def method_added(method_name)
-      define_crystalized_method(method_name, instance_method(method_name)) if @crystalize_next
+      define_crystalized_method(instance_method(method_name)) if should_crystalize_next?
+      expose_ruby_method_to_crystal(instance_method(method_name)) if should_expose_next?
       super
     end
 
     # We trigger attaching of crystalized class methods here.
     # If a method is added after a crystalize annotation we assume it's the target of the crystalize annotation.
+    # @note This method is called when a method is added to the singleton class of the object.
+    # @param [Symbol] method_name The name of the method being added.
     def singleton_method_added(method_name)
-      define_crystalized_method(method_name, singleton_method(method_name)) if @crystalize_next
+      define_crystalized_method(singleton_method(method_name)) if should_crystalize_next?
+      expose_ruby_method_to_crystal(singleton_method(method_name)) if should_expose_next?
       super
     end
 
-    # Use this method to define inline Crystal code that does not need to be bound to a Ruby method.
-    # This is useful for defining classes, modules, performing set-up tasks etc.
-    # See: docs for .crystalize to understand the `raw` and `lib` parameters.
-    def crystal(raw: false, lib: "crystalruby", &block)
-      inline_crystal_body = Template::InlineChunk.render(
-        {
-          module_name: name, body: extract_source(block, raw: raw)
-        }
-      )
-      CrystalRuby::Library[lib].crystalize_chunk(
-        self,
-        Digest::MD5.hexdigest(inline_crystal_body),
-        inline_crystal_body
-      )
+    # Helper method to determine if the next method added should be crystalized.
+    # @return [Boolean] True if the next method added should be crystalized.
+    def should_crystalize_next?
+      defined?(@crystalize_next) && @crystalize_next
+    end
+
+    # Helper method to determine if the next method added should be exposed to Crystal libraries.
+    # @return [Boolean] True if the next method added should be exposed.
+    def should_expose_next?
+      defined?(@expose_next_to_crystal) && @expose_next_to_crystal
+    end
+
+    # This is where we extract the Ruby method metadata and invoke the Crystal::Library functionality
+    # to compile a stub for the Ruby method into the Crystal library.
+    def expose_ruby_method_to_crystal(method)
+      returns, libs = @expose_next_to_crystal.values_at(:returns, :libs)
+      @expose_next_to_crystal = nil
+
+      args, source = SourceReader.extract_args_and_source_from_method(method)
+      returns = args.delete(:returns) if args[:returns] && returns == :void
+      args[:__yield_to] = args.delete(:yield) if args[:yield]
+      src = <<~RUBY
+        def #{method.name} (#{(args.keys - [:__yield_to]).join(", ")})
+          #{source}
+        end
+      RUBY
+
+      owner = method.owner.singleton_class? ? method.owner.attached_object : method.owner
+      owner.class_eval(src)
+      owner.instance_eval(src) unless method.kind_of?(UnboundMethod) && method.owner.ancestors.include?(CrystalRuby::Types::Type)
+      method = owner.send(method.kind_of?(UnboundMethod) ? :instance_method : :method, method.name)
+
+      libs.each do |lib|
+        CrystalRuby::Library[lib].expose_method(
+          method,
+          args,
+          returns,
+        )
+      end
     end
 
     # We attach crystalized class methods here.
@@ -105,29 +167,33 @@ module CrystalRuby
     # - Overwriting the method and class methods by the same name in the caller.
     # - Lazily triggering compilation and attachment of the Ruby method to the Crystal code.
     # - We also optionally prepend a block (if given) to the owner, to allow Ruby code to wrap around Crystal code.
-    def define_crystalized_method(method_name, method)
-      CrystalRuby.log_debug("Defining crystalized method #{name}.#{method_name}")
+    # @param [Symbol] method_name The name of the method being added.
+    # @param [UnboundMethod] method The method being added.
+    def define_crystalized_method(method)
+      CrystalRuby.log_debug("Defining crystalized method #{name}.#{method.name}")
 
-      args, returns, block, async, lib, raw = @crystalize_next.values_at(:args, :returns, :block, :async, :lib, :raw)
+      returns, block, async, lib, raw = @crystalize_next.values_at(:returns, :block, :async, :lib, :raw)
       @crystalize_next = nil
 
+      args, source = SourceReader.extract_args_and_source_from_method(method, raw: raw)
+
+      # We can safely claim the `yield` argument name for typing the yielded block
+      # because this is an illegal identifier in Crystal anyway.
+      args[:__yield_to] = args.delete(:yield) if args[:yield]
+      returns = args.delete(:returns) if args[:returns] && returns == :void
+
       CrystalRuby::Library[lib].crystalize_method(
         method,
         args,
         returns,
-        extract_source(method, raw: raw),
+        source,
         async,
         &block
       )
     end
-
-    # Extract Ruby source to serve as Crystal code directly.
-    # If it's a raw method, we'll strip the string delimiters at either end of the definition.
-    # We need to clear the MethodSource cache here to allow for code reloading.
-    def extract_source(method_or_block, raw: false)
-      method_or_block.source.lines[raw ? 2...-2 : 1...-1].join("\n").tap do
-        MethodSource.instance_variable_get(:@lines_for_file).delete(method_or_block.source_location[0])
-      end
-    end
   end
 end
+
+Module.prepend(CrystalRuby::Adapter)
+BasicObject.prepend(CrystalRuby::Adapter)
+BasicObject.singleton_class.prepend(CrystalRuby::Adapter)

data/lib/crystalruby/arc_mutex.rb

@@ -0,0 +1,47 @@
+module CrystalRuby
+  module LibC
+    extend FFI::Library
+    ffi_lib "c"
+    class PThreadMutexT < FFI::Struct
+      layout :__align, :int64, :__size, :char, 40
+    end
+
+    attach_function :pthread_mutex_init, [PThreadMutexT.by_ref, :pointer], :int
+    attach_function :pthread_mutex_lock, [PThreadMutexT.by_ref], :int
+    attach_function :pthread_mutex_unlock, [PThreadMutexT.by_ref], :int
+  end
+
+  class ArcMutex
+    def phtread_mutex
+      @phtread_mutex ||= init_mutex!
+    end
+
+    def synchronize
+      lock
+      yield
+      unlock
+    end
+
+    def to_ptr
+      phtread_mutex.pointer
+    end
+
+    def init_mutex!
+      mutex = LibC::PThreadMutexT.new
+      res = LibC.pthread_mutex_init(mutex, nil)
+      raise "Failed to initialize mutex" unless res.zero?
+
+      mutex
+    end
+
+    def lock
+      res = LibC.pthread_mutex_lock(phtread_mutex)
+      raise "Failed to lock mutex" unless res.zero?
+    end
+
+    def unlock
+      res = LibC.pthread_mutex_unlock(phtread_mutex)
+      raise "Failed to unlock mutex" unless res.zero?
+    end
+  end
+end

data/lib/crystalruby/compilation.rb

@@ -1,10 +1,18 @@
 require "tmpdir"
 require "shellwords"
+require "timeout"
 
 module CrystalRuby
   module Compilation
     class CompilationFailedError < StandardError; end
 
+    # Simple wrapper around invocation of the Crystal compiler
+    # @param src [String] path to the source file
+    # @param lib [String] path to the library file
+    # @param verbose [Boolean] whether to print the compiler output
+    # @param debug [Boolean] whether to compile in debug mode
+    # @raise [CompilationFailedError] if the compilation fails
+    # @return [void]
     def self.compile!(
       src:,
       lib:,
@@ -13,15 +21,36 @@ module CrystalRuby
     )
       compile_command = build_compile_command(verbose: verbose, debug: debug, lib: lib, src: src)
       CrystalRuby.log_debug "Compiling Crystal code #{verbose ? ": #{compile_command}" : ""}"
-      raise CompilationFailedError, "Compilation failed" unless system(compile_command)
+      IO.popen(compile_command, chdir: File.dirname(src), &:read)
+      raise CompilationFailedError, "Compilation failed in #{src}" unless $?&.success?
     end
 
+    # Builds the command to compile the Crystal source file
+    # @param verbose [Boolean] whether to print the compiler output
+    # @param debug [Boolean] whether to compile in debug mode
+    # @param lib [String] path to the library file
+    # @param src [String] path to the source file
+    # @return [String] the command to compile the Crystal source file
     def self.build_compile_command(verbose:, debug:, lib:, src:)
       verbose_flag = verbose ? "--verbose --progress" : ""
       debug_flag = debug ? "" : "--release --no-debug"
-      redirect_output = " &> /dev/null " unless verbose
-      lib, src, lib_dir = [lib, src, File.dirname(src)].map(&Shellwords.method(:escape))
-      %(cd #{lib_dir} && crystal build #{verbose_flag} #{debug_flag} --single-module --link-flags "-shared" -o #{lib} #{src}#{redirect_output})
+      redirect_output = " > /dev/null " unless verbose
+      lib, src = [lib, src].map(&Shellwords.method(:escape))
+      %(crystal build #{verbose_flag} #{debug_flag} --single-module --link-flags "-shared" -o #{lib} #{src}#{redirect_output})
+    end
+
+    # Trigger the shards install command in the given source directory
+    def self.install_shards!(src_dir)
+      CrystalRuby.log_debug "Running shards install inside #{src_dir}"
+      output = IO.popen("shards update", chdir: src_dir, &:read)
+      CrystalRuby.log_debug output if CrystalRuby.config.verbose
+      raise CompilationFailedError, "Shards install failed" unless $?&.success?
+    end
+
+    # Return whether the shards check command succeeds in the given source directory
+    def self.shard_check?(src_dir)
+      IO.popen("shards check", chdir: src_dir, &:read)
+      $?&.success?
     end
   end
 end

data/lib/crystalruby/config.rb

@@ -17,59 +17,63 @@ module CrystalRuby
   # - CrystalRuby.configure block
   class Configuration
     include Singleton
-    attr_accessor :debug, :verbose, :logger, :colorize_log_output, :single_thread_mode
+    attr_accessor :debug, :verbose, :logger, :colorize_log_output,
+      :single_thread_mode, :crystal_missing_ignore
 
     def initialize
-      @debug = true
       @paths_cache = {}
-      config = File.exist?("crystalruby.yaml") && begin
+      config = read_config || {}
+      @crystal_src_dir        = config.fetch("crystal_src_dir", "./crystalruby")
+      @crystal_codegen_dir    = config.fetch("crystal_codegen_dir", "generated")
+      @crystal_project_root   = config.fetch("crystal_project_root", Pathname.pwd)
+      @crystal_missing_ignore = config.fetch("crystal_missing_ignore", false)
+      @debug                  = config.fetch("debug", false)
+      @verbose                = config.fetch("verbose", false)
+      @single_thread_mode     = config.fetch("single_thread_mode", false)
+      @colorize_log_output    = config.fetch("colorize_log_output", false)
+      @log_level              = config.fetch("log_level", ENV.fetch("CRYSTALRUBY_LOG_LEVEL", "info"))
+      @logger                 = Logger.new($stdout)
+      @logger.level           = Logger.const_get(@log_level.to_s.upcase)
+    end
+
+    def file_config
+      @file_config ||= File.exist?("crystalruby.yaml") && begin
         YAML.safe_load(IO.read("crystalruby.yaml"))
       rescue StandardError
         nil
       end || {}
-      @crystal_src_dir      = config.fetch("crystal_src_dir", "./crystalruby")
-      @crystal_codegen_dir  = config.fetch("crystal_codegen_dir", "generated")
-      @crystal_project_root = config.fetch("crystal_project_root", Pathname.pwd)
-      @debug                = config.fetch("debug", true)
-      @verbose              = config.fetch("verbose", false)
-      @single_thread_mode   = config.fetch("single_thread_mode", false)
-      @colorize_log_output  = config.fetch("colorize_log_output", false)
-      @log_level            = config.fetch("log_level", ENV.fetch("CRYSTALRUBY_LOG_LEVEL", "info"))
-      @logger               = Logger.new(STDOUT)
-      @logger.level         = Logger.const_get(@log_level.to_s.upcase)
     end
 
-    %w[crystal_project_root].each do |method_name|
-      define_method(method_name) do
-        @paths_cache[method_name] ||= Pathname.new(instance_variable_get(:"@#{method_name}"))
+    def self.define_directory_accessors!(parent, directory_node)
+      case directory_node
+      when Array then directory_node.each(&method(:define_directory_accessors!).curry[parent])
+      when Hash
+        directory_node.each do |par, children|
+          define_directory_accessors!(parent, par)
+          define_directory_accessors!(par, children)
+        end
+      else
+        define_method(directory_node) do
+          @paths_cache[directory_node] ||= Pathname.new(instance_variable_get(:"@#{directory_node}"))
+        end
+        define_method("#{directory_node}_abs") do
+          @paths_cache["#{directory_node}_abs"] ||= parent ? send("#{parent}_abs") / Pathname.new(instance_variable_get(:"@#{directory_node}")) : send(directory_node)
+        end
       end
     end
 
-    %w[crystal_codegen_dir].each do |method_name|
-      abs_method_name = "#{method_name}_abs"
-      define_method(abs_method_name) do
-        @paths_cache[abs_method_name] ||= crystal_src_dir_abs / instance_variable_get(:"@#{method_name}")
-      end
+    define_directory_accessors!(nil, { crystal_project_root: { crystal_src_dir: [:crystal_codegen_dir] } })
 
-      define_method(method_name) do
-        @paths_cache[method_name] ||= Pathname.new instance_variable_get(:"@#{method_name}")
-      end
+    def log_level=(level)
+      @logger.level = Logger.const_get(@log_level = level.to_s.upcase)
     end
 
-    %w[crystal_src_dir].each do |method_name|
-      abs_method_name = "#{method_name}_abs"
-      define_method(abs_method_name) do
-        @paths_cache[abs_method_name] ||= crystal_project_root / instance_variable_get(:"@#{method_name}")
-      end
+    private
 
-      define_method(method_name) do
-        @paths_cache[method_name] ||= Pathname.new instance_variable_get(:"@#{method_name}")
-      end
-    end
-
-    def log_level=(level)
-      @log_level = level
-      @logger.level = Logger.const_get(level.to_s.upcase)
+    def read_config
+      YAML.safe_load(IO.read("crystalruby.yaml"))
+    rescue
+      {}
     end
   end