crystalruby 0.2.2 → 0.3.0

data/lib/crystalruby/types/type.rb

@@ -1,66 +1,275 @@
+# frozen_string_literal: true
+
+# Can create complex structs in Ruby
+# E.g.
+# Types::MyType.new({a: 1, b: [2,3, {four: "five"}})
+#
+# Can read complex structs in Crystal.
+# Expose Ruby setters to Crystal.
+#   - These accepts pointers and *copy* the values from them (so that they are governed by Ruby GC)
+#   - Safe to GC on Crystal side after call to setter
+#
+# All Structs are memory managed in Ruby.
+# Parameters are temporarily stored in memory *before* being passed, then nilled (might be GC'd if we don't hold on to it elsewhere)
+# Internal pointers are stored in complex types.
+# Cannot create new structs in Crystal.
+# All information passed from Crystal back to Ruby is either:
+# - Direct memory access (for primitives)
+# - Using setters + copy
+#
+# What if:
+#   - We ask Crystal to implement allocation og memory, and implement all of the setters
+#   - It stores the pointers to the memory in an address indexed hash
+#   - Ruby will reference count using AutoPointer
+#   - If reference count is 0, clear out hash in Crystal (Trigger GC)
+#   - Don't like this because, types can't stand alone in Ruby AND crosses FFI bridge many times to allocate complex struct.
+
+require "forwardable"
+
 module CrystalRuby
+  InvalidCastError = Class.new(StandardError)
+
   module Types
     class Type
-      attr_accessor :name, :error, :inner_types, :inner_keys, :accept_if, :convert
+      # TODO: Replace with pthread primitives and share
+      # with Crystal
+      ARC_MUTEX = CrystalRuby::ArcMutex.new
+
+      include Allocator
+      extend Typemaps
 
-      def initialize(name, error: nil, inner_types: nil, inner_keys: nil, accept_if: [], &convert)
-        self.name = name
-        self.error = error
-        self.inner_types = inner_types
-        self.inner_keys = inner_keys
-        self.accept_if = accept_if
-        self.convert = convert
+      class << self
+        attr_accessor :typename, :ffi_type, :memsize, :convert_if, :inner_types
       end
 
-      def union_types
-        [self]
+      def_delegators :@class, :primitive?, :cast!, :type, :typename, :memsize, :refsize,
+                     :ffi_type, :error, :inner_type, :inner_keys, :inner_types,
+                     :write_mixed_byte_slices_to_uint8_array, :data_offset, :size_offset,
+                     :union_types
+
+      attr_accessor :value, :memory
+
+      def initialize(_rbval)
+        @class = self.class
+        raise error if error
       end
 
-      def valid?
-        !error
+      def self.finalize(_memory)
+        ->(_) {}
       end
 
-      def |(other)
-        raise "Cannot union non-crystal type #{other}" unless other.is_a?(Type) || (
-          other.is_a?(Class) && other.ancestors.include?(Typedef)
-        )
+      def self.inspect_name
+        (name || "#{typename}").to_s.gsub(/^CrystalRuby::Types::[^::]+::/, "")
+      end
 
-        UnionType.new(*union_types, *other.union_types)
+      def self.union_types
+        [self]
       end
 
-      def type_expr
-        inspect
+      def self.valid?
+        true
       end
 
-      def inspect
+      def self.native_type_expr
         if !inner_types
-          name
+          "::#{typename}"
         elsif !inner_keys
-          "#{name}(#{inner_types.map(&:inspect).join(", ")})"
+          "::#{typename}(#{inner_types.map(&:native_type_expr).join(", ")})"
         else
-          "#{name}(#{inner_keys.zip(inner_types).map { |k, v| "#{k}: #{v.inspect}" }.join(", ")})"
+          "::#{typename}(#{inner_keys.zip(inner_types).map { |k, v| "#{k}: #{v.native_type_expr}" }.join(", ")})"
+        end
+      end
+
+      def self.valid_cast?(raw)
+        raw.is_a?(self) || convert_if.any? { |type| raw.is_a?(type) }
+      end
+
+      def self.[](*value)
+        is_list_type = ancestors.any? { |a| a < CrystalRuby::Types::Array || a < CrystalRuby::Types::Tuple }
+        new(is_list_type ? value : value.first)
+      end
+
+      def self.anonymous?
+        name.nil? || name.start_with?("CrystalRuby::Types::")
+      end
+
+      def self.crystal_class_name
+        name || native_type_expr.split(",").join("_and_")
+                                .split("|").join("_or_")
+                                .split("(").join("_of_")
+                                .gsub(/[^a-zA-Z0-9_]/, "")
+                                .split("_")
+                                .map(&:capitalize).join << "_#{type_digest[0..6]}"
+      end
+
+      def self.base_crystal_class_name
+        crystal_class_name.split("::").last
+      end
+
+      def value(native: false)
+        @value
+      end
+
+      def native
+        value(native: true)
+      end
+
+      def self.type_digest
+        Digest::MD5.hexdigest(native_type_expr.to_s)
+      end
+
+      def self.nested_types
+        [self, *(inner_types || []).map(&:nested_types)].flatten.uniq
+      end
+
+      def self.pointer_to_crystal_type_conversion(expr)
+        anonymous? ? "#{crystal_class_name}.new(#{expr}).native" : "#{crystal_class_name}.new(#{expr})"
+      end
+
+      def self.crystal_type_to_pointer_type_conversion(expr)
+        anonymous? ? "#{crystal_class_name}.new(#{expr}).return_value" : "#{expr}.return_value"
+      end
+
+      def self.template_name
+        typename || superclass.template_name
+      end
+
+      def self.type_defn
+        unless Template.const_defined?(template_name) && Template.const_get(template_name).is_a?(Template::Renderer)
+          raise "Template not found for #{template_name}"
         end
+
+        Template.const_get(template_name).render(binding)
+      end
+
+      def self.numeric?
+        false
+      end
+
+      def self.primitive?
+        false
       end
 
-      def interprets?(raw)
-        accept_if.any? { |type| raw.is_a?(type) }
+      def self.variable_width?
+        false
       end
 
-      def interpret!(raw)
-        if interprets?(raw)
-          convert ? convert.call(raw) : raw
+      def self.fixed_width?
+        false
+      end
+
+      def self.cast!(value)
+        value.is_a?(Type) ? value.value : value
+      end
+
+      def ==(other)
+        value(native: true) == (other.is_a?(Type) ? other.value(native: true) : other)
+      end
+
+      def nil?
+        value.nil?
+      end
+
+      def coerce(other)
+        [other, value]
+      end
+
+      def inspect
+        value.inspect
+      end
+
+      def self.from_ffi_array_repr(value)
+        anonymous? ? new(value).value : new(value)
+      end
+
+      def inner_value
+        @value
+      end
+
+      # Create a brand new copy of this object
+      def deep_dup
+        self.class.new(native)
+      end
+
+      # Create a new reference to this object.
+      def dup
+        self.class.new(@memory)
+      end
+
+      def method_missing(method, *args, &block)
+        v = begin
+          native
+        rescue StandardError
+          super
+        end
+        if v.respond_to?(method)
+          hash_before = v.hash
+          result = v.send(method, *args, &block)
+          if v.hash != hash_before
+            self.value = v
+            v.equal?(result) ? self : result
+          else
+            result
+          end
         else
-          raise "Invalid deserialized value #{raw} for type #{inspect}"
+          super
         end
       end
 
+      def self.bind_local_vars!(variable_names, binding)
+        variable_names.each do |name|
+          define_singleton_method(name) do
+            binding.local_variable_get("#{name}")
+          end
+          define_method(name) do
+            binding.local_variable_get("#{name}")
+          end
+        end
+      end
+
+      def self.each_child_address(pointer); end
+
+      def item_size
+        inner_types.map(&:memsize).sum
+      end
+
+      def self.crystal_type
+        lib_type(ffi_type)
+      end
+
+      def self.|(other)
+        raise "Cannot union non-crystal type #{other}" unless other.is_a?(Class) && other.ancestors.include?(Type)
+
+        CrystalRuby::Types::TaggedUnion(*union_types, *other.union_types)
+      end
+
       def self.validate!(type)
-        unless type.is_a?(Types::Type) || (type.is_a?(Class) && type.ancestors.include?(Types::Typedef))
+        unless type.is_a?(Class) && type.ancestors.include?(Types::Type)
           raise "Result #{type} is not a valid CrystalRuby type"
         end
 
         raise "Invalid type: #{type.error}" unless type.valid?
       end
+
+      def self.inner_type
+        inner_types.first
+      end
+
+      def self.type_expr
+        if !inner_types
+          inspect_name
+        elsif !anonymous?
+          name
+        elsif inner_keys
+          "#{inspect_name}(#{inner_keys.zip(inner_types).map { |k, v| "#{k}: #{v.inspect}" }.join(", ")})"
+        else
+          "#{inspect_name}(#{inner_types.map(&:inspect).join(", ")})"
+        end
+      end
+
+      def self.inspect
+        type_expr
+      end
     end
   end
 end

data/lib/crystalruby/types/variable_width/array.cr

@@ -0,0 +1,74 @@
+class <%= base_crystal_class_name %> < CrystalRuby::Types::VariableWidth
+
+  include Enumerable(<%= inner_type.native_type_expr %>)
+
+  def initialize(array : Array(<%= inner_type.native_type_expr %>))
+    @memory = malloc(data_offset + 8)
+    self.value = array
+    increment_ref_count!
+  end
+
+  def each
+    size.times do |i|
+      yield self[i].native
+    end
+  end
+
+  def ==(other : Array(<%= inner_type.native_type_expr %>))
+    native == other
+  end
+
+  def data_pointer
+    Pointer(::UInt8).new((@memory + data_offset).as(Pointer(::UInt64)).value)
+  end
+
+  def value=(array : Array(<%= inner_type.native_type_expr %>))
+    if self.ref_count > 0
+      self.class.decr_inner_ref_counts!(memory)
+    end
+    self.class.copy_to!(array, self.memory)
+  end
+
+  def <<(value : <%= inner_type.native_type_expr %>)
+    self.value = self.native + [value]
+  end
+
+  def []=(index : Int, value : <%= inner_type.native_type_expr %>)
+    index += size if index < 0
+    <%= inner_type.crystal_class_name %>.write_single(data_pointer + index * <%= inner_type.refsize %>, value)
+  end
+
+  def [](index : Int)
+    index += size if index < 0
+    <%= inner_type.crystal_class_name %>.fetch_single(data_pointer + index * <%= inner_type.refsize %>)
+  end
+
+  def self.copy_to!(array : Array(<%= inner_type.native_type_expr %>), memory : Pointer(::UInt8))
+    data_pointer = malloc(array.size * <%= inner_type.refsize %>)
+    array.size.times do |i|
+      <%= inner_type.crystal_class_name %>.write_single(data_pointer + i * <%= inner_type.refsize %>, array[i])
+    end
+    (memory+size_offset).as(Pointer(::UInt32)).value = array.size.to_u32
+    (memory+data_offset).as(Pointer(::UInt64)).value = data_pointer.address
+  end
+
+  def ==(other : <%= native_type_expr %>)
+    native == other
+  end
+
+  def value
+    <%= inner_type.crystal_class_name %>.fetch_multi!(data_pointer, size)
+  end
+
+  def native : ::Array(<%= inner_type.native_type_expr %>)
+    <%= inner_type.crystal_class_name %>.fetch_multi_native!(data_pointer, size)
+  end
+
+  def size
+    (@memory + self.class.size_offset).as(Pointer(::UInt32))[0]
+  end
+
+  def self.memsize
+    <%= memsize %>
+  end
+end

data/lib/crystalruby/types/variable_width/array.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module CrystalRuby::Types
+  Array = VariableWidth.build(error: "Array type must have a type parameter. E.g. Array(Float64)")
+
+  # An array-like, reference counted manually managed memory type.
+  # Shareable between Crystal and Crystal.
+  def self.Array(type)
+    VariableWidth.build(:Array, inner_types: [type], convert_if: [Array, Root::Array], superclass: Array) do
+      include Enumerable
+
+      # Implement the Enumerable interface
+      # Helps this object to act like an Array
+      def each
+        size.times { |i| yield self[i] }
+      end
+
+      # We only accept Array-like values, from which all elements
+      # can successfully be cast to our inner type
+      def self.cast!(value)
+        unless value.is_a?(Array) || value.is_a?(Root::Array) && value.all?(&inner_type.method(:valid_cast?))
+          raise CrystalRuby::InvalidCastError, "Cannot cast #{value} to #{inspect}"
+        end
+
+        if inner_type.primitive?
+          value.map(&inner_type.method(:to_ffi_repr))
+        else
+          value
+        end
+      end
+
+      def self.copy_to!(rbval, memory:)
+        data_pointer = malloc(rbval.size * inner_type.refsize)
+
+        memory[size_offset].write_uint32(rbval.size)
+        memory[data_offset].write_pointer(data_pointer)
+
+        if inner_type.primitive?
+          data_pointer.send("put_array_of_#{inner_type.ffi_type}", 0, rbval)
+        else
+          rbval.each_with_index do |val, i|
+            inner_type.write_single(data_pointer[i * refsize], val)
+          end
+        end
+      end
+
+      def self.each_child_address(pointer)
+        size = pointer[size_offset].get_int32(0)
+        pointer = pointer[data_offset].read_pointer
+        size.times do |i|
+          yield inner_type, pointer[i * inner_type.refsize]
+        end
+      end
+
+      def checked_offset!(index, size)
+        raise "Index out of bounds: #{index} >= #{size}" if index >= size
+
+        if index < 0
+          raise "Index out of bounds: #{index} < -#{size}" if index < -size
+
+          index += size
+        end
+        index
+      end
+
+      # Return the element at the given index.
+      # This will automatically increment
+      # the reference count if not a primitive type.
+      def [](index)
+        inner_type.fetch_single(data_pointer[checked_offset!(index, size) * inner_type.refsize])
+      end
+
+      # Overwrite the element at the given index
+      # The replaced element will have
+      # its reference count decremented.
+      def []=(index, value)
+        inner_type.write_single(data_pointer[checked_offset!(index, size) * inner_type.refsize], value)
+      end
+
+      # Load values stored inside array type.
+      # If it's a primitive type, we can quickly bulk load the values.
+      # Otherwise we need toinstantiate new ref-checked instances.
+      def value(native: false)
+        inner_type.fetch_multi(data_pointer, size, native: native)
+      end
+    end
+  end
+end

data/lib/crystalruby/types/variable_width/hash.cr

@@ -0,0 +1,146 @@
+class <%= base_crystal_class_name %> < CrystalRuby::Types::VariableWidth
+
+  def initialize(hash : Hash(<%= key_type.native_type_expr %>, <%= value_type.native_type_expr %>))
+    @memory = malloc(data_offset + 8)
+    self.value = hash
+    increment_ref_count!
+  end
+
+  def value=(hash : Hash(<%= key_type.native_type_expr %>, <%= value_type.native_type_expr %>))
+    if self.ref_count > 0
+      self.class.decr_inner_ref_counts!(memory)
+    end
+    self.class.copy_to!(hash, self.memory)
+  end
+
+  def ==(other : <%= native_type_expr %>)
+    native == other
+  end
+
+  def self.copy_to!(hash : Hash(<%= key_type.native_type_expr %>, <%= value_type.native_type_expr %>), memory : Pointer(::UInt8))
+    data_pointer = malloc(hash.size * (keysize +  valsize)).as(Pointer(::UInt8))
+
+    hash.keys.each_with_index do |key, i|
+      <%= key_type.crystal_class_name %>.write_single(data_pointer + i * keysize, key)
+    end
+
+    hash.values.each_with_index do |value, i|
+      <%= value_type.crystal_class_name %>.write_single(data_pointer + hash.size * keysize + i * valsize, value)
+    end
+
+    (memory+size_offset).as(Pointer(::UInt32)).value = hash.size.to_u32
+    (memory+data_offset).as(Pointer(::UInt64)).value = data_pointer.address
+  end
+
+  def value
+    ::Hash.zip(keys, values)
+  end
+
+  def native : ::Hash(<%= key_type.native_type_expr %>, <%= value_type.native_type_expr %>)
+    ::Hash.zip(keys_native, values_native)
+  end
+
+  include Enumerable(::Tuple(<%= key_type.native_type_expr %>, <%= value_type.native_type_expr %>))
+
+  def each
+    size.times do |i|
+      yield({keys_native[i], values_native[i]})
+    end
+  end
+
+  def size
+    (memory+size_offset).as(Pointer(::UInt32)).value.to_i
+  end
+
+  def self.keysize
+    <%= key_type.refsize %>
+  end
+
+  def keysize
+    <%= key_type.refsize %>
+  end
+
+  def self.valsize
+    <%= value_type.refsize %>
+  end
+
+  def valsize
+    <%= value_type.refsize %>
+  end
+
+  def []=(key : <%= key_type.native_type_expr %>, value : <%= value_type.native_type_expr %>)
+    index = index_of(key)
+    if index
+      <%= value_type.crystal_class_name %>.write_single(data_pointer + size * keysize + index * valsize, value)
+    else
+      self.value = self.native.merge({key => value})
+    end
+  end
+
+  def key_ptr : Pointer(<%= !key_type.numeric? ? "UInt64" : key_type.native_type_expr %>)
+    data_pointer.as(Pointer(<%= !key_type.numeric? ? "UInt8" : key_type.native_type_expr %>))
+  end
+
+  def value_ptr : Pointer(<%= !value_type.numeric? ? "UInt64" : value_type.native_type_expr %>)
+    (data_pointer + size * keysize).as(Pointer(<%= !value_type.numeric? ? "UInt8" : value_type.native_type_expr %>))
+  end
+
+  def []?(key : <%= key_type.native_type_expr %>)
+    index = index_of(key)
+    if index
+      values[index]
+    else
+      nil
+    end
+  end
+
+  def [](key : <%= key_type.native_type_expr %>)
+    index = index_of(key)
+    if index
+      values[index]
+    else
+      raise "Key not found"
+    end
+  end
+
+
+  def data_pointer
+    Pointer(::UInt8).new((@memory + data_offset).as(::Pointer(UInt64)).value)
+  end
+
+  def keys
+    keys = [] of <%=  key_type.numeric? ? key_type.native_type_expr : key_type.crystal_class_name %>
+    size.times do |i|
+      keys << <%= !key_type.numeric? ? "#{key_type.crystal_class_name}.fetch_single(data_pointer + i * keysize)" : "key_ptr[i]" %>
+    end
+    keys
+  end
+
+  def values
+    values = [] of <%= value_type.numeric? ? value_type.native_type_expr : value_type.crystal_class_name %>
+    size.times do |i|
+      values << <%= !value_type.numeric? ? "#{value_type.crystal_class_name}.fetch_single(data_pointer + size * keysize + i * valsize)" : "value_ptr[i]" %>
+    end
+    values
+  end
+
+  def keys_native
+    keys = [] of <%= key_type.native_type_expr %>
+    size.times do |i|
+      keys << <%= !key_type.numeric? ? "#{key_type.crystal_class_name}.fetch_single(data_pointer + i * keysize).native" : "key_ptr[i]" %>.as(<%= key_type.native_type_expr %>)
+    end
+    keys
+  end
+
+  def values_native
+    values = [] of <%= value_type.native_type_expr %>
+    size.times do |i|
+      values << <%= !value_type.numeric? ? "#{value_type.crystal_class_name}.fetch_single(data_pointer + size * keysize + i * valsize).native" : "value_ptr[i]" %>.as(<%= value_type.native_type_expr %>)
+    end
+    values
+  end
+
+  def index_of(key : <%= key_type.native_type_expr %>)
+    keys.index(key)
+  end
+end