onnxruntime 0.9.1-aarch64-linux → 0.9.3-aarch64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f3931f39e07804b4f3e9a90e79fe43bdffb30a11f9869dda46e37be9272f401a
-  data.tar.gz: d6d70158ffe07c91697a2923ed7c8f4bba838420e18ce7e06f48702eccc16592
+  metadata.gz: e2af4ce55d5564f02da2f6061e3749eaf372db66d5a39f86e3eed82e14c40684
+  data.tar.gz: fd5ae05a129b3eb93950198eaeaa59db4ed65cf0644297147120aa2702752fb4
 SHA512:
-  metadata.gz: 99fdd3e53d952387acb04455b58c36ad0abee5c3bc4fd56a1bd625bda92954187f7a4b16814ea9fbc4d5b250f590e0916f54865c3f927496a5981fd459235082
-  data.tar.gz: 03cb16cecbaf6e8d6855453f58b3f455751ef3795fbace5a25527a5ec3ea4444f30e20413048e653f13e7042e3bdb689075e0de33be8cce035a561684b53f146
+  metadata.gz: 8878ee1a2047552d8acbaa4bb2592c69c7dfc4a559b9d61c13f68fc47bae9b28be307bb2b17a2c8575099d5fdcf54b7acda21fee19a5d040ca89c1038639445a
+  data.tar.gz: e4afdb4c19450a37e535d96d610c93f375104e58593590dd0fc26fb8c6d5af9f34eef19327710dcd038879e59b1219ce2da9d3d1211efd91b6ae350fc1080e6e

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+## 0.9.3 (2024-11-01)
+
+- Updated ONNX Runtime to 1.20.0
+- Added experimental `OrtValue` class
+- Added experimental `run_with_ort_values` method
+
+## 0.9.2 (2024-09-04)
+
+- Updated ONNX Runtime to 1.19.2
+- Added support for CoreML
+
 ## 0.9.1 (2024-05-22)
 
 - Updated ONNX Runtime to 1.18.0

data/README.md

@@ -4,7 +4,7 @@
 
 Check out [an example](https://ankane.org/tensorflow-ruby)
 
-[![Build Status](https://github.com/ankane/onnxruntime-ruby/workflows/build/badge.svg?branch=master)](https://github.com/ankane/onnxruntime-ruby/actions)
+[![Build Status](https://github.com/ankane/onnxruntime-ruby/actions/workflows/build.yml/badge.svg)](https://github.com/ankane/onnxruntime-ruby/actions)
 
 ## Installation
 
@@ -108,7 +108,9 @@ OnnxRuntime::Datasets.example("sigmoid.onnx")
 
 ## GPU Support
 
-To enable GPU support on Linux and Windows, download the appropriate [GPU release](https://github.com/microsoft/onnxruntime/releases) and set:
+### Linux and Windows
+
+Download the appropriate [GPU release](https://github.com/microsoft/onnxruntime/releases) and set:
 
 ```ruby
 OnnxRuntime.ffi_lib = "path/to/lib/libonnxruntime.so" # onnxruntime.dll for Windows
@@ -120,6 +122,14 @@ and use:
 model = OnnxRuntime::Model.new("model.onnx", providers: ["CUDAExecutionProvider"])
 ```
 
+### Mac
+
+Use:
+
+```ruby
+model = OnnxRuntime::Model.new("model.onnx", providers: ["CoreMLExecutionProvider"])
+```
+
 ## History
 
 View the [changelog](https://github.com/ankane/onnxruntime-ruby/blob/master/CHANGELOG.md)

data/lib/onnxruntime/ffi.rb

@@ -11,7 +11,7 @@ module OnnxRuntime
 
     # enums
     TensorElementDataType = enum(:undefined, :float, :uint8, :int8, :uint16, :int16, :int32, :int64, :string, :bool, :float16, :double, :uint32, :uint64, :complex64, :complex128, :bfloat16)
-    OnnxType = enum(:unknown, :tensor, :sequence, :map, :opaque, :sparsetensor)
+    OnnxType = enum(:unknown, :tensor, :sequence, :map, :opaque, :sparsetensor, :optional)
 
     class Api < ::FFI::Struct
       layout \
@@ -144,7 +144,7 @@ module OnnxRuntime
         :ReleaseAvailableProviders, callback(%i[pointer int], :pointer),
         :GetStringTensorElementLength, callback(%i[], :pointer),
         :GetStringTensorElement, callback(%i[], :pointer),
-        :FillStringTensorElement, callback(%i[], :pointer),
+        :FillStringTensorElement, callback(%i[pointer string size_t], :pointer),
         :AddSessionConfigEntry, callback(%i[pointer string string], :pointer),
         :CreateAllocator, callback(%i[], :pointer),
         :ReleaseAllocator, callback(%i[], :pointer),
@@ -246,6 +246,14 @@ module OnnxRuntime
 
     attach_function :OrtGetApiBase, %i[], ApiBase.by_ref
 
+    def self.api
+      @api ||= begin
+        api = self.OrtGetApiBase[:GetApi].call(ORT_API_VERSION)
+        api = Api.by_ref.from_native(api, nil) if RUBY_PLATFORM == "java"
+        api
+      end
+    end
+
     if Gem.win_platform?
       class Libc
         extend ::FFI::Library
@@ -253,5 +261,11 @@ module OnnxRuntime
         attach_function :mbstowcs, %i[pointer string size_t], :size_t
       end
     end
+
+    # https://github.com/microsoft/onnxruntime/blob/main/include/onnxruntime/core/providers/coreml/coreml_provider_factory.h
+    begin
+      attach_function :OrtSessionOptionsAppendExecutionProvider_CoreML, %i[pointer uint32], :pointer
+    rescue ::FFI::NotFoundError
+    end
   end
 end

data/lib/onnxruntime/inference_session.rb

@@ -66,6 +66,13 @@ module OnnxRuntime
           check_status api[:CreateCUDAProviderOptions].call(cuda_options)
           check_status api[:SessionOptionsAppendExecutionProvider_CUDA_V2].call(session_options.read_pointer, cuda_options.read_pointer)
           release :CUDAProviderOptions, cuda_options
+        when "CoreMLExecutionProvider"
+          unless FFI.respond_to?(:OrtSessionOptionsAppendExecutionProvider_CoreML)
+            raise ArgumentError, "Provider not available: #{provider}"
+          end
+
+          coreml_flags = 0
+          check_status FFI.OrtSessionOptionsAppendExecutionProvider_CoreML(session_options.read_pointer, coreml_flags)
         when "CPUExecutionProvider"
           break
         else
@@ -76,23 +83,36 @@ module OnnxRuntime
       @session = load_session(path_or_bytes, session_options)
       ObjectSpace.define_finalizer(@session, self.class.finalize(read_pointer.to_i))
 
-      @allocator = load_allocator
+      @allocator = Utils.allocator
       @inputs = load_inputs
       @outputs = load_outputs
     ensure
       release :SessionOptions, session_options
     end
 
-    # TODO support logid
     def run(output_names, input_feed, log_severity_level: nil, log_verbosity_level: nil, logid: nil, terminate: nil, output_type: :ruby)
-      # pointer references
-      refs = []
+      if ![:ruby, :numo, :ort_value].include?(output_type)
+        raise ArgumentError, "Invalid output type: #{output_type}"
+      end
+
+      ort_values = input_feed.keys.zip(create_input_tensor(input_feed)).to_h
 
-      input_tensor = create_input_tensor(input_feed, refs)
+      outputs = run_with_ort_values(output_names, ort_values, log_severity_level: log_severity_level, log_verbosity_level: log_verbosity_level, logid: logid, terminate: terminate)
+
+      outputs.map { |v| output_type == :numo ? v.numo : (output_type == :ort_value ? v : v.to_ruby) }
+    end
+
+    # TODO support logid
+    def run_with_ort_values(output_names, input_feed, log_severity_level: nil, log_verbosity_level: nil, logid: nil, terminate: nil)
+      input_tensor = ::FFI::MemoryPointer.new(:pointer, input_feed.size)
+      input_feed.each_with_index do |(_, input), i|
+        input_tensor[i].write_pointer(input.to_ptr)
+      end
 
       output_names ||= @outputs.map { |v| v[:name] }
 
       output_tensor = ::FFI::MemoryPointer.new(:pointer, outputs.size)
+      refs = []
       input_node_names = create_node_names(input_feed.keys.map(&:to_s), refs)
       output_node_names = create_node_names(output_names.map(&:to_s), refs)
 
@@ -106,17 +126,9 @@ module OnnxRuntime
 
       check_status api[:Run].call(read_pointer, run_options.read_pointer, input_node_names, input_tensor, input_feed.size, output_node_names, output_names.size, output_tensor)
 
-      output_names.size.times.map do |i|
-        create_from_onnx_value(output_tensor[i].read_pointer, output_type)
-      end
+      output_names.size.times.map { |i| OrtValue.new(output_tensor[i]) }
     ensure
       release :RunOptions, run_options
-      if input_tensor
-        input_feed.size.times do |i|
-          release :Value, input_tensor[i]
-        end
-      end
-      # output values released in create_from_onnx_value
     end
 
     def modelmeta
@@ -214,12 +226,6 @@ module OnnxRuntime
       session
     end
 
-    def load_allocator
-      allocator = ::FFI::MemoryPointer.new(:pointer)
-      check_status api[:GetAllocatorWithDefaultOptions].call(allocator)
-      allocator
-    end
-
     def load_inputs
       inputs = []
       num_input_nodes = ::FFI::MemoryPointer.new(:size_t)
@@ -230,7 +236,7 @@ module OnnxRuntime
         # freed in node_info
         typeinfo = ::FFI::MemoryPointer.new(:pointer)
         check_status api[:SessionGetInputTypeInfo].call(read_pointer, i, typeinfo)
-        inputs << {name: name_ptr.read_pointer.read_string}.merge(node_info(typeinfo))
+        inputs << {name: name_ptr.read_pointer.read_string}.merge(Utils.node_info(typeinfo))
         allocator_free name_ptr
       end
       inputs
@@ -246,87 +252,28 @@ module OnnxRuntime
         # freed in node_info
         typeinfo = ::FFI::MemoryPointer.new(:pointer)
         check_status api[:SessionGetOutputTypeInfo].call(read_pointer, i, typeinfo)
-        outputs << {name: name_ptr.read_pointer.read_string}.merge(node_info(typeinfo))
+        outputs << {name: name_ptr.read_pointer.read_string}.merge(Utils.node_info(typeinfo))
         allocator_free name_ptr
       end
       outputs
     end
 
-    def create_input_tensor(input_feed, refs)
-      allocator_info = ::FFI::MemoryPointer.new(:pointer)
-      check_status api[:CreateCpuMemoryInfo].call(1, 0, allocator_info)
-      input_tensor = ::FFI::MemoryPointer.new(:pointer, input_feed.size)
-
-      input_feed.each_with_index do |(input_name, input), idx|
-        if numo_array?(input)
-          shape = input.shape
-        else
-          input = input.to_a unless input.is_a?(Array)
-
-          shape = []
-          s = input
-          while s.is_a?(Array)
-            shape << s.size
-            s = s.first
-          end
-        end
-
+    def create_input_tensor(input_feed)
+      input_feed.map do |input_name, input|
         # TODO support more types
         inp = @inputs.find { |i| i[:name] == input_name.to_s }
         raise Error, "Unknown input: #{input_name}" unless inp
 
-        input_node_dims = ::FFI::MemoryPointer.new(:int64, shape.size)
-        input_node_dims.write_array_of_int64(shape)
-
-        if inp[:type] == "tensor(string)"
-          str_ptrs =
-            if numo_array?(input)
-              input.size.times.map { |i| ::FFI::MemoryPointer.from_string(input[i]) }
-            else
-              input.flatten.map { |v| ::FFI::MemoryPointer.from_string(v) }
-            end
-
-          input_tensor_values = ::FFI::MemoryPointer.new(:pointer, str_ptrs.size)
-          input_tensor_values.write_array_of_pointer(str_ptrs)
-
-          type_enum = FFI::TensorElementDataType[:string]
-          check_status api[:CreateTensorAsOrtValue].call(@allocator.read_pointer, input_node_dims, shape.size, type_enum, input_tensor[idx])
-          check_status api[:FillStringTensor].call(input_tensor[idx].read_pointer, input_tensor_values, str_ptrs.size)
-
-          refs << str_ptrs
+        if input.is_a?(OrtValue)
+          input
+        elsif inp[:type] == "tensor(string)"
+          OrtValue.from_array(input, element_type: :string)
+        elsif (tensor_type = tensor_types[inp[:type]])
+          OrtValue.from_array(input, element_type: tensor_type)
         else
-          tensor_type = tensor_types[inp[:type]]
-
-          if tensor_type
-            if numo_array?(input)
-              input_tensor_values = input.cast_to(numo_types[tensor_type]).to_binary
-            else
-              flat_input = input.flatten.to_a
-              input_tensor_values = ::FFI::MemoryPointer.new(tensor_type, flat_input.size)
-              if tensor_type == :bool
-                input_tensor_values.write_array_of_uint8(flat_input.map { |v| v ? 1 : 0 })
-              else
-                input_tensor_values.send("write_array_of_#{tensor_type}", flat_input)
-              end
-            end
-
-            type_enum = FFI::TensorElementDataType[tensor_type]
-          else
-            unsupported_type("input", inp[:type])
-          end
-
-          check_status api[:CreateTensorWithDataAsOrtValue].call(allocator_info.read_pointer, input_tensor_values, input_tensor_values.size, input_node_dims, shape.size, type_enum, input_tensor[idx])
-
-          refs << input_node_dims
-          refs << input_tensor_values
+          Utils.unsupported_type("input", inp[:type])
         end
       end
-
-      refs << allocator_info
-
-      input_tensor
-    ensure
-      release :MemoryInfo, allocator_info
     end
 
     def create_node_names(names, refs)
@@ -338,230 +285,18 @@ module OnnxRuntime
       ptr
     end
 
-    def create_from_onnx_value(out_ptr, output_type)
-      out_type = ::FFI::MemoryPointer.new(:int)
-      check_status api[:GetValueType].call(out_ptr, out_type)
-      type = FFI::OnnxType[out_type.read_int]
-
-      case type
-      when :tensor
-        typeinfo = ::FFI::MemoryPointer.new(:pointer)
-        check_status api[:GetTensorTypeAndShape].call(out_ptr, typeinfo)
-
-        type, shape = tensor_type_and_shape(typeinfo)
-
-        tensor_data = ::FFI::MemoryPointer.new(:pointer)
-        check_status api[:GetTensorMutableData].call(out_ptr, tensor_data)
-
-        out_size = ::FFI::MemoryPointer.new(:size_t)
-        check_status api[:GetTensorShapeElementCount].call(typeinfo.read_pointer, out_size)
-        output_tensor_size = out_size.read(:size_t)
-
-        release :TensorTypeAndShapeInfo, typeinfo
-
-        # TODO support more types
-        type = FFI::TensorElementDataType[type]
-
-        case output_type
-        when :numo
-          case type
-          when :string
-            result = Numo::RObject.new(shape)
-            result.allocate
-            create_strings_from_onnx_value(out_ptr, output_tensor_size, result)
-          else
-            numo_type = numo_types[type]
-            unsupported_type("element", type) unless numo_type
-            numo_type.from_binary(tensor_data.read_pointer.read_bytes(output_tensor_size * numo_type::ELEMENT_BYTE_SIZE), shape)
-          end
-        when :ruby
-          arr =
-            case type
-            when :float, :uint8, :int8, :uint16, :int16, :int32, :int64, :double, :uint32, :uint64
-              tensor_data.read_pointer.send("read_array_of_#{type}", output_tensor_size)
-            when :bool
-              tensor_data.read_pointer.read_array_of_uint8(output_tensor_size).map { |v| v == 1 }
-            when :string
-              create_strings_from_onnx_value(out_ptr, output_tensor_size, [])
-            else
-              unsupported_type("element", type)
-            end
-
-          Utils.reshape(arr, shape)
-        else
-          raise ArgumentError, "Invalid output type: #{output_type}"
-        end
-      when :sequence
-        out = ::FFI::MemoryPointer.new(:size_t)
-        check_status api[:GetValueCount].call(out_ptr, out)
-
-        out.read(:size_t).times.map do |i|
-          seq = ::FFI::MemoryPointer.new(:pointer)
-          check_status api[:GetValue].call(out_ptr, i, @allocator.read_pointer, seq)
-          create_from_onnx_value(seq.read_pointer, output_type)
-        end
-      when :map
-        type_shape = ::FFI::MemoryPointer.new(:pointer)
-        map_keys = ::FFI::MemoryPointer.new(:pointer)
-        map_values = ::FFI::MemoryPointer.new(:pointer)
-        elem_type = ::FFI::MemoryPointer.new(:int)
-
-        check_status api[:GetValue].call(out_ptr, 0, @allocator.read_pointer, map_keys)
-        check_status api[:GetValue].call(out_ptr, 1, @allocator.read_pointer, map_values)
-        check_status api[:GetTensorTypeAndShape].call(map_keys.read_pointer, type_shape)
-        check_status api[:GetTensorElementType].call(type_shape.read_pointer, elem_type)
-        release :TensorTypeAndShapeInfo, type_shape
-
-        # TODO support more types
-        elem_type = FFI::TensorElementDataType[elem_type.read_int]
-        case elem_type
-        when :int64
-          ret = {}
-          keys = create_from_onnx_value(map_keys.read_pointer, output_type)
-          values = create_from_onnx_value(map_values.read_pointer, output_type)
-          keys.zip(values).each do |k, v|
-            ret[k] = v
-          end
-          ret
-        else
-          unsupported_type("element", elem_type)
-        end
-      else
-        unsupported_type("ONNX", type)
-      end
-    ensure
-      api[:ReleaseValue].call(out_ptr) unless out_ptr.null?
-    end
-
-    def create_strings_from_onnx_value(out_ptr, output_tensor_size, result)
-      len = ::FFI::MemoryPointer.new(:size_t)
-      check_status api[:GetStringTensorDataLength].call(out_ptr, len)
-
-      s_len = len.read(:size_t)
-      s = ::FFI::MemoryPointer.new(:uchar, s_len)
-      offsets = ::FFI::MemoryPointer.new(:size_t, output_tensor_size)
-      check_status api[:GetStringTensorContent].call(out_ptr, s, s_len, offsets, output_tensor_size)
-
-      offsets = output_tensor_size.times.map { |i| offsets[i].read(:size_t) }
-      offsets << s_len
-      output_tensor_size.times do |i|
-        result[i] = s.get_bytes(offsets[i], offsets[i + 1] - offsets[i])
-      end
-      result
-    end
-
     def read_pointer
       @session.read_pointer
     end
 
     def check_status(status)
-      unless status.null?
-        message = api[:GetErrorMessage].call(status).read_string
-        api[:ReleaseStatus].call(status)
-        raise Error, message
-      end
-    end
-
-    def node_info(typeinfo)
-      onnx_type = ::FFI::MemoryPointer.new(:int)
-      check_status api[:GetOnnxTypeFromTypeInfo].call(typeinfo.read_pointer, onnx_type)
-
-      type = FFI::OnnxType[onnx_type.read_int]
-      case type
-      when :tensor
-        tensor_info = ::FFI::MemoryPointer.new(:pointer)
-        # don't free tensor_info
-        check_status api[:CastTypeInfoToTensorInfo].call(typeinfo.read_pointer, tensor_info)
-
-        type, shape = tensor_type_and_shape(tensor_info)
-        {
-          type: "tensor(#{FFI::TensorElementDataType[type]})",
-          shape: shape
-        }
-      when :sequence
-        sequence_type_info = ::FFI::MemoryPointer.new(:pointer)
-        check_status api[:CastTypeInfoToSequenceTypeInfo].call(typeinfo.read_pointer, sequence_type_info)
-        nested_type_info = ::FFI::MemoryPointer.new(:pointer)
-        check_status api[:GetSequenceElementType].call(sequence_type_info.read_pointer, nested_type_info)
-        v = node_info(nested_type_info)[:type]
-
-        {
-          type: "seq(#{v})",
-          shape: []
-        }
-      when :map
-        map_type_info = ::FFI::MemoryPointer.new(:pointer)
-        check_status api[:CastTypeInfoToMapTypeInfo].call(typeinfo.read_pointer, map_type_info)
-
-        # key
-        key_type = ::FFI::MemoryPointer.new(:int)
-        check_status api[:GetMapKeyType].call(map_type_info.read_pointer, key_type)
-        k = FFI::TensorElementDataType[key_type.read_int]
-
-        # value
-        value_type_info = ::FFI::MemoryPointer.new(:pointer)
-        check_status api[:GetMapValueType].call(map_type_info.read_pointer, value_type_info)
-        v = node_info(value_type_info)[:type]
-
-        {
-          type: "map(#{k},#{v})",
-          shape: []
-        }
-      else
-        unsupported_type("ONNX", type)
-      end
-    ensure
-      release :TypeInfo, typeinfo
-    end
-
-    def tensor_type_and_shape(tensor_info)
-      type = ::FFI::MemoryPointer.new(:int)
-      check_status api[:GetTensorElementType].call(tensor_info.read_pointer, type)
-
-      num_dims_ptr = ::FFI::MemoryPointer.new(:size_t)
-      check_status api[:GetDimensionsCount].call(tensor_info.read_pointer, num_dims_ptr)
-      num_dims = num_dims_ptr.read(:size_t)
-
-      node_dims = ::FFI::MemoryPointer.new(:int64, num_dims)
-      check_status api[:GetDimensions].call(tensor_info.read_pointer, node_dims, num_dims)
-      dims = node_dims.read_array_of_int64(num_dims)
-
-      symbolic_dims = ::FFI::MemoryPointer.new(:pointer, num_dims)
-      check_status api[:GetSymbolicDimensions].call(tensor_info.read_pointer, symbolic_dims, num_dims)
-      named_dims = num_dims.times.map { |i| symbolic_dims[i].read_pointer.read_string }
-      dims = named_dims.zip(dims).map { |n, d| n.empty? ? d : n }
-
-      [type.read_int, dims]
-    end
-
-    def unsupported_type(name, type)
-      raise Error, "Unsupported #{name} type: #{type}"
+      Utils.check_status(status)
     end
 
     def tensor_types
       @tensor_types ||= [:float, :uint8, :int8, :uint16, :int16, :int32, :int64, :bool, :double, :uint32, :uint64].map { |v| ["tensor(#{v})", v] }.to_h
     end
 
-    def numo_array?(obj)
-      defined?(Numo::NArray) && obj.is_a?(Numo::NArray)
-    end
-
-    def numo_types
-      @numo_types ||= {
-        float: Numo::SFloat,
-        uint8: Numo::UInt8,
-        int8: Numo::Int8,
-        uint16: Numo::UInt16,
-        int16: Numo::Int16,
-        int32: Numo::Int32,
-        int64: Numo::Int64,
-        bool: Numo::UInt8,
-        double: Numo::DFloat,
-        uint32: Numo::UInt32,
-        uint64: Numo::UInt64
-      }
-    end
-
     def api
       self.class.api
     end
@@ -575,11 +310,11 @@ module OnnxRuntime
     end
 
     def self.api
-      @api ||= FFI.OrtGetApiBase[:GetApi].call(FFI::ORT_API_VERSION)
+      FFI.api
     end
 
     def self.release(type, pointer)
-      api[:"Release#{type}"].call(pointer.read_pointer) if pointer && !pointer.null?
+      Utils.release(type, pointer)
     end
 
     def self.finalize(addr)

data/lib/onnxruntime/ort_value.rb

@@ -0,0 +1,278 @@
+module OnnxRuntime
+  class OrtValue
+    def initialize(ptr, ref = nil)
+      @ptr = ptr.read_pointer
+      @ref = ref # keep reference to data
+      ObjectSpace.define_finalizer(@ptr, self.class.finalize(@ptr.to_i))
+    end
+
+    def self.from_numo(numo_obj)
+      element_type = numo_obj.is_a?(Numo::Bit) ? :bool : Utils.numo_types.invert[numo_obj.class]
+      Utils.unsupported_type("Numo", numo_obj.class.name) unless element_type
+
+      from_array(numo_obj, element_type: element_type)
+    end
+
+    def self.from_array(input, element_type:)
+      type_enum = FFI::TensorElementDataType[element_type]
+      Utils.unsupported_type("element", element_type) unless type_enum
+
+      input = input.to_a unless input.is_a?(Array) || Utils.numo_array?(input)
+
+      shape = Utils.input_shape(input)
+      input_node_dims = ::FFI::MemoryPointer.new(:int64, shape.size)
+      input_node_dims.write_array_of_int64(shape)
+
+      ptr = ::FFI::MemoryPointer.new(:pointer)
+      if element_type == :string
+        # keep reference to _str_ptrs until FillStringTensor call
+        input_tensor_values, _str_ptrs = create_input_strings(input)
+        Utils.check_status FFI.api[:CreateTensorAsOrtValue].call(Utils.allocator.read_pointer, input_node_dims, shape.size, type_enum, ptr)
+        Utils.check_status FFI.api[:FillStringTensor].call(ptr.read_pointer, input_tensor_values, input_tensor_values.size / input_tensor_values.type_size)
+      else
+        input_tensor_values = create_input_data(input, element_type)
+        Utils.check_status FFI.api[:CreateTensorWithDataAsOrtValue].call(allocator_info.read_pointer, input_tensor_values, input_tensor_values.size, input_node_dims, shape.size, type_enum, ptr)
+      end
+
+      new(ptr, input_tensor_values)
+    end
+
+    def self.from_shape_and_type(shape, element_type)
+      type_enum = FFI::TensorElementDataType[element_type]
+      Utils.unsupported_type("element", element_type) unless type_enum
+
+      input_node_dims = ::FFI::MemoryPointer.new(:int64, shape.size)
+      input_node_dims.write_array_of_int64(shape)
+
+      ptr = ::FFI::MemoryPointer.new(:pointer)
+      Utils.check_status FFI.api[:CreateTensorAsOrtValue].call(Utils.allocator.read_pointer, input_node_dims, shape.size, type_enum, ptr)
+
+      new(ptr)
+    end
+
+    def self.create_input_data(input, tensor_type)
+      if Utils.numo_array?(input)
+        input.cast_to(Utils.numo_types[tensor_type]).to_binary
+      else
+        flat_input = input.flatten.to_a
+        input_tensor_values = ::FFI::MemoryPointer.new(tensor_type, flat_input.size)
+        if tensor_type == :bool
+          input_tensor_values.write_array_of_uint8(flat_input.map { |v| v ? 1 : 0 })
+        else
+          input_tensor_values.send("write_array_of_#{tensor_type}", flat_input)
+        end
+        input_tensor_values
+      end
+    end
+    private_class_method :create_input_data
+
+    def self.create_input_strings(input)
+      str_ptrs =
+        if Utils.numo_array?(input)
+          input.size.times.map { |i| ::FFI::MemoryPointer.from_string(input[i]) }
+        else
+          input.flatten.map { |v| ::FFI::MemoryPointer.from_string(v) }
+        end
+
+      input_tensor_values = ::FFI::MemoryPointer.new(:pointer, str_ptrs.size)
+      input_tensor_values.write_array_of_pointer(str_ptrs)
+      [input_tensor_values, str_ptrs]
+    end
+    private_class_method :create_input_strings
+
+    def tensor?
+      FFI::OnnxType[value_type] == :tensor
+    end
+
+    def data_type
+      @data_type ||= begin
+        typeinfo = ::FFI::MemoryPointer.new(:pointer)
+        Utils.check_status FFI.api[:GetTypeInfo].call(@ptr, typeinfo)
+        Utils.node_info(typeinfo)[:type]
+      end
+    end
+
+    def element_type
+      FFI::TensorElementDataType[type_and_shape_info[0]]
+    end
+
+    def shape
+      type_and_shape_info[1]
+    end
+
+    def device_name
+      "cpu"
+    end
+
+    def numo
+      create_from_onnx_value(@ptr, :numo)
+    end
+
+    def to_ruby
+      create_from_onnx_value(@ptr, :ruby)
+    end
+
+    def to_ptr
+      @ptr
+    end
+
+    def data_ptr
+      tensor_data = ::FFI::MemoryPointer.new(:pointer)
+      FFI.api[:GetTensorMutableData].call(@ptr, tensor_data)
+      tensor_data.read_pointer
+    end
+
+    private
+
+    def value_type
+      @value_type ||= begin
+        out_type = ::FFI::MemoryPointer.new(:int)
+        Utils.check_status FFI.api[:GetValueType].call(@ptr, out_type)
+        out_type.read_int
+      end
+    end
+
+    def type_and_shape_info
+      @type_and_shape_info ||= begin
+        begin
+          typeinfo = ::FFI::MemoryPointer.new(:pointer)
+          Utils.check_status FFI.api[:GetTensorTypeAndShape].call(@ptr, typeinfo)
+          Utils.tensor_type_and_shape(typeinfo)
+        ensure
+          Utils.release :TensorTypeAndShapeInfo, typeinfo
+        end
+      end
+    end
+
+    def create_from_onnx_value(out_ptr, output_type)
+      out_type = ::FFI::MemoryPointer.new(:int)
+      Utils.check_status FFI.api[:GetValueType].call(out_ptr, out_type)
+      type = FFI::OnnxType[out_type.read_int]
+
+      case type
+      when :tensor
+        typeinfo = ::FFI::MemoryPointer.new(:pointer)
+        Utils.check_status FFI.api[:GetTensorTypeAndShape].call(out_ptr, typeinfo)
+
+        type, shape = Utils.tensor_type_and_shape(typeinfo)
+
+        tensor_data = ::FFI::MemoryPointer.new(:pointer)
+        Utils.check_status FFI.api[:GetTensorMutableData].call(out_ptr, tensor_data)
+
+        out_size = ::FFI::MemoryPointer.new(:size_t)
+        Utils.check_status FFI.api[:GetTensorShapeElementCount].call(typeinfo.read_pointer, out_size)
+        output_tensor_size = out_size.read(:size_t)
+
+        Utils.release :TensorTypeAndShapeInfo, typeinfo
+
+        # TODO support more types
+        type = FFI::TensorElementDataType[type]
+
+        case output_type
+        when :numo
+          case type
+          when :string
+            result = Numo::RObject.new(shape)
+            result.allocate
+            create_strings_from_onnx_value(out_ptr, output_tensor_size, result)
+          else
+            numo_type = Utils.numo_types[type]
+            Utils.unsupported_type("element", type) unless numo_type
+            numo_type.from_binary(tensor_data.read_pointer.read_bytes(output_tensor_size * numo_type::ELEMENT_BYTE_SIZE), shape)
+          end
+        when :ruby
+          arr =
+            case type
+            when :float, :uint8, :int8, :uint16, :int16, :int32, :int64, :double, :uint32, :uint64
+              tensor_data.read_pointer.send("read_array_of_#{type}", output_tensor_size)
+            when :bool
+              tensor_data.read_pointer.read_array_of_uint8(output_tensor_size).map { |v| v == 1 }
+            when :string
+              create_strings_from_onnx_value(out_ptr, output_tensor_size, [])
+            else
+              Utils.unsupported_type("element", type)
+            end
+
+          reshape(arr, shape)
+        else
+          raise ArgumentError, "Invalid output type: #{output_type}"
+        end
+      when :sequence
+        out = ::FFI::MemoryPointer.new(:size_t)
+        Utils.check_status FFI.api[:GetValueCount].call(out_ptr, out)
+
+        out.read(:size_t).times.map do |i|
+          seq = ::FFI::MemoryPointer.new(:pointer)
+          Utils.check_status FFI.api[:GetValue].call(out_ptr, i, Utils.allocator.read_pointer, seq)
+          create_from_onnx_value(seq.read_pointer, output_type)
+        end
+      when :map
+        type_shape = ::FFI::MemoryPointer.new(:pointer)
+        map_keys = ::FFI::MemoryPointer.new(:pointer)
+        map_values = ::FFI::MemoryPointer.new(:pointer)
+        elem_type = ::FFI::MemoryPointer.new(:int)
+
+        Utils.check_status FFI.api[:GetValue].call(out_ptr, 0, Utils.allocator.read_pointer, map_keys)
+        Utils.check_status FFI.api[:GetValue].call(out_ptr, 1, Utils.allocator.read_pointer, map_values)
+        Utils.check_status FFI.api[:GetTensorTypeAndShape].call(map_keys.read_pointer, type_shape)
+        Utils.check_status FFI.api[:GetTensorElementType].call(type_shape.read_pointer, elem_type)
+        Utils.release :TensorTypeAndShapeInfo, type_shape
+
+        # TODO support more types
+        elem_type = FFI::TensorElementDataType[elem_type.read_int]
+        case elem_type
+        when :int64
+          ret = {}
+          keys = create_from_onnx_value(map_keys.read_pointer, output_type)
+          values = create_from_onnx_value(map_values.read_pointer, output_type)
+          keys.zip(values).each do |k, v|
+            ret[k] = v
+          end
+          ret
+        else
+          Utils.unsupported_type("element", elem_type)
+        end
+      else
+        Utils.unsupported_type("ONNX", type)
+      end
+    end
+
+    def create_strings_from_onnx_value(out_ptr, output_tensor_size, result)
+      len = ::FFI::MemoryPointer.new(:size_t)
+      Utils.check_status FFI.api[:GetStringTensorDataLength].call(out_ptr, len)
+
+      s_len = len.read(:size_t)
+      s = ::FFI::MemoryPointer.new(:uchar, s_len)
+      offsets = ::FFI::MemoryPointer.new(:size_t, output_tensor_size)
+      Utils.check_status FFI.api[:GetStringTensorContent].call(out_ptr, s, s_len, offsets, output_tensor_size)
+
+      offsets = output_tensor_size.times.map { |i| offsets[i].read(:size_t) }
+      offsets << s_len
+      output_tensor_size.times do |i|
+        result[i] = s.get_bytes(offsets[i], offsets[i + 1] - offsets[i])
+      end
+      result
+    end
+
+    def reshape(arr, dims)
+      arr = arr.flatten
+      dims[1..-1].reverse_each do |dim|
+        arr = arr.each_slice(dim)
+      end
+      arr.to_a
+    end
+
+    def self.finalize(addr)
+      # must use proc instead of stabby lambda
+      proc { FFI.api[:ReleaseValue].call(::FFI::Pointer.new(:pointer, addr)) }
+    end
+
+    def self.allocator_info
+      @allocator_info ||= begin
+        allocator_info = ::FFI::MemoryPointer.new(:pointer)
+        Utils.check_status FFI.api[:CreateCpuMemoryInfo].call(1, 0, allocator_info)
+        allocator_info
+      end
+    end
+  end
+end

data/lib/onnxruntime/utils.rb

@@ -5,12 +5,138 @@ module OnnxRuntime
     end
     self.mutex = Mutex.new
 
-    def self.reshape(arr, dims)
-      arr = arr.flatten
-      dims[1..-1].reverse.each do |dim|
-        arr = arr.each_slice(dim)
+    def self.check_status(status)
+      unless status.null?
+        message = api[:GetErrorMessage].call(status).read_string
+        api[:ReleaseStatus].call(status)
+        raise Error, message
+      end
+    end
+
+    def self.api
+      FFI.api
+    end
+
+    def self.release(type, pointer)
+      FFI.api[:"Release#{type}"].call(pointer.read_pointer) if pointer && !pointer.null?
+    end
+
+    def self.unsupported_type(name, type)
+      raise Error, "Unsupported #{name} type: #{type}"
+    end
+
+    def self.tensor_type_and_shape(tensor_info)
+      type = ::FFI::MemoryPointer.new(:int)
+      check_status api[:GetTensorElementType].call(tensor_info.read_pointer, type)
+
+      num_dims_ptr = ::FFI::MemoryPointer.new(:size_t)
+      check_status api[:GetDimensionsCount].call(tensor_info.read_pointer, num_dims_ptr)
+      num_dims = num_dims_ptr.read(:size_t)
+
+      node_dims = ::FFI::MemoryPointer.new(:int64, num_dims)
+      check_status api[:GetDimensions].call(tensor_info.read_pointer, node_dims, num_dims)
+      dims = node_dims.read_array_of_int64(num_dims)
+
+      symbolic_dims = ::FFI::MemoryPointer.new(:pointer, num_dims)
+      check_status api[:GetSymbolicDimensions].call(tensor_info.read_pointer, symbolic_dims, num_dims)
+      named_dims = num_dims.times.map { |i| symbolic_dims[i].read_pointer.read_string }
+      dims = named_dims.zip(dims).map { |n, d| n.empty? ? d : n }
+
+      [type.read_int, dims]
+    end
+
+    def self.node_info(typeinfo)
+      onnx_type = ::FFI::MemoryPointer.new(:int)
+      check_status api[:GetOnnxTypeFromTypeInfo].call(typeinfo.read_pointer, onnx_type)
+
+      type = FFI::OnnxType[onnx_type.read_int]
+      case type
+      when :tensor
+        tensor_info = ::FFI::MemoryPointer.new(:pointer)
+        # don't free tensor_info
+        check_status api[:CastTypeInfoToTensorInfo].call(typeinfo.read_pointer, tensor_info)
+
+        type, shape = Utils.tensor_type_and_shape(tensor_info)
+        {
+          type: "tensor(#{FFI::TensorElementDataType[type]})",
+          shape: shape
+        }
+      when :sequence
+        sequence_type_info = ::FFI::MemoryPointer.new(:pointer)
+        check_status api[:CastTypeInfoToSequenceTypeInfo].call(typeinfo.read_pointer, sequence_type_info)
+        nested_type_info = ::FFI::MemoryPointer.new(:pointer)
+        check_status api[:GetSequenceElementType].call(sequence_type_info.read_pointer, nested_type_info)
+        v = node_info(nested_type_info)[:type]
+
+        {
+          type: "seq(#{v})",
+          shape: []
+        }
+      when :map
+        map_type_info = ::FFI::MemoryPointer.new(:pointer)
+        check_status api[:CastTypeInfoToMapTypeInfo].call(typeinfo.read_pointer, map_type_info)
+
+        # key
+        key_type = ::FFI::MemoryPointer.new(:int)
+        check_status api[:GetMapKeyType].call(map_type_info.read_pointer, key_type)
+        k = FFI::TensorElementDataType[key_type.read_int]
+
+        # value
+        value_type_info = ::FFI::MemoryPointer.new(:pointer)
+        check_status api[:GetMapValueType].call(map_type_info.read_pointer, value_type_info)
+        v = node_info(value_type_info)[:type]
+
+        {
+          type: "map(#{k},#{v})",
+          shape: []
+        }
+      else
+        Utils.unsupported_type("ONNX", type)
+      end
+    ensure
+      release :TypeInfo, typeinfo
+    end
+
+    def self.numo_array?(obj)
+      defined?(Numo::NArray) && obj.is_a?(Numo::NArray)
+    end
+
+    def self.numo_types
+      @numo_types ||= {
+        float: Numo::SFloat,
+        uint8: Numo::UInt8,
+        int8: Numo::Int8,
+        uint16: Numo::UInt16,
+        int16: Numo::Int16,
+        int32: Numo::Int32,
+        int64: Numo::Int64,
+        bool: Numo::UInt8,
+        double: Numo::DFloat,
+        uint32: Numo::UInt32,
+        uint64: Numo::UInt64
+      }
+    end
+
+    def self.input_shape(input)
+      if numo_array?(input)
+        input.shape
+      else
+        shape = []
+        s = input
+        while s.is_a?(Array)
+          shape << s.size
+          s = s.first
+        end
+        shape
+      end
+    end
+
+    def self.allocator
+      @allocator ||= begin
+        allocator = ::FFI::MemoryPointer.new(:pointer)
+        check_status api[:GetAllocatorWithDefaultOptions].call(allocator)
+        allocator
       end
-      arr.to_a
     end
   end
 end

data/lib/onnxruntime/version.rb

@@ -1,3 +1,3 @@
 module OnnxRuntime
-  VERSION = "0.9.1"
+  VERSION = "0.9.3"
 end

data/lib/onnxruntime.rb

@@ -5,6 +5,7 @@ require "ffi"
 require_relative "onnxruntime/datasets"
 require_relative "onnxruntime/inference_session"
 require_relative "onnxruntime/model"
+require_relative "onnxruntime/ort_value"
 require_relative "onnxruntime/utils"
 require_relative "onnxruntime/version"
 

data/vendor/ThirdPartyNotices.txt

@@ -4820,7 +4820,7 @@ SOFTWARE.
 
 ----------------------------------------------------------------------------
 
-This is the MIT/Expat Licence. For more information see:
+This is the MIT/Expat License. For more information see:
 
 1. http://www.opensource.org/licenses/mit-license.php
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: onnxruntime
 version: !ruby/object:Gem::Version
-  version: 0.9.1
+  version: 0.9.3
 platform: aarch64-linux
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-05-23 00:00:00.000000000 Z
+date: 2024-11-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -38,6 +38,7 @@ files:
 - lib/onnxruntime/ffi.rb
 - lib/onnxruntime/inference_session.rb
 - lib/onnxruntime/model.rb
+- lib/onnxruntime/ort_value.rb
 - lib/onnxruntime/utils.rb
 - lib/onnxruntime/version.rb
 - vendor/LICENSE
@@ -62,7 +63,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.9
+rubygems_version: 3.5.16
 signing_key:
 specification_version: 4
 summary: High performance scoring engine for ML models