onnxruntime 0.7.0-arm64-darwin → 0.7.1-arm64-darwin

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 98aa76296341e9cd08b89f21d8c440e8fb4d2f269238e064060b6c16f7885c71
-  data.tar.gz: d28b0a71f55bed6a8087159069a1805cd54c484f015c77337c96c60ecf78beed
+  metadata.gz: 700408871a528f72933d83422befdec6da7f6f8f4855d309dfb1847d250478ec
+  data.tar.gz: 04c5e12036a4165f231351cde9593de78f9b86778acfe42dcdee95942dad5302
 SHA512:
-  metadata.gz: 53003ffe9559aa9bec701f78f38776aabd27bfb73664c6335009f5041137d4daffe6e1f8061f005df3a36cc276c1d09aee58d175de200e9b616d2f86f6ee5d12
-  data.tar.gz: 90b0fa6faf3b88f7a7ad10825437e2a084ce0b41d9145c3dc3cd8ee515e9bf90a421a662bf1e7a6e4a48da569d9ce40a253f3ee646a4ae82c5cca0292fe6833d
+  metadata.gz: b4402d887fc380b45ce5ae0c0609ba6a776b3dcbac081588cd36a55a671ce2d5d6590fe2fad8e180140d17a80cf1770a1fc82015ad8c1da58d062b48ddf52cc5
+  data.tar.gz: f0c23653db526dc18e34917b497deec2478c89b42f3c1c4a2266dfb9579962c83589e56df92624377042bfb40f8378b2ffc7c7fad4601fed108363fd7104764a

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 0.7.1 (2022-03-28)
+
+- Added `graph_description` to metadata
+- Added `free_dimension_overrides_by_denotation` and `free_dimension_overrides_by_name` options
+- Added `profile_file_prefix` option
+- Added `session_config_entries` option
+- Fixed memory leaks
+- Fixed `enable_cpu_mem_arena: false`
+
 ## 0.7.0 (2022-03-27)
 
 - Added platform-specific gems

data/README.md

@@ -64,13 +64,17 @@ OnnxRuntime::Model.new(path_or_bytes, {
   enable_mem_pattern: true,
   enable_profiling: false,
   execution_mode: :sequential,    # :sequential or :parallel
+  free_dimension_overrides_by_denotation: nil,
+  free_dimension_overrides_by_name: nil,
   graph_optimization_level: nil,  # :none, :basic, :extended, or :all
   inter_op_num_threads: nil,
   intra_op_num_threads: nil,
   log_severity_level: 2,
   log_verbosity_level: 0,
   logid: nil,
-  optimized_model_filepath: nil
+  optimized_model_filepath: nil,
+  profile_file_prefix: "onnxruntime_profile_",
+  session_config_entries: nil
 })
 ```
 

data/lib/onnxruntime/ffi.rb

@@ -2,19 +2,13 @@ module OnnxRuntime
   module FFI
     extend ::FFI::Library
 
-    begin
-      ffi_lib OnnxRuntime.ffi_lib
-    rescue LoadError => e
-      if e.message.include?("Library not loaded: /usr/local/opt/libomp/lib/libomp.dylib") && e.message.include?("Reason: image not found")
-        raise LoadError, "OpenMP not found. Run `brew install libomp`"
-      else
-        raise e
-      end
-    end
+    ffi_lib OnnxRuntime.ffi_lib
 
     # https://github.com/microsoft/onnxruntime/blob/master/include/onnxruntime/core/session/onnxruntime_c_api.h
     # keep same order
 
+    ORT_API_VERSION = 11
+
     # enums
     TensorElementDataType = enum(:undefined, :float, :uint8, :int8, :uint16, :int16, :int32, :int64, :string, :bool, :float16, :double, :uint32, :uint64, :complex64, :complex128, :bfloat16)
     OnnxType = enum(:unknown, :tensor, :sequence, :map, :opaque, :sparsetensor)
@@ -84,7 +78,7 @@ module OnnxRuntime
         :GetTensorElementType, callback(%i[pointer pointer], :pointer),
         :GetDimensionsCount, callback(%i[pointer pointer], :pointer),
         :GetDimensions, callback(%i[pointer pointer size_t], :pointer),
-        :GetSymbolicDimensions, callback(%i[], :pointer),
+        :GetSymbolicDimensions, callback(%i[pointer pointer size_t], :pointer),
         :GetTensorShapeElementCount, callback(%i[pointer pointer], :pointer),
         :GetTensorTypeAndShape, callback(%i[pointer pointer], :pointer),
         :GetTypeInfo, callback(%i[pointer pointer], :pointer),
@@ -97,10 +91,10 @@ module OnnxRuntime
         :MemoryInfoGetMemType, callback(%i[], :pointer),
         :MemoryInfoGetType, callback(%i[], :pointer),
         :AllocatorAlloc, callback(%i[], :pointer),
-        :AllocatorFree, callback(%i[], :pointer),
+        :AllocatorFree, callback(%i[pointer pointer], :pointer),
         :AllocatorGetInfo, callback(%i[], :pointer),
         :GetAllocatorWithDefaultOptions, callback(%i[pointer], :pointer),
-        :AddFreeDimensionOverride, callback(%i[], :pointer),
+        :AddFreeDimensionOverride, callback(%i[pointer string int64], :pointer),
         :GetValue, callback(%i[pointer int pointer pointer], :pointer),
         :GetValueCount, callback(%i[pointer pointer], :pointer),
         :CreateValue, callback(%i[], :pointer),
@@ -145,9 +139,91 @@ module OnnxRuntime
         :CreateThreadingOptions, callback(%i[], :pointer),
         :ReleaseThreadingOptions, callback(%i[], :pointer),
         :ModelMetadataGetCustomMetadataMapKeys, callback(%i[pointer pointer pointer pointer], :pointer),
-        :AddFreeDimensionOverrideByName, callback(%i[], :pointer),
+        :AddFreeDimensionOverrideByName, callback(%i[pointer string int64], :pointer),
         :GetAvailableProviders, callback(%i[pointer pointer], :pointer),
-        :ReleaseAvailableProviders, callback(%i[pointer int], :pointer)
+        :ReleaseAvailableProviders, callback(%i[pointer int], :pointer),
+        :GetStringTensorElementLength, callback(%i[], :pointer),
+        :GetStringTensorElement, callback(%i[], :pointer),
+        :FillStringTensorElement, callback(%i[], :pointer),
+        :AddSessionConfigEntry, callback(%i[pointer string string], :pointer),
+        :CreateAllocator, callback(%i[], :pointer),
+        :ReleaseAllocator, callback(%i[], :pointer),
+        :RunWithBinding, callback(%i[], :pointer),
+        :CreateIoBinding, callback(%i[], :pointer),
+        :ReleaseIoBinding, callback(%i[], :pointer),
+        :BindInput, callback(%i[], :pointer),
+        :BindOutput, callback(%i[], :pointer),
+        :BindOutputToDevice, callback(%i[], :pointer),
+        :GetBoundOutputNames, callback(%i[], :pointer),
+        :GetBoundOutputValues, callback(%i[], :pointer),
+        :ClearBoundInputs, callback(%i[], :pointer),
+        :ClearBoundOutputs, callback(%i[], :pointer),
+        :TensorAt, callback(%i[], :pointer),
+        :CreateAndRegisterAllocator, callback(%i[], :pointer),
+        :SetLanguageProjection, callback(%i[], :pointer),
+        :SessionGetProfilingStartTimeNs, callback(%i[], :pointer),
+        :SetGlobalIntraOpNumThreads, callback(%i[], :pointer),
+        :SetGlobalInterOpNumThreads, callback(%i[], :pointer),
+        :SetGlobalSpinControl, callback(%i[], :pointer),
+        :AddInitializer, callback(%i[], :pointer),
+        :CreateEnvWithCustomLoggerAndGlobalThreadPools, callback(%i[], :pointer),
+        :SessionOptionsAppendExecutionProvider_CUDA, callback(%i[], :pointer),
+        :SessionOptionsAppendExecutionProvider_ROCM, callback(%i[], :pointer),
+        :SessionOptionsAppendExecutionProvider_OpenVINO, callback(%i[], :pointer),
+        :SetGlobalDenormalAsZero, callback(%i[], :pointer),
+        :CreateArenaCfg, callback(%i[], :pointer),
+        :ReleaseArenaCfg, callback(%i[], :pointer),
+        :ModelMetadataGetGraphDescription, callback(%i[pointer pointer pointer], :pointer),
+        :SessionOptionsAppendExecutionProvider_TensorRT, callback(%i[], :pointer),
+        :SetCurrentGpuDeviceId, callback(%i[], :pointer),
+        :GetCurrentGpuDeviceId, callback(%i[], :pointer),
+        :KernelInfoGetAttributeArray_float, callback(%i[], :pointer),
+        :KernelInfoGetAttributeArray_int64, callback(%i[], :pointer),
+        :CreateArenaCfgV2, callback(%i[], :pointer),
+        :AddRunConfigEntry, callback(%i[], :pointer),
+        :CreatePrepackedWeightsContainer, callback(%i[], :pointer),
+        :ReleasePrepackedWeightsContainer, callback(%i[], :pointer),
+        :CreateSessionWithPrepackedWeightsContainer, callback(%i[], :pointer),
+        :CreateSessionFromArrayWithPrepackedWeightsContainer, callback(%i[], :pointer),
+        :SessionOptionsAppendExecutionProvider_TensorRT_V2, callback(%i[], :pointer),
+        :CreateTensorRTProviderOptions, callback(%i[], :pointer),
+        :UpdateTensorRTProviderOptions, callback(%i[], :pointer),
+        :GetTensorRTProviderOptionsAsString, callback(%i[], :pointer),
+        :ReleaseTensorRTProviderOptions, callback(%i[], :pointer),
+        :EnableOrtCustomOps, callback(%i[], :pointer),
+        :RegisterAllocator, callback(%i[], :pointer),
+        :UnregisterAllocator, callback(%i[], :pointer),
+        :IsSparseTensor, callback(%i[], :pointer),
+        :CreateSparseTensorAsOrtValue, callback(%i[], :pointer),
+        :FillSparseTensorCoo, callback(%i[], :pointer),
+        :FillSparseTensorCsr, callback(%i[], :pointer),
+        :FillSparseTensorBlockSparse, callback(%i[], :pointer),
+        :CreateSparseTensorWithValuesAsOrtValue, callback(%i[], :pointer),
+        :UseCsrIndices, callback(%i[], :pointer),
+        :UseBlockSparseIndices, callback(%i[], :pointer),
+        :GetSparseTensorFormat, callback(%i[], :pointer),
+        :GetSparseTensorValuesTypeAndShape, callback(%i[], :pointer),
+        :GetSparseTensorValues, callback(%i[], :pointer),
+        :GetSparseTensorIndicesTypeShape, callback(%i[], :pointer),
+        :GetSparseTensorIndices, callback(%i[], :pointer),
+        :HasValue, callback(%i[], :pointer),
+        :KernelContext_GetGPUComputeStream, callback(%i[], :pointer),
+        :GetTensorMemoryInfo, callback(%i[], :pointer),
+        :GetExecutionProviderApi, callback(%i[], :pointer),
+        :SessionOptionsSetCustomCreateThreadFn, callback(%i[], :pointer),
+        :SessionOptionsSetCustomThreadCreationOptions, callback(%i[], :pointer),
+        :SessionOptionsSetCustomJoinThreadFn, callback(%i[], :pointer),
+        :SetGlobalCustomCreateThreadFn, callback(%i[], :pointer),
+        :SetGlobalCustomThreadCreationOptions, callback(%i[], :pointer),
+        :SetGlobalCustomJoinThreadFn, callback(%i[], :pointer),
+        :SynchronizeBoundInputs, callback(%i[], :pointer),
+        :SynchronizeBoundOutputs, callback(%i[], :pointer),
+        :SessionOptionsAppendExecutionProvider_CUDA_V2, callback(%i[], :pointer),
+        :CreateCUDAProviderOptions, callback(%i[], :pointer),
+        :UpdateCUDAProviderOptions, callback(%i[], :pointer),
+        :GetCUDAProviderOptionsAsString, callback(%i[], :pointer),
+        :ReleaseCUDAProviderOptions, callback(%i[], :pointer),
+        :SessionOptionsAppendExecutionProvider_MIGraphX, callback(%i[], :pointer)
     end
 
     class ApiBase < ::FFI::Struct

data/lib/onnxruntime/inference_session.rb

@@ -2,19 +2,41 @@ module OnnxRuntime
   class InferenceSession
     attr_reader :inputs, :outputs
 
-    def initialize(path_or_bytes, enable_cpu_mem_arena: true, enable_mem_pattern: true, enable_profiling: false, execution_mode: nil, graph_optimization_level: nil, inter_op_num_threads: nil, intra_op_num_threads: nil, log_severity_level: nil, log_verbosity_level: nil, logid: nil, optimized_model_filepath: nil)
+    def initialize(path_or_bytes, enable_cpu_mem_arena: true, enable_mem_pattern: true, enable_profiling: false, execution_mode: nil, free_dimension_overrides_by_denotation: nil, free_dimension_overrides_by_name: nil, graph_optimization_level: nil, inter_op_num_threads: nil, intra_op_num_threads: nil, log_severity_level: nil, log_verbosity_level: nil, logid: nil, optimized_model_filepath: nil, profile_file_prefix: nil, session_config_entries: nil)
       # session options
       session_options = ::FFI::MemoryPointer.new(:pointer)
       check_status api[:CreateSessionOptions].call(session_options)
-      check_status api[:EnableCpuMemArena].call(session_options.read_pointer) if enable_cpu_mem_arena
-      check_status api[:EnableMemPattern].call(session_options.read_pointer) if enable_mem_pattern
-      check_status api[:EnableProfiling].call(session_options.read_pointer, ort_string("onnxruntime_profile_")) if enable_profiling
+      if enable_cpu_mem_arena
+        check_status api[:EnableCpuMemArena].call(session_options.read_pointer)
+      else
+        check_status api[:DisableCpuMemArena].call(session_options.read_pointer)
+      end
+      if enable_mem_pattern
+        check_status api[:EnableMemPattern].call(session_options.read_pointer)
+      else
+        check_status api[:DisableMemPattern].call(session_options.read_pointer)
+      end
+      if enable_profiling
+        check_status api[:EnableProfiling].call(session_options.read_pointer, ort_string(profile_file_prefix || "onnxruntime_profile_"))
+      else
+        check_status api[:DisableProfiling].call(session_options.read_pointer)
+      end
       if execution_mode
         execution_modes = {sequential: 0, parallel: 1}
         mode = execution_modes[execution_mode]
         raise ArgumentError, "Invalid execution mode" unless mode
         check_status api[:SetSessionExecutionMode].call(session_options.read_pointer, mode)
       end
+      if free_dimension_overrides_by_denotation
+        free_dimension_overrides_by_denotation.each do |k, v|
+          check_status api[:AddFreeDimensionOverride].call(session_options.read_pointer, k.to_s, v)
+        end
+      end
+      if free_dimension_overrides_by_name
+        free_dimension_overrides_by_name.each do |k, v|
+          check_status api[:AddFreeDimensionOverrideByName].call(session_options.read_pointer, k.to_s, v)
+        end
+      end
       if graph_optimization_level
         optimization_levels = {none: 0, basic: 1, extended: 2, all: 99}
         level = optimization_levels[graph_optimization_level]
@@ -27,6 +49,11 @@ module OnnxRuntime
       check_status api[:SetSessionLogVerbosityLevel].call(session_options.read_pointer, log_verbosity_level) if log_verbosity_level
       check_status api[:SetSessionLogId].call(session_options.read_pointer, logid) if logid
       check_status api[:SetOptimizedModelFilePath].call(session_options.read_pointer, ort_string(optimized_model_filepath)) if optimized_model_filepath
+      if session_config_entries
+        session_config_entries.each do |k, v|
+          check_status api[:AddSessionConfigEntry].call(session_options.read_pointer, k.to_s, v.to_s)
+        end
+      end
 
       # session
       @session = ::FFI::MemoryPointer.new(:pointer)
@@ -47,6 +74,7 @@ module OnnxRuntime
       ObjectSpace.define_finalizer(self, self.class.finalize(@session))
 
       # input info
+      # don't free allocator
       allocator = ::FFI::MemoryPointer.new(:pointer)
       check_status api[:GetAllocatorWithDefaultOptions].call(allocator)
       @allocator = allocator
@@ -60,9 +88,11 @@ module OnnxRuntime
       num_input_nodes.read(:size_t).times do |i|
         name_ptr = ::FFI::MemoryPointer.new(:string)
         check_status api[:SessionGetInputName].call(read_pointer, i, @allocator.read_pointer, name_ptr)
+        # freed in node_info
         typeinfo = ::FFI::MemoryPointer.new(:pointer)
         check_status api[:SessionGetInputTypeInfo].call(read_pointer, i, typeinfo)
         @inputs << {name: name_ptr.read_pointer.read_string}.merge(node_info(typeinfo))
+        allocator_free name_ptr
       end
 
       # output
@@ -71,12 +101,14 @@ module OnnxRuntime
       num_output_nodes.read(:size_t).times do |i|
         name_ptr = ::FFI::MemoryPointer.new(:string)
         check_status api[:SessionGetOutputName].call(read_pointer, i, allocator.read_pointer, name_ptr)
+        # freed in node_info
         typeinfo = ::FFI::MemoryPointer.new(:pointer)
         check_status api[:SessionGetOutputTypeInfo].call(read_pointer, i, typeinfo)
         @outputs << {name: name_ptr.read_pointer.read_string}.merge(node_info(typeinfo))
+        allocator_free name_ptr
       end
     ensure
-      # release :SessionOptions, session_options
+      release :SessionOptions, session_options
     end
 
     # TODO support logid
@@ -112,6 +144,7 @@ module OnnxRuntime
           release :Value, input_tensor[i]
         end
       end
+      # output values released in create_from_onnx_value
     end
 
     def modelmeta
@@ -120,6 +153,7 @@ module OnnxRuntime
       description = ::FFI::MemoryPointer.new(:string)
       domain = ::FFI::MemoryPointer.new(:string)
       graph_name = ::FFI::MemoryPointer.new(:string)
+      graph_description = ::FFI::MemoryPointer.new(:string)
       producer_name = ::FFI::MemoryPointer.new(:string)
       version = ::FFI::MemoryPointer.new(:int64_t)
 
@@ -129,15 +163,21 @@ module OnnxRuntime
       custom_metadata_map = {}
       check_status api[:ModelMetadataGetCustomMetadataMapKeys].call(metadata.read_pointer, @allocator.read_pointer, keys, num_keys)
       num_keys.read(:int64_t).times do |i|
-        key = keys.read_pointer[i * ::FFI::Pointer.size].read_pointer.read_string
+        key_ptr = keys.read_pointer[i * ::FFI::Pointer.size]
+        key = key_ptr.read_pointer.read_string
         value = ::FFI::MemoryPointer.new(:string)
         check_status api[:ModelMetadataLookupCustomMetadataMap].call(metadata.read_pointer, @allocator.read_pointer, key, value)
         custom_metadata_map[key] = value.read_pointer.read_string
+
+        allocator_free key_ptr
+        allocator_free value
       end
+      allocator_free keys
 
       check_status api[:ModelMetadataGetDescription].call(metadata.read_pointer, @allocator.read_pointer, description)
       check_status api[:ModelMetadataGetDomain].call(metadata.read_pointer, @allocator.read_pointer, domain)
       check_status api[:ModelMetadataGetGraphName].call(metadata.read_pointer, @allocator.read_pointer, graph_name)
+      check_status api[:ModelMetadataGetGraphDescription].call(metadata.read_pointer, @allocator.read_pointer, graph_description)
       check_status api[:ModelMetadataGetProducerName].call(metadata.read_pointer, @allocator.read_pointer, producer_name)
       check_status api[:ModelMetadataGetVersion].call(metadata.read_pointer, version)
 
@@ -146,11 +186,17 @@ module OnnxRuntime
         description: description.read_pointer.read_string,
         domain: domain.read_pointer.read_string,
         graph_name: graph_name.read_pointer.read_string,
+        graph_description: graph_description.read_pointer.read_string,
         producer_name: producer_name.read_pointer.read_string,
         version: version.read(:int64_t)
       }
     ensure
       release :ModelMetadata, metadata
+      allocator_free description
+      allocator_free domain
+      allocator_free graph_name
+      allocator_free graph_description
+      allocator_free producer_name
     end
 
     # return value has double underscore like Python
@@ -250,6 +296,8 @@ module OnnxRuntime
       refs << allocator_info
 
       input_tensor
+    ensure
+      release :MemoryInfo, allocator_info
     end
 
     def create_node_names(names, refs)
@@ -352,6 +400,8 @@ module OnnxRuntime
       else
         unsupported_type("ONNX", type)
       end
+    ensure
+      api[:ReleaseValue].call(out_ptr) unless out_ptr.null?
     end
 
     def create_strings_from_onnx_value(out_ptr, output_tensor_size, result)
@@ -445,8 +495,15 @@ module OnnxRuntime
 
       node_dims = ::FFI::MemoryPointer.new(:int64, num_dims)
       check_status api[:GetDimensions].call(tensor_info.read_pointer, node_dims, num_dims)
+      dims = node_dims.read_array_of_int64(num_dims)
+
+      # TODO uncomment in 0.8.0
+      # symbolic_dims = ::FFI::MemoryPointer.new(:pointer, num_dims)
+      # check_status api[:GetSymbolicDimensions].call(tensor_info.read_pointer, symbolic_dims, num_dims)
+      # named_dims = num_dims.times.map { |i| symbolic_dims[i].read_pointer.read_string }
+      # dims = named_dims.zip(dims).map { |n, d| n.empty? ? d : n }
 
-      [type.read_int, node_dims.read_array_of_int64(num_dims)]
+      [type.read_int, dims]
     end
 
     def unsupported_type(name, type)
@@ -485,8 +542,12 @@ module OnnxRuntime
       self.class.release(*args)
     end
 
+    def allocator_free(ptr)
+      api[:AllocatorFree].call(@allocator.read_pointer, ptr.read_pointer)
+    end
+
     def self.api
-      @api ||= FFI.OrtGetApiBase[:GetApi].call(4)
+      @api ||= FFI.OrtGetApiBase[:GetApi].call(FFI::ORT_API_VERSION)
     end
 
     def self.release(type, pointer)

data/lib/onnxruntime/version.rb

@@ -1,3 +1,3 @@
 module OnnxRuntime
-  VERSION = "0.7.0"
+  VERSION = "0.7.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: onnxruntime
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.7.1
 platform: arm64-darwin
 authors:
 - Andrew Kane