onnxruntime 0.7.0 → 0.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c477d6cadf767f4d29b6d097c3ae558d2fe31145ccef6a97ae5ec19e84090da0
-  data.tar.gz: 9b0e98d3e5fd8dcd7b3c55819690c4f60c03d4edf3a3beb75459ca7e70e92dd4
+  metadata.gz: 74295d3afa2b3642e1dac491b3c6ffaf7f5aa25dd349aaf14e0562730700de1c
+  data.tar.gz: ee3ec2f1304b139c7e7cf6fa445502c0b9246ae15ca1134e834618b676527cef
 SHA512:
-  metadata.gz: 7805929c5e5e29e9418753dc18bd0c74eb3bd3062b7402960e09668134e89c6660ab3d44bb0915435b40cc6a8cff13bfae4aa7f40138b20f0d8f06949ea0a400
-  data.tar.gz: d11b6a32348bfffca58634009393e9282149bb3009bc66be82f7ec3eab887c4ccb017c4a2ee8e36eebe98434febbce2bd67210a9f56895e1b34b7f2205003914
+  metadata.gz: 988f4aded844974c674d2fd19a9c305a00deb32bcb9812b190e4cd17a005aed5cb39a3b3d67ad47ee8ad63082b42646ac356067c17def9a67769a80ee0f7d1e8
+  data.tar.gz: 3317c92f82bee23a4a518c8f097e7e8f73c68019b89e9846da09f05d1fb3cac64751b6c71c93bd9e80215631056b3267a2f3dba93e54ab5f551840f7f1840e8f

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 0.7.1 (2022-03-28)
+
+- Added `graph_description` to metadata
+- Added `free_dimension_overrides_by_denotation` and `free_dimension_overrides_by_name` options
+- Added `profile_file_prefix` option
+- Added `session_config_entries` option
+- Fixed memory leaks
+- Fixed `enable_cpu_mem_arena: false`
+
 ## 0.7.0 (2022-03-27)
 
 - Added platform-specific gems

data/README.md

@@ -64,13 +64,17 @@ OnnxRuntime::Model.new(path_or_bytes, {
   enable_mem_pattern: true,
   enable_profiling: false,
   execution_mode: :sequential,    # :sequential or :parallel
+  free_dimension_overrides_by_denotation: nil,
+  free_dimension_overrides_by_name: nil,
   graph_optimization_level: nil,  # :none, :basic, :extended, or :all
   inter_op_num_threads: nil,
   intra_op_num_threads: nil,
   log_severity_level: 2,
   log_verbosity_level: 0,
   logid: nil,
-  optimized_model_filepath: nil
+  optimized_model_filepath: nil,
+  profile_file_prefix: "onnxruntime_profile_",
+  session_config_entries: nil
 })
 ```
 

data/lib/onnxruntime/ffi.rb

@@ -2,19 +2,13 @@ module OnnxRuntime
   module FFI
     extend ::FFI::Library
 
-    begin
-      ffi_lib OnnxRuntime.ffi_lib
-    rescue LoadError => e
-      if e.message.include?("Library not loaded: /usr/local/opt/libomp/lib/libomp.dylib") && e.message.include?("Reason: image not found")
-        raise LoadError, "OpenMP not found. Run `brew install libomp`"
-      else
-        raise e
-      end
-    end
+    ffi_lib OnnxRuntime.ffi_lib
 
     # https://github.com/microsoft/onnxruntime/blob/master/include/onnxruntime/core/session/onnxruntime_c_api.h
     # keep same order
 
+    ORT_API_VERSION = 11
+
     # enums
     TensorElementDataType = enum(:undefined, :float, :uint8, :int8, :uint16, :int16, :int32, :int64, :string, :bool, :float16, :double, :uint32, :uint64, :complex64, :complex128, :bfloat16)
     OnnxType = enum(:unknown, :tensor, :sequence, :map, :opaque, :sparsetensor)
@@ -84,7 +78,7 @@ module OnnxRuntime
         :GetTensorElementType, callback(%i[pointer pointer], :pointer),
         :GetDimensionsCount, callback(%i[pointer pointer], :pointer),
         :GetDimensions, callback(%i[pointer pointer size_t], :pointer),
-        :GetSymbolicDimensions, callback(%i[], :pointer),
+        :GetSymbolicDimensions, callback(%i[pointer pointer size_t], :pointer),
         :GetTensorShapeElementCount, callback(%i[pointer pointer], :pointer),
         :GetTensorTypeAndShape, callback(%i[pointer pointer], :pointer),
         :GetTypeInfo, callback(%i[pointer pointer], :pointer),
@@ -97,10 +91,10 @@ module OnnxRuntime
         :MemoryInfoGetMemType, callback(%i[], :pointer),
         :MemoryInfoGetType, callback(%i[], :pointer),
         :AllocatorAlloc, callback(%i[], :pointer),
-        :AllocatorFree, callback(%i[], :pointer),
+        :AllocatorFree, callback(%i[pointer pointer], :pointer),
         :AllocatorGetInfo, callback(%i[], :pointer),
         :GetAllocatorWithDefaultOptions, callback(%i[pointer], :pointer),
-        :AddFreeDimensionOverride, callback(%i[], :pointer),
+        :AddFreeDimensionOverride, callback(%i[pointer string int64], :pointer),
         :GetValue, callback(%i[pointer int pointer pointer], :pointer),
         :GetValueCount, callback(%i[pointer pointer], :pointer),
         :CreateValue, callback(%i[], :pointer),
@@ -145,9 +139,91 @@ module OnnxRuntime
         :CreateThreadingOptions, callback(%i[], :pointer),
         :ReleaseThreadingOptions, callback(%i[], :pointer),
         :ModelMetadataGetCustomMetadataMapKeys, callback(%i[pointer pointer pointer pointer], :pointer),
-        :AddFreeDimensionOverrideByName, callback(%i[], :pointer),
+        :AddFreeDimensionOverrideByName, callback(%i[pointer string int64], :pointer),
         :GetAvailableProviders, callback(%i[pointer pointer], :pointer),
-        :ReleaseAvailableProviders, callback(%i[pointer int], :pointer)
+        :ReleaseAvailableProviders, callback(%i[pointer int], :pointer),
+        :GetStringTensorElementLength, callback(%i[], :pointer),
+        :GetStringTensorElement, callback(%i[], :pointer),
+        :FillStringTensorElement, callback(%i[], :pointer),
+        :AddSessionConfigEntry, callback(%i[pointer string string], :pointer),
+        :CreateAllocator, callback(%i[], :pointer),
+        :ReleaseAllocator, callback(%i[], :pointer),
+        :RunWithBinding, callback(%i[], :pointer),
+        :CreateIoBinding, callback(%i[], :pointer),
+        :ReleaseIoBinding, callback(%i[], :pointer),
+        :BindInput, callback(%i[], :pointer),
+        :BindOutput, callback(%i[], :pointer),
+        :BindOutputToDevice, callback(%i[], :pointer),
+        :GetBoundOutputNames, callback(%i[], :pointer),
+        :GetBoundOutputValues, callback(%i[], :pointer),
+        :ClearBoundInputs, callback(%i[], :pointer),
+        :ClearBoundOutputs, callback(%i[], :pointer),
+        :TensorAt, callback(%i[], :pointer),
+        :CreateAndRegisterAllocator, callback(%i[], :pointer),
+        :SetLanguageProjection, callback(%i[], :pointer),
+        :SessionGetProfilingStartTimeNs, callback(%i[], :pointer),
+        :SetGlobalIntraOpNumThreads, callback(%i[], :pointer),
+        :SetGlobalInterOpNumThreads, callback(%i[], :pointer),
+        :SetGlobalSpinControl, callback(%i[], :pointer),
+        :AddInitializer, callback(%i[], :pointer),
+        :CreateEnvWithCustomLoggerAndGlobalThreadPools, callback(%i[], :pointer),
+        :SessionOptionsAppendExecutionProvider_CUDA, callback(%i[], :pointer),
+        :SessionOptionsAppendExecutionProvider_ROCM, callback(%i[], :pointer),
+        :SessionOptionsAppendExecutionProvider_OpenVINO, callback(%i[], :pointer),
+        :SetGlobalDenormalAsZero, callback(%i[], :pointer),
+        :CreateArenaCfg, callback(%i[], :pointer),
+        :ReleaseArenaCfg, callback(%i[], :pointer),
+        :ModelMetadataGetGraphDescription, callback(%i[pointer pointer pointer], :pointer),
+        :SessionOptionsAppendExecutionProvider_TensorRT, callback(%i[], :pointer),
+        :SetCurrentGpuDeviceId, callback(%i[], :pointer),
+        :GetCurrentGpuDeviceId, callback(%i[], :pointer),
+        :KernelInfoGetAttributeArray_float, callback(%i[], :pointer),
+        :KernelInfoGetAttributeArray_int64, callback(%i[], :pointer),
+        :CreateArenaCfgV2, callback(%i[], :pointer),
+        :AddRunConfigEntry, callback(%i[], :pointer),
+        :CreatePrepackedWeightsContainer, callback(%i[], :pointer),
+        :ReleasePrepackedWeightsContainer, callback(%i[], :pointer),
+        :CreateSessionWithPrepackedWeightsContainer, callback(%i[], :pointer),
+        :CreateSessionFromArrayWithPrepackedWeightsContainer, callback(%i[], :pointer),
+        :SessionOptionsAppendExecutionProvider_TensorRT_V2, callback(%i[], :pointer),
+        :CreateTensorRTProviderOptions, callback(%i[], :pointer),
+        :UpdateTensorRTProviderOptions, callback(%i[], :pointer),
+        :GetTensorRTProviderOptionsAsString, callback(%i[], :pointer),
+        :ReleaseTensorRTProviderOptions, callback(%i[], :pointer),
+        :EnableOrtCustomOps, callback(%i[], :pointer),
+        :RegisterAllocator, callback(%i[], :pointer),
+        :UnregisterAllocator, callback(%i[], :pointer),
+        :IsSparseTensor, callback(%i[], :pointer),
+        :CreateSparseTensorAsOrtValue, callback(%i[], :pointer),
+        :FillSparseTensorCoo, callback(%i[], :pointer),
+        :FillSparseTensorCsr, callback(%i[], :pointer),
+        :FillSparseTensorBlockSparse, callback(%i[], :pointer),
+        :CreateSparseTensorWithValuesAsOrtValue, callback(%i[], :pointer),
+        :UseCsrIndices, callback(%i[], :pointer),
+        :UseBlockSparseIndices, callback(%i[], :pointer),
+        :GetSparseTensorFormat, callback(%i[], :pointer),
+        :GetSparseTensorValuesTypeAndShape, callback(%i[], :pointer),
+        :GetSparseTensorValues, callback(%i[], :pointer),
+        :GetSparseTensorIndicesTypeShape, callback(%i[], :pointer),
+        :GetSparseTensorIndices, callback(%i[], :pointer),
+        :HasValue, callback(%i[], :pointer),
+        :KernelContext_GetGPUComputeStream, callback(%i[], :pointer),
+        :GetTensorMemoryInfo, callback(%i[], :pointer),
+        :GetExecutionProviderApi, callback(%i[], :pointer),
+        :SessionOptionsSetCustomCreateThreadFn, callback(%i[], :pointer),
+        :SessionOptionsSetCustomThreadCreationOptions, callback(%i[], :pointer),
+        :SessionOptionsSetCustomJoinThreadFn, callback(%i[], :pointer),
+        :SetGlobalCustomCreateThreadFn, callback(%i[], :pointer),
+        :SetGlobalCustomThreadCreationOptions, callback(%i[], :pointer),
+        :SetGlobalCustomJoinThreadFn, callback(%i[], :pointer),
+        :SynchronizeBoundInputs, callback(%i[], :pointer),
+        :SynchronizeBoundOutputs, callback(%i[], :pointer),
+        :SessionOptionsAppendExecutionProvider_CUDA_V2, callback(%i[], :pointer),
+        :CreateCUDAProviderOptions, callback(%i[], :pointer),
+        :UpdateCUDAProviderOptions, callback(%i[], :pointer),
+        :GetCUDAProviderOptionsAsString, callback(%i[], :pointer),
+        :ReleaseCUDAProviderOptions, callback(%i[], :pointer),
+        :SessionOptionsAppendExecutionProvider_MIGraphX, callback(%i[], :pointer)
     end
 
     class ApiBase < ::FFI::Struct

data/lib/onnxruntime/inference_session.rb

@@ -2,19 +2,41 @@ module OnnxRuntime
   class InferenceSession
     attr_reader :inputs, :outputs
 
-    def initialize(path_or_bytes, enable_cpu_mem_arena: true, enable_mem_pattern: true, enable_profiling: false, execution_mode: nil, graph_optimization_level: nil, inter_op_num_threads: nil, intra_op_num_threads: nil, log_severity_level: nil, log_verbosity_level: nil, logid: nil, optimized_model_filepath: nil)
+    def initialize(path_or_bytes, enable_cpu_mem_arena: true, enable_mem_pattern: true, enable_profiling: false, execution_mode: nil, free_dimension_overrides_by_denotation: nil, free_dimension_overrides_by_name: nil, graph_optimization_level: nil, inter_op_num_threads: nil, intra_op_num_threads: nil, log_severity_level: nil, log_verbosity_level: nil, logid: nil, optimized_model_filepath: nil, profile_file_prefix: nil, session_config_entries: nil)
       # session options
       session_options = ::FFI::MemoryPointer.new(:pointer)
       check_status api[:CreateSessionOptions].call(session_options)
-      check_status api[:EnableCpuMemArena].call(session_options.read_pointer) if enable_cpu_mem_arena
-      check_status api[:EnableMemPattern].call(session_options.read_pointer) if enable_mem_pattern
-      check_status api[:EnableProfiling].call(session_options.read_pointer, ort_string("onnxruntime_profile_")) if enable_profiling
+      if enable_cpu_mem_arena
+        check_status api[:EnableCpuMemArena].call(session_options.read_pointer)
+      else
+        check_status api[:DisableCpuMemArena].call(session_options.read_pointer)
+      end
+      if enable_mem_pattern
+        check_status api[:EnableMemPattern].call(session_options.read_pointer)
+      else
+        check_status api[:DisableMemPattern].call(session_options.read_pointer)
+      end
+      if enable_profiling
+        check_status api[:EnableProfiling].call(session_options.read_pointer, ort_string(profile_file_prefix || "onnxruntime_profile_"))
+      else
+        check_status api[:DisableProfiling].call(session_options.read_pointer)
+      end
       if execution_mode
         execution_modes = {sequential: 0, parallel: 1}
         mode = execution_modes[execution_mode]
         raise ArgumentError, "Invalid execution mode" unless mode
         check_status api[:SetSessionExecutionMode].call(session_options.read_pointer, mode)
       end
+      if free_dimension_overrides_by_denotation
+        free_dimension_overrides_by_denotation.each do |k, v|
+          check_status api[:AddFreeDimensionOverride].call(session_options.read_pointer, k.to_s, v)
+        end
+      end
+      if free_dimension_overrides_by_name
+        free_dimension_overrides_by_name.each do |k, v|
+          check_status api[:AddFreeDimensionOverrideByName].call(session_options.read_pointer, k.to_s, v)
+        end
+      end
       if graph_optimization_level
         optimization_levels = {none: 0, basic: 1, extended: 2, all: 99}
         level = optimization_levels[graph_optimization_level]
@@ -27,6 +49,11 @@ module OnnxRuntime
       check_status api[:SetSessionLogVerbosityLevel].call(session_options.read_pointer, log_verbosity_level) if log_verbosity_level
       check_status api[:SetSessionLogId].call(session_options.read_pointer, logid) if logid
       check_status api[:SetOptimizedModelFilePath].call(session_options.read_pointer, ort_string(optimized_model_filepath)) if optimized_model_filepath
+      if session_config_entries
+        session_config_entries.each do |k, v|
+          check_status api[:AddSessionConfigEntry].call(session_options.read_pointer, k.to_s, v.to_s)
+        end
+      end
 
       # session
       @session = ::FFI::MemoryPointer.new(:pointer)
@@ -47,6 +74,7 @@ module OnnxRuntime
       ObjectSpace.define_finalizer(self, self.class.finalize(@session))
 
       # input info
+      # don't free allocator
       allocator = ::FFI::MemoryPointer.new(:pointer)
       check_status api[:GetAllocatorWithDefaultOptions].call(allocator)
       @allocator = allocator
@@ -60,9 +88,11 @@ module OnnxRuntime
       num_input_nodes.read(:size_t).times do |i|
         name_ptr = ::FFI::MemoryPointer.new(:string)
         check_status api[:SessionGetInputName].call(read_pointer, i, @allocator.read_pointer, name_ptr)
+        # freed in node_info
         typeinfo = ::FFI::MemoryPointer.new(:pointer)
         check_status api[:SessionGetInputTypeInfo].call(read_pointer, i, typeinfo)
         @inputs << {name: name_ptr.read_pointer.read_string}.merge(node_info(typeinfo))
+        allocator_free name_ptr
       end
 
       # output
@@ -71,12 +101,14 @@ module OnnxRuntime
       num_output_nodes.read(:size_t).times do |i|
         name_ptr = ::FFI::MemoryPointer.new(:string)
         check_status api[:SessionGetOutputName].call(read_pointer, i, allocator.read_pointer, name_ptr)
+        # freed in node_info
         typeinfo = ::FFI::MemoryPointer.new(:pointer)
         check_status api[:SessionGetOutputTypeInfo].call(read_pointer, i, typeinfo)
         @outputs << {name: name_ptr.read_pointer.read_string}.merge(node_info(typeinfo))
+        allocator_free name_ptr
       end
     ensure
-      # release :SessionOptions, session_options
+      release :SessionOptions, session_options
     end
 
     # TODO support logid
@@ -112,6 +144,7 @@ module OnnxRuntime
           release :Value, input_tensor[i]
         end
       end
+      # output values released in create_from_onnx_value
     end
 
     def modelmeta
@@ -120,6 +153,7 @@ module OnnxRuntime
       description = ::FFI::MemoryPointer.new(:string)
       domain = ::FFI::MemoryPointer.new(:string)
       graph_name = ::FFI::MemoryPointer.new(:string)
+      graph_description = ::FFI::MemoryPointer.new(:string)
       producer_name = ::FFI::MemoryPointer.new(:string)
       version = ::FFI::MemoryPointer.new(:int64_t)
 
@@ -129,15 +163,21 @@ module OnnxRuntime
       custom_metadata_map = {}
       check_status api[:ModelMetadataGetCustomMetadataMapKeys].call(metadata.read_pointer, @allocator.read_pointer, keys, num_keys)
       num_keys.read(:int64_t).times do |i|
-        key = keys.read_pointer[i * ::FFI::Pointer.size].read_pointer.read_string
+        key_ptr = keys.read_pointer[i * ::FFI::Pointer.size]
+        key = key_ptr.read_pointer.read_string
         value = ::FFI::MemoryPointer.new(:string)
         check_status api[:ModelMetadataLookupCustomMetadataMap].call(metadata.read_pointer, @allocator.read_pointer, key, value)
         custom_metadata_map[key] = value.read_pointer.read_string
+
+        allocator_free key_ptr
+        allocator_free value
       end
+      allocator_free keys
 
       check_status api[:ModelMetadataGetDescription].call(metadata.read_pointer, @allocator.read_pointer, description)
       check_status api[:ModelMetadataGetDomain].call(metadata.read_pointer, @allocator.read_pointer, domain)
       check_status api[:ModelMetadataGetGraphName].call(metadata.read_pointer, @allocator.read_pointer, graph_name)
+      check_status api[:ModelMetadataGetGraphDescription].call(metadata.read_pointer, @allocator.read_pointer, graph_description)
       check_status api[:ModelMetadataGetProducerName].call(metadata.read_pointer, @allocator.read_pointer, producer_name)
       check_status api[:ModelMetadataGetVersion].call(metadata.read_pointer, version)
 
@@ -146,11 +186,17 @@ module OnnxRuntime
         description: description.read_pointer.read_string,
         domain: domain.read_pointer.read_string,
         graph_name: graph_name.read_pointer.read_string,
+        graph_description: graph_description.read_pointer.read_string,
         producer_name: producer_name.read_pointer.read_string,
         version: version.read(:int64_t)
       }
     ensure
       release :ModelMetadata, metadata
+      allocator_free description
+      allocator_free domain
+      allocator_free graph_name
+      allocator_free graph_description
+      allocator_free producer_name
     end
 
     # return value has double underscore like Python
@@ -250,6 +296,8 @@ module OnnxRuntime
       refs << allocator_info
 
       input_tensor
+    ensure
+      release :MemoryInfo, allocator_info
     end
 
     def create_node_names(names, refs)
@@ -352,6 +400,8 @@ module OnnxRuntime
       else
         unsupported_type("ONNX", type)
       end
+    ensure
+      api[:ReleaseValue].call(out_ptr) unless out_ptr.null?
     end
 
     def create_strings_from_onnx_value(out_ptr, output_tensor_size, result)
@@ -445,8 +495,15 @@ module OnnxRuntime
 
       node_dims = ::FFI::MemoryPointer.new(:int64, num_dims)
       check_status api[:GetDimensions].call(tensor_info.read_pointer, node_dims, num_dims)
+      dims = node_dims.read_array_of_int64(num_dims)
+
+      # TODO uncomment in 0.8.0
+      # symbolic_dims = ::FFI::MemoryPointer.new(:pointer, num_dims)
+      # check_status api[:GetSymbolicDimensions].call(tensor_info.read_pointer, symbolic_dims, num_dims)
+      # named_dims = num_dims.times.map { |i| symbolic_dims[i].read_pointer.read_string }
+      # dims = named_dims.zip(dims).map { |n, d| n.empty? ? d : n }
 
-      [type.read_int, node_dims.read_array_of_int64(num_dims)]
+      [type.read_int, dims]
     end
 
     def unsupported_type(name, type)
@@ -485,8 +542,12 @@ module OnnxRuntime
       self.class.release(*args)
     end
 
+    def allocator_free(ptr)
+      api[:AllocatorFree].call(@allocator.read_pointer, ptr.read_pointer)
+    end
+
     def self.api
-      @api ||= FFI.OrtGetApiBase[:GetApi].call(4)
+      @api ||= FFI.OrtGetApiBase[:GetApi].call(FFI::ORT_API_VERSION)
     end
 
     def self.release(type, pointer)

data/lib/onnxruntime/version.rb

@@ -1,3 +1,3 @@
 module OnnxRuntime
-  VERSION = "0.7.0"
+  VERSION = "0.7.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: onnxruntime
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.7.1
 platform: ruby
 authors:
 - Andrew Kane