onnxruntime 0.9.1-x86_64-linux → 0.9.3-x86_64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d6420c30204279c92682f6a323b82ca4b0918f4cc38c3dd5ecbc0078316dff06
4
- data.tar.gz: 77bc593d96ee862b414a5056633f1632c6be112f4878108820f91285e511f836
3
+ metadata.gz: a569d35fb99b55ea5827b2ed431ae041f306a37e4379d29b71d2c46f94319c0d
4
+ data.tar.gz: 150d205dea67483dbe46f349c59796f0e3768b5bbbab59cad3687adf2151025b
5
5
  SHA512:
6
- metadata.gz: 1666588bccef5788102d4ac9fec70e6846f6c2b1d6fc7c787b5eec5e300fb2519beccbf0e45d98d0ab391cc86fe8489025a456cfb8ed69b9e501744453da837f
7
- data.tar.gz: 9b8e7bd216994ec0d190881f21785918d031a353646e5c1cb14d747fae91475cfad6fb67ecf595b738d3ee6259c2d246202169d5cb5a3009fe2d73f5802c09be
6
+ metadata.gz: e91449cbfcb7b901bfc6e56f3f17bbb94115d742e9af47c13c56717273a09f9ad5b119507d7f1a6b3171955604a9c47bc6dc0c6563052229c4a149cf5d85b65d
7
+ data.tar.gz: 97719176a1d55f818b1c5dea42f75de59fdad552f12e5ae9c42659bcf10e9d563870b3df7f874634a2179aadf0910b8e1b4d7aa037fa7e70b584c9c1340f9a65
data/CHANGELOG.md CHANGED
@@ -1,3 +1,14 @@
1
+ ## 0.9.3 (2024-11-01)
2
+
3
+ - Updated ONNX Runtime to 1.20.0
4
+ - Added experimental `OrtValue` class
5
+ - Added experimental `run_with_ort_values` method
6
+
7
+ ## 0.9.2 (2024-09-04)
8
+
9
+ - Updated ONNX Runtime to 1.19.2
10
+ - Added support for CoreML
11
+
1
12
  ## 0.9.1 (2024-05-22)
2
13
 
3
14
  - Updated ONNX Runtime to 1.18.0
data/README.md CHANGED
@@ -4,7 +4,7 @@
4
4
 
5
5
  Check out [an example](https://ankane.org/tensorflow-ruby)
6
6
 
7
- [![Build Status](https://github.com/ankane/onnxruntime-ruby/workflows/build/badge.svg?branch=master)](https://github.com/ankane/onnxruntime-ruby/actions)
7
+ [![Build Status](https://github.com/ankane/onnxruntime-ruby/actions/workflows/build.yml/badge.svg)](https://github.com/ankane/onnxruntime-ruby/actions)
8
8
 
9
9
  ## Installation
10
10
 
@@ -108,7 +108,9 @@ OnnxRuntime::Datasets.example("sigmoid.onnx")
108
108
 
109
109
  ## GPU Support
110
110
 
111
- To enable GPU support on Linux and Windows, download the appropriate [GPU release](https://github.com/microsoft/onnxruntime/releases) and set:
111
+ ### Linux and Windows
112
+
113
+ Download the appropriate [GPU release](https://github.com/microsoft/onnxruntime/releases) and set:
112
114
 
113
115
  ```ruby
114
116
  OnnxRuntime.ffi_lib = "path/to/lib/libonnxruntime.so" # onnxruntime.dll for Windows
@@ -120,6 +122,14 @@ and use:
120
122
  model = OnnxRuntime::Model.new("model.onnx", providers: ["CUDAExecutionProvider"])
121
123
  ```
122
124
 
125
+ ### Mac
126
+
127
+ Use:
128
+
129
+ ```ruby
130
+ model = OnnxRuntime::Model.new("model.onnx", providers: ["CoreMLExecutionProvider"])
131
+ ```
132
+
123
133
  ## History
124
134
 
125
135
  View the [changelog](https://github.com/ankane/onnxruntime-ruby/blob/master/CHANGELOG.md)
@@ -11,7 +11,7 @@ module OnnxRuntime
11
11
 
12
12
  # enums
13
13
  TensorElementDataType = enum(:undefined, :float, :uint8, :int8, :uint16, :int16, :int32, :int64, :string, :bool, :float16, :double, :uint32, :uint64, :complex64, :complex128, :bfloat16)
14
- OnnxType = enum(:unknown, :tensor, :sequence, :map, :opaque, :sparsetensor)
14
+ OnnxType = enum(:unknown, :tensor, :sequence, :map, :opaque, :sparsetensor, :optional)
15
15
 
16
16
  class Api < ::FFI::Struct
17
17
  layout \
@@ -144,7 +144,7 @@ module OnnxRuntime
144
144
  :ReleaseAvailableProviders, callback(%i[pointer int], :pointer),
145
145
  :GetStringTensorElementLength, callback(%i[], :pointer),
146
146
  :GetStringTensorElement, callback(%i[], :pointer),
147
- :FillStringTensorElement, callback(%i[], :pointer),
147
+ :FillStringTensorElement, callback(%i[pointer string size_t], :pointer),
148
148
  :AddSessionConfigEntry, callback(%i[pointer string string], :pointer),
149
149
  :CreateAllocator, callback(%i[], :pointer),
150
150
  :ReleaseAllocator, callback(%i[], :pointer),
@@ -246,6 +246,14 @@ module OnnxRuntime
246
246
 
247
247
  attach_function :OrtGetApiBase, %i[], ApiBase.by_ref
248
248
 
249
+ def self.api
250
+ @api ||= begin
251
+ api = self.OrtGetApiBase[:GetApi].call(ORT_API_VERSION)
252
+ api = Api.by_ref.from_native(api, nil) if RUBY_PLATFORM == "java"
253
+ api
254
+ end
255
+ end
256
+
249
257
  if Gem.win_platform?
250
258
  class Libc
251
259
  extend ::FFI::Library
@@ -253,5 +261,11 @@ module OnnxRuntime
253
261
  attach_function :mbstowcs, %i[pointer string size_t], :size_t
254
262
  end
255
263
  end
264
+
265
+ # https://github.com/microsoft/onnxruntime/blob/main/include/onnxruntime/core/providers/coreml/coreml_provider_factory.h
266
+ begin
267
+ attach_function :OrtSessionOptionsAppendExecutionProvider_CoreML, %i[pointer uint32], :pointer
268
+ rescue ::FFI::NotFoundError
269
+ end
256
270
  end
257
271
  end
@@ -66,6 +66,13 @@ module OnnxRuntime
66
66
  check_status api[:CreateCUDAProviderOptions].call(cuda_options)
67
67
  check_status api[:SessionOptionsAppendExecutionProvider_CUDA_V2].call(session_options.read_pointer, cuda_options.read_pointer)
68
68
  release :CUDAProviderOptions, cuda_options
69
+ when "CoreMLExecutionProvider"
70
+ unless FFI.respond_to?(:OrtSessionOptionsAppendExecutionProvider_CoreML)
71
+ raise ArgumentError, "Provider not available: #{provider}"
72
+ end
73
+
74
+ coreml_flags = 0
75
+ check_status FFI.OrtSessionOptionsAppendExecutionProvider_CoreML(session_options.read_pointer, coreml_flags)
69
76
  when "CPUExecutionProvider"
70
77
  break
71
78
  else
@@ -76,23 +83,36 @@ module OnnxRuntime
76
83
  @session = load_session(path_or_bytes, session_options)
77
84
  ObjectSpace.define_finalizer(@session, self.class.finalize(read_pointer.to_i))
78
85
 
79
- @allocator = load_allocator
86
+ @allocator = Utils.allocator
80
87
  @inputs = load_inputs
81
88
  @outputs = load_outputs
82
89
  ensure
83
90
  release :SessionOptions, session_options
84
91
  end
85
92
 
86
- # TODO support logid
87
93
  def run(output_names, input_feed, log_severity_level: nil, log_verbosity_level: nil, logid: nil, terminate: nil, output_type: :ruby)
88
- # pointer references
89
- refs = []
94
+ if ![:ruby, :numo, :ort_value].include?(output_type)
95
+ raise ArgumentError, "Invalid output type: #{output_type}"
96
+ end
97
+
98
+ ort_values = input_feed.keys.zip(create_input_tensor(input_feed)).to_h
90
99
 
91
- input_tensor = create_input_tensor(input_feed, refs)
100
+ outputs = run_with_ort_values(output_names, ort_values, log_severity_level: log_severity_level, log_verbosity_level: log_verbosity_level, logid: logid, terminate: terminate)
101
+
102
+ outputs.map { |v| output_type == :numo ? v.numo : (output_type == :ort_value ? v : v.to_ruby) }
103
+ end
104
+
105
+ # TODO support logid
106
+ def run_with_ort_values(output_names, input_feed, log_severity_level: nil, log_verbosity_level: nil, logid: nil, terminate: nil)
107
+ input_tensor = ::FFI::MemoryPointer.new(:pointer, input_feed.size)
108
+ input_feed.each_with_index do |(_, input), i|
109
+ input_tensor[i].write_pointer(input.to_ptr)
110
+ end
92
111
 
93
112
  output_names ||= @outputs.map { |v| v[:name] }
94
113
 
95
114
  output_tensor = ::FFI::MemoryPointer.new(:pointer, outputs.size)
115
+ refs = []
96
116
  input_node_names = create_node_names(input_feed.keys.map(&:to_s), refs)
97
117
  output_node_names = create_node_names(output_names.map(&:to_s), refs)
98
118
 
@@ -106,17 +126,9 @@ module OnnxRuntime
106
126
 
107
127
  check_status api[:Run].call(read_pointer, run_options.read_pointer, input_node_names, input_tensor, input_feed.size, output_node_names, output_names.size, output_tensor)
108
128
 
109
- output_names.size.times.map do |i|
110
- create_from_onnx_value(output_tensor[i].read_pointer, output_type)
111
- end
129
+ output_names.size.times.map { |i| OrtValue.new(output_tensor[i]) }
112
130
  ensure
113
131
  release :RunOptions, run_options
114
- if input_tensor
115
- input_feed.size.times do |i|
116
- release :Value, input_tensor[i]
117
- end
118
- end
119
- # output values released in create_from_onnx_value
120
132
  end
121
133
 
122
134
  def modelmeta
@@ -214,12 +226,6 @@ module OnnxRuntime
214
226
  session
215
227
  end
216
228
 
217
- def load_allocator
218
- allocator = ::FFI::MemoryPointer.new(:pointer)
219
- check_status api[:GetAllocatorWithDefaultOptions].call(allocator)
220
- allocator
221
- end
222
-
223
229
  def load_inputs
224
230
  inputs = []
225
231
  num_input_nodes = ::FFI::MemoryPointer.new(:size_t)
@@ -230,7 +236,7 @@ module OnnxRuntime
230
236
  # freed in node_info
231
237
  typeinfo = ::FFI::MemoryPointer.new(:pointer)
232
238
  check_status api[:SessionGetInputTypeInfo].call(read_pointer, i, typeinfo)
233
- inputs << {name: name_ptr.read_pointer.read_string}.merge(node_info(typeinfo))
239
+ inputs << {name: name_ptr.read_pointer.read_string}.merge(Utils.node_info(typeinfo))
234
240
  allocator_free name_ptr
235
241
  end
236
242
  inputs
@@ -246,87 +252,28 @@ module OnnxRuntime
246
252
  # freed in node_info
247
253
  typeinfo = ::FFI::MemoryPointer.new(:pointer)
248
254
  check_status api[:SessionGetOutputTypeInfo].call(read_pointer, i, typeinfo)
249
- outputs << {name: name_ptr.read_pointer.read_string}.merge(node_info(typeinfo))
255
+ outputs << {name: name_ptr.read_pointer.read_string}.merge(Utils.node_info(typeinfo))
250
256
  allocator_free name_ptr
251
257
  end
252
258
  outputs
253
259
  end
254
260
 
255
- def create_input_tensor(input_feed, refs)
256
- allocator_info = ::FFI::MemoryPointer.new(:pointer)
257
- check_status api[:CreateCpuMemoryInfo].call(1, 0, allocator_info)
258
- input_tensor = ::FFI::MemoryPointer.new(:pointer, input_feed.size)
259
-
260
- input_feed.each_with_index do |(input_name, input), idx|
261
- if numo_array?(input)
262
- shape = input.shape
263
- else
264
- input = input.to_a unless input.is_a?(Array)
265
-
266
- shape = []
267
- s = input
268
- while s.is_a?(Array)
269
- shape << s.size
270
- s = s.first
271
- end
272
- end
273
-
261
+ def create_input_tensor(input_feed)
262
+ input_feed.map do |input_name, input|
274
263
  # TODO support more types
275
264
  inp = @inputs.find { |i| i[:name] == input_name.to_s }
276
265
  raise Error, "Unknown input: #{input_name}" unless inp
277
266
 
278
- input_node_dims = ::FFI::MemoryPointer.new(:int64, shape.size)
279
- input_node_dims.write_array_of_int64(shape)
280
-
281
- if inp[:type] == "tensor(string)"
282
- str_ptrs =
283
- if numo_array?(input)
284
- input.size.times.map { |i| ::FFI::MemoryPointer.from_string(input[i]) }
285
- else
286
- input.flatten.map { |v| ::FFI::MemoryPointer.from_string(v) }
287
- end
288
-
289
- input_tensor_values = ::FFI::MemoryPointer.new(:pointer, str_ptrs.size)
290
- input_tensor_values.write_array_of_pointer(str_ptrs)
291
-
292
- type_enum = FFI::TensorElementDataType[:string]
293
- check_status api[:CreateTensorAsOrtValue].call(@allocator.read_pointer, input_node_dims, shape.size, type_enum, input_tensor[idx])
294
- check_status api[:FillStringTensor].call(input_tensor[idx].read_pointer, input_tensor_values, str_ptrs.size)
295
-
296
- refs << str_ptrs
267
+ if input.is_a?(OrtValue)
268
+ input
269
+ elsif inp[:type] == "tensor(string)"
270
+ OrtValue.from_array(input, element_type: :string)
271
+ elsif (tensor_type = tensor_types[inp[:type]])
272
+ OrtValue.from_array(input, element_type: tensor_type)
297
273
  else
298
- tensor_type = tensor_types[inp[:type]]
299
-
300
- if tensor_type
301
- if numo_array?(input)
302
- input_tensor_values = input.cast_to(numo_types[tensor_type]).to_binary
303
- else
304
- flat_input = input.flatten.to_a
305
- input_tensor_values = ::FFI::MemoryPointer.new(tensor_type, flat_input.size)
306
- if tensor_type == :bool
307
- input_tensor_values.write_array_of_uint8(flat_input.map { |v| v ? 1 : 0 })
308
- else
309
- input_tensor_values.send("write_array_of_#{tensor_type}", flat_input)
310
- end
311
- end
312
-
313
- type_enum = FFI::TensorElementDataType[tensor_type]
314
- else
315
- unsupported_type("input", inp[:type])
316
- end
317
-
318
- check_status api[:CreateTensorWithDataAsOrtValue].call(allocator_info.read_pointer, input_tensor_values, input_tensor_values.size, input_node_dims, shape.size, type_enum, input_tensor[idx])
319
-
320
- refs << input_node_dims
321
- refs << input_tensor_values
274
+ Utils.unsupported_type("input", inp[:type])
322
275
  end
323
276
  end
324
-
325
- refs << allocator_info
326
-
327
- input_tensor
328
- ensure
329
- release :MemoryInfo, allocator_info
330
277
  end
331
278
 
332
279
  def create_node_names(names, refs)
@@ -338,230 +285,18 @@ module OnnxRuntime
338
285
  ptr
339
286
  end
340
287
 
341
- def create_from_onnx_value(out_ptr, output_type)
342
- out_type = ::FFI::MemoryPointer.new(:int)
343
- check_status api[:GetValueType].call(out_ptr, out_type)
344
- type = FFI::OnnxType[out_type.read_int]
345
-
346
- case type
347
- when :tensor
348
- typeinfo = ::FFI::MemoryPointer.new(:pointer)
349
- check_status api[:GetTensorTypeAndShape].call(out_ptr, typeinfo)
350
-
351
- type, shape = tensor_type_and_shape(typeinfo)
352
-
353
- tensor_data = ::FFI::MemoryPointer.new(:pointer)
354
- check_status api[:GetTensorMutableData].call(out_ptr, tensor_data)
355
-
356
- out_size = ::FFI::MemoryPointer.new(:size_t)
357
- check_status api[:GetTensorShapeElementCount].call(typeinfo.read_pointer, out_size)
358
- output_tensor_size = out_size.read(:size_t)
359
-
360
- release :TensorTypeAndShapeInfo, typeinfo
361
-
362
- # TODO support more types
363
- type = FFI::TensorElementDataType[type]
364
-
365
- case output_type
366
- when :numo
367
- case type
368
- when :string
369
- result = Numo::RObject.new(shape)
370
- result.allocate
371
- create_strings_from_onnx_value(out_ptr, output_tensor_size, result)
372
- else
373
- numo_type = numo_types[type]
374
- unsupported_type("element", type) unless numo_type
375
- numo_type.from_binary(tensor_data.read_pointer.read_bytes(output_tensor_size * numo_type::ELEMENT_BYTE_SIZE), shape)
376
- end
377
- when :ruby
378
- arr =
379
- case type
380
- when :float, :uint8, :int8, :uint16, :int16, :int32, :int64, :double, :uint32, :uint64
381
- tensor_data.read_pointer.send("read_array_of_#{type}", output_tensor_size)
382
- when :bool
383
- tensor_data.read_pointer.read_array_of_uint8(output_tensor_size).map { |v| v == 1 }
384
- when :string
385
- create_strings_from_onnx_value(out_ptr, output_tensor_size, [])
386
- else
387
- unsupported_type("element", type)
388
- end
389
-
390
- Utils.reshape(arr, shape)
391
- else
392
- raise ArgumentError, "Invalid output type: #{output_type}"
393
- end
394
- when :sequence
395
- out = ::FFI::MemoryPointer.new(:size_t)
396
- check_status api[:GetValueCount].call(out_ptr, out)
397
-
398
- out.read(:size_t).times.map do |i|
399
- seq = ::FFI::MemoryPointer.new(:pointer)
400
- check_status api[:GetValue].call(out_ptr, i, @allocator.read_pointer, seq)
401
- create_from_onnx_value(seq.read_pointer, output_type)
402
- end
403
- when :map
404
- type_shape = ::FFI::MemoryPointer.new(:pointer)
405
- map_keys = ::FFI::MemoryPointer.new(:pointer)
406
- map_values = ::FFI::MemoryPointer.new(:pointer)
407
- elem_type = ::FFI::MemoryPointer.new(:int)
408
-
409
- check_status api[:GetValue].call(out_ptr, 0, @allocator.read_pointer, map_keys)
410
- check_status api[:GetValue].call(out_ptr, 1, @allocator.read_pointer, map_values)
411
- check_status api[:GetTensorTypeAndShape].call(map_keys.read_pointer, type_shape)
412
- check_status api[:GetTensorElementType].call(type_shape.read_pointer, elem_type)
413
- release :TensorTypeAndShapeInfo, type_shape
414
-
415
- # TODO support more types
416
- elem_type = FFI::TensorElementDataType[elem_type.read_int]
417
- case elem_type
418
- when :int64
419
- ret = {}
420
- keys = create_from_onnx_value(map_keys.read_pointer, output_type)
421
- values = create_from_onnx_value(map_values.read_pointer, output_type)
422
- keys.zip(values).each do |k, v|
423
- ret[k] = v
424
- end
425
- ret
426
- else
427
- unsupported_type("element", elem_type)
428
- end
429
- else
430
- unsupported_type("ONNX", type)
431
- end
432
- ensure
433
- api[:ReleaseValue].call(out_ptr) unless out_ptr.null?
434
- end
435
-
436
- def create_strings_from_onnx_value(out_ptr, output_tensor_size, result)
437
- len = ::FFI::MemoryPointer.new(:size_t)
438
- check_status api[:GetStringTensorDataLength].call(out_ptr, len)
439
-
440
- s_len = len.read(:size_t)
441
- s = ::FFI::MemoryPointer.new(:uchar, s_len)
442
- offsets = ::FFI::MemoryPointer.new(:size_t, output_tensor_size)
443
- check_status api[:GetStringTensorContent].call(out_ptr, s, s_len, offsets, output_tensor_size)
444
-
445
- offsets = output_tensor_size.times.map { |i| offsets[i].read(:size_t) }
446
- offsets << s_len
447
- output_tensor_size.times do |i|
448
- result[i] = s.get_bytes(offsets[i], offsets[i + 1] - offsets[i])
449
- end
450
- result
451
- end
452
-
453
288
  def read_pointer
454
289
  @session.read_pointer
455
290
  end
456
291
 
457
292
  def check_status(status)
458
- unless status.null?
459
- message = api[:GetErrorMessage].call(status).read_string
460
- api[:ReleaseStatus].call(status)
461
- raise Error, message
462
- end
463
- end
464
-
465
- def node_info(typeinfo)
466
- onnx_type = ::FFI::MemoryPointer.new(:int)
467
- check_status api[:GetOnnxTypeFromTypeInfo].call(typeinfo.read_pointer, onnx_type)
468
-
469
- type = FFI::OnnxType[onnx_type.read_int]
470
- case type
471
- when :tensor
472
- tensor_info = ::FFI::MemoryPointer.new(:pointer)
473
- # don't free tensor_info
474
- check_status api[:CastTypeInfoToTensorInfo].call(typeinfo.read_pointer, tensor_info)
475
-
476
- type, shape = tensor_type_and_shape(tensor_info)
477
- {
478
- type: "tensor(#{FFI::TensorElementDataType[type]})",
479
- shape: shape
480
- }
481
- when :sequence
482
- sequence_type_info = ::FFI::MemoryPointer.new(:pointer)
483
- check_status api[:CastTypeInfoToSequenceTypeInfo].call(typeinfo.read_pointer, sequence_type_info)
484
- nested_type_info = ::FFI::MemoryPointer.new(:pointer)
485
- check_status api[:GetSequenceElementType].call(sequence_type_info.read_pointer, nested_type_info)
486
- v = node_info(nested_type_info)[:type]
487
-
488
- {
489
- type: "seq(#{v})",
490
- shape: []
491
- }
492
- when :map
493
- map_type_info = ::FFI::MemoryPointer.new(:pointer)
494
- check_status api[:CastTypeInfoToMapTypeInfo].call(typeinfo.read_pointer, map_type_info)
495
-
496
- # key
497
- key_type = ::FFI::MemoryPointer.new(:int)
498
- check_status api[:GetMapKeyType].call(map_type_info.read_pointer, key_type)
499
- k = FFI::TensorElementDataType[key_type.read_int]
500
-
501
- # value
502
- value_type_info = ::FFI::MemoryPointer.new(:pointer)
503
- check_status api[:GetMapValueType].call(map_type_info.read_pointer, value_type_info)
504
- v = node_info(value_type_info)[:type]
505
-
506
- {
507
- type: "map(#{k},#{v})",
508
- shape: []
509
- }
510
- else
511
- unsupported_type("ONNX", type)
512
- end
513
- ensure
514
- release :TypeInfo, typeinfo
515
- end
516
-
517
- def tensor_type_and_shape(tensor_info)
518
- type = ::FFI::MemoryPointer.new(:int)
519
- check_status api[:GetTensorElementType].call(tensor_info.read_pointer, type)
520
-
521
- num_dims_ptr = ::FFI::MemoryPointer.new(:size_t)
522
- check_status api[:GetDimensionsCount].call(tensor_info.read_pointer, num_dims_ptr)
523
- num_dims = num_dims_ptr.read(:size_t)
524
-
525
- node_dims = ::FFI::MemoryPointer.new(:int64, num_dims)
526
- check_status api[:GetDimensions].call(tensor_info.read_pointer, node_dims, num_dims)
527
- dims = node_dims.read_array_of_int64(num_dims)
528
-
529
- symbolic_dims = ::FFI::MemoryPointer.new(:pointer, num_dims)
530
- check_status api[:GetSymbolicDimensions].call(tensor_info.read_pointer, symbolic_dims, num_dims)
531
- named_dims = num_dims.times.map { |i| symbolic_dims[i].read_pointer.read_string }
532
- dims = named_dims.zip(dims).map { |n, d| n.empty? ? d : n }
533
-
534
- [type.read_int, dims]
535
- end
536
-
537
- def unsupported_type(name, type)
538
- raise Error, "Unsupported #{name} type: #{type}"
293
+ Utils.check_status(status)
539
294
  end
540
295
 
541
296
  def tensor_types
542
297
  @tensor_types ||= [:float, :uint8, :int8, :uint16, :int16, :int32, :int64, :bool, :double, :uint32, :uint64].map { |v| ["tensor(#{v})", v] }.to_h
543
298
  end
544
299
 
545
- def numo_array?(obj)
546
- defined?(Numo::NArray) && obj.is_a?(Numo::NArray)
547
- end
548
-
549
- def numo_types
550
- @numo_types ||= {
551
- float: Numo::SFloat,
552
- uint8: Numo::UInt8,
553
- int8: Numo::Int8,
554
- uint16: Numo::UInt16,
555
- int16: Numo::Int16,
556
- int32: Numo::Int32,
557
- int64: Numo::Int64,
558
- bool: Numo::UInt8,
559
- double: Numo::DFloat,
560
- uint32: Numo::UInt32,
561
- uint64: Numo::UInt64
562
- }
563
- end
564
-
565
300
  def api
566
301
  self.class.api
567
302
  end
@@ -575,11 +310,11 @@ module OnnxRuntime
575
310
  end
576
311
 
577
312
  def self.api
578
- @api ||= FFI.OrtGetApiBase[:GetApi].call(FFI::ORT_API_VERSION)
313
+ FFI.api
579
314
  end
580
315
 
581
316
  def self.release(type, pointer)
582
- api[:"Release#{type}"].call(pointer.read_pointer) if pointer && !pointer.null?
317
+ Utils.release(type, pointer)
583
318
  end
584
319
 
585
320
  def self.finalize(addr)
@@ -0,0 +1,278 @@
1
+ module OnnxRuntime
2
+ class OrtValue
3
+ def initialize(ptr, ref = nil)
4
+ @ptr = ptr.read_pointer
5
+ @ref = ref # keep reference to data
6
+ ObjectSpace.define_finalizer(@ptr, self.class.finalize(@ptr.to_i))
7
+ end
8
+
9
+ def self.from_numo(numo_obj)
10
+ element_type = numo_obj.is_a?(Numo::Bit) ? :bool : Utils.numo_types.invert[numo_obj.class]
11
+ Utils.unsupported_type("Numo", numo_obj.class.name) unless element_type
12
+
13
+ from_array(numo_obj, element_type: element_type)
14
+ end
15
+
16
+ def self.from_array(input, element_type:)
17
+ type_enum = FFI::TensorElementDataType[element_type]
18
+ Utils.unsupported_type("element", element_type) unless type_enum
19
+
20
+ input = input.to_a unless input.is_a?(Array) || Utils.numo_array?(input)
21
+
22
+ shape = Utils.input_shape(input)
23
+ input_node_dims = ::FFI::MemoryPointer.new(:int64, shape.size)
24
+ input_node_dims.write_array_of_int64(shape)
25
+
26
+ ptr = ::FFI::MemoryPointer.new(:pointer)
27
+ if element_type == :string
28
+ # keep reference to _str_ptrs until FillStringTensor call
29
+ input_tensor_values, _str_ptrs = create_input_strings(input)
30
+ Utils.check_status FFI.api[:CreateTensorAsOrtValue].call(Utils.allocator.read_pointer, input_node_dims, shape.size, type_enum, ptr)
31
+ Utils.check_status FFI.api[:FillStringTensor].call(ptr.read_pointer, input_tensor_values, input_tensor_values.size / input_tensor_values.type_size)
32
+ else
33
+ input_tensor_values = create_input_data(input, element_type)
34
+ Utils.check_status FFI.api[:CreateTensorWithDataAsOrtValue].call(allocator_info.read_pointer, input_tensor_values, input_tensor_values.size, input_node_dims, shape.size, type_enum, ptr)
35
+ end
36
+
37
+ new(ptr, input_tensor_values)
38
+ end
39
+
40
+ def self.from_shape_and_type(shape, element_type)
41
+ type_enum = FFI::TensorElementDataType[element_type]
42
+ Utils.unsupported_type("element", element_type) unless type_enum
43
+
44
+ input_node_dims = ::FFI::MemoryPointer.new(:int64, shape.size)
45
+ input_node_dims.write_array_of_int64(shape)
46
+
47
+ ptr = ::FFI::MemoryPointer.new(:pointer)
48
+ Utils.check_status FFI.api[:CreateTensorAsOrtValue].call(Utils.allocator.read_pointer, input_node_dims, shape.size, type_enum, ptr)
49
+
50
+ new(ptr)
51
+ end
52
+
53
+ def self.create_input_data(input, tensor_type)
54
+ if Utils.numo_array?(input)
55
+ input.cast_to(Utils.numo_types[tensor_type]).to_binary
56
+ else
57
+ flat_input = input.flatten.to_a
58
+ input_tensor_values = ::FFI::MemoryPointer.new(tensor_type, flat_input.size)
59
+ if tensor_type == :bool
60
+ input_tensor_values.write_array_of_uint8(flat_input.map { |v| v ? 1 : 0 })
61
+ else
62
+ input_tensor_values.send("write_array_of_#{tensor_type}", flat_input)
63
+ end
64
+ input_tensor_values
65
+ end
66
+ end
67
+ private_class_method :create_input_data
68
+
69
+ def self.create_input_strings(input)
70
+ str_ptrs =
71
+ if Utils.numo_array?(input)
72
+ input.size.times.map { |i| ::FFI::MemoryPointer.from_string(input[i]) }
73
+ else
74
+ input.flatten.map { |v| ::FFI::MemoryPointer.from_string(v) }
75
+ end
76
+
77
+ input_tensor_values = ::FFI::MemoryPointer.new(:pointer, str_ptrs.size)
78
+ input_tensor_values.write_array_of_pointer(str_ptrs)
79
+ [input_tensor_values, str_ptrs]
80
+ end
81
+ private_class_method :create_input_strings
82
+
83
+ def tensor?
84
+ FFI::OnnxType[value_type] == :tensor
85
+ end
86
+
87
+ def data_type
88
+ @data_type ||= begin
89
+ typeinfo = ::FFI::MemoryPointer.new(:pointer)
90
+ Utils.check_status FFI.api[:GetTypeInfo].call(@ptr, typeinfo)
91
+ Utils.node_info(typeinfo)[:type]
92
+ end
93
+ end
94
+
95
+ def element_type
96
+ FFI::TensorElementDataType[type_and_shape_info[0]]
97
+ end
98
+
99
+ def shape
100
+ type_and_shape_info[1]
101
+ end
102
+
103
+ def device_name
104
+ "cpu"
105
+ end
106
+
107
+ def numo
108
+ create_from_onnx_value(@ptr, :numo)
109
+ end
110
+
111
+ def to_ruby
112
+ create_from_onnx_value(@ptr, :ruby)
113
+ end
114
+
115
+ def to_ptr
116
+ @ptr
117
+ end
118
+
119
+ def data_ptr
120
+ tensor_data = ::FFI::MemoryPointer.new(:pointer)
121
+ FFI.api[:GetTensorMutableData].call(@ptr, tensor_data)
122
+ tensor_data.read_pointer
123
+ end
124
+
125
+ private
126
+
127
+ def value_type
128
+ @value_type ||= begin
129
+ out_type = ::FFI::MemoryPointer.new(:int)
130
+ Utils.check_status FFI.api[:GetValueType].call(@ptr, out_type)
131
+ out_type.read_int
132
+ end
133
+ end
134
+
135
+ def type_and_shape_info
136
+ @type_and_shape_info ||= begin
137
+ begin
138
+ typeinfo = ::FFI::MemoryPointer.new(:pointer)
139
+ Utils.check_status FFI.api[:GetTensorTypeAndShape].call(@ptr, typeinfo)
140
+ Utils.tensor_type_and_shape(typeinfo)
141
+ ensure
142
+ Utils.release :TensorTypeAndShapeInfo, typeinfo
143
+ end
144
+ end
145
+ end
146
+
147
+ def create_from_onnx_value(out_ptr, output_type)
148
+ out_type = ::FFI::MemoryPointer.new(:int)
149
+ Utils.check_status FFI.api[:GetValueType].call(out_ptr, out_type)
150
+ type = FFI::OnnxType[out_type.read_int]
151
+
152
+ case type
153
+ when :tensor
154
+ typeinfo = ::FFI::MemoryPointer.new(:pointer)
155
+ Utils.check_status FFI.api[:GetTensorTypeAndShape].call(out_ptr, typeinfo)
156
+
157
+ type, shape = Utils.tensor_type_and_shape(typeinfo)
158
+
159
+ tensor_data = ::FFI::MemoryPointer.new(:pointer)
160
+ Utils.check_status FFI.api[:GetTensorMutableData].call(out_ptr, tensor_data)
161
+
162
+ out_size = ::FFI::MemoryPointer.new(:size_t)
163
+ Utils.check_status FFI.api[:GetTensorShapeElementCount].call(typeinfo.read_pointer, out_size)
164
+ output_tensor_size = out_size.read(:size_t)
165
+
166
+ Utils.release :TensorTypeAndShapeInfo, typeinfo
167
+
168
+ # TODO support more types
169
+ type = FFI::TensorElementDataType[type]
170
+
171
+ case output_type
172
+ when :numo
173
+ case type
174
+ when :string
175
+ result = Numo::RObject.new(shape)
176
+ result.allocate
177
+ create_strings_from_onnx_value(out_ptr, output_tensor_size, result)
178
+ else
179
+ numo_type = Utils.numo_types[type]
180
+ Utils.unsupported_type("element", type) unless numo_type
181
+ numo_type.from_binary(tensor_data.read_pointer.read_bytes(output_tensor_size * numo_type::ELEMENT_BYTE_SIZE), shape)
182
+ end
183
+ when :ruby
184
+ arr =
185
+ case type
186
+ when :float, :uint8, :int8, :uint16, :int16, :int32, :int64, :double, :uint32, :uint64
187
+ tensor_data.read_pointer.send("read_array_of_#{type}", output_tensor_size)
188
+ when :bool
189
+ tensor_data.read_pointer.read_array_of_uint8(output_tensor_size).map { |v| v == 1 }
190
+ when :string
191
+ create_strings_from_onnx_value(out_ptr, output_tensor_size, [])
192
+ else
193
+ Utils.unsupported_type("element", type)
194
+ end
195
+
196
+ reshape(arr, shape)
197
+ else
198
+ raise ArgumentError, "Invalid output type: #{output_type}"
199
+ end
200
+ when :sequence
201
+ out = ::FFI::MemoryPointer.new(:size_t)
202
+ Utils.check_status FFI.api[:GetValueCount].call(out_ptr, out)
203
+
204
+ out.read(:size_t).times.map do |i|
205
+ seq = ::FFI::MemoryPointer.new(:pointer)
206
+ Utils.check_status FFI.api[:GetValue].call(out_ptr, i, Utils.allocator.read_pointer, seq)
207
+ create_from_onnx_value(seq.read_pointer, output_type)
208
+ end
209
+ when :map
210
+ type_shape = ::FFI::MemoryPointer.new(:pointer)
211
+ map_keys = ::FFI::MemoryPointer.new(:pointer)
212
+ map_values = ::FFI::MemoryPointer.new(:pointer)
213
+ elem_type = ::FFI::MemoryPointer.new(:int)
214
+
215
+ Utils.check_status FFI.api[:GetValue].call(out_ptr, 0, Utils.allocator.read_pointer, map_keys)
216
+ Utils.check_status FFI.api[:GetValue].call(out_ptr, 1, Utils.allocator.read_pointer, map_values)
217
+ Utils.check_status FFI.api[:GetTensorTypeAndShape].call(map_keys.read_pointer, type_shape)
218
+ Utils.check_status FFI.api[:GetTensorElementType].call(type_shape.read_pointer, elem_type)
219
+ Utils.release :TensorTypeAndShapeInfo, type_shape
220
+
221
+ # TODO support more types
222
+ elem_type = FFI::TensorElementDataType[elem_type.read_int]
223
+ case elem_type
224
+ when :int64
225
+ ret = {}
226
+ keys = create_from_onnx_value(map_keys.read_pointer, output_type)
227
+ values = create_from_onnx_value(map_values.read_pointer, output_type)
228
+ keys.zip(values).each do |k, v|
229
+ ret[k] = v
230
+ end
231
+ ret
232
+ else
233
+ Utils.unsupported_type("element", elem_type)
234
+ end
235
+ else
236
+ Utils.unsupported_type("ONNX", type)
237
+ end
238
+ end
239
+
240
+ def create_strings_from_onnx_value(out_ptr, output_tensor_size, result)
241
+ len = ::FFI::MemoryPointer.new(:size_t)
242
+ Utils.check_status FFI.api[:GetStringTensorDataLength].call(out_ptr, len)
243
+
244
+ s_len = len.read(:size_t)
245
+ s = ::FFI::MemoryPointer.new(:uchar, s_len)
246
+ offsets = ::FFI::MemoryPointer.new(:size_t, output_tensor_size)
247
+ Utils.check_status FFI.api[:GetStringTensorContent].call(out_ptr, s, s_len, offsets, output_tensor_size)
248
+
249
+ offsets = output_tensor_size.times.map { |i| offsets[i].read(:size_t) }
250
+ offsets << s_len
251
+ output_tensor_size.times do |i|
252
+ result[i] = s.get_bytes(offsets[i], offsets[i + 1] - offsets[i])
253
+ end
254
+ result
255
+ end
256
+
257
+ def reshape(arr, dims)
258
+ arr = arr.flatten
259
+ dims[1..-1].reverse_each do |dim|
260
+ arr = arr.each_slice(dim)
261
+ end
262
+ arr.to_a
263
+ end
264
+
265
+ def self.finalize(addr)
266
+ # must use proc instead of stabby lambda
267
+ proc { FFI.api[:ReleaseValue].call(::FFI::Pointer.new(:pointer, addr)) }
268
+ end
269
+
270
+ def self.allocator_info
271
+ @allocator_info ||= begin
272
+ allocator_info = ::FFI::MemoryPointer.new(:pointer)
273
+ Utils.check_status FFI.api[:CreateCpuMemoryInfo].call(1, 0, allocator_info)
274
+ allocator_info
275
+ end
276
+ end
277
+ end
278
+ end
@@ -5,12 +5,138 @@ module OnnxRuntime
5
5
  end
6
6
  self.mutex = Mutex.new
7
7
 
8
- def self.reshape(arr, dims)
9
- arr = arr.flatten
10
- dims[1..-1].reverse.each do |dim|
11
- arr = arr.each_slice(dim)
8
+ def self.check_status(status)
9
+ unless status.null?
10
+ message = api[:GetErrorMessage].call(status).read_string
11
+ api[:ReleaseStatus].call(status)
12
+ raise Error, message
13
+ end
14
+ end
15
+
16
+ def self.api
17
+ FFI.api
18
+ end
19
+
20
+ def self.release(type, pointer)
21
+ FFI.api[:"Release#{type}"].call(pointer.read_pointer) if pointer && !pointer.null?
22
+ end
23
+
24
+ def self.unsupported_type(name, type)
25
+ raise Error, "Unsupported #{name} type: #{type}"
26
+ end
27
+
28
+ def self.tensor_type_and_shape(tensor_info)
29
+ type = ::FFI::MemoryPointer.new(:int)
30
+ check_status api[:GetTensorElementType].call(tensor_info.read_pointer, type)
31
+
32
+ num_dims_ptr = ::FFI::MemoryPointer.new(:size_t)
33
+ check_status api[:GetDimensionsCount].call(tensor_info.read_pointer, num_dims_ptr)
34
+ num_dims = num_dims_ptr.read(:size_t)
35
+
36
+ node_dims = ::FFI::MemoryPointer.new(:int64, num_dims)
37
+ check_status api[:GetDimensions].call(tensor_info.read_pointer, node_dims, num_dims)
38
+ dims = node_dims.read_array_of_int64(num_dims)
39
+
40
+ symbolic_dims = ::FFI::MemoryPointer.new(:pointer, num_dims)
41
+ check_status api[:GetSymbolicDimensions].call(tensor_info.read_pointer, symbolic_dims, num_dims)
42
+ named_dims = num_dims.times.map { |i| symbolic_dims[i].read_pointer.read_string }
43
+ dims = named_dims.zip(dims).map { |n, d| n.empty? ? d : n }
44
+
45
+ [type.read_int, dims]
46
+ end
47
+
48
+ def self.node_info(typeinfo)
49
+ onnx_type = ::FFI::MemoryPointer.new(:int)
50
+ check_status api[:GetOnnxTypeFromTypeInfo].call(typeinfo.read_pointer, onnx_type)
51
+
52
+ type = FFI::OnnxType[onnx_type.read_int]
53
+ case type
54
+ when :tensor
55
+ tensor_info = ::FFI::MemoryPointer.new(:pointer)
56
+ # don't free tensor_info
57
+ check_status api[:CastTypeInfoToTensorInfo].call(typeinfo.read_pointer, tensor_info)
58
+
59
+ type, shape = Utils.tensor_type_and_shape(tensor_info)
60
+ {
61
+ type: "tensor(#{FFI::TensorElementDataType[type]})",
62
+ shape: shape
63
+ }
64
+ when :sequence
65
+ sequence_type_info = ::FFI::MemoryPointer.new(:pointer)
66
+ check_status api[:CastTypeInfoToSequenceTypeInfo].call(typeinfo.read_pointer, sequence_type_info)
67
+ nested_type_info = ::FFI::MemoryPointer.new(:pointer)
68
+ check_status api[:GetSequenceElementType].call(sequence_type_info.read_pointer, nested_type_info)
69
+ v = node_info(nested_type_info)[:type]
70
+
71
+ {
72
+ type: "seq(#{v})",
73
+ shape: []
74
+ }
75
+ when :map
76
+ map_type_info = ::FFI::MemoryPointer.new(:pointer)
77
+ check_status api[:CastTypeInfoToMapTypeInfo].call(typeinfo.read_pointer, map_type_info)
78
+
79
+ # key
80
+ key_type = ::FFI::MemoryPointer.new(:int)
81
+ check_status api[:GetMapKeyType].call(map_type_info.read_pointer, key_type)
82
+ k = FFI::TensorElementDataType[key_type.read_int]
83
+
84
+ # value
85
+ value_type_info = ::FFI::MemoryPointer.new(:pointer)
86
+ check_status api[:GetMapValueType].call(map_type_info.read_pointer, value_type_info)
87
+ v = node_info(value_type_info)[:type]
88
+
89
+ {
90
+ type: "map(#{k},#{v})",
91
+ shape: []
92
+ }
93
+ else
94
+ Utils.unsupported_type("ONNX", type)
95
+ end
96
+ ensure
97
+ release :TypeInfo, typeinfo
98
+ end
99
+
100
+ def self.numo_array?(obj)
101
+ defined?(Numo::NArray) && obj.is_a?(Numo::NArray)
102
+ end
103
+
104
+ def self.numo_types
105
+ @numo_types ||= {
106
+ float: Numo::SFloat,
107
+ uint8: Numo::UInt8,
108
+ int8: Numo::Int8,
109
+ uint16: Numo::UInt16,
110
+ int16: Numo::Int16,
111
+ int32: Numo::Int32,
112
+ int64: Numo::Int64,
113
+ bool: Numo::UInt8,
114
+ double: Numo::DFloat,
115
+ uint32: Numo::UInt32,
116
+ uint64: Numo::UInt64
117
+ }
118
+ end
119
+
120
+ def self.input_shape(input)
121
+ if numo_array?(input)
122
+ input.shape
123
+ else
124
+ shape = []
125
+ s = input
126
+ while s.is_a?(Array)
127
+ shape << s.size
128
+ s = s.first
129
+ end
130
+ shape
131
+ end
132
+ end
133
+
134
+ def self.allocator
135
+ @allocator ||= begin
136
+ allocator = ::FFI::MemoryPointer.new(:pointer)
137
+ check_status api[:GetAllocatorWithDefaultOptions].call(allocator)
138
+ allocator
12
139
  end
13
- arr.to_a
14
140
  end
15
141
  end
16
142
  end
@@ -1,3 +1,3 @@
1
1
  module OnnxRuntime
2
- VERSION = "0.9.1"
2
+ VERSION = "0.9.3"
3
3
  end
data/lib/onnxruntime.rb CHANGED
@@ -5,6 +5,7 @@ require "ffi"
5
5
  require_relative "onnxruntime/datasets"
6
6
  require_relative "onnxruntime/inference_session"
7
7
  require_relative "onnxruntime/model"
8
+ require_relative "onnxruntime/ort_value"
8
9
  require_relative "onnxruntime/utils"
9
10
  require_relative "onnxruntime/version"
10
11
 
@@ -4820,7 +4820,7 @@ SOFTWARE.
4820
4820
 
4821
4821
  ----------------------------------------------------------------------------
4822
4822
 
4823
- This is the MIT/Expat Licence. For more information see:
4823
+ This is the MIT/Expat License. For more information see:
4824
4824
 
4825
4825
  1. http://www.opensource.org/licenses/mit-license.php
4826
4826
 
Binary file
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: onnxruntime
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.9.1
4
+ version: 0.9.3
5
5
  platform: x86_64-linux
6
6
  authors:
7
7
  - Andrew Kane
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2024-05-23 00:00:00.000000000 Z
11
+ date: 2024-11-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: ffi
@@ -38,6 +38,7 @@ files:
38
38
  - lib/onnxruntime/ffi.rb
39
39
  - lib/onnxruntime/inference_session.rb
40
40
  - lib/onnxruntime/model.rb
41
+ - lib/onnxruntime/ort_value.rb
41
42
  - lib/onnxruntime/utils.rb
42
43
  - lib/onnxruntime/version.rb
43
44
  - vendor/LICENSE
@@ -62,7 +63,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
62
63
  - !ruby/object:Gem::Version
63
64
  version: '0'
64
65
  requirements: []
65
- rubygems_version: 3.5.9
66
+ rubygems_version: 3.5.16
66
67
  signing_key:
67
68
  specification_version: 4
68
69
  summary: High performance scoring engine for ML models