onnxruntime 0.9.2-aarch64-linux → 0.9.4-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 52e865156423eb866f07a8e2e13961a7376da543f93ed1b5dbcff08bfc2bf88d
4
- data.tar.gz: a5d82e576c5f8a4528dd14efe4ec8bd508933da2a954118f7ff105a8193bd389
3
+ metadata.gz: 9544eb4f8df1fd611ba42ce3b3687af10d226434202e42caac7e03dd3926162d
4
+ data.tar.gz: 9176e3659227243408ed5f5eab6508b719ccfbdddb37a2d173f6ebcb26ce490d
5
5
  SHA512:
6
- metadata.gz: 655c0c752e56372ae2b37710d4e8e2864f67431be48f834588160f61b1a6ecb93874de75f5f892115925d3fbfd78a820df08f5d3cde2405b9807908d80b19e21
7
- data.tar.gz: 2b64c39e0ed3c09072b2f70691a4b1ce805616d2b397a563cbf2ed9270e46fb0422cbe3b5b385311c8f8f9ef384ee86545558185049ac1652a9c01dc1f71aa55
6
+ metadata.gz: 255f60ce409099e1ef8b4bbf820360903785b0690c26fd540511c839185ba8abfde9eee1a67ffb3ff1c38d1a36c1910b9b05626da98eb1b6119eddc1b18036ee
7
+ data.tar.gz: 177f73d9d97856c34c9f5dccb19a9428776421a54edf2b65349411eae556783ce24d5375f06b1a6ff12449c5c894ce12f0ffd566880e0263ae9ea67aa474ac3b
data/CHANGELOG.md CHANGED
@@ -1,3 +1,13 @@
1
+ ## 0.9.4 (2025-03-08)
2
+
3
+ - Updated ONNX Runtime to 1.21.0
4
+
5
+ ## 0.9.3 (2024-11-01)
6
+
7
+ - Updated ONNX Runtime to 1.20.0
8
+ - Added experimental `OrtValue` class
9
+ - Added experimental `run_with_ort_values` method
10
+
1
11
  ## 0.9.2 (2024-09-04)
2
12
 
3
13
  - Updated ONNX Runtime to 1.19.2
data/README.md CHANGED
@@ -4,6 +4,8 @@
4
4
 
5
5
  Check out [an example](https://ankane.org/tensorflow-ruby)
6
6
 
7
+ For transformer models, check out [Informers](https://github.com/ankane/informers)
8
+
7
9
  [![Build Status](https://github.com/ankane/onnxruntime-ruby/actions/workflows/build.yml/badge.svg)](https://github.com/ankane/onnxruntime-ruby/actions)
8
10
 
9
11
  ## Installation
@@ -11,7 +11,7 @@ module OnnxRuntime
11
11
 
12
12
  # enums
13
13
  TensorElementDataType = enum(:undefined, :float, :uint8, :int8, :uint16, :int16, :int32, :int64, :string, :bool, :float16, :double, :uint32, :uint64, :complex64, :complex128, :bfloat16)
14
- OnnxType = enum(:unknown, :tensor, :sequence, :map, :opaque, :sparsetensor)
14
+ OnnxType = enum(:unknown, :tensor, :sequence, :map, :opaque, :sparsetensor, :optional)
15
15
 
16
16
  class Api < ::FFI::Struct
17
17
  layout \
@@ -247,7 +247,11 @@ module OnnxRuntime
247
247
  attach_function :OrtGetApiBase, %i[], ApiBase.by_ref
248
248
 
249
249
  def self.api
250
- @api ||= self.OrtGetApiBase[:GetApi].call(ORT_API_VERSION)
250
+ @api ||= begin
251
+ api = self.OrtGetApiBase[:GetApi].call(ORT_API_VERSION)
252
+ api = Api.by_ref.from_native(api, nil) if RUBY_PLATFORM == "java"
253
+ api
254
+ end
251
255
  end
252
256
 
253
257
  if Gem.win_platform?
@@ -83,23 +83,36 @@ module OnnxRuntime
83
83
  @session = load_session(path_or_bytes, session_options)
84
84
  ObjectSpace.define_finalizer(@session, self.class.finalize(read_pointer.to_i))
85
85
 
86
- @allocator = load_allocator
86
+ @allocator = Utils.allocator
87
87
  @inputs = load_inputs
88
88
  @outputs = load_outputs
89
89
  ensure
90
90
  release :SessionOptions, session_options
91
91
  end
92
92
 
93
- # TODO support logid
94
93
  def run(output_names, input_feed, log_severity_level: nil, log_verbosity_level: nil, logid: nil, terminate: nil, output_type: :ruby)
95
- # pointer references
96
- refs = []
94
+ if ![:ruby, :numo, :ort_value].include?(output_type)
95
+ raise ArgumentError, "Invalid output type: #{output_type}"
96
+ end
97
97
 
98
- input_tensor = create_input_tensor(input_feed, refs)
98
+ ort_values = input_feed.keys.zip(create_input_tensor(input_feed)).to_h
99
+
100
+ outputs = run_with_ort_values(output_names, ort_values, log_severity_level: log_severity_level, log_verbosity_level: log_verbosity_level, logid: logid, terminate: terminate)
101
+
102
+ outputs.map { |v| output_type == :numo ? v.numo : (output_type == :ort_value ? v : v.to_ruby) }
103
+ end
104
+
105
+ # TODO support logid
106
+ def run_with_ort_values(output_names, input_feed, log_severity_level: nil, log_verbosity_level: nil, logid: nil, terminate: nil)
107
+ input_tensor = ::FFI::MemoryPointer.new(:pointer, input_feed.size)
108
+ input_feed.each_with_index do |(_, input), i|
109
+ input_tensor[i].write_pointer(input.to_ptr)
110
+ end
99
111
 
100
112
  output_names ||= @outputs.map { |v| v[:name] }
101
113
 
102
114
  output_tensor = ::FFI::MemoryPointer.new(:pointer, outputs.size)
115
+ refs = []
103
116
  input_node_names = create_node_names(input_feed.keys.map(&:to_s), refs)
104
117
  output_node_names = create_node_names(output_names.map(&:to_s), refs)
105
118
 
@@ -113,17 +126,9 @@ module OnnxRuntime
113
126
 
114
127
  check_status api[:Run].call(read_pointer, run_options.read_pointer, input_node_names, input_tensor, input_feed.size, output_node_names, output_names.size, output_tensor)
115
128
 
116
- output_names.size.times.map do |i|
117
- create_from_onnx_value(output_tensor[i].read_pointer, output_type)
118
- end
129
+ output_names.size.times.map { |i| OrtValue.new(output_tensor[i]) }
119
130
  ensure
120
131
  release :RunOptions, run_options
121
- if input_tensor
122
- input_feed.size.times do |i|
123
- release :Value, input_tensor[i]
124
- end
125
- end
126
- # output values released in create_from_onnx_value
127
132
  end
128
133
 
129
134
  def modelmeta
@@ -221,12 +226,6 @@ module OnnxRuntime
221
226
  session
222
227
  end
223
228
 
224
- def load_allocator
225
- allocator = ::FFI::MemoryPointer.new(:pointer)
226
- check_status api[:GetAllocatorWithDefaultOptions].call(allocator)
227
- allocator
228
- end
229
-
230
229
  def load_inputs
231
230
  inputs = []
232
231
  num_input_nodes = ::FFI::MemoryPointer.new(:size_t)
@@ -237,7 +236,7 @@ module OnnxRuntime
237
236
  # freed in node_info
238
237
  typeinfo = ::FFI::MemoryPointer.new(:pointer)
239
238
  check_status api[:SessionGetInputTypeInfo].call(read_pointer, i, typeinfo)
240
- inputs << {name: name_ptr.read_pointer.read_string}.merge(node_info(typeinfo))
239
+ inputs << {name: name_ptr.read_pointer.read_string}.merge(Utils.node_info(typeinfo))
241
240
  allocator_free name_ptr
242
241
  end
243
242
  inputs
@@ -253,91 +252,28 @@ module OnnxRuntime
253
252
  # freed in node_info
254
253
  typeinfo = ::FFI::MemoryPointer.new(:pointer)
255
254
  check_status api[:SessionGetOutputTypeInfo].call(read_pointer, i, typeinfo)
256
- outputs << {name: name_ptr.read_pointer.read_string}.merge(node_info(typeinfo))
255
+ outputs << {name: name_ptr.read_pointer.read_string}.merge(Utils.node_info(typeinfo))
257
256
  allocator_free name_ptr
258
257
  end
259
258
  outputs
260
259
  end
261
260
 
262
- def create_input_tensor(input_feed, refs)
263
- allocator_info = ::FFI::MemoryPointer.new(:pointer)
264
- check_status api[:CreateCpuMemoryInfo].call(1, 0, allocator_info)
265
- input_tensor = ::FFI::MemoryPointer.new(:pointer, input_feed.size)
266
-
267
- input_feed.each_with_index do |(input_name, input), idx|
261
+ def create_input_tensor(input_feed)
262
+ input_feed.map do |input_name, input|
268
263
  # TODO support more types
269
264
  inp = @inputs.find { |i| i[:name] == input_name.to_s }
270
265
  raise Error, "Unknown input: #{input_name}" unless inp
271
266
 
272
- input = input.to_a unless input.is_a?(Array) || numo_array?(input)
273
- shape = input_shape(input)
274
-
275
- input_node_dims = ::FFI::MemoryPointer.new(:int64, shape.size)
276
- input_node_dims.write_array_of_int64(shape)
277
-
278
- if inp[:type] == "tensor(string)"
279
- type_enum = FFI::TensorElementDataType[:string]
280
- check_status api[:CreateTensorAsOrtValue].call(@allocator.read_pointer, input_node_dims, shape.size, type_enum, input_tensor[idx])
281
-
282
- # keep reference to _str_ptrs until FillStringTensor call
283
- input_tensor_values, _str_ptrs = create_input_strings(input)
284
- check_status api[:FillStringTensor].call(input_tensor[idx].read_pointer, input_tensor_values, input_tensor_values.size / input_tensor_values.type_size)
267
+ if input.is_a?(OrtValue)
268
+ input
269
+ elsif inp[:type] == "tensor(string)"
270
+ OrtValue.from_array(input, element_type: :string)
285
271
  elsif (tensor_type = tensor_types[inp[:type]])
286
- input_tensor_values = create_input_data(input, tensor_type)
287
- type_enum = FFI::TensorElementDataType[tensor_type]
288
- check_status api[:CreateTensorWithDataAsOrtValue].call(allocator_info.read_pointer, input_tensor_values, input_tensor_values.size, input_node_dims, shape.size, type_enum, input_tensor[idx])
289
-
290
- refs << input_tensor_values
272
+ OrtValue.from_array(input, element_type: tensor_type)
291
273
  else
292
- unsupported_type("input", inp[:type])
274
+ Utils.unsupported_type("input", inp[:type])
293
275
  end
294
276
  end
295
-
296
- input_tensor
297
- ensure
298
- release :MemoryInfo, allocator_info
299
- end
300
-
301
- def input_shape(input)
302
- if numo_array?(input)
303
- input.shape
304
- else
305
- shape = []
306
- s = input
307
- while s.is_a?(Array)
308
- shape << s.size
309
- s = s.first
310
- end
311
- shape
312
- end
313
- end
314
-
315
- def create_input_strings(input)
316
- str_ptrs =
317
- if numo_array?(input)
318
- input.size.times.map { |i| ::FFI::MemoryPointer.from_string(input[i]) }
319
- else
320
- input.flatten.map { |v| ::FFI::MemoryPointer.from_string(v) }
321
- end
322
-
323
- input_tensor_values = ::FFI::MemoryPointer.new(:pointer, str_ptrs.size)
324
- input_tensor_values.write_array_of_pointer(str_ptrs)
325
- [input_tensor_values, str_ptrs]
326
- end
327
-
328
- def create_input_data(input, tensor_type)
329
- if numo_array?(input)
330
- input.cast_to(numo_types[tensor_type]).to_binary
331
- else
332
- flat_input = input.flatten.to_a
333
- input_tensor_values = ::FFI::MemoryPointer.new(tensor_type, flat_input.size)
334
- if tensor_type == :bool
335
- input_tensor_values.write_array_of_uint8(flat_input.map { |v| v ? 1 : 0 })
336
- else
337
- input_tensor_values.send("write_array_of_#{tensor_type}", flat_input)
338
- end
339
- input_tensor_values
340
- end
341
277
  end
342
278
 
343
279
  def create_node_names(names, refs)
@@ -349,230 +285,18 @@ module OnnxRuntime
349
285
  ptr
350
286
  end
351
287
 
352
- def create_from_onnx_value(out_ptr, output_type)
353
- out_type = ::FFI::MemoryPointer.new(:int)
354
- check_status api[:GetValueType].call(out_ptr, out_type)
355
- type = FFI::OnnxType[out_type.read_int]
356
-
357
- case type
358
- when :tensor
359
- typeinfo = ::FFI::MemoryPointer.new(:pointer)
360
- check_status api[:GetTensorTypeAndShape].call(out_ptr, typeinfo)
361
-
362
- type, shape = tensor_type_and_shape(typeinfo)
363
-
364
- tensor_data = ::FFI::MemoryPointer.new(:pointer)
365
- check_status api[:GetTensorMutableData].call(out_ptr, tensor_data)
366
-
367
- out_size = ::FFI::MemoryPointer.new(:size_t)
368
- check_status api[:GetTensorShapeElementCount].call(typeinfo.read_pointer, out_size)
369
- output_tensor_size = out_size.read(:size_t)
370
-
371
- release :TensorTypeAndShapeInfo, typeinfo
372
-
373
- # TODO support more types
374
- type = FFI::TensorElementDataType[type]
375
-
376
- case output_type
377
- when :numo
378
- case type
379
- when :string
380
- result = Numo::RObject.new(shape)
381
- result.allocate
382
- create_strings_from_onnx_value(out_ptr, output_tensor_size, result)
383
- else
384
- numo_type = numo_types[type]
385
- unsupported_type("element", type) unless numo_type
386
- numo_type.from_binary(tensor_data.read_pointer.read_bytes(output_tensor_size * numo_type::ELEMENT_BYTE_SIZE), shape)
387
- end
388
- when :ruby
389
- arr =
390
- case type
391
- when :float, :uint8, :int8, :uint16, :int16, :int32, :int64, :double, :uint32, :uint64
392
- tensor_data.read_pointer.send("read_array_of_#{type}", output_tensor_size)
393
- when :bool
394
- tensor_data.read_pointer.read_array_of_uint8(output_tensor_size).map { |v| v == 1 }
395
- when :string
396
- create_strings_from_onnx_value(out_ptr, output_tensor_size, [])
397
- else
398
- unsupported_type("element", type)
399
- end
400
-
401
- Utils.reshape(arr, shape)
402
- else
403
- raise ArgumentError, "Invalid output type: #{output_type}"
404
- end
405
- when :sequence
406
- out = ::FFI::MemoryPointer.new(:size_t)
407
- check_status api[:GetValueCount].call(out_ptr, out)
408
-
409
- out.read(:size_t).times.map do |i|
410
- seq = ::FFI::MemoryPointer.new(:pointer)
411
- check_status api[:GetValue].call(out_ptr, i, @allocator.read_pointer, seq)
412
- create_from_onnx_value(seq.read_pointer, output_type)
413
- end
414
- when :map
415
- type_shape = ::FFI::MemoryPointer.new(:pointer)
416
- map_keys = ::FFI::MemoryPointer.new(:pointer)
417
- map_values = ::FFI::MemoryPointer.new(:pointer)
418
- elem_type = ::FFI::MemoryPointer.new(:int)
419
-
420
- check_status api[:GetValue].call(out_ptr, 0, @allocator.read_pointer, map_keys)
421
- check_status api[:GetValue].call(out_ptr, 1, @allocator.read_pointer, map_values)
422
- check_status api[:GetTensorTypeAndShape].call(map_keys.read_pointer, type_shape)
423
- check_status api[:GetTensorElementType].call(type_shape.read_pointer, elem_type)
424
- release :TensorTypeAndShapeInfo, type_shape
425
-
426
- # TODO support more types
427
- elem_type = FFI::TensorElementDataType[elem_type.read_int]
428
- case elem_type
429
- when :int64
430
- ret = {}
431
- keys = create_from_onnx_value(map_keys.read_pointer, output_type)
432
- values = create_from_onnx_value(map_values.read_pointer, output_type)
433
- keys.zip(values).each do |k, v|
434
- ret[k] = v
435
- end
436
- ret
437
- else
438
- unsupported_type("element", elem_type)
439
- end
440
- else
441
- unsupported_type("ONNX", type)
442
- end
443
- ensure
444
- api[:ReleaseValue].call(out_ptr) unless out_ptr.null?
445
- end
446
-
447
- def create_strings_from_onnx_value(out_ptr, output_tensor_size, result)
448
- len = ::FFI::MemoryPointer.new(:size_t)
449
- check_status api[:GetStringTensorDataLength].call(out_ptr, len)
450
-
451
- s_len = len.read(:size_t)
452
- s = ::FFI::MemoryPointer.new(:uchar, s_len)
453
- offsets = ::FFI::MemoryPointer.new(:size_t, output_tensor_size)
454
- check_status api[:GetStringTensorContent].call(out_ptr, s, s_len, offsets, output_tensor_size)
455
-
456
- offsets = output_tensor_size.times.map { |i| offsets[i].read(:size_t) }
457
- offsets << s_len
458
- output_tensor_size.times do |i|
459
- result[i] = s.get_bytes(offsets[i], offsets[i + 1] - offsets[i])
460
- end
461
- result
462
- end
463
-
464
288
  def read_pointer
465
289
  @session.read_pointer
466
290
  end
467
291
 
468
292
  def check_status(status)
469
- unless status.null?
470
- message = api[:GetErrorMessage].call(status).read_string
471
- api[:ReleaseStatus].call(status)
472
- raise Error, message
473
- end
474
- end
475
-
476
- def node_info(typeinfo)
477
- onnx_type = ::FFI::MemoryPointer.new(:int)
478
- check_status api[:GetOnnxTypeFromTypeInfo].call(typeinfo.read_pointer, onnx_type)
479
-
480
- type = FFI::OnnxType[onnx_type.read_int]
481
- case type
482
- when :tensor
483
- tensor_info = ::FFI::MemoryPointer.new(:pointer)
484
- # don't free tensor_info
485
- check_status api[:CastTypeInfoToTensorInfo].call(typeinfo.read_pointer, tensor_info)
486
-
487
- type, shape = tensor_type_and_shape(tensor_info)
488
- {
489
- type: "tensor(#{FFI::TensorElementDataType[type]})",
490
- shape: shape
491
- }
492
- when :sequence
493
- sequence_type_info = ::FFI::MemoryPointer.new(:pointer)
494
- check_status api[:CastTypeInfoToSequenceTypeInfo].call(typeinfo.read_pointer, sequence_type_info)
495
- nested_type_info = ::FFI::MemoryPointer.new(:pointer)
496
- check_status api[:GetSequenceElementType].call(sequence_type_info.read_pointer, nested_type_info)
497
- v = node_info(nested_type_info)[:type]
498
-
499
- {
500
- type: "seq(#{v})",
501
- shape: []
502
- }
503
- when :map
504
- map_type_info = ::FFI::MemoryPointer.new(:pointer)
505
- check_status api[:CastTypeInfoToMapTypeInfo].call(typeinfo.read_pointer, map_type_info)
506
-
507
- # key
508
- key_type = ::FFI::MemoryPointer.new(:int)
509
- check_status api[:GetMapKeyType].call(map_type_info.read_pointer, key_type)
510
- k = FFI::TensorElementDataType[key_type.read_int]
511
-
512
- # value
513
- value_type_info = ::FFI::MemoryPointer.new(:pointer)
514
- check_status api[:GetMapValueType].call(map_type_info.read_pointer, value_type_info)
515
- v = node_info(value_type_info)[:type]
516
-
517
- {
518
- type: "map(#{k},#{v})",
519
- shape: []
520
- }
521
- else
522
- unsupported_type("ONNX", type)
523
- end
524
- ensure
525
- release :TypeInfo, typeinfo
526
- end
527
-
528
- def tensor_type_and_shape(tensor_info)
529
- type = ::FFI::MemoryPointer.new(:int)
530
- check_status api[:GetTensorElementType].call(tensor_info.read_pointer, type)
531
-
532
- num_dims_ptr = ::FFI::MemoryPointer.new(:size_t)
533
- check_status api[:GetDimensionsCount].call(tensor_info.read_pointer, num_dims_ptr)
534
- num_dims = num_dims_ptr.read(:size_t)
535
-
536
- node_dims = ::FFI::MemoryPointer.new(:int64, num_dims)
537
- check_status api[:GetDimensions].call(tensor_info.read_pointer, node_dims, num_dims)
538
- dims = node_dims.read_array_of_int64(num_dims)
539
-
540
- symbolic_dims = ::FFI::MemoryPointer.new(:pointer, num_dims)
541
- check_status api[:GetSymbolicDimensions].call(tensor_info.read_pointer, symbolic_dims, num_dims)
542
- named_dims = num_dims.times.map { |i| symbolic_dims[i].read_pointer.read_string }
543
- dims = named_dims.zip(dims).map { |n, d| n.empty? ? d : n }
544
-
545
- [type.read_int, dims]
546
- end
547
-
548
- def unsupported_type(name, type)
549
- raise Error, "Unsupported #{name} type: #{type}"
293
+ Utils.check_status(status)
550
294
  end
551
295
 
552
296
  def tensor_types
553
297
  @tensor_types ||= [:float, :uint8, :int8, :uint16, :int16, :int32, :int64, :bool, :double, :uint32, :uint64].map { |v| ["tensor(#{v})", v] }.to_h
554
298
  end
555
299
 
556
- def numo_array?(obj)
557
- defined?(Numo::NArray) && obj.is_a?(Numo::NArray)
558
- end
559
-
560
- def numo_types
561
- @numo_types ||= {
562
- float: Numo::SFloat,
563
- uint8: Numo::UInt8,
564
- int8: Numo::Int8,
565
- uint16: Numo::UInt16,
566
- int16: Numo::Int16,
567
- int32: Numo::Int32,
568
- int64: Numo::Int64,
569
- bool: Numo::UInt8,
570
- double: Numo::DFloat,
571
- uint32: Numo::UInt32,
572
- uint64: Numo::UInt64
573
- }
574
- end
575
-
576
300
  def api
577
301
  self.class.api
578
302
  end
@@ -590,7 +314,7 @@ module OnnxRuntime
590
314
  end
591
315
 
592
316
  def self.release(type, pointer)
593
- api[:"Release#{type}"].call(pointer.read_pointer) if pointer && !pointer.null?
317
+ Utils.release(type, pointer)
594
318
  end
595
319
 
596
320
  def self.finalize(addr)